chore: modify the directory of release bucket and remove unused files (#2059)

* docs: rfc of metric engine

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* add drawback section

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* add sections about physical impl and meta routing

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* add chart about region id group

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix typos

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

.github/actions/build-dev-builder-image/action.yml

@@ -0,0 +1,70 @@
+name: Build and push dev-builder image
+description: Build and push dev-builder image to DockerHub and ACR
+inputs:
+  dockerhub-image-registry:
+    description: The dockerhub image registry to store the images
+    required: false
+    default: docker.io
+  dockerhub-image-registry-username:
+    description: The dockerhub username to login to the image registry
+    required: true
+  dockerhub-image-registry-password:
+    description: The dockerhub password to login to the image registry
+    required: true
+  dockerhub-image-namespace:
+    description: The dockerhub namespace of the image registry to store the images
+    required: false
+    default: greptime
+  acr-image-registry:
+    description: The ACR image registry to store the images
+    required: true
+  acr-image-registry-username:
+    description: The ACR username to login to the image registry
+    required: true
+  acr-image-registry-password:
+    description: The ACR password to login to the image registry
+    required: true
+  acr-image-namespace:
+    description: The ACR namespace of the image registry to store the images
+    required: false
+    default: greptime
+  version:
+    description: Version of the dev-builder
+    required: false
+    default: latest
+runs:
+  using: composite
+  steps:
+    - name: Login to Dockerhub
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ inputs.dockerhub-image-registry }}
+        username: ${{ inputs.dockerhub-image-registry-username }}
+        password: ${{ inputs.dockerhub-image-registry-password }}
+
+    - name: Build and push dev builder image to dockerhub
+      shell: bash
+      run:
+        make dev-builder \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}
+
+    - name: Login to ACR
+      uses: docker/login-action@v2
+      continue-on-error: true
+      with:
+        registry: ${{ inputs.acr-image-registry }}
+        username: ${{ inputs.acr-image-registry-username }}
+        password: ${{ inputs.acr-image-registry-password }}
+
+    - name: Build and push dev builder image to ACR
+      shell: bash
+      continue-on-error: true
+      run: # buildx will cache the images that already built, so it will not take long time to build the images again.
+        make dev-builder \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}

.github/actions/build-greptime-binary/action.yml

@@ -0,0 +1,51 @@
+name: Build greptime binary
+description: Build and upload the single linux artifact
+inputs:
+  base-image:
+    description: Base image to build greptime
+    required: true
+  features:
+    description: Cargo features to build
+    required: true
+  cargo-profile:
+    description: Cargo profile to build
+    required: true
+  artifacts-dir:
+    description: Directory to store artifacts
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Build greptime binary
+      shell: bash
+      run: |
+        make build-greptime-by-buildx \
+          CARGO_PROFILE=${{ inputs.cargo-profile }} \
+          FEATURES=${{ inputs.features }} \
+          BASE_IMAGE=${{ inputs.base-image }}
+
+    - name: Upload artifacts
+      uses: ./.github/actions/upload-artifacts
+      with:
+        artifacts-dir: ${{ inputs.artifacts-dir }}
+        target-file: ./greptime
+        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}

.github/actions/build-greptime-images/action.yml

@@ -0,0 +1,88 @@
+name: Build greptime images
+description: Build and push greptime images
+inputs:
+  image-registry:
+    description: The image registry to store the images
+    required: true
+  image-registry-username:
+    description: The username to login to the image registry
+    required: true
+  image-registry-password:
+    description: The password to login to the image registry
+    required: true
+  amd64-artifact-name:
+    description: The name of the amd64 artifact for building images
+    required: true
+  arm64-artifact-name:
+    description: The name of the arm64 artifact for building images
+    required: false
+    default: ""
+  image-namespace:
+    description: The namespace of the image registry to store the images
+    required: true
+  image-name:
+    description: The name of the image to build
+    required: true
+  image-tag:
+    description: The tag of the image to build
+    required: true
+  docker-file:
+    description: The path to the Dockerfile to build
+    required: true
+  platforms:
+    description: The supported platforms to build the image
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Login to image registry
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ inputs.image-registry }}
+        username: ${{ inputs.image-registry-username }}
+        password: ${{ inputs.image-registry-password }}
+
+    - name: Set up qemu for multi-platform builds
+      uses: docker/setup-qemu-action@v2
+
+    - name: Set up buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Download amd64 artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: ${{ inputs.amd64-artifact-name }}
+
+    - name: Unzip the amd64 artifacts
+      shell: bash
+      run: |
+        tar xvf ${{ inputs.amd64-artifact-name }}.tar.gz && \
+        rm ${{ inputs.amd64-artifact-name }}.tar.gz && \
+        rm -rf amd64 && \
+        mv ${{ inputs.amd64-artifact-name }} amd64
+
+    - name: Download arm64 artifacts
+      uses: actions/download-artifact@v3
+      if: ${{ inputs.arm64-artifact-name }}
+      with:
+        name: ${{ inputs.arm64-artifact-name }}
+
+    - name: Unzip the arm64 artifacts
+      shell: bash
+      if: ${{ inputs.arm64-artifact-name }}
+      run: |
+        tar xvf ${{ inputs.arm64-artifact-name }}.tar.gz && \
+        rm ${{ inputs.arm64-artifact-name }}.tar.gz && \
+        rm -rf arm64 && \
+        mv ${{ inputs.arm64-artifact-name }} arm64
+
+    - name: Build and push images for amd64 and arm64
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: ${{ inputs.docker-file }}
+        push: true
+        platforms: ${{ inputs.platforms }}
+        tags: |
+          ${{ inputs.image-registry }}/${{ inputs.image-namespace }}/${{ inputs.image-name }}:latest
+          ${{ inputs.image-registry }}/${{ inputs.image-namespace }}/${{ inputs.image-name }}:${{ inputs.image-tag }}

.github/actions/build-images/action.yml

@@ -0,0 +1,47 @@
+name: Group for building greptimedb images
+description: Group for building greptimedb images
+inputs:
+  image-registry:
+    description: The image registry to store the images
+    required: true
+  image-namespace:
+    description: The namespace of the image registry to store the images
+    required: true
+  image-registry-username:
+    description: The username to login to the image registry
+    required: true
+  image-registry-password:
+    description: The password to login to the image registry
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Build and push standard images to dockerhub
+      uses: ./.github/actions/build-greptime-images
+      with:
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}
+        image-registry-username: ${{ inputs.image-registry-username }}
+        image-registry-password: ${{ inputs.image-registry-password }}
+        image-name: greptimedb
+        image-tag: ${{ inputs.version }}
+        docker-file: docker/ci/Dockerfile
+        amd64-artifact-name: greptime-linux-amd64-pyo3-${{ inputs.version }}
+        arm64-artifact-name: greptime-linux-arm64-pyo3-${{ inputs.version }}
+        platforms: linux/amd64,linux/arm64
+
+    - name: Build and push centos images to dockerhub
+      uses: ./.github/actions/build-greptime-images
+      with:
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}
+        image-registry-username: ${{ inputs.image-registry-username }}
+        image-registry-password: ${{ inputs.image-registry-password }}
+        image-name: greptimedb-centos
+        image-tag: ${{ inputs.version }}
+        docker-file: docker/ci/Dockerfile-centos
+        amd64-artifact-name: greptime-linux-amd64-centos-${{ inputs.version }}
+        platforms: linux/amd64

.github/actions/build-linux-artifacts/action.yml

@@ -0,0 +1,84 @@
+name: Build linux artifacts
+description: Build linux artifacts
+inputs:
+  arch:
+    description: Architecture to build
+    required: true
+  cargo-profile:
+    description: Cargo profile to build
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  disable-run-tests:
+    description: Disable running integration tests
+    required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Run integration test
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      shell: bash
+      # NOTE: If the BUILD_JOBS > 4, it's always OOM in EC2 instance.
+      run: |
+        make run-it-in-container BUILD_JOBS=4
+
+    - name: Upload sqlness logs
+      if: ${{ failure() && inputs.disable-run-tests == 'false' }} # Only upload logs when the integration tests failed.
+      uses: actions/upload-artifact@v3
+      with:
+        name: sqlness-logs
+        path: /tmp/greptime-*.log
+        retention-days: 3
+
+    - name: Build standard greptime
+      uses: ./.github/actions/build-greptime-binary
+      with:
+        base-image: ubuntu
+        features: pyo3_backend,servers/dashboard
+        cargo-profile: ${{ inputs.cargo-profile }}
+        artifacts-dir: greptime-linux-${{ inputs.arch }}-pyo3-${{ inputs.version }}
+        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+
+    - name: Build greptime without pyo3
+      uses: ./.github/actions/build-greptime-binary
+      with:
+        base-image: ubuntu
+        features: servers/dashboard
+        cargo-profile: ${{ inputs.cargo-profile }}
+        artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
+        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+
+    - name: Build greptime on centos base image
+      uses: ./.github/actions/build-greptime-binary
+      if: ${{ inputs.arch == 'amd64' }} # Only build centos7 base image for amd64.
+      with:
+        base-image: centos
+        features: servers/dashboard
+        cargo-profile: ${{ inputs.cargo-profile }}
+        artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
+        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}

.github/actions/build-macos-artifacts/action.yml

@@ -0,0 +1,105 @@
+name: Build macos artifacts
+description: Build macos artifacts
+inputs:
+  arch:
+    description: Architecture to build
+    required: true
+  rust-toolchain:
+    description: Rust toolchain to use
+    required: true
+  cargo-profile:
+    description: Cargo profile to build
+    required: true
+  features:
+    description: Cargo features to build
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  disable-run-tests:
+    description: Disable running integration tests
+    required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  artifacts-dir:
+    description: Directory to store artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Cache cargo assets
+      id: cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+          target/
+        key: ${{ inputs.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Install protoc
+      shell: bash
+      run: |
+        brew install protobuf
+
+    - name: Install rust toolchain
+      uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ inputs.rust-toolchain }}
+        targets: ${{ inputs.arch }}
+
+    - name: Start etcd # For integration tests.
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      shell: bash
+      run: |
+        brew install etcd && \
+        brew services start etcd
+
+    - name: Install latest nextest release # For integration tests.
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      uses: taiki-e/install-action@nextest
+
+    - name: Run integration tests
+      if: ${{ inputs.disable_run_tests == 'false' }}
+      shell: bash
+      run: |
+        make test sqlness-test
+
+    - name: Upload sqlness logs
+      if: ${{ failure() }} # Only upload logs when the integration tests failed.
+      uses: actions/upload-artifact@v3
+      with:
+        name: sqlness-logs
+        path: /tmp/greptime-*.log
+        retention-days: 3
+
+    - name: Build greptime binary
+      shell: bash
+      run: |
+        make build \
+        CARGO_PROFILE=${{ inputs.cargo-profile }} \
+        FEATURES=${{ inputs.features }} \
+        TARGET=${{ inputs.arch }}
+
+    - name: Upload artifacts
+      uses: ./.github/actions/upload-artifacts
+      with:
+        artifacts-dir: ${{ inputs.artifacts-dir }}
+        target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
+        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}

.github/actions/release-artifacts/action.yml

@@ -0,0 +1,50 @@
+name: Release artifacts
+description: Release artifacts
+inputs:
+  version:
+    description: Version to release
+    required: true
+runs:
+  using: composite
+  steps:
+    # Download artifacts from previous jobs, the artifacts will be downloaded to:
+    # ${WORKING_DIR}
+    #   |- greptime-darwin-amd64-pyo3-v0.5.0/greptime-darwin-amd64-pyo3-v0.5.0.tar.gz
+    #   |- greptime-darwin-amd64-pyo3-v0.5.0.sha256sum/greptime-darwin-amd64-pyo3-v0.5.0.sha256sum
+    #   |- greptime-darwin-amd64-v0.5.0/greptime-darwin-amd64-v0.5.0.tar.gz
+    #   |- greptime-darwin-amd64-v0.5.0.sha256sum/greptime-darwin-amd64-v0.5.0.sha256sum
+    #   ...
+    - name: Download artifacts
+      uses: actions/download-artifact@v3
+
+    - name: Create git tag for release
+      if: ${{ github.event_name != 'push' }} # Meaning this is a scheduled or manual workflow.
+      shell: bash
+      run: |
+        git tag ${{ inputs.version }}
+
+    # Only publish release when the release tag is like v1.0.0, v1.0.1, v1.0.2, etc.
+    - name: Set release arguments
+      shell: bash
+      run: |
+        if [[ "${{ inputs.version }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          echo "prerelease=false" >> $GITHUB_ENV
+          echo "makeLatest=true" >> $GITHUB_ENV
+          echo "generateReleaseNotes=false" >> $GITHUB_ENV
+        else
+          echo "prerelease=true" >> $GITHUB_ENV
+          echo "makeLatest=false" >> $GITHUB_ENV
+          echo "generateReleaseNotes=true" >> $GITHUB_ENV
+        fi
+
+    - name: Publish release
+      uses: ncipollo/release-action@v1
+      with:
+        name: "Release ${{ inputs.version }}"
+        prerelease: ${{ env.prerelease }}
+        makeLatest: ${{ env.makeLatest }}
+        tag: ${{ inputs.version }}
+        generateReleaseNotes: ${{ env.generateReleaseNotes }}
+        allowUpdates: true
+        artifacts: |
+          **/greptime-*/*

.github/actions/start-runner/action.yml

@@ -0,0 +1,67 @@
+name: Start EC2 runner
+description: Start EC2 runner
+inputs:
+  runner:
+    description: The linux runner name
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+  github-token:
+    description: The GitHub token to clone private repository
+    required: false
+    default: ""
+  image-id:
+    description: The EC2 image id
+    required: true
+  security-group-id:
+    description: The EC2 security group id
+    required: true
+  subnet-id:
+    description: The EC2 subnet id
+    required: true
+outputs:
+  label:
+    description: "label"
+    value: ${{ steps.start-linux-arm64-ec2-runner.outputs.label || inputs.runner }}
+  ec2-instance-id:
+    description: "ec2-instance-id"
+    value: ${{ steps.start-linux-arm64-ec2-runner.outputs.ec2-instance-id }}
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials
+      if: startsWith(inputs.runner, 'ec2')
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+
+    # The EC2 runner will use the following format:
+    # <vm-type>-<instance-type>-<arch>
+    # like 'ec2-c6a.4xlarge-amd64'.
+    - name: Get EC2 instance type
+      if: startsWith(inputs.runner, 'ec2')
+      id: get-ec2-instance-type
+      shell: bash
+      run: |
+        echo "instance-type=$(echo ${{ inputs.runner }} | cut -d'-' -f2)" >> $GITHUB_OUTPUT
+
+    - name: Start EC2 runner
+      if: startsWith(inputs.runner, 'ec2')
+      uses: machulav/ec2-github-runner@v2
+      id: start-linux-arm64-ec2-runner
+      with:
+        mode: start
+        ec2-image-id: ${{ inputs.image-id }}
+        ec2-instance-type: ${{ steps.get-ec2-instance-type.outputs.instance-type }}
+        subnet-id: ${{ inputs.subnet-id }}
+        security-group-id: ${{ inputs.security-group-id }}
+        github-token: ${{ inputs.github-token }}

.github/actions/stop-runner/action.yml

@@ -0,0 +1,41 @@
+name: Stop EC2 runner
+description: Stop EC2 runner
+inputs:
+  label:
+    description: The linux runner name
+    required: true
+  ec2-instance-id:
+    description: The EC2 instance id
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+  github-token:
+    description: The GitHub token to clone private repository
+    required: false
+    default: ""
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials
+      if: ${{ inputs.label && inputs.ec2-instance-id }}
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+
+    - name: Stop EC2 runner
+      if: ${{ inputs.label && inputs.ec2-instance-id }}
+      uses: machulav/ec2-github-runner@v2
+      with:
+        mode: stop
+        label: ${{ inputs.label }}
+        ec2-instance-id: ${{ inputs.ec2-instance-id }}
+        github-token: ${{ inputs.github-token }}

.github/actions/upload-artifacts/action.yml

@@ -0,0 +1,82 @@
+name: Upload artifacts
+description: Upload artifacts
+inputs:
+  artifacts-dir:
+    description: Directory to store artifacts
+    required: true
+  target-file:
+    description: The path of the target artifact
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Create artifacts directory
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.artifacts-dir }} && \
+        mv ${{ inputs.target-file }} ${{ inputs.artifacts-dir }}
+
+    # The compressed artifacts will use the following layout:
+    # greptime-linux-amd64-pyo3-v0.3.0sha256sum
+    # greptime-linux-amd64-pyo3-v0.3.0.tar.gz
+    #   greptime-linux-amd64-pyo3-v0.3.0
+    #   └── greptime
+    - name: Compress artifacts and calculate checksum
+      shell: bash
+      run: |
+        tar -zcvf ${{ inputs.artifacts-dir }}.tar.gz ${{ inputs.artifacts-dir }} && \
+        echo $(shasum -a 256 ${{ inputs.artifacts-dir }}.tar.gz | cut -f1 -d' ') > ${{ inputs.artifacts-dir }}.sha256sum
+
+    # Note: The artifacts will be double zip compressed(related issue: https://github.com/actions/upload-artifact/issues/39).
+    # However, when we use 'actions/download-artifact@v3' to download the artifacts, it will be automatically unzipped.
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.artifacts-dir }}
+        path: ${{ inputs.artifacts-dir }}.tar.gz
+
+    - name: Upload checksum
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.artifacts-dir }}.sha256sum
+        path: ${{ inputs.artifacts-dir }}.sha256sum
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+
+    - name: Upload artifacts to S3
+      shell: bash
+      # The bucket layout will be:
+      # releases/greptimedb
+      # ├── v0.1.0
+      # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
+      # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
+      # └── v0.2.0
+      #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
+      #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
+      run: |
+        aws s3 cp \
+          ${{ inputs.artifacts-dir }}.tar.gz \
+          s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.tar.gz && \
+        aws s3 cp \
+          ${{ inputs.artifacts-dir }}.sha256sum \
+          s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.sha256sum

.github/scripts/create-version.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -e
+
+# - If it's a tag push release, the version is the tag name(${{ github.ref_name }});
+# - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like v0.2.0-nigthly-20230313;
+# - If it's a manual release, the version is '${{ env.NEXT_RELEASE_VERSION }}-$(git rev-parse --short HEAD)-YYYYMMDDSS', like v0.2.0-e5b243c-2023071245;
+# create_version ${GIHUB_EVENT_NAME} ${NEXT_RELEASE_VERSION} ${NIGHTLY_RELEASE_PREFIX}
+function create_version() {
+  # Read from envrionment variables.
+  if [ -z "$GITHUB_EVENT_NAME" ]; then
+      echo "GITHUB_EVENT_NAME is empty"
+      exit 1
+  fi
+
+  if [ -z "$NEXT_RELEASE_VERSION" ]; then
+      echo "NEXT_RELEASE_VERSION is empty"
+      exit 1
+  fi
+
+  if [ -z "$NIGHTLY_RELEASE_PREFIX" ]; then
+      echo "NIGHTLY_RELEASE_PREFIX is empty"
+      exit 1
+  fi
+
+  # Note: Only output 'version=xxx' to stdout when everything is ok, so that it can be used in GitHub Actions Outputs.
+  if [ "$GITHUB_EVENT_NAME" = push ]; then
+    if [ -z "$GITHUB_REF_NAME" ]; then
+      echo "GITHUB_REF_NAME is empty in push event"
+      exit 1
+    fi
+    echo "$GITHUB_REF_NAME"
+  elif [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then
+    echo "$NEXT_RELEASE_VERSION-$(git rev-parse --short HEAD)-$(date "+%Y%m%d%S")"
+  elif [ "$GITHUB_EVENT_NAME" = schedule ]; then
+    echo "$NEXT_RELEASE_VERSION-$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")"
+  else
+    echo "Unsupported GITHUB_EVENT_NAME: $GITHUB_EVENT_NAME"
+    exit 1
+  fi
+}
+
+# You can run as: GITHUB_EVENT_NAME=push NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigthly GITHUB_REF_NAME=v0.3.0 ./create-version.sh
+create_version

.github/workflows/develop.yml

@@ -1,4 +1,5 @@
 on:
+  merge_group:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
     paths-ignore:
@@ -116,7 +117,6 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest-8-cores
     timeout-minutes: 60
-    needs: [clippy]
     steps:
       - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
@@ -190,7 +190,6 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest-8-cores
     timeout-minutes: 60
-    needs: [clippy]
     steps:
       - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1

.github/workflows/docs.yml

@@ -1,4 +1,5 @@
 on:
+  merge_group:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
     paths:

.github/workflows/release.yml

@@ -1,3 +1,8 @@
+name: Release
+
+# There are two kinds of formal release:
+# 1. The tag('v*.*.*') push release: the release workflow will be triggered by the tag push event.
+# 2. The scheduled release(the version will be '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD'): the release workflow will be triggered by the schedule event.
 on:
   push:
     tags:
@@ -5,506 +10,386 @@ on:
   schedule:
     # At 00:00 on Monday.
     - cron: '0 0 * * 1'
-  # Mannually trigger only builds binaries.
-  workflow_dispatch:
+  workflow_dispatch: # Allows you to run this workflow manually.
+    # Notes: The GitHub Actions ONLY support 10 inputs, and it's already used up.
     inputs:
-      dry_run:
-        description: 'Skip docker push and release steps'
+      linux_amd64_runner:
+        type: choice
+        description: The runner uses to build linux-amd64 artifacts
+        default: ec2-c6i.4xlarge-amd64
+        options:
+          - ubuntu-latest
+          - ubuntu-latest-8-cores
+          - ubuntu-latest-16-cores
+          - ubuntu-latest-32-cores
+          - ubuntu-latest-64-cores
+          - ec2-c6i.xlarge-amd64 # 4C8G
+          - ec2-c6i.2xlarge-amd64 # 8C16G
+          - ec2-c6i.4xlarge-amd64 # 16C32G
+          - ec2-c6i.8xlarge-amd64 # 32C64G
+          - ec2-c6i.16xlarge-amd64 # 64C128G
+      linux_arm64_runner:
+        type: choice
+        description: The runner uses to build linux-arm64 artifacts
+        default: ec2-c6g.4xlarge-arm64
+        options:
+          - ec2-c6g.xlarge-arm64 # 4C8G
+          - ec2-c6g.2xlarge-arm64 # 8C16G
+          - ec2-c6g.4xlarge-arm64 # 16C32G
+          - ec2-c6g.8xlarge-arm64 # 32C64G
+          - ec2-c6g.16xlarge-arm64 # 64C128G
+      macos_runner:
+        type: choice
+        description: The runner uses to build macOS artifacts
+        default: macos-latest
+        options:
+          - macos-latest
+      skip_test:
+        description: Do not run integration tests during the build
         type: boolean
         default: true
-      skip_test:
-        description: 'Do not run tests during build'
+      build_linux_amd64_artifacts:
         type: boolean
+        description: Build linux-amd64 artifacts
+        required: false
+        default: false
+      build_linux_arm64_artifacts:
+        type: boolean
+        description: Build linux-arm64 artifacts
+        required: false
+        default: false
+      build_macos_artifacts:
+        type: boolean
+        description: Build macos artifacts
+        required: false
+        default: false
+      release_artifacts:
+        type: boolean
+        description: Create GitHub release and upload artifacts
+        required: false
+        default: false
+      release_images:
+        type: boolean
+        description: Build and push images to DockerHub and ACR
+        required: false
+        default: false
+      release_dev_builder_image:
+        type: boolean
+        description: Release dev-builder image
+        required: false
         default: false
 
-name: Release
-
+# Use env variables to control all the release process.
 env:
+  # The arguments of building greptime.
   RUST_TOOLCHAIN: nightly-2023-05-03
-
-  SCHEDULED_BUILD_VERSION_PREFIX: v0.4.0
-
-  SCHEDULED_PERIOD: nightly
-
   CARGO_PROFILE: nightly
 
   # Controls whether to run tests, include unit-test, integration-test and sqlness.
-  DISABLE_RUN_TESTS: ${{ inputs.skip_test || false }}
+  DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}
+
+  # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
+  NIGHTLY_RELEASE_PREFIX: nightly
+  # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
+  NEXT_RELEASE_VERSION: v0.4.0
 
 jobs:
-  build-macos:
-    name: Build macOS binary
+  allocate-runners:
+    name: Allocate runners
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-latest
+    outputs:
+      linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
+      macos-runner: ${{ inputs.macos_runner || vars.DEFAULT_MACOS_RUNNER }}
+
+      # The following EC2 resource id will be used for resource releasing.
+      linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-amd64-ec2-runner-instance-id: ${{ steps.start-linux-amd64-runner.outputs.ec2-instance-id }}
+      linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }}
+      linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }}
+
+      # The 'version' use as the global tag name of the release workflow.
+      version: ${{ steps.create-version.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      # The create-version will create a global variable named 'version' in the global workflows.
+      # - If it's a tag push release, the version is the tag name(${{ github.ref_name }});
+      # - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like v0.2.0-nigthly-20230313;
+      # - If it's a manual release, the version is '${{ env.NEXT_RELEASE_VERSION }}-<short-git-sha>-YYYYMMDDSS', like v0.2.0-e5b243c-2023071245;
+      - name: Create version
+        id: create-version
+        run: |
+          echo "version=$(./.github/scripts/create-version.sh)" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_REF_NAME: ${{ github.ref_name }}
+          NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }}
+          NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}
+
+      - name: Allocate linux-amd64 runner
+        if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-amd64-runner
+        with:
+          runner: ${{ inputs.linux_amd64_runner || vars.DEFAULT_AMD64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_AMD64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+      - name: Allocate linux-arm64 runner
+        if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-arm64-runner
+        with:
+          runner: ${{ inputs.linux_arm64_runner || vars.DEFAULT_ARM64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+  build-linux-amd64-artifacts:
+    name: Build linux-amd64 artifacts
+    if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-amd64-runner }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: amd64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+
+  build-linux-arm64-artifacts:
+    name: Build linux-arm64 artifacts
+    if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-arm64-runner }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: arm64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+
+  build-macos-artifacts:
+    name: Build macOS artifacts
     strategy:
       fail-fast: false
       matrix:
-        # The file format is greptime-<os>-<arch>
         include:
-          - arch: aarch64-apple-darwin
-            os: macos-latest
-            file: greptime-darwin-arm64
-            continue-on-error: false
-            opts: "-F servers/dashboard"
-          - arch: x86_64-apple-darwin
-            os: macos-latest
-            file: greptime-darwin-amd64
-            continue-on-error: false
-            opts: "-F servers/dashboard"
-          - arch: aarch64-apple-darwin
-            os: macos-latest
-            file: greptime-darwin-arm64-pyo3
-            continue-on-error: false
-            opts: "-F pyo3_backend,servers/dashboard"
-          - arch: x86_64-apple-darwin
-            os: macos-latest
-            file: greptime-darwin-amd64-pyo3
-            continue-on-error: false
-            opts: "-F pyo3_backend,servers/dashboard"
+          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
+            arch: aarch64-apple-darwin
+            features: servers/dashboard
+            artifacts-dir-prefix: greptime-darwin-arm64
+          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
+            arch: aarch64-apple-darwin
+            features: pyo3_backend,servers/dashboard
+            artifacts-dir-prefix: greptime-darwin-arm64-pyo3
+          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
+            features: servers/dashboard
+            arch: x86_64-apple-darwin
+            artifacts-dir-prefix: greptime-darwin-amd64
+          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
+            features: pyo3_backend,servers/dashboard
+            arch: x86_64-apple-darwin
+            artifacts-dir-prefix: greptime-darwin-amd64-pyo3
     runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.continue-on-error }}
-    if: github.repository == 'GreptimeTeam/greptimedb'
+    needs: [
+      allocate-runners,
+    ]
+    if: ${{ inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Cache cargo assets
-        id: cache
-        uses: actions/cache@v3
+      - uses: actions/checkout@v3
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ matrix.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }}
+          fetch-depth: 0
 
-      - name: Install Protoc for macos
-        if: contains(matrix.arch, 'darwin')
-        run: |
-          brew install protobuf
-
-      - name: Install etcd for macos
-        if: contains(matrix.arch, 'darwin')
-        run: |
-          brew install etcd
-          brew services start etcd
-
-      - name: Install rust toolchain
-        uses: dtolnay/rust-toolchain@master
+      - uses: ./.github/actions/build-macos-artifacts
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          targets: ${{ matrix.arch }}
-      - name: Install latest nextest release
-        uses: taiki-e/install-action@nextest
-      - name: Output package versions
-        run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version
+          arch: ${{ matrix.arch }}
+          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          features: ${{ matrix.features }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          artifacts-dir: ${{ matrix.artifacts-dir-prefix }}-${{ needs.allocate-runners.outputs.version }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
 
-      - name: Run tests
-        if: env.DISABLE_RUN_TESTS == 'false'
-        run: make test sqlness-test
-
-      - name: Run cargo build
-        if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false
-        run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
-
-      - name: Calculate checksum and rename binary
-        shell: bash
-        run: |
-          cd target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}
-          chmod +x greptime
-          tar -zcvf ${{ matrix.file }}.tgz greptime
-          echo $(shasum -a 256 ${{ matrix.file }}.tgz | cut -f1 -d' ') > ${{ matrix.file }}.sha256sum
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.file }}
-          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.tgz
-
-      - name: Upload checksum of artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.file }}.sha256sum
-          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.sha256sum
-
-      - name: Configure tag
-        shell: bash
-        if: github.event_name == 'push'
-        run: |
-          VERSION=${{ github.ref_name }}
-          echo "TAG=${VERSION:1}" >> $GITHUB_ENV
-
-      - name: Upload to S3
-        run: |
-          aws s3 cp target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }} s3://${{ secrets.GREPTIMEDB_RELEASE_BUCKET_NAME }}/releases/${TAG} --recursive --exclude "*" --include "*.tgz"
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: ${{ secrets.AWS_CN_REGION }}
-
-  build-linux:
-    name: Build linux binary
-    strategy:
-      fail-fast: false
-      matrix:
-        # The file format is greptime-<os>-<arch>
-        include:
-          - arch: x86_64-unknown-linux-gnu
-            os: ubuntu-2004-16-cores
-            file: greptime-linux-amd64
-            continue-on-error: false
-            opts: "-F servers/dashboard"
-          - arch: aarch64-unknown-linux-gnu
-            os: ubuntu-2004-16-cores
-            file: greptime-linux-arm64
-            continue-on-error: false
-            opts: "-F servers/dashboard"
-          - arch: x86_64-unknown-linux-gnu
-            os: ubuntu-2004-16-cores
-            file: greptime-linux-amd64-pyo3
-            continue-on-error: false
-            opts: "-F pyo3_backend,servers/dashboard"
-          - arch: aarch64-unknown-linux-gnu
-            os: ubuntu-2004-16-cores
-            file: greptime-linux-arm64-pyo3
-            continue-on-error: false
-            opts: "-F pyo3_backend,servers/dashboard"
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.continue-on-error }}
-    if: github.repository == 'GreptimeTeam/greptimedb'
+  release-images-to-dockerhub:
+    name: Build and push images to DockerHub
+    if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-2004-16-cores
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Cache cargo assets
-        id: cache
-        uses: actions/cache@v3
+      - uses: actions/checkout@v3
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ matrix.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }}
+          fetch-depth: 0
 
-      - name: Install Protoc for linux
-        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
-        run: | # Make sure the protoc is >= 3.15
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip
-          unzip protoc-21.9-linux-x86_64.zip -d protoc
-          sudo cp protoc/bin/protoc /usr/local/bin/
-          sudo cp -r protoc/include/google /usr/local/include/
-
-      - name: Install etcd for linux
-        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
-        run: |
-          ETCD_VER=v3.5.7
-          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
-          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-          mkdir -p /tmp/etcd-download
-          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1
-          rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-
-          sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/
-          nohup etcd >/tmp/etcd.log 2>&1 &
-
-      - name: Install dependencies for linux
-        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
-        run: |
-          sudo apt-get -y update
-          sudo apt-get -y install libssl-dev pkg-config g++-aarch64-linux-gnu gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu wget
-
-      # FIXME(zyy17): Should we specify the version of python when building binary for darwin?
-      - name: Compile Python 3.10.10 from source for linux
-        if: contains(matrix.arch, 'linux') && contains(matrix.opts, 'pyo3_backend')
-        run: |
-          sudo chmod +x ./docker/aarch64/compile-python.sh
-          sudo ./docker/aarch64/compile-python.sh ${{ matrix.arch }}
-
-      - name: Install rust toolchain
-        uses: dtolnay/rust-toolchain@master
+      - name: Build and push images to dockerhub
+        uses: ./.github/actions/build-images
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          targets: ${{ matrix.arch }}
-      - name: Install latest nextest release
-        uses: taiki-e/install-action@nextest
-      - name: Output package versions
-        run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version
+          image-registry: docker.io
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
+          version: ${{ needs.allocate-runners.outputs.version }}
 
-      - name: Run tests
-        if: env.DISABLE_RUN_TESTS == 'false'
-        run: make test sqlness-test
-
-      - name: Run cargo build
-        if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false
-        run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
-
-      - name: Run cargo build with pyo3 for aarch64-linux
-        if: contains(matrix.arch, 'aarch64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend')
-        run: |
-          # TODO(zyy17): We should make PYO3_CROSS_LIB_DIR configurable.
-          export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64
-          export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
-          export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
-          export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
-
-          export PYO3_CROSS_LIB_DIR=${PWD}/python-3.10.10/aarch64
-          echo "PYO3_CROSS_LIB_DIR: $PYO3_CROSS_LIB_DIR"
-          alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3
-          alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip
-
-          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
-
-      - name: Run cargo build with pyo3 for amd64-linux
-        if: contains(matrix.arch, 'x86_64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend')
-        run: |
-          export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64
-          export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
-          export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
-          export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
-
-          echo "implementation=CPython" >> pyo3.config
-          echo "version=3.10" >> pyo3.config
-          echo "implementation=CPython" >> pyo3.config
-          echo "shared=true" >> pyo3.config
-          echo "abi3=true" >> pyo3.config
-          echo "lib_name=python3.10" >> pyo3.config
-          echo "lib_dir=$PYTHON_INSTALL_PATH_AMD64/lib" >> pyo3.config
-          echo "executable=$PYTHON_INSTALL_PATH_AMD64/bin/python3" >> pyo3.config
-          echo "pointer_width=64" >> pyo3.config
-          echo "build_flags=" >> pyo3.config
-          echo "suppress_build_script_link_lines=false" >> pyo3.config
-
-          cat pyo3.config
-          export PYO3_CONFIG_FILE=${PWD}/pyo3.config
-          alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3
-          alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip
-
-          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
-
-      - name: Calculate checksum and rename binary
-        shell: bash
-        run: |
-          cd target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}
-          chmod +x greptime
-          tar -zcvf ${{ matrix.file }}.tgz greptime
-          echo $(shasum -a 256 ${{ matrix.file }}.tgz | cut -f1 -d' ') > ${{ matrix.file }}.sha256sum
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.file }}
-          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.tgz
-
-      - name: Upload checksum of artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.file }}.sha256sum
-          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.sha256sum
-
-      - name: Configure tag
-        shell: bash
-        if: github.event_name == 'push'
-        run: |
-          VERSION=${{ github.ref_name }}
-          echo "TAG=${VERSION:1}" >> $GITHUB_ENV
-
-      - name: Upload to S3
-        run: |
-          aws s3 cp target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }} s3://${{ secrets.GREPTIMEDB_RELEASE_BUCKET_NAME }}/releases/${TAG} --recursive --exclude "*" --include "*.tgz"
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: ${{ secrets.AWS_CN_REGION }}
-
-  docker:
-    name: Build docker image
-    needs: [build-linux, build-macos]
-    runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false)
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Login to Dockerhub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
-        shell: bash
-        if: github.event_name != 'push'
-        run: |
-          buildTime=`date "+%Y%m%d"`
-          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
-          echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV
-
-      - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
-        shell: bash
-        if: github.event_name == 'push'
-        run: |
-          VERSION=${{ github.ref_name }}
-          echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Download amd64 binary
-        uses: actions/download-artifact@v3
-        with:
-          name: greptime-linux-amd64-pyo3
-          path: amd64
-
-      - name: Unzip the amd64 artifacts
-        run: |
-          tar xvf amd64/greptime-linux-amd64-pyo3.tgz -C amd64/ && rm amd64/greptime-linux-amd64-pyo3.tgz
-          cp -r amd64 docker/ci
-
-      - name: Download arm64 binary
-        id: download-arm64
-        uses: actions/download-artifact@v3
-        with:
-          name: greptime-linux-arm64-pyo3
-          path: arm64
-
-      - name: Unzip the arm64 artifacts
-        id: unzip-arm64
-        if: success() || steps.download-arm64.conclusion == 'success'
-        run: |
-          tar xvf arm64/greptime-linux-arm64-pyo3.tgz -C arm64/ && rm arm64/greptime-linux-arm64-pyo3.tgz
-          cp -r arm64 docker/ci
-
-      - name: Build and push all
-        uses: docker/build-push-action@v3
-        if: success() || steps.unzip-arm64.conclusion == 'success' # Build and push all platform if unzip-arm64 succeeds
-        with:
-          context: ./docker/ci/
-          file: ./docker/ci/Dockerfile
-          push: true
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            greptime/greptimedb:latest
-            greptime/greptimedb:${{ env.IMAGE_TAG }}
-
-      - name: Build and push amd64 only
-        uses: docker/build-push-action@v3
-        if: success() || steps.download-arm64.conclusion == 'failure' # Only build and push amd64 platform if download-arm64 fails
-        with:
-          context: ./docker/ci/
-          file: ./docker/ci/Dockerfile
-          push: true
-          platforms: linux/amd64
-          tags: |
-            greptime/greptimedb:latest
-            greptime/greptimedb:${{ env.IMAGE_TAG }}
-
-  release:
-    name: Release artifacts
-    # Release artifacts only when all the artifacts are built successfully.
-    needs: [build-linux, build-macos, docker]
-    runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false)
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Download artifacts
-        uses: actions/download-artifact@v3
-
-      - name: Configure scheduled build version # the version would be ${SCHEDULED_BUILD_VERSION_PREFIX}-${SCHEDULED_PERIOD}-YYYYMMDD, like v0.2.0-nigthly-20230313.
-        shell: bash
-        if: github.event_name != 'push'
-        run: |
-          buildTime=`date "+%Y%m%d"`
-          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-${{ env.SCHEDULED_PERIOD }}-$buildTime
-          echo "SCHEDULED_BUILD_VERSION=${SCHEDULED_BUILD_VERSION}" >> $GITHUB_ENV
-
-      # Only publish release when the release tag is like v1.0.0, v1.0.1, v1.0.2, etc.
-      - name: Set whether it is the latest release
-        run: |
-          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo "prerelease=false" >> $GITHUB_ENV
-            echo "makeLatest=true" >> $GITHUB_ENV
-          else
-            echo "prerelease=true" >> $GITHUB_ENV
-            echo "makeLatest=false" >> $GITHUB_ENV
-          fi
-
-      - name: Create scheduled build git tag
-        if: github.event_name != 'push'
-        run: |
-          git tag ${{ env.SCHEDULED_BUILD_VERSION }}
-
-      - name: Publish scheduled release # configure the different release title and tags.
-        uses: ncipollo/release-action@v1
-        if: github.event_name != 'push'
-        with:
-          name: "Release ${{ env.SCHEDULED_BUILD_VERSION }}"
-          prerelease: ${{ env.prerelease }}
-          makeLatest: ${{ env.makeLatest }}
-          tag: ${{ env.SCHEDULED_BUILD_VERSION }}
-          generateReleaseNotes: true
-          artifacts: |
-            **/greptime-*
-
-      - name: Publish release
-        uses: ncipollo/release-action@v1
-        if: github.event_name == 'push'
-        with:
-          name: "${{ github.ref_name }}"
-          prerelease: ${{ env.prerelease }}
-          makeLatest: ${{ env.makeLatest }}
-          generateReleaseNotes: false
-          allowUpdates: true
-          artifacts: |
-            **/greptime-*
-
-  docker-push-acr:
-    name: Push docker image to alibaba cloud container registry
-    needs: [docker]
-    runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false)
+  release-images-to-acr:
+    name: Build and push images to ACR
+    if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-2004-16-cores
+    # When we push to ACR, it's easy to fail due to some unknown network issues.
+    # However, we don't want to fail the whole workflow because of this.
+    # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
     continue-on-error: true
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to alibaba cloud container registry
-        uses: docker/login-action@v2
+      - uses: actions/checkout@v3
         with:
-          registry: greptime-registry.cn-hangzhou.cr.aliyuncs.com
-          username: ${{ secrets.ALICLOUD_USERNAME }}
-          password: ${{ secrets.ALICLOUD_PASSWORD }}
+          fetch-depth: 0
 
-      - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
-        shell: bash
-        if: github.event_name != 'push'
-        run: |
-          buildTime=`date "+%Y%m%d"`
-          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
-          echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV
+      - name: Build and push images to ACR
+        uses: ./.github/actions/build-images
+        with:
+          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          version: ${{ needs.allocate-runners.outputs.version }}
 
-      - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
-        shell: bash
-        if: github.event_name == 'push'
-        run: |
-          VERSION=${{ github.ref_name }}
-          echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV
+  release-artifacts:
+    name: Create GitHub release and upload artifacts
+    if: ${{ inputs.release_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+      build-macos-artifacts,
+      release-images-to-dockerhub,
+    ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
-      - name: Push image to alibaba cloud container registry # Use 'docker buildx imagetools create' to create a new image base on source image.
-        run: |
-          docker buildx imagetools create \
-            --tag greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb:latest \
-            --tag greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb:${{ env.IMAGE_TAG }} \
-            greptime/greptimedb:${{ env.IMAGE_TAG }}
+      - name: Release artifacts
+        uses: ./.github/actions/release-artifacts
+        with:
+          version: ${{ needs.allocate-runners.outputs.version }}
+
+  release-dev-builder-image:
+    name: Release dev builder image
+    if: ${{ inputs.release_dev_builder_image }} # Only manually trigger this job.
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push dev builder image
+        uses: ./.github/actions/build-dev-builder-image
+        with:
+          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub-image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
+          acr-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          acr-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          acr-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+
+  ### Stop runners ###
+  # It's very necessary to split the job of releasing runners into 'stop-linux-amd64-runner' and 'stop-linux-arm64-runner'.
+  # Because we can terminate the specified EC2 instance immediately after the job is finished without uncessary waiting.
+  stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-amd64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+  stop-linux-arm64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-arm64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-arm64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}

Cargo.toml

@@ -56,28 +56,30 @@ edition = "2021"
 license = "Apache-2.0"
 
 [workspace.dependencies]
-arrow = { version = "40.0" }
-arrow-array = "40.0"
-arrow-flight = "40.0"
-arrow-schema = { version = "40.0", features = ["serde"] }
+arrow = { version = "43.0" }
+etcd-client = "0.11"
+arrow-array = "43.0"
+arrow-flight = "43.0"
+arrow-schema = { version = "43.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
-# TODO(ruihang): use arrow-datafusion when it contains https://github.com/apache/arrow-datafusion/pull/6032
-datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-etcd-client = "0.11"
+datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
+datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
+datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
+datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
+datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
+datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
+datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
+derive_builder = "0.12"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "917ead6274b4dccbaf33c59a7360646ba2f285a9" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "3d8ac534a0c8fd1c6ec66d129345b44c95665ebc" }
 itertools = "0.10"
 lazy_static = "1.4"
-parquet = "40.0"
+once_cell = "1.18"
+opentelemetry-proto = { version = "0.2", features = ["gen-tonic", "metrics"] }
+parquet = "43.0"
 paste = "1.0"
 prost = "0.11"
 rand = "0.8"
@@ -85,10 +87,11 @@ regex = "1.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }
-sqlparser = "0.34"
+sqlparser = "0.35"
 tempfile = "3"
 tokio = { version = "1.28", features = ["full"] }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
+toml = "0.7"
 tonic = { version = "0.9", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
 metrics = "0.20"

Makefile

@@ -1,15 +1,72 @@
-IMAGE_REGISTRY ?= greptimedb
+# The arguments for building images.
+CARGO_PROFILE ?=
+FEATURES ?=
+TARGET_DIR ?=
+CARGO_BUILD_OPTS := --locked
+IMAGE_REGISTRY ?= docker.io
+IMAGE_NAMESPACE ?= greptime
 IMAGE_TAG ?= latest
+BUILDX_MULTI_PLATFORM_BUILD ?= false
+BUILDX_BUILDER_NAME ?= gtbuilder
+BASE_IMAGE ?= ubuntu
+RUST_TOOLCHAIN ?= $(shell cat rust-toolchain.toml | grep channel | cut -d'"' -f2)
+CARGO_REGISTRY_CACHE ?= ${HOME}/.cargo/registry
+
+# The arguments for running integration tests.
+ETCD_VERSION ?= v3.5.9
+ETCD_IMAGE ?= quay.io/coreos/etcd:${ETCD_VERSION}
+RETRY_COUNT ?= 3
+NEXTEST_OPTS := --retries ${RETRY_COUNT}
+BUILD_JOBS ?= $(shell which nproc 1>/dev/null && expr $$(nproc) / 2) # If nproc is not available, we don't set the build jobs.
+ifeq ($(BUILD_JOBS), 0) # If the number of cores is less than 2, set the build jobs to 1.
+  BUILD_JOBS := 1
+endif
+
+ifneq ($(strip $(BUILD_JOBS)),)
+	NEXTEST_OPTS += --build-jobs=${BUILD_JOBS}
+endif
+
+ifneq ($(strip $(CARGO_PROFILE)),)
+	CARGO_BUILD_OPTS += --profile ${CARGO_PROFILE}
+endif
+
+ifneq ($(strip $(FEATURES)),)
+	CARGO_BUILD_OPTS += --features ${FEATURES}
+endif
+
+ifneq ($(strip $(TARGET_DIR)),)
+	CARGO_BUILD_OPTS += --target-dir ${TARGET_DIR}
+endif
+
+ifeq ($(BUILDX_MULTI_PLATFORM_BUILD), true)
+	BUILDX_MULTI_PLATFORM_BUILD_OPTS := --platform linux/amd64,linux/arm64 --push
+else
+	BUILDX_MULTI_PLATFORM_BUILD_OPTS := -o type=docker
+endif
 
 ##@ Build
 
 .PHONY: build
-build: ## Build debug version greptime.
-	cargo build
+build: ## Build debug version greptime. If USE_DEV_BUILDER is true, the binary will be built in dev-builder.
+ifeq ($(USE_DEV_BUILDER), true)
+	docker run --network=host \
+	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \
+	make build CARGO_PROFILE=${CARGO_PROFILE} FEATURES=${FEATURES} TARGET_DIR=${TARGET_DIR}
+else
+	cargo build ${CARGO_BUILD_OPTS}
+endif
 
 .PHONY: release
-release:  ## Build release version greptime.
-	cargo build --release
+release:  ## Build release version greptime. If USE_DEV_BUILDER is true, the binary will be built in dev-builder.
+ifeq ($(USE_DEV_BUILDER), true)
+	docker run --network=host \
+	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \
+	make release CARGO_PROFILE=${CARGO_PROFILE} FEATURES=${FEATURES} TARGET_DIR=${TARGET_DIR}
+else
+	cargo build --release ${CARGO_BUILD_OPTS}
+endif
 
 .PHONY: clean
 clean: ## Clean the project.
@@ -28,13 +85,38 @@ check-toml: ## Check all TOML files.
 	taplo format --check --option "indent_string=    "
 
 .PHONY: docker-image
-docker-image: ## Build docker image.
-	docker build --network host -f docker/Dockerfile -t ${IMAGE_REGISTRY}:${IMAGE_TAG} .
+docker-image: multi-platform-buildx ## Build docker image.
+	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
+	  --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" --build-arg="FEATURES=${FEATURES}" \
+	  -f docker/${BASE_IMAGE}/Dockerfile \
+	  -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
+
+.PHONY: build-greptime-by-buildx
+build-greptime-by-buildx: multi-platform-buildx ## Build greptime binary by docker buildx. The binary will be copied to the current directory.
+	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
+	  --target=builder \
+	  --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" --build-arg="FEATURES=${FEATURES}" \
+	  -f docker/${BASE_IMAGE}/Dockerfile \
+	  -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb-builder:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
+
+	docker run --rm -v ${PWD}:/data \
+      --entrypoint cp ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb-builder:${IMAGE_TAG} \
+      /out/target/${CARGO_PROFILE}/greptime /data/greptime
+
+.PHONY: dev-builder
+dev-builder: multi-platform-buildx ## Build dev-builder image.
+	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
+	--build-arg="RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" \
+	-f docker/dev-builder/Dockerfile \
+	-t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
+
+.PHONY: multi-platform-buildx
+multi-platform-buildx: ## Create buildx multi-platform builder.
+	docker buildx inspect ${BUILDX_BUILDER_NAME} || docker buildx create --name ${BUILDX_BUILDER_NAME} --driver docker-container --bootstrap --use
 
 ##@ Test
-
 test: nextest ## Run unit and integration tests.
-	cargo nextest run --retries 3
+	cargo nextest run ${NEXTEST_OPTS}
 
 .PHONY: nextest ## Install nextest tools.
 nextest:
@@ -56,6 +138,21 @@ clippy: ## Check clippy rules.
 fmt-check: ## Check code format.
 	cargo fmt --all -- --check
 
+.PHONY: start-etcd
+start-etcd: ## Start single node etcd for testing purpose.
+	docker run --rm -d --network=host -p 2379-2380:2379-2380 ${ETCD_IMAGE}
+
+.PHONY: stop-etcd
+stop-etcd: ## Stop single node etcd for testing purpose.
+	docker stop $$(docker ps -q --filter ancestor=${ETCD_IMAGE})
+
+.PHONY: run-it-in-container
+run-it-in-container: start-etcd ## Run integration tests in dev-builder.
+	docker run --network=host \
+	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v /tmp:/tmp \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \
+	make test sqlness-test BUILD_JOBS=${BUILD_JOBS}
+
 ##@ General
 
 # The help target prints out all targets with their descriptions organized
@@ -71,4 +168,4 @@ fmt-check: ## Check code format.
 
 .PHONY: help
 help: ## Display help messages.
-	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-30s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

README.md

@@ -47,14 +47,10 @@ for years. Based on their best-practices, GreptimeDB is born to give you:
 
 ## Quick Start
 
-### GreptimePlay
+### [GreptimePlay](https://greptime.com/playground)
 
 Try out the features of GreptimeDB right from your browser.
 
-<a href="https://greptime.com/playground" target="_blank"><img
-src="https://www.greptime.com/assets/greptime_play_button_colorful.1bbe2746.png"
-alt="GreptimePlay" width="200px" /></a>
-
 ### Build
 
 #### Build from Source
@@ -106,7 +102,7 @@ Please see [the online document site](https://docs.greptime.com/getting-started/
 
 Read the [complete getting started guide](https://docs.greptime.com/getting-started/overview#connect) on our [official document site](https://docs.greptime.com/).
 
-To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/client/overview).
+To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/clients/overview).
 
 ## Resources
 

config/datanode.example.toml

@@ -10,8 +10,10 @@ rpc_addr = "127.0.0.1:3001"
 rpc_hostname = "127.0.0.1"
 # The number of gRPC server worker threads, 8 by default.
 rpc_runtime_size = 8
+
+[heartbeat]
 # Interval for sending heartbeat messages to the Metasrv in milliseconds, 5000 by default.
-heartbeat_interval_millis = 5000
+interval_millis = 5000
 
 # Metasrv client options.
 [meta_client_options]

config/frontend.example.toml

@@ -1,7 +1,9 @@
 # Node running mode, see `standalone.example.toml`.
 mode = "distributed"
+
+[heartbeat]
 # Interval for sending heartbeat task to the Metasrv in milliseconds, 5000 by default.
-heartbeat_interval_millis = 5000
+interval_millis = 5000
 # Interval for retry sending heartbeat task in milliseconds, 5000 by default.
 retry_interval_millis = 5000
 
@@ -47,18 +49,20 @@ runtime_size = 2
 [influxdb_options]
 enable = true
 
-# Prometheus protocol options, see `standalone.example.toml`.
-[prometheus_options]
+# Prometheus remote storage options, see `standalone.example.toml`.
+[prom_store_options]
 enable = true
 
 # Prometheus protocol options, see `standalone.example.toml`.
-[prom_options]
+[prometheus_options]
 addr = "127.0.0.1:4004"
 
 # Metasrv client options, see `datanode.example.toml`.
 [meta_client_options]
 metasrv_addrs = ["127.0.0.1:3002"]
 timeout_millis = 3000
+# DDL timeouts options.
+ddl_timeout_millis = 10000
 connect_timeout_millis = 5000
 tcp_nodelay = true
 
@@ -66,3 +70,11 @@ tcp_nodelay = true
 # [logging]
 # dir = "/tmp/greptimedb/logs"
 # level = "info"
+
+# Datanode options.
+[datanode]
+# Datanode client options.
+[datanode.client]
+timeout = "10s"
+connect_timeout = "10s"
+tcp_nodelay = true

config/metasrv.example.toml

@@ -18,3 +18,18 @@ use_memory_store = false
 # [logging]
 # dir = "/tmp/greptimedb/logs"
 # level = "info"
+
+# Procedure storage options.
+[procedure]
+# Procedure max retry time.
+max_retry_times = 3
+# Initial retry delay of procedures, increases exponentially
+retry_delay = "500ms"
+
+# # Datanode options.
+# [datanode]
+# # Datanode client options.
+# [datanode.client_options]
+# timeout_millis = 10000
+# connect_timeout_millis = 10000
+# tcp_nodelay = true

config/standalone.example.toml

@@ -69,13 +69,13 @@ runtime_size = 2
 # Whether to enable InfluxDB protocol in HTTP API, true by default.
 enable = true
 
-# Prometheus protocol options.
-[prometheus_options]
+# Prometheus remote storage options
+[prom_store_options]
 # Whether to enable Prometheus remote write and read in HTTP API, true by default.
 enable = true
 
-# Prom protocol options.
-[prom_options]
+# Prometheus protocol options
+[prometheus_options]
 # Prometheus API server address, "127.0.0.1:4004" by default.
 addr = "127.0.0.1:4004"
 

docker/Dockerfile-centos7-builder

@@ -1,29 +0,0 @@
-FROM centos:7
-
-ENV LANG en_US.utf8
-WORKDIR /greptimedb
-
-RUN sed -e 's|^mirrorlist=|#mirrorlist=|g' \
-             -e 's|^#baseurl=http://mirror.centos.org/centos|baseurl=http://mirrors.tuna.tsinghua.edu.cn/centos|g' \
-             -i.bak \
-             /etc/yum.repos.d/CentOS-*.repo
-
-# Install dependencies
-RUN RUN ulimit -n 1024000 && yum groupinstall -y 'Development Tools'
-RUN yum install -y epel-release  \
-    openssl \
-    openssl-devel  \
-    centos-release-scl  \
-    rh-python38  \
-    rh-python38-python-devel
-
-# Install protoc
-RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
-RUN unzip protoc-3.15.8-linux-x86_64.zip -d /usr/local/
-
-# Install Rust
-SHELL ["/bin/bash", "-c"]
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
-ENV PATH /opt/rh/rh-python38/root/usr/bin:/usr/local/bin:/root/.cargo/bin/:$PATH
-
-CMD ["cargo", "build", "--release"]

docker/aarch64/Dockerfile

@@ -1,60 +0,0 @@
-FROM ubuntu:22.04 as builder
-
-ENV LANG en_US.utf8
-WORKDIR /greptimedb
-
-# Install dependencies.
-RUN apt-get update && apt-get install -y \
-    libssl-dev \
-    protobuf-compiler \
-    curl \
-    git \
-    build-essential \
-    pkg-config \
-    wget
-
-# Install Rust.
-SHELL ["/bin/bash", "-c"]
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
-ENV PATH /root/.cargo/bin/:$PATH
-
-# Install cross platform toolchain
-RUN apt-get -y update && \
-    apt-get -y install g++-aarch64-linux-gnu gcc-aarch64-linux-gnu && \
-    apt-get install binutils-aarch64-linux-gnu
-
-COPY ./docker/aarch64/compile-python.sh ./docker/aarch64/
-RUN chmod +x ./docker/aarch64/compile-python.sh && \
-    ./docker/aarch64/compile-python.sh
-
-COPY ./rust-toolchain.toml .
-# Install rustup target for cross compiling.
-RUN rustup target add aarch64-unknown-linux-gnu
-COPY . .
-# Update dependency, using separate `RUN` to separate cache
-RUN cargo fetch
-
-# This three env var is set in script, so I set it manually in dockerfile.
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
-ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib/
-ENV PY_INSTALL_PATH=/greptimedb/python_arm64_build
-
-# Set the environment variable for cross compiling and compile it
-# cross compiled python is `python3` in path, but pyo3 need `python` in path so alias it
-# Build the project in release mode.
-RUN export PYO3_CROSS_LIB_DIR=$PY_INSTALL_PATH/lib && \ 
-    alias python=python3 && \
-    cargo build --target aarch64-unknown-linux-gnu --release -F pyo3_backend
-
-# Exporting the binary to the clean image
-FROM ubuntu:22.04 as base
-
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get \
-    -y install ca-certificates \
-    curl
-
-WORKDIR /greptime
-COPY --from=builder /greptimedb/target/aarch64-unknown-linux-gnu/release/greptime /greptime/bin/
-ENV PATH /greptime/bin/:$PATH
-
-ENTRYPOINT ["greptime"]

docker/aarch64/compile-python.sh

@@ -1,87 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-# this script will download Python source code, compile it, and install it to /usr/local/lib
-# then use this python to compile cross-compiled python for aarch64
-ARCH=$1
-PYTHON_VERSION=3.10.10
-PYTHON_SOURCE_DIR=Python-${PYTHON_VERSION}
-PYTHON_INSTALL_PATH_AMD64=${PWD}/python-${PYTHON_VERSION}/amd64
-PYTHON_INSTALL_PATH_AARCH64=${PWD}/python-${PYTHON_VERSION}/aarch64
-
-function download_python_source_code() {
-  wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz
-  tar -xvf Python-$PYTHON_VERSION.tgz
-}
-
-function compile_for_amd64_platform() {
-  mkdir -p "$PYTHON_INSTALL_PATH_AMD64"
-
-  echo "Compiling for amd64 platform..."
-
-  ./configure \
-    --prefix="$PYTHON_INSTALL_PATH_AMD64" \
-    --enable-shared \
-    ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \
-    ac_cv_have_long_long_format=yes \
-    --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no
-
-  make
-  make install
-}
-
-# explain Python compile options here a bit:s
-# --enable-shared: enable building a shared Python library (default is no) but we do need it for calling from rust
-# CC, CXX, AR, LD, RANLIB: set the compiler, archiver, linker, and ranlib programs to use
-# build: the machine you are building on, host: the machine you will run the compiled program on
-# --with-system-ffi: build _ctypes module using an installed ffi library, see Doc/library/ctypes.rst, not used in here TODO: could remove
-# ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes:
-# allow cross-compiled python to have -pthread set for CXX, see https://github.com/python/cpython/pull/22525
-# ac_cv_have_long_long_format=yes: target platform supports long long type
-# disable-ipv6: disable ipv6 support, we don't need it in here
-# ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no: disable pty support, we don't need it in here
-function compile_for_aarch64_platform() {
-  export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
-  export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
-  export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
-
-  mkdir -p "$PYTHON_INSTALL_PATH_AARCH64"
-
-  echo "Compiling for aarch64 platform..."
-  echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
-  echo "LIBRARY_PATH: $LIBRARY_PATH"
-  echo "PATH: $PATH"
-
-  ./configure --build=x86_64-linux-gnu --host=aarch64-linux-gnu \
-    --prefix="$PYTHON_INSTALL_PATH_AARCH64" --enable-optimizations \
-    CC=aarch64-linux-gnu-gcc \
-    CXX=aarch64-linux-gnu-g++ \
-    AR=aarch64-linux-gnu-ar \
-    LD=aarch64-linux-gnu-ld \
-    RANLIB=aarch64-linux-gnu-ranlib \
-    --enable-shared \
-    ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \
-    ac_cv_have_long_long_format=yes \
-    --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no
-
-  make
-  make altinstall
-}
-
-# Main script starts here.
-download_python_source_code
-
-# Enter the python source code directory.
-cd $PYTHON_SOURCE_DIR || exit 1
-
-# Build local python first, then build cross-compiled python.
-compile_for_amd64_platform
-
-# Clean the build directory.
-make clean && make distclean
-
-# Cross compile python for aarch64.
-if [ "$ARCH" = "aarch64-unknown-linux-gnu" ]; then
-  compile_for_aarch64_platform
-fi

docker/centos/Dockerfile

@@ -0,0 +1,51 @@
+FROM centos:7 as builder
+
+ARG CARGO_PROFILE
+ARG FEATURES
+
+ENV LANG en_US.utf8
+WORKDIR /greptimedb
+
+# Install dependencies
+RUN ulimit -n 1024000 && yum groupinstall -y 'Development Tools'
+RUN yum install -y epel-release  \
+    openssl \
+    openssl-devel  \
+    centos-release-scl  \
+    rh-python38  \
+    rh-python38-python-devel
+
+# Install protoc
+RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
+RUN unzip protoc-3.15.8-linux-x86_64.zip -d /usr/local/
+
+# Install Rust
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /opt/rh/rh-python38/root/usr/bin:/usr/local/bin:/root/.cargo/bin/:$PATH
+
+# Build the project in release mode.
+RUN --mount=target=.,rw \
+    --mount=type=cache,target=/root/.cargo/registry \
+    make build \
+    CARGO_PROFILE=${CARGO_PROFILE} \
+    FEATURES=${FEATURES} \
+    TARGET_DIR=/out/target
+
+# Export the binary to the clean image.
+FROM centos:7 as base
+
+ARG CARGO_PROFILE
+
+RUN yum install -y epel-release \
+    openssl \
+    openssl-devel  \
+    centos-release-scl  \
+    rh-python38  \
+    rh-python38-python-devel
+
+WORKDIR /greptime
+COPY --from=builder /out/target/${CARGO_PROFILE}/greptime /greptime/bin/
+ENV PATH /greptime/bin/:$PATH
+
+ENTRYPOINT ["greptime"]

docker/ci/Dockerfile

@@ -7,7 +7,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     python3-pip \
     curl
 
-COPY requirements.txt /etc/greptime/requirements.txt
+COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt
 
 RUN python3 -m pip install -r /etc/greptime/requirements.txt
 

docker/ci/Dockerfile-centos

@@ -0,0 +1,16 @@
+FROM centos:7
+
+RUN yum install -y epel-release \
+    openssl \
+    openssl-devel  \
+    centos-release-scl  \
+    rh-python38  \
+    rh-python38-python-devel
+
+ARG TARGETARCH
+
+ADD $TARGETARCH/greptime /greptime/bin/
+
+ENV PATH /greptime/bin/:$PATH
+
+ENTRYPOINT ["greptime"]

docker/dev-builder/Dockerfile

@@ -0,0 +1,36 @@
+FROM ubuntu:22.04
+
+ENV LANG en_US.utf8
+WORKDIR /greptimedb
+
+# Install dependencies.
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    libssl-dev \
+    tzdata \
+    protobuf-compiler \
+    curl \
+    ca-certificates \
+    git \
+    build-essential \
+    pkg-config \
+    python3.10 \
+    python3.10-dev \
+    python3-pip
+
+RUN git config --global --add safe.directory /greptimedb
+
+# Install Python dependencies.
+COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt
+RUN python3 -m pip install -r /etc/greptime/requirements.txt
+
+# Install Rust.
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /root/.cargo/bin/:$PATH
+
+# Install Rust toolchains.
+ARG RUST_TOOLCHAIN
+RUN rustup toolchain install ${RUST_TOOLCHAIN}
+
+# Install nextest.
+RUN cargo install cargo-nextest --locked

docker/ubuntu/Dockerfile

@@ -1,21 +1,23 @@
 FROM ubuntu:22.04 as builder
 
+ARG CARGO_PROFILE
+ARG FEATURES
+
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
 
 # Install dependencies.
-RUN apt-get update && apt-get install -y \
+RUN --mount=type=cache,target=/var/cache/apt \
+    apt-get update && apt-get install -y \
     libssl-dev \
     protobuf-compiler \
     curl \
     git \
     build-essential \
     pkg-config \
-    python3 \
-    python3-dev \
-    python3-pip \
-    && pip3 install --upgrade pip \
-    && pip3 install pyarrow
+    python3.10 \
+    python3.10-dev \
+    python3-pip
 
 # Install Rust.
 SHELL ["/bin/bash", "-c"]
@@ -23,19 +25,32 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-mo
 ENV PATH /root/.cargo/bin/:$PATH
 
 # Build the project in release mode.
-COPY . .
-RUN cargo build --release
+RUN --mount=target=.,rw \
+    --mount=type=cache,target=/root/.cargo/registry \
+    make build \
+    CARGO_PROFILE=${CARGO_PROFILE} \
+    FEATURES=${FEATURES} \
+    TARGET_DIR=/out/target
 
 # Export the binary to the clean image.
 # TODO(zyy17): Maybe should use the more secure container image.
 FROM ubuntu:22.04 as base
 
+ARG CARGO_PROFILE
+
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get \
     -y install ca-certificates \
+    python3.10 \
+    python3.10-dev \
+    python3-pip \
     curl
 
+COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt
+
+RUN python3 -m pip install -r /etc/greptime/requirements.txt
+
 WORKDIR /greptime
-COPY --from=builder /greptimedb/target/release/greptime /greptime/bin/
+COPY --from=builder /out/target/${CARGO_PROFILE}/greptime /greptime/bin/
 ENV PATH /greptime/bin/:$PATH
 
 ENTRYPOINT ["greptime"]

docs/rfcs/2023-07-10-metric-engine.md

@@ -0,0 +1,202 @@
+---
+Feature Name: metric-engine
+Tracking Issue: TBD
+Date: 2023-07-10
+Author: "Ruihang Xia <waynestxia@gmail.com>"
+---
+
+# Summary
+
+A new metric engine that can significantly enhance our ability to handle the tremendous number of small tables in scenarios like Prometheus metrics, by leveraging a synthetic wide table that offers storage and metadata multiplexing capabilities over the existing engine.
+
+# Motivation
+
+The concept "Table" in GreptimeDB is a bit "heavy" compared to other time-series storage like Prometheus or VictoriaMetrics. This has lots of disadvantages in aspects from performance, footprint, and storage to cost.
+
+# Details
+
+## Top level description
+
+- User Interface
+
+    This feature will add a new type of storage engine. It might be available to be an option like `with ENGINE=mito` or an internal interface like auto create table on Prometheus remote write. From the user side, there is no difference from tables in mito engine. All the DDL like `CREATE`, `ALTER` and DML like `SELECT` should be supported.
+
+- Implementation Overlook
+
+    This new engine doesn't re-implement low level components like file R/W etc. It's a wrapper layer over the existing mito engine, with extra storage and metadata multiplexing capabilities. I.e., it expose multiple table based on one mito engine table like this:
+	``` plaintext
+	   ┌───────────────┐ ┌───────────────┐ ┌───────────────┐
+	   │ Metric Engine │ │ Metric Engine │ │ Metric Engine │
+	   │   Table 1     │ │   Table 2     │ │   Table 3     │
+	   └───────────────┘ └───────────────┘ └───────────────┘
+	           ▲               ▲                   ▲
+	           │               │                   │
+	           └───────────────┼───────────────────┘
+	                           │
+	                 ┌─────────┴────────┐
+	                 │ Metric Region    │
+	                 │   Engine         │
+	                 │    ┌─────────────┤
+	                 │    │ Mito Region │
+	                 │    │   Engine    │
+	                 └────▲─────────────┘
+	                      │
+	                      │
+	                ┌─────┴───────────────┐
+	                │                     │
+	                │  Mito Engine Table  │
+	                │                     │
+	                └─────────────────────┘
+	```
+
+The following parts will describe these implementation details:
+    - How to route these metric region tables and how those table are distributed
+    - How to maintain the schema and other metadata of the underlying mito engine table
+    - How to maintain the schema of metric engine table
+    - How the query goes
+
+## Routing
+
+Before this change, the region route rule was based on a group of partition keys. Relation of physical table to region is one-to-many.
+
+``` rust
+  pub struct PartitionDef {
+      partition_columns: Vec<String>,
+      partition_bounds: Vec<PartitionBound>,
+  }
+```
+
+And for metric engine tables, the key difference is we split the concept of "physical table" and "logical table". Like the previous ASCII chart, multiple logical tables are based on one physical table. The relationship of logical table to region becomes many-to-many. Thus, we must include the table name (of logical table) into partition rules.
+
+Consider the partition/route interface is a generic map of string array to region id, all we need to do is to insert logical table name into the request:
+
+``` rust
+  fn route(request: Vec<String>) -> RegionId;
+```
+
+The next question is, where to do this conversion? The basic idea is to dispatch different routing behavior based on the engine type. Since we have all the necessary information in frontend, it's a good place to do that. And can leave meta server untouched. The essential change is to associate engine type with route rule.
+
+## Physical Region Schema
+
+The idea "physical wide table" is to perform column-level multiplexing. I.e., map all logical columns to physical columns by their names.
+
+```
+   ┌────────────┐      ┌────────────┐         ┌────────────┐
+   │   Table 1  │      │   Table 2  │         │   Table 3  │
+   ├───┬────┬───┤      ├───┬────┬───┤         ├───┬────┬───┤
+   │C1 │ C2 │ C3│      │C1 │ C3 │ C5├──────┐  │C2 │ C4 │ C6│
+   └─┬─┴──┬─┴─┬─┘ ┌────┴───┴──┬─┴───┘      │  └─┬─┴──┬─┴─┬─┘
+     │    │   │   │           │            │    │    │   │
+     │    │   │   │           └──────────┐ │    │    │   │
+     │    │   │   │                      │ │    │    │   │
+     │    │   │   │  ┌─────────────────┐ │ │    │    │   │
+     │    │   │   │  │ Physical Table  │ │ │    │    │   │
+     │    │   │   │  ├──┬──┬──┬──┬──┬──┘ │ │    │    │   │
+     └────x───x───┴─►│C1│C2│C3│C4│C5│C6◄─┼─x────x────x───┘
+          │   │      └──┘▲─┘▲─┴─▲└─▲└──┘ │ │    │    │
+          │   │          │  │   │  │     │ │    │    │
+          ├───x──────────┘  ├───x──x─────┘ │    │    │
+          │   │             │   │  │       │    │    │
+          │   └─────────────┘   │  └───────┘    │    │
+          │                     │               │    │
+          └─────────────────────x───────────────┘    │
+                                │                    │
+                                └────────────────────┘
+```
+
+This approach is very straightforward but has one problem. It only works when two columns have different semantic type (time index, tag or field) or data types but with the same name. E.g., `CREATE TABLE t1 (c1 timestamp(3) TIME INDEX)` and `CREATE TABLE t2 (c1 STRING PRIMARY KEY)`.
+
+One possible workaround is to prefix each column with its data type and semantic type, like `_STRING_PK_c1`. However, considering the primary goal at present is to support data from monitoring metrics like Prometheus remote write, it's acceptable not to support this at first because data types are often simple and limited here.
+
+
+The next point is changing the physical table's schema. This is only needed when creating a new logical table or altering the existing table. Typically speaking, table creating and altering are explicit. We only need to emit an add column request to underlying physical table on processing logical table's DDL. GreptimeDB can create or alter table automatically on some protocols, but the internal logic is the same.
+
+Also for simplicity, we don't support shrinking the underlying table at first. This can be achieved by introducing mechanism on the physical column.
+
+Frontend needs not to keep physical table's schema.
+
+## Metadata of physical regions
+
+Those metric engine regions need to store extra metadata like the schema of logical table or all logical table's name. That information is relatively simple and can be stored in a format like key-value pair. For now, we have to use another physical mito region for metadata. This involves an issue with region scheduling. Since we don't have the ability to perform affinity scheduling, the initial version will just assume the data region and metadata region are in the same instance. See alternatives - other storage for physical region's metadata for possible future improvement.
+
+Here is the schema of metadata region and how we would use it. The `CREATE TABLE` clause of metadata region looks like the following. Notice that it wouldn't be actually created by SQL.
+
+``` sql
+  CREATE TABLE metadata(
+  	ts timestamp time index,
+    	key string primary key,
+    	value string
+  );
+```
+
+The `ts` field is just a placeholder -- for the constraints that a mito region must contain a time index field. It will always be `0`. The other two fields `key` and `value` will be used as a k-v storage. It contains two group of key
+    - `__table_<TABLE_NAME>` is used for marking table existence. It doesn't have value.
+    - `__column_<TABLE_NAME>_<COLUMN_NAME>` is used for marking table existence, the value is column's semantic type.
+
+## Physical region implementation
+
+This RFC proposes to add a new region implementation named "MetricRegion". As showed in the first chart, it's wrapped over the existing mito region. This section will describe the implementation details. Firstly, here is a chart shows how the region hierarchy looks like:
+
+```plaintext
+ ┌───────────────────────┐
+ │ Metric Region         │
+ │                       │
+ │   ┌────────┬──────────┤
+ │   │ Mito   │ Mito     │
+ │   │ Region │ Region   │
+ │   │ for    │ for      │
+ │   │ Data   │ Metadata │
+ └───┴────────┴──────────┘
+```
+
+All upper levels only see the Metric Region. E.g., Meta Server schedules on this region, or Frontend routes requests to this Metrics Region's id. To be scheduled (open or close etc.), Metric Region needs to implement its own procedures. Most of those procedures can be simply assembled from underlying Mito Regions', but those related to data like alter or drop will have its own new logic.
+
+Another point is region id. Since the region id is used widely from meta server to persisted state, it's better to keep it unchanged. This means we can't use the same id for two regions, but one for each. To achieve this, this RFC proposes a concept named "region id group". A region id group is a group of region ids that are bound for different purposes. Like the two underlying regions here. 
+
+This preserves the first 8 bits of the `u32` region number for grouping. Each group has one main id (the first one) and other sub ids (the rest non-zero ids). All components other than the region implementation itself doesn't aware of the existence of region id group. They only see the main id. The region implementation is in response of managing and using the region id group.
+
+```plaintext
+63                                  31         23                  0
+┌────────────────────────────────────┬──────────┬──────────────────┐
+│          Table Id(32)              │ Group(8) │ Region Number(24)│
+└────────────────────────────────────┴──────────┴──────────────────┘
+                                            Region Id(32)
+```
+
+## Routing in meta server
+
+From previous sections, we can conclude the following points about routing:
+- Each "logical table" has its own, universe unique table id.
+- Logical table doesn't have physical region, they share the same physical region with other logical tables.
+- Route rule of logical table's is a strict subset of physical table's.
+
+To associate the logical table with physical region, we need to specify necessary information in the create table request. Specifically, the table type and its parent table. This require to change our gRPC proto's definition. And once meta recognize the table to create is a logical table, it will use the parent table's region to create route entry.
+
+And to reduce the consumption of region failover (which need to update the physical table route info), we'd better to split the current route table structure into two parts:
+
+```rust
+region_route: Map<TableName, [RegionId]>,
+node_route: Map<RegionId, NodeId>,
+```
+
+By doing this on each failover the meta server only needs to update the second `node_route` map and leave the first one untouched.
+
+## Query
+
+Like other existing components, a user query always starts in the frontend. In the planning phase, frontend needs to fetch related schemas of the queried table. This part is the same. I.e., changes in this RFC don't affect components above the `Table` abstraction.
+
+# Alternatives
+
+## Other routing method
+
+We can also do this "special" route rule in the meta server. But there is no difference with the proposed method.
+
+## Other storage for physical region's metadata
+
+Once we have implemented the "region family" that allows multiple physical schemas exist in one region, we can store the metadata and table data into one region.
+
+Before that, we can also let the `MetricRegion` holds a `KvBackend` to access the storage layer directly. But this breaks the abstraction in some way.
+
+# Drawbacks
+
+Since the physical storage is mixed together. It's hard to do fine-grained operations in table level. Like configuring TTL, memtable size or compaction strategy in table level. Or define different partition rules for different tables. For scenarios like this, it's better to move the table out of metrics engine and "upgrade" it to a normal mito engine table. This requires a migration process in a low cost. And we have to ensure data consistency during the migration, which may require a out-of-service period.

src/api/Cargo.toml

@@ -5,7 +5,6 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-arrow-flight.workspace = true
 common-base = { path = "../common/base" }
 common-error = { path = "../common/error" }
 common-time = { path = "../common/time" }

src/api/src/error.rs

@@ -15,7 +15,7 @@
 use std::any::Any;
 
 use common_error::ext::ErrorExt;
-use common_error::prelude::StatusCode;
+use common_error::status_code::StatusCode;
 use datatypes::prelude::ConcreteDataType;
 use snafu::prelude::*;
 use snafu::Location;

src/api/src/helper.rs

@@ -15,7 +15,7 @@
 use common_base::BitVec;
 use common_time::timestamp::TimeUnit;
 use datatypes::prelude::ConcreteDataType;
-use datatypes::types::TimestampType;
+use datatypes::types::{TimeType, TimestampType};
 use datatypes::value::Value;
 use datatypes::vectors::VectorRef;
 use greptime_proto::v1::ddl_request::Expr;
@@ -71,6 +71,10 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
             ColumnDataType::TimestampNanosecond => {
                 ConcreteDataType::timestamp_nanosecond_datatype()
             }
+            ColumnDataType::TimeSecond => ConcreteDataType::time_second_datatype(),
+            ColumnDataType::TimeMillisecond => ConcreteDataType::time_millisecond_datatype(),
+            ColumnDataType::TimeMicrosecond => ConcreteDataType::time_microsecond_datatype(),
+            ColumnDataType::TimeNanosecond => ConcreteDataType::time_nanosecond_datatype(),
         }
     }
 }
@@ -95,13 +99,20 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
             ConcreteDataType::String(_) => ColumnDataType::String,
             ConcreteDataType::Date(_) => ColumnDataType::Date,
             ConcreteDataType::DateTime(_) => ColumnDataType::Datetime,
-            ConcreteDataType::Timestamp(unit) => match unit {
+            ConcreteDataType::Timestamp(t) => match t {
                 TimestampType::Second(_) => ColumnDataType::TimestampSecond,
                 TimestampType::Millisecond(_) => ColumnDataType::TimestampMillisecond,
                 TimestampType::Microsecond(_) => ColumnDataType::TimestampMicrosecond,
                 TimestampType::Nanosecond(_) => ColumnDataType::TimestampNanosecond,
             },
-            ConcreteDataType::Null(_)
+            ConcreteDataType::Time(t) => match t {
+                TimeType::Second(_) => ColumnDataType::TimeSecond,
+                TimeType::Millisecond(_) => ColumnDataType::TimeMillisecond,
+                TimeType::Microsecond(_) => ColumnDataType::TimeMicrosecond,
+                TimeType::Nanosecond(_) => ColumnDataType::TimeNanosecond,
+            },
+            ConcreteDataType::Interval(_)
+            | ConcreteDataType::Null(_)
             | ConcreteDataType::List(_)
             | ConcreteDataType::Dictionary(_) => {
                 return error::IntoColumnDataTypeSnafu { from: datatype }.fail()
@@ -189,6 +200,22 @@ pub fn values_with_capacity(datatype: ColumnDataType, capacity: usize) -> Values
             ts_nanosecond_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
+        ColumnDataType::TimeSecond => Values {
+            time_second_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::TimeMillisecond => Values {
+            time_millisecond_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::TimeMicrosecond => Values {
+            time_microsecond_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::TimeNanosecond => Values {
+            time_nanosecond_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
     }
 }
 
@@ -223,7 +250,13 @@ pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
             TimeUnit::Microsecond => values.ts_microsecond_values.push(val.value()),
             TimeUnit::Nanosecond => values.ts_nanosecond_values.push(val.value()),
         },
-        Value::List(_) => unreachable!(),
+        Value::Time(val) => match val.unit() {
+            TimeUnit::Second => values.time_second_values.push(val.value()),
+            TimeUnit::Millisecond => values.time_millisecond_values.push(val.value()),
+            TimeUnit::Microsecond => values.time_microsecond_values.push(val.value()),
+            TimeUnit::Nanosecond => values.time_nanosecond_values.push(val.value()),
+        },
+        Value::Interval(_) | Value::List(_) => unreachable!(),
     });
     column.null_mask = null_mask.into_vec();
 }
@@ -256,6 +289,8 @@ fn ddl_request_type(request: &DdlRequest) -> &'static str {
         Some(Expr::Alter(_)) => "ddl.alter",
         Some(Expr::DropTable(_)) => "ddl.drop_table",
         Some(Expr::FlushTable(_)) => "ddl.flush_table",
+        Some(Expr::CompactTable(_)) => "ddl.compact_table",
+        Some(Expr::TruncateTable(_)) => "ddl.truncate_table",
         None => "ddl.empty",
     }
 }
@@ -265,7 +300,8 @@ mod tests {
     use std::sync::Arc;
 
     use datatypes::vectors::{
-        BooleanVector, TimestampMicrosecondVector, TimestampMillisecondVector,
+        BooleanVector, TimeMicrosecondVector, TimeMillisecondVector, TimeNanosecondVector,
+        TimeSecondVector, TimestampMicrosecondVector, TimestampMillisecondVector,
         TimestampNanosecondVector, TimestampSecondVector,
     };
 
@@ -328,6 +364,10 @@ mod tests {
         let values = values_with_capacity(ColumnDataType::TimestampMillisecond, 2);
         let values = values.ts_millisecond_values;
         assert_eq!(2, values.capacity());
+
+        let values = values_with_capacity(ColumnDataType::TimeMillisecond, 2);
+        let values = values.time_millisecond_values;
+        assert_eq!(2, values.capacity());
     }
 
     #[test]
@@ -396,6 +436,10 @@ mod tests {
             ConcreteDataType::timestamp_millisecond_datatype(),
             ColumnDataTypeWrapper(ColumnDataType::TimestampMillisecond).into()
         );
+        assert_eq!(
+            ConcreteDataType::time_datatype(TimeUnit::Millisecond),
+            ColumnDataTypeWrapper(ColumnDataType::TimeMillisecond).into()
+        );
     }
 
     #[test]
@@ -524,9 +568,50 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_column_put_time_values() {
+        let mut column = Column {
+            column_name: "test".to_string(),
+            semantic_type: 0,
+            values: Some(Values {
+                ..Default::default()
+            }),
+            null_mask: vec![],
+            datatype: 0,
+        };
+
+        let vector = Arc::new(TimeNanosecondVector::from_vec(vec![1, 2, 3]));
+        push_vals(&mut column, 3, vector);
+        assert_eq!(
+            vec![1, 2, 3],
+            column.values.as_ref().unwrap().time_nanosecond_values
+        );
+
+        let vector = Arc::new(TimeMillisecondVector::from_vec(vec![4, 5, 6]));
+        push_vals(&mut column, 3, vector);
+        assert_eq!(
+            vec![4, 5, 6],
+            column.values.as_ref().unwrap().time_millisecond_values
+        );
+
+        let vector = Arc::new(TimeMicrosecondVector::from_vec(vec![7, 8, 9]));
+        push_vals(&mut column, 3, vector);
+        assert_eq!(
+            vec![7, 8, 9],
+            column.values.as_ref().unwrap().time_microsecond_values
+        );
+
+        let vector = Arc::new(TimeSecondVector::from_vec(vec![10, 11, 12]));
+        push_vals(&mut column, 3, vector);
+        assert_eq!(
+            vec![10, 11, 12],
+            column.values.as_ref().unwrap().time_second_values
+        );
+    }
+
     #[test]
     fn test_column_put_vector() {
-        use crate::v1::column::SemanticType;
+        use crate::v1::SemanticType;
         // Some(false), None, Some(true), Some(true)
         let mut column = Column {
             column_name: "test".to_string(),

src/api/src/lib.rs

@@ -15,7 +15,7 @@
 pub mod error;
 pub mod helper;
 
-pub mod prometheus {
+pub mod prom_store {
     pub mod remote {
         pub use greptime_proto::prometheus::remote::*;
     }

src/catalog/Cargo.toml

@@ -13,7 +13,6 @@ arc-swap = "1.0"
 arrow-schema.workspace = true
 async-stream.workspace = true
 async-trait = "0.1"
-backoff = { version = "0.4", features = ["tokio"] }
 common-catalog = { path = "../common/catalog" }
 common-error = { path = "../common/error" }
 common-grpc = { path = "../common/grpc" }
@@ -28,18 +27,16 @@ datafusion.workspace = true
 datatypes = { path = "../datatypes" }
 futures = "0.3"
 futures-util.workspace = true
-key-lock = "0.1"
-lazy_static = "1.4"
+lazy_static.workspace = true
 meta-client = { path = "../meta-client" }
 metrics.workspace = true
 moka = { version = "0.11", features = ["future"] }
 parking_lot = "0.12"
-regex = "1.6"
+regex.workspace = true
 serde = "1.0"
 serde_json = "1.0"
 session = { path = "../session" }
 snafu = { version = "0.7", features = ["backtraces"] }
-storage = { path = "../storage" }
 store-api = { path = "../store-api" }
 table = { path = "../table" }
 tokio.workspace = true

src/catalog/src/error.rs

@@ -16,10 +16,10 @@ use std::any::Any;
 use std::fmt::Debug;
 
 use common_error::ext::{BoxedError, ErrorExt};
-use common_error::prelude::{Snafu, StatusCode};
+use common_error::status_code::StatusCode;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
-use snafu::Location;
+use snafu::{Location, Snafu};
 use tokio::task::JoinError;
 
 use crate::DeregisterTableRequest;

src/catalog/src/helper.rs

@@ -1,420 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use std::fmt::{Display, Formatter};
-
-use common_catalog::error::{
-    DeserializeCatalogEntryValueSnafu, Error, InvalidCatalogSnafu, SerializeCatalogEntryValueSnafu,
-};
-use lazy_static::lazy_static;
-use regex::Regex;
-use serde::{Deserialize, Serialize, Serializer};
-use snafu::{ensure, OptionExt, ResultExt};
-use table::metadata::{RawTableInfo, TableId, TableVersion};
-
-pub const CATALOG_KEY_PREFIX: &str = "__c";
-pub const SCHEMA_KEY_PREFIX: &str = "__s";
-pub const TABLE_GLOBAL_KEY_PREFIX: &str = "__tg";
-pub const TABLE_REGIONAL_KEY_PREFIX: &str = "__tr";
-
-const ALPHANUMERICS_NAME_PATTERN: &str = "[a-zA-Z_][a-zA-Z0-9_]*";
-const TABLE_NAME_PATTERN: &str = "[a-zA-Z_:][a-zA-Z0-9_:]*";
-
-lazy_static! {
-    static ref CATALOG_KEY_PATTERN: Regex = Regex::new(&format!(
-        "^{CATALOG_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})$"
-    ))
-    .unwrap();
-}
-
-lazy_static! {
-    static ref SCHEMA_KEY_PATTERN: Regex = Regex::new(&format!(
-        "^{SCHEMA_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})$"
-    ))
-    .unwrap();
-}
-
-lazy_static! {
-    static ref TABLE_GLOBAL_KEY_PATTERN: Regex = Regex::new(&format!(
-        "^{TABLE_GLOBAL_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})-({TABLE_NAME_PATTERN})$"
-    ))
-    .unwrap();
-}
-
-lazy_static! {
-    static ref TABLE_REGIONAL_KEY_PATTERN: Regex = Regex::new(&format!(
-        "^{TABLE_REGIONAL_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})-({TABLE_NAME_PATTERN})-([0-9]+)$"
-    ))
-    .unwrap();
-}
-
-pub fn build_catalog_prefix() -> String {
-    format!("{CATALOG_KEY_PREFIX}-")
-}
-
-pub fn build_schema_prefix(catalog_name: impl AsRef<str>) -> String {
-    format!("{SCHEMA_KEY_PREFIX}-{}-", catalog_name.as_ref())
-}
-
-/// Global table info has only one key across all datanodes so it does not have `node_id` field.
-pub fn build_table_global_prefix(
-    catalog_name: impl AsRef<str>,
-    schema_name: impl AsRef<str>,
-) -> String {
-    format!(
-        "{TABLE_GLOBAL_KEY_PREFIX}-{}-{}-",
-        catalog_name.as_ref(),
-        schema_name.as_ref()
-    )
-}
-
-/// Regional table info varies between datanode, so it contains a `node_id` field.
-pub fn build_table_regional_prefix(
-    catalog_name: impl AsRef<str>,
-    schema_name: impl AsRef<str>,
-) -> String {
-    format!(
-        "{}-{}-{}-",
-        TABLE_REGIONAL_KEY_PREFIX,
-        catalog_name.as_ref(),
-        schema_name.as_ref()
-    )
-}
-
-/// Table global info has only one key across all datanodes so it does not have `node_id` field.
-#[derive(Clone, Hash, Eq, PartialEq)]
-pub struct TableGlobalKey {
-    pub catalog_name: String,
-    pub schema_name: String,
-    pub table_name: String,
-}
-
-impl Display for TableGlobalKey {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.write_str(TABLE_GLOBAL_KEY_PREFIX)?;
-        f.write_str("-")?;
-        f.write_str(&self.catalog_name)?;
-        f.write_str("-")?;
-        f.write_str(&self.schema_name)?;
-        f.write_str("-")?;
-        f.write_str(&self.table_name)
-    }
-}
-
-impl TableGlobalKey {
-    pub fn parse<S: AsRef<str>>(s: S) -> Result<Self, Error> {
-        let key = s.as_ref();
-        let captures = TABLE_GLOBAL_KEY_PATTERN
-            .captures(key)
-            .context(InvalidCatalogSnafu { key })?;
-        ensure!(captures.len() == 4, InvalidCatalogSnafu { key });
-
-        Ok(Self {
-            catalog_name: captures[1].to_string(),
-            schema_name: captures[2].to_string(),
-            table_name: captures[3].to_string(),
-        })
-    }
-
-    pub fn to_raw_key(&self) -> Vec<u8> {
-        self.to_string().into_bytes()
-    }
-
-    pub fn try_from_raw_key(key: &[u8]) -> Result<Self, Error> {
-        Self::parse(String::from_utf8_lossy(key))
-    }
-}
-
-/// Table global info contains necessary info for a datanode to create table regions, including
-/// table id, table meta(schema...), region id allocation across datanodes.
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct TableGlobalValue {
-    /// Id of datanode that created the global table info kv. only for debugging.
-    pub node_id: u64,
-    /// Allocation of region ids across all datanodes.
-    pub regions_id_map: HashMap<u64, Vec<u32>>,
-    pub table_info: RawTableInfo,
-}
-
-impl TableGlobalValue {
-    pub fn table_id(&self) -> TableId {
-        self.table_info.ident.table_id
-    }
-
-    pub fn engine(&self) -> &str {
-        &self.table_info.meta.engine
-    }
-}
-
-/// Table regional info that varies between datanode, so it contains a `node_id` field.
-pub struct TableRegionalKey {
-    pub catalog_name: String,
-    pub schema_name: String,
-    pub table_name: String,
-    pub node_id: u64,
-}
-
-impl Display for TableRegionalKey {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.write_str(TABLE_REGIONAL_KEY_PREFIX)?;
-        f.write_str("-")?;
-        f.write_str(&self.catalog_name)?;
-        f.write_str("-")?;
-        f.write_str(&self.schema_name)?;
-        f.write_str("-")?;
-        f.write_str(&self.table_name)?;
-        f.write_str("-")?;
-        f.serialize_u64(self.node_id)
-    }
-}
-
-impl TableRegionalKey {
-    pub fn parse<S: AsRef<str>>(s: S) -> Result<Self, Error> {
-        let key = s.as_ref();
-        let captures = TABLE_REGIONAL_KEY_PATTERN
-            .captures(key)
-            .context(InvalidCatalogSnafu { key })?;
-        ensure!(captures.len() == 5, InvalidCatalogSnafu { key });
-        let node_id = captures[4]
-            .to_string()
-            .parse()
-            .map_err(|_| InvalidCatalogSnafu { key }.build())?;
-        Ok(Self {
-            catalog_name: captures[1].to_string(),
-            schema_name: captures[2].to_string(),
-            table_name: captures[3].to_string(),
-            node_id,
-        })
-    }
-}
-
-/// Regional table info of specific datanode, including table version on that datanode and
-/// region ids allocated by metasrv.
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct TableRegionalValue {
-    // We can remove the `Option` from the table id once all regional values
-    // stored in meta have table ids.
-    pub table_id: Option<TableId>,
-    pub version: TableVersion,
-    pub regions_ids: Vec<u32>,
-    pub engine_name: Option<String>,
-}
-
-pub struct CatalogKey {
-    pub catalog_name: String,
-}
-
-impl Display for CatalogKey {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.write_str(CATALOG_KEY_PREFIX)?;
-        f.write_str("-")?;
-        f.write_str(&self.catalog_name)
-    }
-}
-
-impl CatalogKey {
-    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-        let key = s.as_ref();
-        let captures = CATALOG_KEY_PATTERN
-            .captures(key)
-            .context(InvalidCatalogSnafu { key })?;
-        ensure!(captures.len() == 2, InvalidCatalogSnafu { key });
-        Ok(Self {
-            catalog_name: captures[1].to_string(),
-        })
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct CatalogValue;
-
-pub struct SchemaKey {
-    pub catalog_name: String,
-    pub schema_name: String,
-}
-
-impl Display for SchemaKey {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.write_str(SCHEMA_KEY_PREFIX)?;
-        f.write_str("-")?;
-        f.write_str(&self.catalog_name)?;
-        f.write_str("-")?;
-        f.write_str(&self.schema_name)
-    }
-}
-
-impl SchemaKey {
-    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-        let key = s.as_ref();
-        let captures = SCHEMA_KEY_PATTERN
-            .captures(key)
-            .context(InvalidCatalogSnafu { key })?;
-        ensure!(captures.len() == 3, InvalidCatalogSnafu { key });
-        Ok(Self {
-            catalog_name: captures[1].to_string(),
-            schema_name: captures[2].to_string(),
-        })
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct SchemaValue;
-
-macro_rules! define_catalog_value {
-    ( $($val_ty: ty), *) => {
-            $(
-                impl $val_ty {
-                    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-                        serde_json::from_str(s.as_ref())
-                            .context(DeserializeCatalogEntryValueSnafu { raw: s.as_ref() })
-                    }
-
-                    pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result<Self, Error> {
-                         Self::parse(&String::from_utf8_lossy(bytes.as_ref()))
-                    }
-
-                    pub fn as_bytes(&self) -> Result<Vec<u8>, Error> {
-                        Ok(serde_json::to_string(self)
-                            .context(SerializeCatalogEntryValueSnafu)?
-                            .into_bytes())
-                    }
-                }
-            )*
-        }
-}
-
-define_catalog_value!(
-    TableRegionalValue,
-    TableGlobalValue,
-    CatalogValue,
-    SchemaValue
-);
-
-#[cfg(test)]
-mod tests {
-    use datatypes::prelude::ConcreteDataType;
-    use datatypes::schema::{ColumnSchema, RawSchema, Schema};
-    use table::metadata::{RawTableMeta, TableIdent, TableType};
-
-    use super::*;
-
-    #[test]
-    fn test_parse_catalog_key() {
-        let key = "__c-C";
-        let catalog_key = CatalogKey::parse(key).unwrap();
-        assert_eq!("C", catalog_key.catalog_name);
-        assert_eq!(key, catalog_key.to_string());
-    }
-
-    #[test]
-    fn test_parse_schema_key() {
-        let key = "__s-C-S";
-        let schema_key = SchemaKey::parse(key).unwrap();
-        assert_eq!("C", schema_key.catalog_name);
-        assert_eq!("S", schema_key.schema_name);
-        assert_eq!(key, schema_key.to_string());
-    }
-
-    #[test]
-    fn test_parse_table_key() {
-        let key = "__tg-C-S-T";
-        let entry = TableGlobalKey::parse(key).unwrap();
-        assert_eq!("C", entry.catalog_name);
-        assert_eq!("S", entry.schema_name);
-        assert_eq!("T", entry.table_name);
-        assert_eq!(key, &entry.to_string());
-    }
-
-    #[test]
-    fn test_build_prefix() {
-        assert_eq!("__c-", build_catalog_prefix());
-        assert_eq!("__s-CATALOG-", build_schema_prefix("CATALOG"));
-        assert_eq!(
-            "__tg-CATALOG-SCHEMA-",
-            build_table_global_prefix("CATALOG", "SCHEMA")
-        );
-    }
-
-    #[test]
-    fn test_serialize_schema() {
-        let schema = Schema::new(vec![ColumnSchema::new(
-            "name",
-            ConcreteDataType::string_datatype(),
-            true,
-        )]);
-
-        let meta = RawTableMeta {
-            schema: RawSchema::from(&schema),
-            engine: "mito".to_string(),
-            created_on: chrono::DateTime::default(),
-            primary_key_indices: vec![0, 1],
-            next_column_id: 3,
-            engine_options: Default::default(),
-            value_indices: vec![2, 3],
-            options: Default::default(),
-            region_numbers: vec![1],
-        };
-
-        let table_info = RawTableInfo {
-            ident: TableIdent {
-                table_id: 42,
-                version: 1,
-            },
-            name: "table_1".to_string(),
-            desc: Some("blah".to_string()),
-            catalog_name: "catalog_1".to_string(),
-            schema_name: "schema_1".to_string(),
-            meta,
-            table_type: TableType::Base,
-        };
-
-        let value = TableGlobalValue {
-            node_id: 0,
-            regions_id_map: HashMap::from([(0, vec![1, 2, 3])]),
-            table_info,
-        };
-        let serialized = serde_json::to_string(&value).unwrap();
-        let deserialized = TableGlobalValue::parse(serialized).unwrap();
-        assert_eq!(value, deserialized);
-    }
-
-    #[test]
-    fn test_table_global_value_compatibility() {
-        let s = r#"{"node_id":1,"regions_id_map":{"1":[0]},"table_info":{"ident":{"table_id":1098,"version":1},"name":"container_cpu_limit","desc":"Created on insertion","catalog_name":"greptime","schema_name":"dd","meta":{"schema":{"column_schemas":[{"name":"container_id","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"container_name","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"docker_image","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"host","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"image_name","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"image_tag","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"interval","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"runtime","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"short_image","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"type","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"dd_value","data_type":{"Float64":{}},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"ts","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":true,"default_constraint":null,"metadata":{"greptime:time_index":"true"}},{"name":"git.repository_url","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}}],"timestamp_index":11,"version":1},"primary_key_indices":[0,1,2,3,4,5,6,7,8,9,12],"value_indices":[10,11],"engine":"mito","next_column_id":12,"region_numbers":[],"engine_options":{},"options":{},"created_on":"1970-01-01T00:00:00Z"},"table_type":"Base"}}"#;
-        let _ = TableGlobalValue::parse(s).unwrap();
-    }
-
-    fn test_valid_table_patterns(table_name: &str) {
-        assert_eq!(
-            table_name,
-            TableGlobalKey::parse(format!("__tg-catalog-schema-{}", table_name))
-                .unwrap()
-                .table_name
-        );
-
-        assert_eq!(
-            table_name,
-            TableRegionalKey::parse(format!("__tr-catalog-schema-{}-0", table_name))
-                .unwrap()
-                .table_name
-        );
-    }
-
-    #[test]
-    fn test_table_name_pattern() {
-        test_valid_table_patterns("cpu:metrics");
-        test_valid_table_patterns(":cpu:metrics");
-    }
-}

src/catalog/src/information_schema.rs

@@ -19,7 +19,7 @@ use std::any::Any;
 use std::sync::{Arc, Weak};
 
 use async_trait::async_trait;
-use common_error::prelude::BoxedError;
+use common_error::ext::BoxedError;
 use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
 use datatypes::schema::SchemaRef;
 use futures_util::StreamExt;

src/catalog/src/information_schema/columns.rs

@@ -18,12 +18,12 @@ use arrow_schema::SchemaRef as ArrowSchemaRef;
 use common_catalog::consts::{
     SEMANTIC_TYPE_FIELD, SEMANTIC_TYPE_PRIMARY_KEY, SEMANTIC_TYPE_TIME_INDEX,
 };
-use common_error::prelude::BoxedError;
+use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
 use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
-use datafusion::datasource::streaming::PartitionStream as DfPartitionStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
 use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
 use datatypes::prelude::{ConcreteDataType, DataType};
 use datatypes::scalars::ScalarVectorBuilder;

src/catalog/src/information_schema/tables.rs

@@ -16,12 +16,12 @@ use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
 use common_catalog::consts::INFORMATION_SCHEMA_NAME;
-use common_error::prelude::BoxedError;
+use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
 use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
-use datafusion::datasource::streaming::PartitionStream as DfPartitionStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
 use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
 use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};

src/catalog/src/lib.rs

@@ -21,7 +21,7 @@ use std::collections::HashMap;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
-use api::v1::meta::{RegionStat, TableName};
+use api::v1::meta::{RegionStat, TableIdent, TableName};
 use common_telemetry::{info, warn};
 use snafu::ResultExt;
 use table::engine::{EngineContext, TableEngineRef};
@@ -32,7 +32,6 @@ use table::TableRef;
 use crate::error::{CreateTableSnafu, Result};
 
 pub mod error;
-pub mod helper;
 pub mod information_schema;
 pub mod local;
 mod metrics;
@@ -226,19 +225,25 @@ pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<Reg
             for table_name in table_names {
                 let Ok(Some(table)) = catalog_manager.table(&catalog_name, &schema_name, &table_name).await else { continue };
 
-                let region_numbers = &table.table_info().meta.region_numbers;
+                let table_info = table.table_info();
+                let region_numbers = &table_info.meta.region_numbers;
                 region_number += region_numbers.len() as u64;
 
-                let engine = &table.table_info().meta.engine;
+                let engine = &table_info.meta.engine;
+                let table_id = table_info.ident.table_id;
 
                 match table.region_stats() {
                     Ok(stats) => {
                         let stats = stats.into_iter().map(|stat| RegionStat {
                             region_id: stat.region_id,
-                            table_name: Some(TableName {
-                                catalog_name: catalog_name.clone(),
-                                schema_name: schema_name.clone(),
-                                table_name: table_name.clone(),
+                            table_ident: Some(TableIdent {
+                                table_id,
+                                table_name: Some(TableName {
+                                    catalog_name: catalog_name.clone(),
+                                    schema_name: schema_name.clone(),
+                                    table_name: table_name.clone(),
+                                }),
+                                engine: engine.clone(),
                             }),
                             approximate_bytes: stat.disk_usage_bytes as i64,
                             attrs: HashMap::from([("engine_name".to_owned(), engine.clone())]),

src/catalog/src/local/manager.rs

@@ -525,8 +525,6 @@ impl CatalogManager for LocalCatalogManager {
     }
 
     async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
-        self.check_state().await?;
-
         let catalog_name = request.create_table_request.catalog_name.clone();
         let schema_name = request.create_table_request.schema_name.clone();
 

src/catalog/src/local/memory.rs

@@ -75,15 +75,7 @@ impl CatalogManager for MemoryCatalogManager {
     }
 
     async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        let catalog = request.catalog.clone();
-        let schema = request.schema.clone();
-        let result = self.register_table_sync(request);
-        increment_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&catalog, &schema)],
-        );
-        result
+        self.register_table_sync(request)
     }
 
     async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
@@ -143,11 +135,7 @@ impl CatalogManager for MemoryCatalogManager {
     }
 
     async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        let registered = self.register_schema_sync(request)?;
-        if registered {
-            increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
-        }
-        Ok(registered)
+        self.register_schema_sync(request)
     }
 
     async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool> {
@@ -263,11 +251,7 @@ impl CatalogManager for MemoryCatalogManager {
     }
 
     async fn register_catalog(&self, name: String) -> Result<bool> {
-        let registered = self.register_catalog_sync(name)?;
-        if registered {
-            increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
-        }
-        Ok(registered)
+        self.register_catalog_sync(name)
     }
 
     fn as_any(&self) -> &dyn Any {
@@ -291,7 +275,15 @@ impl MemoryCatalogManager {
 
     pub fn register_catalog_sync(&self, name: String) -> Result<bool> {
         let mut catalogs = self.catalogs.write().unwrap();
-        Ok(catalogs.insert(name, HashMap::new()).is_some())
+
+        match catalogs.entry(name) {
+            Entry::Vacant(e) => {
+                e.insert(HashMap::new());
+                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
+                Ok(true)
+            }
+            Entry::Occupied(_) => Ok(false),
+        }
     }
 
     pub fn register_schema_sync(&self, request: RegisterSchemaRequest) -> Result<bool> {
@@ -301,11 +293,15 @@ impl MemoryCatalogManager {
             .with_context(|| CatalogNotFoundSnafu {
                 catalog_name: &request.catalog,
             })?;
-        if catalog.contains_key(&request.schema) {
-            return Ok(false);
+
+        match catalog.entry(request.schema) {
+            Entry::Vacant(e) => {
+                e.insert(HashMap::new());
+                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
+                Ok(true)
+            }
+            Entry::Occupied(_) => Ok(false),
         }
-        let _ = catalog.insert(request.schema, HashMap::new());
-        Ok(true)
     }
 
     pub fn register_table_sync(&self, request: RegisterTableRequest) -> Result<bool> {
@@ -327,8 +323,13 @@ impl MemoryCatalogManager {
             }
             .fail();
         }
-
-        Ok(schema.insert(request.table_name, request.table).is_none())
+        schema.insert(request.table_name, request.table);
+        increment_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
+        Ok(true)
     }
 
     #[cfg(any(test, feature = "testing"))]
@@ -355,7 +356,7 @@ pub fn new_memory_catalog_manager() -> Result<Arc<MemoryCatalogManager>> {
 mod tests {
     use common_catalog::consts::*;
     use common_error::ext::ErrorExt;
-    use common_error::prelude::StatusCode;
+    use common_error::status_code::StatusCode;
     use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
 
     use super::*;

src/catalog/src/remote/client.rs

@@ -17,7 +17,7 @@ use std::fmt::Debug;
 use std::sync::Arc;
 use std::time::Duration;
 
-use common_error::prelude::BoxedError;
+use common_error::ext::BoxedError;
 use common_meta::error::Error::{CacheNotGet, GetKvCache};
 use common_meta::error::{CacheNotGetSnafu, Error, MetaSrvSnafu, Result};
 use common_meta::kv_backend::{KvBackend, KvBackendRef, TxnService};
@@ -58,35 +58,12 @@ impl KvBackend for CachedMetaKvBackend {
         &self.name
     }
 
-    async fn range(&self, req: RangeRequest) -> Result<RangeResponse> {
-        self.kv_backend.range(req).await
+    fn as_any(&self) -> &dyn Any {
+        self
     }
 
-    async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
-        let _timer = timer!(METRIC_CATALOG_KV_GET);
-
-        let init = async {
-            let _timer = timer!(METRIC_CATALOG_KV_REMOTE_GET);
-            self.kv_backend.get(key).await.map(|val| {
-                val.with_context(|| CacheNotGetSnafu {
-                    key: String::from_utf8_lossy(key),
-                })
-            })?
-        };
-
-        // currently moka doesn't have `optionally_try_get_with_by_ref`
-        // TODO(fys): change to moka method when available
-        // https://github.com/moka-rs/moka/issues/254
-        match self.cache.try_get_with_by_ref(key, init).await {
-            Ok(val) => Ok(Some(val)),
-            Err(e) => match e.as_ref() {
-                CacheNotGet { .. } => Ok(None),
-                _ => Err(e),
-            },
-        }
-        .map_err(|e| GetKvCache {
-            err_msg: e.to_string(),
-        })
+    async fn range(&self, req: RangeRequest) -> Result<RangeResponse> {
+        self.kv_backend.range(req).await
     }
 
     async fn put(&self, req: PutRequest) -> Result<PutResponse> {
@@ -119,6 +96,22 @@ impl KvBackend for CachedMetaKvBackend {
         resp
     }
 
+    async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
+        self.kv_backend.batch_get(req).await
+    }
+
+    async fn compare_and_put(&self, req: CompareAndPutRequest) -> Result<CompareAndPutResponse> {
+        let key = &req.key.clone();
+
+        let ret = self.kv_backend.compare_and_put(req).await;
+
+        if ret.is_ok() {
+            self.invalidate_key(key).await;
+        }
+
+        ret
+    }
+
     async fn delete_range(&self, mut req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
         let prev_kv = req.prev_kv;
 
@@ -159,22 +152,6 @@ impl KvBackend for CachedMetaKvBackend {
         }
     }
 
-    async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
-        self.kv_backend.batch_get(req).await
-    }
-
-    async fn compare_and_put(&self, req: CompareAndPutRequest) -> Result<CompareAndPutResponse> {
-        let key = &req.key.clone();
-
-        let ret = self.kv_backend.compare_and_put(req).await;
-
-        if ret.is_ok() {
-            self.invalidate_key(key).await;
-        }
-
-        ret
-    }
-
     async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
         let from_key = &req.from_key.clone();
         let to_key = &req.to_key.clone();
@@ -189,8 +166,31 @@ impl KvBackend for CachedMetaKvBackend {
         ret
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
+    async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
+        let _timer = timer!(METRIC_CATALOG_KV_GET);
+
+        let init = async {
+            let _timer = timer!(METRIC_CATALOG_KV_REMOTE_GET);
+            self.kv_backend.get(key).await.map(|val| {
+                val.with_context(|| CacheNotGetSnafu {
+                    key: String::from_utf8_lossy(key),
+                })
+            })?
+        };
+
+        // currently moka doesn't have `optionally_try_get_with_by_ref`
+        // TODO(fys): change to moka method when available
+        // https://github.com/moka-rs/moka/issues/254
+        match self.cache.try_get_with_by_ref(key, init).await {
+            Ok(val) => Ok(Some(val)),
+            Err(e) => match e.as_ref() {
+                CacheNotGet { .. } => Ok(None),
+                _ => Err(e),
+            },
+        }
+        .map_err(|e| GetKvCache {
+            err_msg: e.to_string(),
+        })
     }
 }
 

src/catalog/src/remote/region_alive_keeper.rs

@@ -312,6 +312,10 @@ impl RegionAliveKeeper {
         }
         deadline
     }
+
+    pub fn table_ident(&self) -> &TableIdent {
+        &self.table_ident
+    }
 }
 
 #[derive(Debug)]

src/catalog/tests/remote_catalog_tests.rs

@@ -22,13 +22,14 @@ mod tests {
     use std::time::Duration;
 
     use catalog::error::Error;
-    use catalog::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
     use catalog::remote::mock::MockTableEngine;
     use catalog::remote::region_alive_keeper::RegionAliveKeepers;
     use catalog::remote::{CachedMetaKvBackend, RemoteCatalogManager};
     use catalog::{CatalogManager, RegisterSchemaRequest, RegisterTableRequest};
     use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
+    use common_meta::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
     use common_meta::ident::TableIdent;
+    use common_meta::key::TableMetadataManager;
     use common_meta::kv_backend::memory::MemoryKvBackend;
     use common_meta::kv_backend::KvBackend;
     use common_meta::rpc::store::{CompareAndPutRequest, PutRequest, RangeRequest};
@@ -155,6 +156,7 @@ mod tests {
             node_id,
             cached_backend.clone(),
             region_alive_keepers.clone(),
+            Arc::new(TableMetadataManager::new(cached_backend)),
         );
         catalog_manager.start().await.unwrap();
 

src/client/Cargo.toml

@@ -15,7 +15,6 @@ common-base = { path = "../common/base" }
 common-catalog = { path = "../common/catalog" }
 common-error = { path = "../common/error" }
 common-grpc = { path = "../common/grpc" }
-common-grpc-expr = { path = "../common/grpc-expr" }
 common-query = { path = "../common/query" }
 common-recordbatch = { path = "../common/recordbatch" }
 common-time = { path = "../common/time" }
@@ -35,6 +34,7 @@ tokio.workspace = true
 tonic.workspace = true
 
 [dev-dependencies]
+common-grpc-expr = { path = "../common/grpc-expr" }
 datanode = { path = "../datanode" }
 derive-new = "0.5"
 substrait = { path = "../common/substrait" }

src/client/examples/logical.rs

@@ -73,7 +73,7 @@ async fn run() {
 
     let logical = mock_logical_plan();
     event!(Level::INFO, "plan size: {:#?}", logical.len());
-    let result = db.logical_plan(logical).await.unwrap();
+    let result = db.logical_plan(logical, None).await.unwrap();
 
     event!(Level::INFO, "result: {:#?}", result);
 }

src/client/examples/stream_ingest.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::column::*;
 use api::v1::*;
 use client::{Client, Database, DEFAULT_SCHEMA_NAME};
 use derive_new::new;

src/client/src/client_manager.rs

@@ -13,12 +13,10 @@
 // limitations under the License.
 
 use std::fmt::{Debug, Formatter};
-use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
 use common_meta::peer::Peer;
-use common_telemetry::info;
 use moka::future::{Cache, CacheBuilder};
 
 use crate::Client;
@@ -26,21 +24,11 @@ use crate::Client;
 pub struct DatanodeClients {
     channel_manager: ChannelManager,
     clients: Cache<Peer, Client>,
-    started: Arc<Mutex<bool>>,
 }
 
 impl Default for DatanodeClients {
     fn default() -> Self {
-        let config = ChannelConfig::new().timeout(Duration::from_secs(8));
-
-        Self {
-            channel_manager: ChannelManager::with_config(config),
-            clients: CacheBuilder::new(1024)
-                .time_to_live(Duration::from_secs(30 * 60))
-                .time_to_idle(Duration::from_secs(5 * 60))
-                .build(),
-            started: Arc::new(Mutex::new(false)),
-        }
+        Self::new(ChannelConfig::new())
     }
 }
 
@@ -53,16 +41,14 @@ impl Debug for DatanodeClients {
 }
 
 impl DatanodeClients {
-    pub fn start(&self) {
-        let mut started = self.started.lock().unwrap();
-        if *started {
-            return;
+    pub fn new(config: ChannelConfig) -> Self {
+        Self {
+            channel_manager: ChannelManager::with_config(config),
+            clients: CacheBuilder::new(1024)
+                .time_to_live(Duration::from_secs(30 * 60))
+                .time_to_idle(Duration::from_secs(5 * 60))
+                .build(),
         }
-
-        self.channel_manager.start_channel_recycle();
-
-        info!("Datanode clients manager is started!");
-        *started = true;
     }
 
     pub async fn get_client(&self, datanode: &Peer) -> Client {

src/client/src/database.rs

@@ -17,12 +17,12 @@ use api::v1::ddl_request::Expr as DdlExpr;
 use api::v1::greptime_request::Request;
 use api::v1::query_request::Query;
 use api::v1::{
-    greptime_response, AffectedRows, AlterExpr, AuthHeader, CreateTableExpr, DdlRequest,
-    DeleteRequest, DropTableExpr, FlushTableExpr, GreptimeRequest, InsertRequests, PromRangeQuery,
-    QueryRequest, RequestHeader,
+    AlterExpr, AuthHeader, CompactTableExpr, CreateTableExpr, DdlRequest, DeleteRequest,
+    DropTableExpr, FlushTableExpr, GreptimeRequest, InsertRequests, PromRangeQuery, QueryRequest,
+    RequestHeader, TruncateTableExpr,
 };
 use arrow_flight::{FlightData, Ticket};
-use common_error::prelude::*;
+use common_error::ext::{BoxedError, ErrorExt};
 use common_grpc::flight::{flight_messages_to_recordbatches, FlightDecoder, FlightMessage};
 use common_query::Output;
 use common_telemetry::{logging, timer};
@@ -30,10 +30,8 @@ use futures_util::{TryFutureExt, TryStreamExt};
 use prost::Message;
 use snafu::{ensure, ResultExt};
 
-use crate::error::{
-    ConvertFlightDataSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
-};
-use crate::{error, metrics, Client, Result, StreamInserter};
+use crate::error::{ConvertFlightDataSnafu, IllegalFlightMessagesSnafu, ServerSnafu};
+use crate::{error, from_grpc_response, metrics, Client, Result, StreamInserter};
 
 #[derive(Clone, Debug, Default)]
 pub struct Database {
@@ -141,27 +139,21 @@ impl Database {
 
     async fn handle(&self, request: Request) -> Result<u32> {
         let mut client = self.client.make_database_client()?.inner;
-        let request = self.to_rpc_request(request);
-        let response = client
-            .handle(request)
-            .await?
-            .into_inner()
-            .response
-            .context(IllegalDatabaseResponseSnafu {
-                err_msg: "GreptimeResponse is empty",
-            })?;
-        let greptime_response::Response::AffectedRows(AffectedRows { value }) = response;
-        Ok(value)
+        let request = self.to_rpc_request(request, None);
+        let response = client.handle(request).await?.into_inner();
+        from_grpc_response(response)
     }
 
     #[inline]
-    fn to_rpc_request(&self, request: Request) -> GreptimeRequest {
+    fn to_rpc_request(&self, request: Request, trace_id: Option<u64>) -> GreptimeRequest {
         GreptimeRequest {
             header: Some(RequestHeader {
                 catalog: self.catalog.clone(),
                 schema: self.schema.clone(),
                 authorization: self.ctx.auth_header.clone(),
                 dbname: self.dbname.clone(),
+                trace_id,
+                span_id: None,
             }),
             request: Some(request),
         }
@@ -169,17 +161,27 @@ impl Database {
 
     pub async fn sql(&self, sql: &str) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_SQL);
-        self.do_get(Request::Query(QueryRequest {
-            query: Some(Query::Sql(sql.to_string())),
-        }))
+        self.do_get(
+            Request::Query(QueryRequest {
+                query: Some(Query::Sql(sql.to_string())),
+            }),
+            None,
+        )
         .await
     }
 
-    pub async fn logical_plan(&self, logical_plan: Vec<u8>) -> Result<Output> {
+    pub async fn logical_plan(
+        &self,
+        logical_plan: Vec<u8>,
+        trace_id: Option<u64>,
+    ) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_LOGICAL_PLAN);
-        self.do_get(Request::Query(QueryRequest {
-            query: Some(Query::LogicalPlan(logical_plan)),
-        }))
+        self.do_get(
+            Request::Query(QueryRequest {
+                query: Some(Query::LogicalPlan(logical_plan)),
+            }),
+            trace_id,
+        )
         .await
     }
 
@@ -191,53 +193,90 @@ impl Database {
         step: &str,
     ) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_PROMQL_RANGE_QUERY);
-        self.do_get(Request::Query(QueryRequest {
-            query: Some(Query::PromRangeQuery(PromRangeQuery {
-                query: promql.to_string(),
-                start: start.to_string(),
-                end: end.to_string(),
-                step: step.to_string(),
-            })),
-        }))
+        self.do_get(
+            Request::Query(QueryRequest {
+                query: Some(Query::PromRangeQuery(PromRangeQuery {
+                    query: promql.to_string(),
+                    start: start.to_string(),
+                    end: end.to_string(),
+                    step: step.to_string(),
+                })),
+            }),
+            None,
+        )
         .await
     }
 
     pub async fn create(&self, expr: CreateTableExpr) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_CREATE_TABLE);
-        self.do_get(Request::Ddl(DdlRequest {
-            expr: Some(DdlExpr::CreateTable(expr)),
-        }))
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::CreateTable(expr)),
+            }),
+            None,
+        )
         .await
     }
 
     pub async fn alter(&self, expr: AlterExpr) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_ALTER);
-        self.do_get(Request::Ddl(DdlRequest {
-            expr: Some(DdlExpr::Alter(expr)),
-        }))
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::Alter(expr)),
+            }),
+            None,
+        )
         .await
     }
 
     pub async fn drop_table(&self, expr: DropTableExpr) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_DROP_TABLE);
-        self.do_get(Request::Ddl(DdlRequest {
-            expr: Some(DdlExpr::DropTable(expr)),
-        }))
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::DropTable(expr)),
+            }),
+            None,
+        )
         .await
     }
 
     pub async fn flush_table(&self, expr: FlushTableExpr) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_FLUSH_TABLE);
-        self.do_get(Request::Ddl(DdlRequest {
-            expr: Some(DdlExpr::FlushTable(expr)),
-        }))
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::FlushTable(expr)),
+            }),
+            None,
+        )
         .await
     }
 
-    async fn do_get(&self, request: Request) -> Result<Output> {
+    pub async fn compact_table(&self, expr: CompactTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_COMPACT_TABLE);
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::CompactTable(expr)),
+            }),
+            None,
+        )
+        .await
+    }
+
+    pub async fn truncate_table(&self, expr: TruncateTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_TRUNCATE_TABLE);
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::TruncateTable(expr)),
+            }),
+            None,
+        )
+        .await
+    }
+
+    async fn do_get(&self, request: Request, trace_id: Option<u64>) -> Result<Output> {
         // FIXME(paomian): should be added some labels for metrics
         let _timer = timer!(metrics::METRIC_GRPC_DO_GET);
-        let request = self.to_rpc_request(request);
+        let request = self.to_rpc_request(request, trace_id);
         let request = Ticket {
             ticket: request.encode_to_vec().into(),
         };
@@ -254,7 +293,7 @@ impl Database {
                 let e: error::Error = e.into();
                 let code = e.status_code();
                 let msg = e.to_string();
-                error::ServerSnafu { code, msg }
+                ServerSnafu { code, msg }
                     .fail::<()>()
                     .map_err(BoxedError::new)
                     .context(error::FlightGetSnafu {

src/client/src/error.rs

@@ -15,8 +15,10 @@
 use std::any::Any;
 use std::str::FromStr;
 
-use common_error::prelude::*;
-use snafu::Location;
+use common_error::ext::{BoxedError, ErrorExt};
+use common_error::status_code::StatusCode;
+use common_error::{INNER_ERROR_CODE, INNER_ERROR_MSG};
+use snafu::{Location, Snafu};
 use tonic::{Code, Status};
 
 #[derive(Debug, Snafu)]

src/client/src/lib.rs

@@ -21,9 +21,42 @@ mod metrics;
 mod stream_insert;
 
 pub use api;
+use api::v1::greptime_response::Response;
+use api::v1::{AffectedRows, GreptimeResponse};
 pub use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use common_error::status_code::StatusCode;
+use snafu::OptionExt;
 
 pub use self::client::Client;
 pub use self::database::Database;
 pub use self::error::{Error, Result};
 pub use self::stream_insert::StreamInserter;
+use crate::error::{IllegalDatabaseResponseSnafu, ServerSnafu};
+
+pub fn from_grpc_response(response: GreptimeResponse) -> Result<u32> {
+    let header = response.header.context(IllegalDatabaseResponseSnafu {
+        err_msg: "missing header",
+    })?;
+    let status = header.status.context(IllegalDatabaseResponseSnafu {
+        err_msg: "missing status",
+    })?;
+
+    if StatusCode::is_success(status.status_code) {
+        let res = response.response.context(IllegalDatabaseResponseSnafu {
+            err_msg: "missing response",
+        })?;
+        match res {
+            Response::AffectedRows(AffectedRows { value }) => Ok(value),
+        }
+    } else {
+        let status_code =
+            StatusCode::from_u32(status.status_code).context(IllegalDatabaseResponseSnafu {
+                err_msg: format!("invalid status: {:?}", status),
+            })?;
+        ServerSnafu {
+            code: status_code,
+            msg: status.err_msg,
+        }
+        .fail()
+    }
+}

src/client/src/metrics.rs

@@ -22,4 +22,6 @@ pub const METRIC_GRPC_LOGICAL_PLAN: &str = "grpc.logical_plan";
 pub const METRIC_GRPC_ALTER: &str = "grpc.alter";
 pub const METRIC_GRPC_DROP_TABLE: &str = "grpc.drop_table";
 pub const METRIC_GRPC_FLUSH_TABLE: &str = "grpc.flush_table";
+pub const METRIC_GRPC_COMPACT_TABLE: &str = "grpc.compact_table";
+pub const METRIC_GRPC_TRUNCATE_TABLE: &str = "grpc.truncate_table";
 pub const METRIC_GRPC_DO_GET: &str = "grpc.do_get";

src/client/src/stream_insert.rs

@@ -15,17 +15,16 @@
 use api::v1::greptime_database_client::GreptimeDatabaseClient;
 use api::v1::greptime_request::Request;
 use api::v1::{
-    greptime_response, AffectedRows, AuthHeader, GreptimeRequest, GreptimeResponse, InsertRequest,
-    InsertRequests, RequestHeader,
+    AuthHeader, GreptimeRequest, GreptimeResponse, InsertRequest, InsertRequests, RequestHeader,
 };
-use snafu::OptionExt;
 use tokio::sync::mpsc;
 use tokio::task::JoinHandle;
 use tokio_stream::wrappers::ReceiverStream;
 use tonic::transport::Channel;
 use tonic::{Response, Status};
 
-use crate::error::{self, IllegalDatabaseResponseSnafu, Result};
+use crate::error::{self, Result};
+use crate::from_grpc_response;
 
 /// A structure that provides some methods for streaming data insert.
 ///
@@ -89,17 +88,8 @@ impl StreamInserter {
         drop(self.sender);
 
         let response = self.join.await.unwrap()?;
-
-        let response = response
-            .into_inner()
-            .response
-            .context(IllegalDatabaseResponseSnafu {
-                err_msg: "GreptimeResponse is empty",
-            })?;
-
-        let greptime_response::Response::AffectedRows(AffectedRows { value }) = response;
-
-        Ok(value)
+        let response = response.into_inner();
+        from_grpc_response(response)
     }
 
     fn to_rpc_request(&self, request: Request) -> GreptimeRequest {

src/cmd/Cargo.toml

@@ -13,22 +13,31 @@ path = "src/bin/greptime.rs"
 default = ["metrics-process"]
 tokio-console = ["common-telemetry/tokio-console"]
 metrics-process = ["servers/metrics-process"]
+greptimedb-telemetry = [
+    "datanode/greptimedb-telemetry",
+    "meta-srv/greptimedb-telemetry",
+]
 
 [dependencies]
 anymap = "1.0.0-beta.2"
+async-trait.workspace = true
 catalog = { path = "../catalog" }
+chrono.workspace = true
 clap = { version = "3.1", features = ["derive"] }
 client = { path = "../client" }
 common-base = { path = "../common/base" }
 common-error = { path = "../common/error" }
 common-query = { path = "../common/query" }
+common-meta = { path = "../common/meta" }
 common-recordbatch = { path = "../common/recordbatch" }
 common-telemetry = { path = "../common/telemetry", features = [
     "deadlock_detection",
 ] }
 config = "0.13"
 datanode = { path = "../datanode" }
+datatypes = { path = "../datatypes" }
 either = "1.8"
+etcd-client.workspace = true
 frontend = { path = "../frontend" }
 futures.workspace = true
 meta-client = { path = "../meta-client" }
@@ -37,12 +46,14 @@ metrics.workspace = true
 nu-ansi-term = "0.46"
 partition = { path = "../partition" }
 query = { path = "../query" }
+rand.workspace = true
 rustyline = "10.1"
 serde.workspace = true
 servers = { path = "../servers" }
 session = { path = "../session" }
 snafu.workspace = true
 substrait = { path = "../common/substrait" }
+table = { path = "../table" }
 tikv-jemallocator = "0.5"
 tokio.workspace = true
 
@@ -51,7 +62,7 @@ common-test-util = { path = "../common/test-util" }
 rexpect = "0.5"
 temp-env = "0.3"
 serde.workspace = true
-toml = "0.5"
+toml.workspace = true
 
 [build-dependencies]
-build-data = "0.1.4"
+common-version = { path = "../common/version" }

src/cmd/build.rs

@@ -12,22 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-const DEFAULT_VALUE: &str = "unknown";
 fn main() {
-    println!(
-        "cargo:rustc-env=GIT_COMMIT={}",
-        build_data::get_git_commit().unwrap_or_else(|_| DEFAULT_VALUE.to_string())
-    );
-    println!(
-        "cargo:rustc-env=GIT_COMMIT_SHORT={}",
-        build_data::get_git_commit_short().unwrap_or_else(|_| DEFAULT_VALUE.to_string())
-    );
-    println!(
-        "cargo:rustc-env=GIT_BRANCH={}",
-        build_data::get_git_branch().unwrap_or_else(|_| DEFAULT_VALUE.to_string())
-    );
-    println!(
-        "cargo:rustc-env=GIT_DIRTY={}",
-        build_data::get_git_dirty().map_or(DEFAULT_VALUE.to_string(), |v| v.to_string())
-    );
+    common_version::setup_git_versions();
 }

src/cmd/src/cli.rs

@@ -12,24 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod bench;
 mod cmd;
 mod helper;
 mod repl;
+mod upgrade;
 
+use async_trait::async_trait;
+use bench::BenchTableMetadataCommand;
 use clap::Parser;
 use common_telemetry::logging::LoggingOptions;
 pub use repl::Repl;
+use upgrade::UpgradeCommand;
 
 use crate::error::Result;
 use crate::options::{Options, TopLevelOptions};
 
-pub struct Instance {
-    repl: Repl,
+#[async_trait]
+pub trait Tool {
+    async fn do_work(&self) -> Result<()>;
+}
+
+pub enum Instance {
+    Repl(Repl),
+    Tool(Box<dyn Tool>),
 }
 
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
-        self.repl.run().await
+        match self {
+            Instance::Repl(repl) => repl.run().await,
+            Instance::Tool(tool) => tool.do_work().await,
+        }
     }
 
     pub async fn stop(&self) -> Result<()> {
@@ -63,12 +77,16 @@ impl Command {
 #[derive(Parser)]
 enum SubCommand {
     Attach(AttachCommand),
+    Upgrade(UpgradeCommand),
+    Bench(BenchTableMetadataCommand),
 }
 
 impl SubCommand {
     async fn build(self) -> Result<Instance> {
         match self {
             SubCommand::Attach(cmd) => cmd.build().await,
+            SubCommand::Upgrade(cmd) => cmd.build().await,
+            SubCommand::Bench(cmd) => cmd.build().await,
         }
     }
 }
@@ -86,7 +104,7 @@ pub(crate) struct AttachCommand {
 impl AttachCommand {
     async fn build(self) -> Result<Instance> {
         let repl = Repl::try_new(&self).await?;
-        Ok(Instance { repl })
+        Ok(Instance::Repl(repl))
     }
 }
 

src/cmd/src/cli/bench.rs

@@ -0,0 +1,199 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod datanode_table;
+mod table_info;
+mod table_name;
+mod table_region;
+
+use std::future::Future;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use async_trait::async_trait;
+use clap::Parser;
+use common_meta::key::table_region::RegionDistribution;
+use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
+use common_meta::table_name::TableName;
+use common_telemetry::info;
+use datatypes::data_type::ConcreteDataType;
+use datatypes::schema::{ColumnSchema, RawSchema};
+use meta_srv::service::store::etcd::EtcdStore;
+use meta_srv::service::store::kv::KvBackendAdapter;
+use rand::prelude::SliceRandom;
+use table::metadata::{RawTableInfo, RawTableMeta, TableId, TableIdent, TableType};
+
+use crate::cli::bench::datanode_table::DatanodeTableBencher;
+use crate::cli::bench::table_info::TableInfoBencher;
+use crate::cli::bench::table_name::TableNameBencher;
+use crate::cli::bench::table_region::TableRegionBencher;
+use crate::cli::{Instance, Tool};
+use crate::error::Result;
+
+async fn bench<F, Fut>(desc: &str, f: F, count: u32)
+where
+    F: Fn(u32) -> Fut,
+    Fut: Future<Output = ()>,
+{
+    let mut total = Duration::default();
+
+    for i in 1..=count {
+        let start = Instant::now();
+
+        f(i).await;
+
+        total += start.elapsed();
+    }
+
+    let cost = total.as_millis() as f64 / count as f64;
+    info!("{desc}, average operation cost: {cost:.2} ms");
+}
+
+async fn bench_self_recorded<F, Fut>(desc: &str, f: F, count: u32)
+where
+    F: Fn(u32) -> Fut,
+    Fut: Future<Output = Duration>,
+{
+    let mut total = Duration::default();
+
+    for i in 1..=count {
+        total += f(i).await;
+    }
+
+    let cost = total.as_millis() as f64 / count as f64;
+    info!("{desc}, average operation cost: {cost:.2} ms");
+}
+
+#[derive(Debug, Default, Parser)]
+pub struct BenchTableMetadataCommand {
+    #[clap(long)]
+    etcd_addr: String,
+    #[clap(long)]
+    count: u32,
+}
+
+impl BenchTableMetadataCommand {
+    pub async fn build(&self) -> Result<Instance> {
+        let etcd_store = EtcdStore::with_endpoints([&self.etcd_addr]).await.unwrap();
+
+        let table_metadata_manager = Arc::new(TableMetadataManager::new(KvBackendAdapter::wrap(
+            etcd_store,
+        )));
+
+        let tool = BenchTableMetadata {
+            table_metadata_manager,
+            count: self.count,
+        };
+        Ok(Instance::Tool(Box::new(tool)))
+    }
+}
+
+struct BenchTableMetadata {
+    table_metadata_manager: TableMetadataManagerRef,
+    count: u32,
+}
+
+#[async_trait]
+impl Tool for BenchTableMetadata {
+    async fn do_work(&self) -> Result<()> {
+        info!("Start benching table name manager ...");
+        TableNameBencher::new(self.table_metadata_manager.table_name_manager(), self.count)
+            .start()
+            .await;
+
+        info!("Start benching table info manager ...");
+        TableInfoBencher::new(self.table_metadata_manager.table_info_manager(), self.count)
+            .start()
+            .await;
+
+        info!("Start benching table region manager ...");
+        TableRegionBencher::new(
+            self.table_metadata_manager.table_region_manager(),
+            self.count,
+        )
+        .start()
+        .await;
+
+        info!("Start benching datanode table manager ...");
+        DatanodeTableBencher::new(
+            self.table_metadata_manager.datanode_table_manager(),
+            self.count,
+        )
+        .start()
+        .await;
+        Ok(())
+    }
+}
+
+fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
+    let columns = 100;
+    let mut column_schemas = Vec::with_capacity(columns);
+    column_schemas.push(
+        ColumnSchema::new(
+            "ts",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            true,
+        )
+        .with_time_index(true),
+    );
+
+    for i in 1..columns {
+        let column_name = format!("my_column_{i}");
+        column_schemas.push(ColumnSchema::new(
+            column_name,
+            ConcreteDataType::string_datatype(),
+            true,
+        ));
+    }
+
+    let meta = RawTableMeta {
+        schema: RawSchema::new(column_schemas),
+        engine: "mito".to_string(),
+        created_on: chrono::DateTime::default(),
+        primary_key_indices: vec![],
+        next_column_id: columns as u32 + 1,
+        engine_options: Default::default(),
+        value_indices: vec![],
+        options: Default::default(),
+        region_numbers: (1..=100).collect(),
+        partition_key_indices: vec![],
+    };
+
+    RawTableInfo {
+        ident: TableIdent {
+            table_id,
+            version: 1,
+        },
+        name: table_name.table_name,
+        desc: Some("blah".to_string()),
+        catalog_name: table_name.catalog_name,
+        schema_name: table_name.schema_name,
+        meta,
+        table_type: TableType::Base,
+    }
+}
+
+fn create_region_distribution() -> RegionDistribution {
+    let mut regions = (1..=100).collect::<Vec<u32>>();
+    regions.shuffle(&mut rand::thread_rng());
+
+    let mut region_distribution = RegionDistribution::new();
+    for datanode_id in 0..10 {
+        region_distribution.insert(
+            datanode_id as u64,
+            regions[datanode_id * 10..(datanode_id + 1) * 10].to_vec(),
+        );
+    }
+    region_distribution
+}

src/cmd/src/cli/bench/datanode_table.rs

@@ -0,0 +1,131 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableManager};
+
+use super::bench;
+
+pub struct DatanodeTableBencher<'a> {
+    datanode_table_manager: &'a DatanodeTableManager,
+    count: u32,
+}
+
+impl<'a> DatanodeTableBencher<'a> {
+    pub fn new(datanode_table_manager: &'a DatanodeTableManager, count: u32) -> Self {
+        Self {
+            datanode_table_manager,
+            count,
+        }
+    }
+
+    pub async fn start(&self) {
+        self.bench_create().await;
+        self.bench_get().await;
+        self.bench_move_region().await;
+        self.bench_tables().await;
+        self.bench_remove().await;
+    }
+
+    async fn bench_create(&self) {
+        let desc = format!(
+            "DatanodeTableBencher: create {} datanode table keys",
+            self.count
+        );
+        bench(
+            &desc,
+            |i| async move {
+                self.datanode_table_manager
+                    .create(1, i, vec![1, 2, 3, 4])
+                    .await
+                    .unwrap();
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_get(&self) {
+        let desc = format!(
+            "DatanodeTableBencher: get {} datanode table keys",
+            self.count
+        );
+        bench(
+            &desc,
+            |i| async move {
+                let key = DatanodeTableKey::new(1, i);
+                assert!(self
+                    .datanode_table_manager
+                    .get(&key)
+                    .await
+                    .unwrap()
+                    .is_some());
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_move_region(&self) {
+        let desc = format!(
+            "DatanodeTableBencher: move {} datanode table regions",
+            self.count
+        );
+        bench(
+            &desc,
+            |i| async move {
+                self.datanode_table_manager
+                    .move_region(1, 2, i, 1)
+                    .await
+                    .unwrap();
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_tables(&self) {
+        let desc = format!(
+            "DatanodeTableBencher: list {} datanode table keys",
+            self.count
+        );
+        bench(
+            &desc,
+            |_| async move {
+                assert!(!self
+                    .datanode_table_manager
+                    .tables(1)
+                    .await
+                    .unwrap()
+                    .is_empty());
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_remove(&self) {
+        let desc = format!(
+            "DatanodeTableBencher: remove {} datanode table keys",
+            self.count
+        );
+        bench(
+            &desc,
+            |i| async move {
+                self.datanode_table_manager.remove(1, i).await.unwrap();
+            },
+            self.count,
+        )
+        .await;
+    }
+}

src/cmd/src/cli/bench/table_info.rs

@@ -0,0 +1,111 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Instant;
+
+use common_meta::key::table_info::TableInfoManager;
+use common_meta::table_name::TableName;
+
+use super::{bench, bench_self_recorded, create_table_info};
+
+pub struct TableInfoBencher<'a> {
+    table_info_manager: &'a TableInfoManager,
+    count: u32,
+}
+
+impl<'a> TableInfoBencher<'a> {
+    pub fn new(table_info_manager: &'a TableInfoManager, count: u32) -> Self {
+        Self {
+            table_info_manager,
+            count,
+        }
+    }
+
+    pub async fn start(&self) {
+        self.bench_create().await;
+        self.bench_get().await;
+        self.bench_compare_and_put().await;
+        self.bench_remove().await;
+    }
+
+    async fn bench_create(&self) {
+        let desc = format!("TableInfoBencher: create {} table infos", self.count);
+        bench(
+            &desc,
+            |i| async move {
+                let table_name = format!("bench_table_name_{}", i);
+                let table_name = TableName::new("bench_catalog", "bench_schema", table_name);
+                let table_info = create_table_info(i, table_name);
+                self.table_info_manager
+                    .create(i, &table_info)
+                    .await
+                    .unwrap();
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_get(&self) {
+        let desc = format!("TableInfoBencher: get {} table infos", self.count);
+        bench(
+            &desc,
+            |i| async move {
+                assert!(self.table_info_manager.get(i).await.unwrap().is_some());
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_compare_and_put(&self) {
+        let desc = format!(
+            "TableInfoBencher: compare_and_put {} table infos",
+            self.count
+        );
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let table_info_value = self.table_info_manager.get(i).await.unwrap().unwrap();
+
+                let mut new_table_info = table_info_value.table_info.clone();
+                new_table_info.ident.version += 1;
+
+                let start = Instant::now();
+
+                self.table_info_manager
+                    .compare_and_put(i, Some(table_info_value), new_table_info)
+                    .await
+                    .unwrap()
+                    .unwrap();
+
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_remove(&self) {
+        let desc = format!("TableInfoBencher: remove {} table infos", self.count);
+        bench(
+            &desc,
+            |i| async move {
+                self.table_info_manager.remove(i).await.unwrap();
+            },
+            self.count,
+        )
+        .await;
+    }
+}

src/cmd/src/cli/bench/table_name.rs

@@ -0,0 +1,131 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_meta::key::table_name::{TableNameKey, TableNameManager};
+
+use super::bench;
+
+pub struct TableNameBencher<'a> {
+    table_name_manager: &'a TableNameManager,
+    count: u32,
+}
+
+impl<'a> TableNameBencher<'a> {
+    pub fn new(table_name_manager: &'a TableNameManager, count: u32) -> Self {
+        Self {
+            table_name_manager,
+            count,
+        }
+    }
+
+    pub async fn start(&self) {
+        self.bench_create().await;
+        self.bench_rename().await;
+        self.bench_get().await;
+        self.bench_tables().await;
+        self.bench_remove().await;
+    }
+
+    async fn bench_create(&self) {
+        let desc = format!("TableNameBencher: create {} table names", self.count);
+        bench(
+            &desc,
+            |i| async move {
+                let table_name = format!("bench_table_name_{}", i);
+                let table_name_key = create_table_name_key(&table_name);
+                self.table_name_manager
+                    .create(&table_name_key, i)
+                    .await
+                    .unwrap();
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_rename(&self) {
+        let desc = format!("TableNameBencher: rename {} table names", self.count);
+        bench(
+            &desc,
+            |i| async move {
+                let table_name = format!("bench_table_name_{}", i);
+                let new_table_name = format!("bench_table_name_new_{}", i);
+                let table_name_key = create_table_name_key(&table_name);
+                self.table_name_manager
+                    .rename(table_name_key, i, &new_table_name)
+                    .await
+                    .unwrap();
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_get(&self) {
+        let desc = format!("TableNameBencher: get {} table names", self.count);
+        bench(
+            &desc,
+            |i| async move {
+                let table_name = format!("bench_table_name_new_{}", i);
+                let table_name_key = create_table_name_key(&table_name);
+                assert!(self
+                    .table_name_manager
+                    .get(table_name_key)
+                    .await
+                    .unwrap()
+                    .is_some());
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_tables(&self) {
+        let desc = format!("TableNameBencher: list all {} table names", self.count);
+        bench(
+            &desc,
+            |_| async move {
+                assert!(!self
+                    .table_name_manager
+                    .tables("bench_catalog", "bench_schema")
+                    .await
+                    .unwrap()
+                    .is_empty());
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_remove(&self) {
+        let desc = format!("TableNameBencher: remove {} table names", self.count);
+        bench(
+            &desc,
+            |i| async move {
+                let table_name = format!("bench_table_name_new_{}", i);
+                let table_name_key = create_table_name_key(&table_name);
+                self.table_name_manager
+                    .remove(table_name_key)
+                    .await
+                    .unwrap();
+            },
+            self.count,
+        )
+        .await;
+    }
+}
+
+fn create_table_name_key(table_name: &str) -> TableNameKey {
+    TableNameKey::new("bench_catalog", "bench_schema", table_name)
+}

src/cmd/src/cli/bench/table_region.rs

@@ -0,0 +1,112 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Instant;
+
+use common_meta::key::table_region::TableRegionManager;
+
+use super::{bench, bench_self_recorded, create_region_distribution};
+
+pub struct TableRegionBencher<'a> {
+    table_region_manager: &'a TableRegionManager,
+    count: u32,
+}
+
+impl<'a> TableRegionBencher<'a> {
+    pub fn new(table_region_manager: &'a TableRegionManager, count: u32) -> Self {
+        Self {
+            table_region_manager,
+            count,
+        }
+    }
+
+    pub async fn start(&self) {
+        self.bench_create().await;
+        self.bench_get().await;
+        self.bench_compare_and_put().await;
+        self.bench_remove().await;
+    }
+
+    async fn bench_create(&self) {
+        let desc = format!("TableRegionBencher: create {} table regions", self.count);
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let region_distribution = create_region_distribution();
+
+                let start = Instant::now();
+
+                self.table_region_manager
+                    .create(i, &region_distribution)
+                    .await
+                    .unwrap();
+
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_get(&self) {
+        let desc = format!("TableRegionBencher: get {} table regions", self.count);
+        bench(
+            &desc,
+            |i| async move {
+                assert!(self.table_region_manager.get(i).await.unwrap().is_some());
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_compare_and_put(&self) {
+        let desc = format!(
+            "TableRegionBencher: compare_and_put {} table regions",
+            self.count
+        );
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let table_region_value = self.table_region_manager.get(i).await.unwrap().unwrap();
+
+                let new_region_distribution = create_region_distribution();
+
+                let start = Instant::now();
+
+                self.table_region_manager
+                    .compare_and_put(i, Some(table_region_value), new_region_distribution)
+                    .await
+                    .unwrap()
+                    .unwrap();
+
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    async fn bench_remove(&self) {
+        let desc = format!("TableRegionBencher: remove {} table regions", self.count);
+        bench(
+            &desc,
+            |i| async move {
+                assert!(self.table_region_manager.remove(i).await.unwrap().is_some());
+            },
+            self.count,
+        )
+        .await;
+    }
+}

src/cmd/src/cli/repl.rs

@@ -20,7 +20,8 @@ use catalog::remote::CachedMetaKvBackend;
 use client::client_manager::DatanodeClients;
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_base::Plugins;
-use common_error::prelude::ErrorExt;
+use common_error::ext::ErrorExt;
+use common_meta::key::TableMetadataManager;
 use common_query::Output;
 use common_recordbatch::RecordBatches;
 use common_telemetry::logging;
@@ -164,10 +165,7 @@ impl Repl {
             let stmt = QueryLanguageParser::parse_sql(&sql)
                 .with_context(|_| ParseSqlSnafu { sql: sql.clone() })?;
 
-            let query_ctx = Arc::new(QueryContext::with(
-                self.database.catalog(),
-                self.database.schema(),
-            ));
+            let query_ctx = QueryContext::with(self.database.catalog(), self.database.schema());
 
             let plan = query_engine
                 .planner()
@@ -179,10 +177,10 @@ impl Repl {
                 query_engine.optimize(&plan).context(PlanStatementSnafu)?;
 
             let plan = DFLogicalSubstraitConvertor {}
-                .encode(plan)
+                .encode(&plan)
                 .context(SubstraitEncodeLogicalPlanSnafu)?;
 
-            self.database.logical_plan(plan.to_vec()).await
+            self.database.logical_plan(plan.to_vec(), None).await
         } else {
             self.database.sql(&sql).await
         }
@@ -263,9 +261,10 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
 
     let catalog_list = Arc::new(FrontendCatalogManager::new(
         cached_meta_backend.clone(),
-        cached_meta_backend,
+        cached_meta_backend.clone(),
         partition_manager,
         datanode_clients,
+        Arc::new(TableMetadataManager::new(cached_meta_backend)),
     ));
     let plugins: Arc<Plugins> = Default::default();
     let state = Arc::new(QueryEngineState::new(

src/cmd/src/cli/upgrade.rs

@@ -0,0 +1,202 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_trait::async_trait;
+use clap::Parser;
+use common_meta::helper::TableGlobalValue;
+use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue};
+use common_meta::key::table_info::{TableInfoKey, TableInfoValue};
+use common_meta::key::table_name::{TableNameKey, TableNameValue};
+use common_meta::key::table_region::{RegionDistribution, TableRegionKey, TableRegionValue};
+use common_meta::key::TableMetaKey;
+use common_meta::rpc::store::{
+    BatchDeleteRequest, BatchPutRequest, PutRequest, RangeRequest, RangeResponse,
+};
+use common_meta::util::get_prefix_end_key;
+use common_telemetry::info;
+use etcd_client::Client;
+use meta_srv::service::store::etcd::EtcdStore;
+use meta_srv::service::store::kv::KvStoreRef;
+use snafu::ResultExt;
+
+use crate::cli::{Instance, Tool};
+use crate::error::{ConnectEtcdSnafu, Result};
+
+#[derive(Debug, Default, Parser)]
+pub struct UpgradeCommand {
+    #[clap(long)]
+    etcd_addr: String,
+    #[clap(long)]
+    dryrun: bool,
+}
+
+impl UpgradeCommand {
+    pub async fn build(&self) -> Result<Instance> {
+        let client = Client::connect([&self.etcd_addr], None)
+            .await
+            .context(ConnectEtcdSnafu {
+                etcd_addr: &self.etcd_addr,
+            })?;
+        let tool = MigrateTableMetadata {
+            etcd_store: EtcdStore::with_etcd_client(client),
+            dryrun: self.dryrun,
+        };
+        Ok(Instance::Tool(Box::new(tool)))
+    }
+}
+
+struct MigrateTableMetadata {
+    etcd_store: KvStoreRef,
+    dryrun: bool,
+}
+
+#[async_trait]
+impl Tool for MigrateTableMetadata {
+    async fn do_work(&self) -> Result<()> {
+        let mut key = b"__tg".to_vec();
+        let range_end = get_prefix_end_key(&key);
+
+        let mut processed_keys = 0;
+        loop {
+            info!("Start scanning key from: {}", String::from_utf8_lossy(&key));
+
+            let req = RangeRequest::new()
+                .with_range(key, range_end.clone())
+                .with_limit(1000);
+            let resp = self.etcd_store.range(req).await.unwrap();
+            for kv in resp.kvs.iter() {
+                let key = String::from_utf8_lossy(kv.key());
+                let value = TableGlobalValue::from_bytes(kv.value())
+                    .unwrap_or_else(|e| panic!("table global value is corrupted: {e}, key: {key}"));
+
+                self.create_table_name_key(&value).await;
+
+                self.create_datanode_table_keys(&value).await;
+
+                self.split_table_global_value(&key, value).await;
+            }
+
+            self.delete_migrated_keys(&resp).await;
+
+            processed_keys += resp.kvs.len();
+
+            if resp.more {
+                key = get_prefix_end_key(resp.kvs.last().unwrap().key());
+            } else {
+                break;
+            }
+        }
+        info!("Total migrated TableGlobalKeys: {processed_keys}");
+        Ok(())
+    }
+}
+
+impl MigrateTableMetadata {
+    async fn delete_migrated_keys(&self, resp: &RangeResponse) {
+        info!("Deleting {} TableGlobalKeys", resp.kvs.len());
+        let req = BatchDeleteRequest {
+            keys: resp.kvs.iter().map(|kv| kv.key().to_vec()).collect(),
+            prev_kv: false,
+        };
+        if self.dryrun {
+            info!("Dryrun: do nothing");
+        } else {
+            self.etcd_store.batch_delete(req).await.unwrap();
+        }
+    }
+
+    async fn split_table_global_value(&self, key: &str, value: TableGlobalValue) {
+        let table_id = value.table_id();
+        let region_distribution: RegionDistribution = value.regions_id_map.into_iter().collect();
+
+        let table_info_key = TableInfoKey::new(table_id);
+        let table_info_value = TableInfoValue::new(value.table_info);
+
+        let table_region_key = TableRegionKey::new(table_id);
+        let table_region_value = TableRegionValue::new(region_distribution);
+
+        info!("Splitting TableGlobalKey '{key}' into '{table_info_key}' and '{table_region_key}'");
+
+        if self.dryrun {
+            info!("Dryrun: do nothing");
+        } else {
+            self.etcd_store
+                .batch_put(
+                    BatchPutRequest::new()
+                        .add_kv(
+                            table_info_key.as_raw_key(),
+                            table_info_value.try_as_raw_value().unwrap(),
+                        )
+                        .add_kv(
+                            table_region_key.as_raw_key(),
+                            table_region_value.try_as_raw_value().unwrap(),
+                        ),
+                )
+                .await
+                .unwrap();
+        }
+    }
+
+    async fn create_table_name_key(&self, value: &TableGlobalValue) {
+        let table_info = &value.table_info;
+        let table_id = value.table_id();
+
+        let table_name_key = TableNameKey::new(
+            &table_info.catalog_name,
+            &table_info.schema_name,
+            &table_info.name,
+        );
+        let table_name_value = TableNameValue::new(table_id);
+
+        info!("Creating '{table_name_key}' => {table_id}");
+
+        if self.dryrun {
+            info!("Dryrun: do nothing");
+        } else {
+            self.etcd_store
+                .put(
+                    PutRequest::new()
+                        .with_key(table_name_key.as_raw_key())
+                        .with_value(table_name_value.try_as_raw_value().unwrap()),
+                )
+                .await
+                .unwrap();
+        }
+    }
+
+    async fn create_datanode_table_keys(&self, value: &TableGlobalValue) {
+        let table_id = value.table_id();
+        let region_distribution: RegionDistribution =
+            value.regions_id_map.clone().into_iter().collect();
+
+        let datanode_table_kvs = region_distribution
+            .into_iter()
+            .map(|(datanode_id, regions)| {
+                let k = DatanodeTableKey::new(datanode_id, table_id);
+                info!("Creating DatanodeTableKey '{k}' => {regions:?}");
+                (k, DatanodeTableValue::new(table_id, regions))
+            })
+            .collect::<Vec<_>>();
+
+        if self.dryrun {
+            info!("Dryrun: do nothing");
+        } else {
+            let mut req = BatchPutRequest::new();
+            for (key, value) in datanode_table_kvs {
+                req = req.add_kv(key.as_raw_key(), value.try_as_raw_value().unwrap());
+            }
+            self.etcd_store.batch_put(req).await.unwrap();
+        }
+    }
+}

src/cmd/src/datanode.rs

@@ -206,6 +206,7 @@ mod tests {
             metasrv_addrs = ["127.0.0.1:3002"]
             timeout_millis = 3000
             connect_timeout_millis = 5000
+            ddl_timeout_millis= 10000
             tcp_nodelay = true
 
             [wal]
@@ -259,10 +260,12 @@ mod tests {
             timeout_millis,
             connect_timeout_millis,
             tcp_nodelay,
+            ddl_timeout_millis,
         } = options.meta_client_options.unwrap();
 
         assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);
         assert_eq!(5000, connect_timeout_millis);
+        assert_eq!(10000, ddl_timeout_millis);
         assert_eq!(3000, timeout_millis);
         assert!(tcp_nodelay);
 

src/cmd/src/error.rs

@@ -14,10 +14,11 @@
 
 use std::any::Any;
 
-use common_error::prelude::*;
+use common_error::ext::ErrorExt;
+use common_error::status_code::StatusCode;
 use config::ConfigError;
 use rustyline::error::ReadlineError;
-use snafu::Location;
+use snafu::{Location, Snafu};
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
@@ -153,6 +154,13 @@ pub enum Error {
         location: Location,
         source: catalog::error::Error,
     },
+
+    #[snafu(display("Failed to connect to Etcd at {etcd_addr}, source: {}", source))]
+    ConnectEtcd {
+        etcd_addr: String,
+        source: etcd_client::Error,
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -172,7 +180,9 @@ impl ErrorExt for Error {
             | Error::LoadLayeredConfig { .. }
             | Error::IllegalConfig { .. }
             | Error::InvalidReplCommand { .. }
-            | Error::IllegalAuthConfig { .. } => StatusCode::InvalidArguments,
+            | Error::IllegalAuthConfig { .. }
+            | Error::ConnectEtcd { .. } => StatusCode::InvalidArguments,
+
             Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,
             Error::RequestDatabase { source, .. } => source.status_code(),
             Error::CollectRecordBatches { source, .. }

src/cmd/src/frontend.rs

@@ -19,7 +19,7 @@ use common_base::Plugins;
 use common_telemetry::logging;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
-use frontend::service_config::{InfluxdbOptions, PromOptions};
+use frontend::service_config::{InfluxdbOptions, PrometheusOptions};
 use meta_client::MetaClientOptions;
 use servers::auth::UserProviderRef;
 use servers::tls::{TlsMode, TlsOption};
@@ -172,7 +172,7 @@ impl StartCommand {
         }
 
         if let Some(addr) = &self.prom_addr {
-            opts.prom_options = Some(PromOptions { addr: addr.clone() });
+            opts.prometheus_options = Some(PrometheusOptions { addr: addr.clone() });
         }
 
         if let Some(addr) = &self.postgres_addr {
@@ -274,7 +274,10 @@ mod tests {
             opts.opentsdb_options.as_ref().unwrap().addr,
             "127.0.0.1:4321"
         );
-        assert_eq!(opts.prom_options.as_ref().unwrap().addr, "127.0.0.1:4444");
+        assert_eq!(
+            opts.prometheus_options.as_ref().unwrap().addr,
+            "127.0.0.1:4444"
+        );
 
         let default_opts = FrontendOptions::default();
         assert_eq!(

src/cmd/src/standalone.rs

@@ -23,7 +23,7 @@ use datanode::instance::InstanceRef;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
 use frontend::service_config::{
-    GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromOptions,
+    GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
     PrometheusOptions,
 };
 use serde::{Deserialize, Serialize};
@@ -89,8 +89,8 @@ pub struct StandaloneOptions {
     pub postgres_options: Option<PostgresOptions>,
     pub opentsdb_options: Option<OpentsdbOptions>,
     pub influxdb_options: Option<InfluxdbOptions>,
+    pub prom_store_options: Option<PromStoreOptions>,
     pub prometheus_options: Option<PrometheusOptions>,
-    pub prom_options: Option<PromOptions>,
     pub wal: WalConfig,
     pub storage: StorageConfig,
     pub procedure: ProcedureConfig,
@@ -108,8 +108,8 @@ impl Default for StandaloneOptions {
             postgres_options: Some(PostgresOptions::default()),
             opentsdb_options: Some(OpentsdbOptions::default()),
             influxdb_options: Some(InfluxdbOptions::default()),
+            prom_store_options: Some(PromStoreOptions::default()),
             prometheus_options: Some(PrometheusOptions::default()),
-            prom_options: Some(PromOptions::default()),
             wal: WalConfig::default(),
             storage: StorageConfig::default(),
             procedure: ProcedureConfig::default(),
@@ -128,8 +128,8 @@ impl StandaloneOptions {
             postgres_options: self.postgres_options,
             opentsdb_options: self.opentsdb_options,
             influxdb_options: self.influxdb_options,
+            prom_store_options: self.prom_store_options,
             prometheus_options: self.prometheus_options,
-            prom_options: self.prom_options,
             meta_client_options: None,
             logging: self.logging,
             ..Default::default()
@@ -269,7 +269,7 @@ impl StartCommand {
         }
 
         if let Some(addr) = &self.prom_addr {
-            opts.prom_options = Some(PromOptions { addr: addr.clone() })
+            opts.prometheus_options = Some(PrometheusOptions { addr: addr.clone() })
         }
 
         if let Some(addr) = &self.postgres_addr {

src/common/base/Cargo.toml

@@ -14,4 +14,4 @@ serde = { version = "1.0", features = ["derive"] }
 snafu.workspace = true
 
 [dev-dependencies]
-toml = "0.5"
+toml.workspace = true

src/common/base/src/buffer.rs

@@ -16,7 +16,7 @@ use std::any::Any;
 use std::io::{Read, Write};
 
 use bytes::{Buf, BufMut, BytesMut};
-use common_error::prelude::ErrorExt;
+use common_error::ext::ErrorExt;
 use paste::paste;
 use snafu::{ensure, Location, ResultExt, Snafu};
 

src/common/catalog/Cargo.toml

@@ -5,12 +5,7 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-async-trait = "0.1"
 common-error = { path = "../error" }
-common-telemetry = { path = "../telemetry" }
-datatypes = { path = "../../datatypes" }
-lazy_static = "1.4"
-regex = "1.6"
 serde.workspace = true
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }

src/common/catalog/src/error.rs

@@ -15,8 +15,8 @@
 use std::any::Any;
 
 use common_error::ext::ErrorExt;
-use common_error::prelude::{Snafu, StatusCode};
-use snafu::Location;
+use common_error::status_code::StatusCode;
+use snafu::{Location, Snafu};
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]

src/common/datasource/Cargo.toml

@@ -17,11 +17,10 @@ async-compression = { version = "0.3", features = [
 ] }
 async-trait.workspace = true
 bytes = "1.1"
-common-base = { path = "../base" }
 common-error = { path = "../error" }
 common-runtime = { path = "../runtime" }
 datafusion.workspace = true
-derive_builder = "0.12"
+derive_builder.workspace = true
 futures.workspace = true
 object-store = { path = "../../object-store" }
 orc-rust = "0.2"

src/common/datasource/src/error.rs

@@ -15,9 +15,10 @@
 use std::any::Any;
 
 use arrow_schema::ArrowError;
-use common_error::prelude::*;
+use common_error::ext::ErrorExt;
+use common_error::status_code::StatusCode;
 use datafusion::parquet::errors::ParquetError;
-use snafu::Location;
+use snafu::{Location, Snafu};
 use url::ParseError;
 
 #[derive(Debug, Snafu)]

src/common/datasource/src/file_format.rs

@@ -30,8 +30,8 @@ use arrow::record_batch::RecordBatch;
 use arrow_schema::{ArrowError, Schema as ArrowSchema};
 use async_trait::async_trait;
 use bytes::{Buf, Bytes};
+use datafusion::datasource::physical_plan::FileOpenFuture;
 use datafusion::error::{DataFusionError, Result as DataFusionResult};
-use datafusion::physical_plan::file_format::FileOpenFuture;
 use datafusion::physical_plan::SendableRecordBatchStream;
 use futures::StreamExt;
 use object_store::ObjectStore;

src/common/datasource/src/file_format/csv.rs

@@ -23,8 +23,8 @@ use arrow::record_batch::RecordBatch;
 use arrow_schema::{Schema, SchemaRef};
 use async_trait::async_trait;
 use common_runtime;
+use datafusion::datasource::physical_plan::{FileMeta, FileOpenFuture, FileOpener};
 use datafusion::error::Result as DataFusionResult;
-use datafusion::physical_plan::file_format::{FileMeta, FileOpenFuture, FileOpener};
 use datafusion::physical_plan::SendableRecordBatchStream;
 use derive_builder::Builder;
 use object_store::ObjectStore;

src/common/datasource/src/file_format/json.rs

@@ -26,8 +26,8 @@ use arrow::record_batch::RecordBatch;
 use arrow_schema::Schema;
 use async_trait::async_trait;
 use common_runtime;
+use datafusion::datasource::physical_plan::{FileMeta, FileOpenFuture, FileOpener};
 use datafusion::error::{DataFusionError, Result as DataFusionResult};
-use datafusion::physical_plan::file_format::{FileMeta, FileOpenFuture, FileOpener};
 use datafusion::physical_plan::SendableRecordBatchStream;
 use object_store::ObjectStore;
 use snafu::ResultExt;

src/common/datasource/src/file_format/orc.rs

@@ -13,18 +13,20 @@
 // limitations under the License.
 
 use std::pin::Pin;
+use std::sync::Arc;
 use std::task::{Context, Poll};
 
-use arrow_schema::{Schema, SchemaRef};
+use arrow::compute::cast;
+use arrow_schema::{ArrowError, Schema, SchemaRef};
 use async_trait::async_trait;
 use datafusion::arrow::record_batch::RecordBatch as DfRecordBatch;
+use datafusion::datasource::physical_plan::{FileMeta, FileOpenFuture, FileOpener};
 use datafusion::error::{DataFusionError, Result as DfResult};
 use datafusion::physical_plan::RecordBatchStream;
-use futures::Stream;
+use futures::{Stream, StreamExt, TryStreamExt};
 use object_store::ObjectStore;
 use orc_rust::arrow_reader::{create_arrow_schema, Cursor};
 use orc_rust::async_arrow_reader::ArrowStreamReader;
-pub use orc_rust::error::Error as OrcError;
 use orc_rust::reader::Reader;
 use snafu::ResultExt;
 use tokio::io::{AsyncRead, AsyncSeek};
@@ -60,12 +62,28 @@ pub async fn infer_orc_schema<R: AsyncRead + AsyncSeek + Unpin + Send + 'static>
 }
 
 pub struct OrcArrowStreamReaderAdapter<T: AsyncRead + AsyncSeek + Unpin + Send + 'static> {
+    output_schema: SchemaRef,
+    projection: Vec<usize>,
     stream: ArrowStreamReader<T>,
 }
 
 impl<T: AsyncRead + AsyncSeek + Unpin + Send + 'static> OrcArrowStreamReaderAdapter<T> {
-    pub fn new(stream: ArrowStreamReader<T>) -> Self {
-        Self { stream }
+    pub fn new(
+        output_schema: SchemaRef,
+        stream: ArrowStreamReader<T>,
+        projection: Option<Vec<usize>>,
+    ) -> Self {
+        let projection = if let Some(projection) = projection {
+            projection
+        } else {
+            (0..output_schema.fields().len()).collect()
+        };
+
+        Self {
+            output_schema,
+            projection,
+            stream,
+        }
     }
 }
 
@@ -73,7 +91,7 @@ impl<T: AsyncRead + AsyncSeek + Unpin + Send + 'static> RecordBatchStream
     for OrcArrowStreamReaderAdapter<T>
 {
     fn schema(&self) -> SchemaRef {
-        self.stream.schema()
+        self.output_schema.clone()
     }
 }
 
@@ -83,6 +101,29 @@ impl<T: AsyncRead + AsyncSeek + Unpin + Send + 'static> Stream for OrcArrowStrea
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
         let batch = futures::ready!(Pin::new(&mut self.stream).poll_next(cx))
             .map(|r| r.map_err(|e| DataFusionError::External(Box::new(e))));
+
+        let projected_schema = self.output_schema.project(&self.projection)?;
+        let batch = batch.map(|b| {
+            b.and_then(|b| {
+                let mut columns = Vec::with_capacity(self.projection.len());
+                for idx in self.projection.iter() {
+                    let column = b.column(*idx);
+                    let field = self.output_schema.field(*idx);
+
+                    if column.data_type() != field.data_type() {
+                        let output = cast(&column, field.data_type())?;
+                        columns.push(output)
+                    } else {
+                        columns.push(column.clone())
+                    }
+                }
+
+                let record_batch = DfRecordBatch::try_new(projected_schema.into(), columns)?;
+
+                Ok(record_batch)
+            })
+        });
+
         Poll::Ready(batch)
     }
 }
@@ -100,3 +141,92 @@ impl FileFormat for OrcFormat {
         Ok(schema)
     }
 }
+
+#[derive(Debug, Clone)]
+pub struct OrcOpener {
+    object_store: Arc<ObjectStore>,
+    output_schema: SchemaRef,
+    projection: Option<Vec<usize>>,
+}
+
+impl OrcOpener {
+    pub fn new(
+        object_store: ObjectStore,
+        output_schema: SchemaRef,
+        projection: Option<Vec<usize>>,
+    ) -> Self {
+        Self {
+            object_store: Arc::from(object_store),
+            output_schema,
+            projection,
+        }
+    }
+}
+
+impl FileOpener for OrcOpener {
+    fn open(&self, meta: FileMeta) -> DfResult<FileOpenFuture> {
+        let object_store = self.object_store.clone();
+        let output_schema = self.output_schema.clone();
+        let projection = self.projection.clone();
+        Ok(Box::pin(async move {
+            let reader = object_store
+                .reader(meta.location().to_string().as_str())
+                .await
+                .map_err(|e| DataFusionError::External(Box::new(e)))?;
+
+            let stream_reader = new_orc_stream_reader(reader)
+                .await
+                .map_err(|e| DataFusionError::External(Box::new(e)))?;
+
+            let stream = OrcArrowStreamReaderAdapter::new(output_schema, stream_reader, projection);
+
+            let adopted = stream.map_err(|e| ArrowError::ExternalError(Box::new(e)));
+            Ok(adopted.boxed())
+        }))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::file_format::FileFormat;
+    use crate::test_util::{self, format_schema, test_store};
+
+    fn test_data_root() -> String {
+        test_util::get_data_dir("tests/orc").display().to_string()
+    }
+
+    #[tokio::test]
+    async fn test_orc_infer_schema() {
+        let orc = OrcFormat::default();
+        let store = test_store(&test_data_root());
+        let schema = orc.infer_schema(&store, "test.orc").await.unwrap();
+        let formatted: Vec<_> = format_schema(schema);
+
+        assert_eq!(
+            vec![
+                "double_a: Float64: NULL",
+                "a: Float32: NULL",
+                "b: Boolean: NULL",
+                "str_direct: Utf8: NULL",
+                "d: Utf8: NULL",
+                "e: Utf8: NULL",
+                "f: Utf8: NULL",
+                "int_short_repeated: Int32: NULL",
+                "int_neg_short_repeated: Int32: NULL",
+                "int_delta: Int32: NULL",
+                "int_neg_delta: Int32: NULL",
+                "int_direct: Int32: NULL",
+                "int_neg_direct: Int32: NULL",
+                "bigint_direct: Int64: NULL",
+                "bigint_neg_direct: Int64: NULL",
+                "bigint_other: Int64: NULL",
+                "utf8_increase: Utf8: NULL",
+                "utf8_decrease: Utf8: NULL",
+                "timestamp_simple: Timestamp(Nanosecond, None): NULL",
+                "date_simple: Date32: NULL"
+            ],
+            formatted
+        );
+    }
+}

src/common/datasource/src/file_format/parquet.rs

@@ -18,13 +18,13 @@ use std::sync::Arc;
 use arrow::record_batch::RecordBatch;
 use arrow_schema::Schema;
 use async_trait::async_trait;
+use datafusion::datasource::physical_plan::{FileMeta, ParquetFileReaderFactory};
 use datafusion::error::Result as DatafusionResult;
 use datafusion::parquet::arrow::async_reader::AsyncFileReader;
 use datafusion::parquet::arrow::{parquet_to_arrow_schema, ArrowWriter};
 use datafusion::parquet::errors::{ParquetError, Result as ParquetResult};
 use datafusion::parquet::file::metadata::ParquetMetaData;
 use datafusion::parquet::format::FileMetaData;
-use datafusion::physical_plan::file_format::{FileMeta, ParquetFileReaderFactory};
 use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
 use futures::future::BoxFuture;
 use object_store::{ObjectStore, Reader};

src/common/datasource/src/file_format/tests.rs

@@ -18,8 +18,8 @@ use std::sync::Arc;
 use std::vec;
 
 use datafusion::assert_batches_eq;
+use datafusion::datasource::physical_plan::{FileOpener, FileScanConfig, FileStream, ParquetExec};
 use datafusion::execution::context::TaskContext;
-use datafusion::physical_plan::file_format::{FileOpener, FileScanConfig, FileStream, ParquetExec};
 use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::SessionContext;
@@ -30,8 +30,9 @@ use crate::compression::CompressionType;
 use crate::error;
 use crate::file_format::csv::{CsvConfigBuilder, CsvOpener};
 use crate::file_format::json::JsonOpener;
+use crate::file_format::orc::{OrcFormat, OrcOpener};
 use crate::file_format::parquet::DefaultParquetFileReaderFactory;
-use crate::file_format::Format;
+use crate::file_format::{FileFormat, Format};
 use crate::test_util::{self, scan_config, test_basic_schema, test_store};
 
 struct Test<'a, T: FileOpener> {
@@ -193,6 +194,51 @@ async fn test_parquet_exec() {
     );
 }
 
+#[tokio::test]
+async fn test_orc_opener() {
+    let root = test_util::get_data_dir("tests/orc").display().to_string();
+    let store = test_store(&root);
+    let orc = OrcFormat::default();
+    let schema = orc.infer_schema(&store, "test.orc").await.unwrap();
+    let schema = Arc::new(schema);
+
+    let orc_opener = OrcOpener::new(store.clone(), schema.clone(), None);
+    let path = &test_util::get_data_dir("/test.orc").display().to_string();
+
+    let tests = [
+        Test {
+            config: scan_config(schema.clone(), None, path),
+            opener: orc_opener.clone(),
+            expected: vec![
+            "+----------+-----+-------+------------+-----+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+----------------------------+-------------+",
+            "| double_a | a   | b     | str_direct | d   | e   | f     | int_short_repeated | int_neg_short_repeated | int_delta | int_neg_delta | int_direct | int_neg_direct | bigint_direct | bigint_neg_direct | bigint_other | utf8_increase | utf8_decrease | timestamp_simple           | date_simple |",
+            "+----------+-----+-------+------------+-----+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+----------------------------+-------------+",
+            "| 1.0      | 1.0 | true  | a          | a   | ddd | aaaaa | 5                  | -5                     | 1         | 5             | 1          | -1             | 1             | -1                | 5            | a             | eeeee         | 2023-04-01T20:15:30.002    | 2023-04-01  |",
+            "| 2.0      | 2.0 | false | cccccc     | bb  | cc  | bbbbb | 5                  | -5                     | 2         | 4             | 6          | -6             | 6             | -6                | -5           | bb            | dddd          | 2021-08-22T07:26:44.525777 | 2023-03-01  |",
+            "| 3.0      |     |       |            |     |     |       |                    |                        |           |               |            |                |               |                   | 1            | ccc           | ccc           | 2023-01-01T00:00:00        | 2023-01-01  |",
+            "| 4.0      | 4.0 | true  | ddd        | ccc | bb  | ccccc | 5                  | -5                     | 4         | 2             | 3          | -3             | 3             | -3                | 5            | dddd          | bb            | 2023-02-01T00:00:00        | 2023-02-01  |",
+            "| 5.0      | 5.0 | false | ee         | ddd | a   | ddddd | 5                  | -5                     | 5         | 1             | 2          | -2             | 2             | -2                | 5            | eeeee         | a             | 2023-03-01T00:00:00        | 2023-03-01  |",
+            "+----------+-----+-------+------------+-----+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+----------------------------+-------------+",
+            ],
+        },
+        Test {
+            config: scan_config(schema.clone(), Some(1), path),
+            opener: orc_opener.clone(),
+            expected: vec![
+                "+----------+-----+------+------------+---+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+-------------------------+-------------+",
+                "| double_a | a   | b    | str_direct | d | e   | f     | int_short_repeated | int_neg_short_repeated | int_delta | int_neg_delta | int_direct | int_neg_direct | bigint_direct | bigint_neg_direct | bigint_other | utf8_increase | utf8_decrease | timestamp_simple        | date_simple |",
+                "+----------+-----+------+------------+---+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+-------------------------+-------------+",
+                "| 1.0      | 1.0 | true | a          | a | ddd | aaaaa | 5                  | -5                     | 1         | 5             | 1          | -1             | 1             | -1                | 5            | a             | eeeee         | 2023-04-01T20:15:30.002 | 2023-04-01  |",
+                "+----------+-----+------+------------+---+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+-------------------------+-------------+",
+            ],
+        },
+    ];
+
+    for test in tests {
+        test.run().await;
+    }
+}
+
 #[test]
 fn test_format() {
     let value = [(FORMAT_TYPE.to_string(), "csv".to_string())]
@@ -213,6 +259,12 @@ fn test_format() {
 
     assert_matches!(Format::try_from(&value).unwrap(), Format::Json(_));
 
+    let value = [(FORMAT_TYPE.to_string(), "ORC".to_string())]
+        .into_iter()
+        .collect::<HashMap<_, _>>();
+
+    assert_matches!(Format::try_from(&value).unwrap(), Format::Orc(_));
+
     let value = [(FORMAT_TYPE.to_string(), "Foobar".to_string())]
         .into_iter()
         .collect::<HashMap<_, _>>();

src/common/datasource/src/test_util.rs

@@ -19,7 +19,7 @@ use arrow_schema::{DataType, Field, Schema, SchemaRef};
 use common_test_util::temp_dir::{create_temp_dir, TempDir};
 use datafusion::datasource::listing::PartitionedFile;
 use datafusion::datasource::object_store::ObjectStoreUrl;
-use datafusion::physical_plan::file_format::{FileScanConfig, FileStream};
+use datafusion::datasource::physical_plan::{FileScanConfig, FileStream};
 use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
 use object_store::services::Fs;
 use object_store::ObjectStore;
@@ -86,7 +86,7 @@ pub fn scan_config(file_schema: SchemaRef, limit: Option<usize>, filename: &str)
         projection: None,
         limit,
         table_partition_cols: vec![],
-        output_ordering: None,
+        output_ordering: vec![],
         infinite_source: false,
     }
 }

src/common/datasource/tests/orc/README.md

@@ -9,3 +9,46 @@ venv/bin/pip install -U pyorc
 
 cargo test
 ```
+
+Schema:
+```
++------------------------+-----------------------------+-------------+
+| column_name            | data_type                   | is_nullable |
++------------------------+-----------------------------+-------------+
+| double_a               | Float64                     | YES         |
+| a                      | Float32                     | YES         |
+| b                      | Boolean                     | YES         |
+| str_direct             | Utf8                        | YES         |
+| d                      | Utf8                        | YES         |
+| e                      | Utf8                        | YES         |
+| f                      | Utf8                        | YES         |
+| int_short_repeated     | Int32                       | YES         |
+| int_neg_short_repeated | Int32                       | YES         |
+| int_delta              | Int32                       | YES         |
+| int_neg_delta          | Int32                       | YES         |
+| int_direct             | Int32                       | YES         |
+| int_neg_direct         | Int32                       | YES         |
+| bigint_direct          | Int64                       | YES         |
+| bigint_neg_direct      | Int64                       | YES         |
+| bigint_other           | Int64                       | YES         |
+| utf8_increase          | Utf8                        | YES         |
+| utf8_decrease          | Utf8                        | YES         |
+| timestamp_simple       | Timestamp(Nanosecond, None) | YES         |
+| date_simple            | Date32                      | YES         |
++------------------------+-----------------------------+-------------+
+```
+
+
+Data:
+```
+"+----------+-----+-------+------------+-----+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+----------------------------+-------------+",
+"| double_a | a   | b     | str_direct | d   | e   | f     | int_short_repeated | int_neg_short_repeated | int_delta | int_neg_delta | int_direct | int_neg_direct | bigint_direct | bigint_neg_direct | bigint_other | utf8_increase | utf8_decrease | timestamp_simple           | date_simple |",
+"+----------+-----+-------+------------+-----+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+----------------------------+-------------+",
+"| 1.0      | 1.0 | true  | a          | a   | ddd | aaaaa | 5                  | -5                     | 1         | 5             | 1          | -1             | 1             | -1                | 5            | a             | eeeee         | 2023-04-01T20:15:30.002    | 2023-04-01  |",
+"| 2.0      | 2.0 | false | cccccc     | bb  | cc  | bbbbb | 5                  | -5                     | 2         | 4             | 6          | -6             | 6             | -6                | -5           | bb            | dddd          | 2021-08-22T07:26:44.525777 | 2023-03-01  |",
+"| 3.0      |     |       |            |     |     |       |                    |                        |           |               |            |                |               |                   | 1            | ccc           | ccc           | 2023-01-01T00:00:00        | 2023-01-01  |",
+"| 4.0      | 4.0 | true  | ddd        | ccc | bb  | ccccc | 5                  | -5                     | 4         | 2             | 3          | -3             | 3             | -3                | 5            | dddd          | bb            | 2023-02-01T00:00:00        | 2023-02-01  |",
+"| 5.0      | 5.0 | false | ee         | ddd | a   | ddddd | 5                  | -5                     | 5         | 1             | 2          | -2             | 2             | -2                | 5            | eeeee         | a             | 2023-03-01T00:00:00        | 2023-03-01  |",
+"+----------+-----+-------+------------+-----+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+----------------------------+-------------+",
+
+```

src/common/error/src/lib.rs

@@ -17,16 +17,7 @@ pub mod format;
 pub mod mock;
 pub mod status_code;
 
-pub mod prelude {
-    pub use snafu::prelude::*;
-    pub use snafu::{Backtrace, ErrorCompat};
-
-    pub use crate::ext::{BoxedError, ErrorExt};
-    pub use crate::format::DebugFormat;
-    pub use crate::status_code::StatusCode;
-
-    pub const INNER_ERROR_CODE: &str = "INNER_ERROR_CODE";
-    pub const INNER_ERROR_MSG: &str = "INNER_ERROR_MSG";
-}
+pub const INNER_ERROR_CODE: &str = "INNER_ERROR_CODE";
+pub const INNER_ERROR_MSG: &str = "INNER_ERROR_MSG";
 
 pub use snafu;

src/common/error/src/mock.rs

@@ -19,7 +19,8 @@ use std::fmt;
 
 use snafu::Location;
 
-use crate::prelude::*;
+use crate::ext::ErrorExt;
+use crate::status_code::StatusCode;
 
 /// A mock error mainly for test.
 #[derive(Debug)]

src/common/error/src/status_code.rs

@@ -66,6 +66,9 @@ pub enum StatusCode {
     // ====== Begin of server related status code =====
     /// Runtime resources exhausted, like creating threads failed.
     RuntimeResourcesExhausted = 6000,
+
+    /// Rate limit exceeded
+    RateLimited = 6001,
     // ====== End of server related status code =======
 
     // ====== Begin of auth related status code =====
@@ -111,6 +114,7 @@ impl StatusCode {
             | StatusCode::TableColumnNotFound
             | StatusCode::TableColumnExists
             | StatusCode::DatabaseNotFound
+            | StatusCode::RateLimited
             | StatusCode::UserNotFound
             | StatusCode::UnsupportedPasswordType
             | StatusCode::UserPasswordMismatch
@@ -141,6 +145,7 @@ impl StatusCode {
             | StatusCode::TableColumnNotFound
             | StatusCode::TableColumnExists
             | StatusCode::DatabaseNotFound
+            | StatusCode::RateLimited
             | StatusCode::UserNotFound
             | StatusCode::UnsupportedPasswordType
             | StatusCode::UserPasswordMismatch
@@ -149,6 +154,44 @@ impl StatusCode {
             | StatusCode::AccessDenied => false,
         }
     }
+
+    pub fn from_u32(value: u32) -> Option<Self> {
+        match value {
+            v if v == StatusCode::Success as u32 => Some(StatusCode::Success),
+            v if v == StatusCode::Unknown as u32 => Some(StatusCode::Unknown),
+            v if v == StatusCode::Unsupported as u32 => Some(StatusCode::Unsupported),
+            v if v == StatusCode::Unexpected as u32 => Some(StatusCode::Unexpected),
+            v if v == StatusCode::Internal as u32 => Some(StatusCode::Internal),
+            v if v == StatusCode::InvalidArguments as u32 => Some(StatusCode::InvalidArguments),
+            v if v == StatusCode::Cancelled as u32 => Some(StatusCode::Cancelled),
+            v if v == StatusCode::InvalidSyntax as u32 => Some(StatusCode::InvalidSyntax),
+            v if v == StatusCode::PlanQuery as u32 => Some(StatusCode::PlanQuery),
+            v if v == StatusCode::EngineExecuteQuery as u32 => Some(StatusCode::EngineExecuteQuery),
+            v if v == StatusCode::TableAlreadyExists as u32 => Some(StatusCode::TableAlreadyExists),
+            v if v == StatusCode::TableNotFound as u32 => Some(StatusCode::TableNotFound),
+            v if v == StatusCode::TableColumnNotFound as u32 => {
+                Some(StatusCode::TableColumnNotFound)
+            }
+            v if v == StatusCode::TableColumnExists as u32 => Some(StatusCode::TableColumnExists),
+            v if v == StatusCode::DatabaseNotFound as u32 => Some(StatusCode::DatabaseNotFound),
+            v if v == StatusCode::StorageUnavailable as u32 => Some(StatusCode::StorageUnavailable),
+            v if v == StatusCode::RuntimeResourcesExhausted as u32 => {
+                Some(StatusCode::RuntimeResourcesExhausted)
+            }
+            v if v == StatusCode::RateLimited as u32 => Some(StatusCode::RateLimited),
+            v if v == StatusCode::UserNotFound as u32 => Some(StatusCode::UserNotFound),
+            v if v == StatusCode::UnsupportedPasswordType as u32 => {
+                Some(StatusCode::UnsupportedPasswordType)
+            }
+            v if v == StatusCode::UserPasswordMismatch as u32 => {
+                Some(StatusCode::UserPasswordMismatch)
+            }
+            v if v == StatusCode::AuthHeaderNotFound as u32 => Some(StatusCode::AuthHeaderNotFound),
+            v if v == StatusCode::InvalidAuthHeader as u32 => Some(StatusCode::InvalidAuthHeader),
+            v if v == StatusCode::AccessDenied as u32 => Some(StatusCode::AccessDenied),
+            _ => None,
+        }
+    }
 }
 
 impl fmt::Display for StatusCode {

src/common/function/Cargo.toml

@@ -16,7 +16,7 @@ datatypes = { path = "../../datatypes" }
 libc = "0.2"
 num = "0.4"
 num-traits = "0.2"
-once_cell = "1.10"
+once_cell.workspace = true
 paste = "1.0"
 snafu.workspace = true
 statrs = "0.16"

src/common/greptimedb-telemetry/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "common-greptimedb-telemetry"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+async-trait.workspace = true
+common-error = { path = "../error" }
+common-runtime = { path = "../runtime" }
+common-telemetry = { path = "../telemetry" }
+serde.workspace = true
+serde_json.workspace = true
+tokio.workspace = true
+reqwest = { version = "0.11", features = ["json"] }
+uuid.workspace = true
+once_cell = "1.17.0"
+
+[dev-dependencies]
+hyper = { version = "0.14", features = ["full"] }
+common-test-util = { path = "../test-util" }
+
+[build-dependencies]
+common-version = { path = "../version" }

src/common/greptimedb-telemetry/build.rs

@@ -0,0 +1,17 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+fn main() {
+    common_version::setup_git_versions();
+}

src/common/greptimedb-telemetry/src/lib.rs

@@ -0,0 +1,371 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::env;
+use std::io::ErrorKind;
+use std::path::PathBuf;
+use std::time::Duration;
+
+use common_runtime::error::{Error, Result};
+use common_runtime::{BoxedTaskFunction, RepeatedTask, Runtime, TaskFunction};
+use common_telemetry::debug;
+use once_cell::sync::Lazy;
+use reqwest::{Client, Response};
+use serde::{Deserialize, Serialize};
+
+pub const TELEMETRY_URL: &str = "https://api-preview.greptime.cloud/db/otel/statistics";
+
+// Getting the right path when running on windows
+static TELEMETRY_UUID_FILE_NAME: Lazy<PathBuf> = Lazy::new(|| {
+    let mut path = PathBuf::new();
+    path.push(env::temp_dir());
+    path.push(".greptimedb-telemetry-uuid");
+    path
+});
+
+pub static TELEMETRY_INTERVAL: Duration = Duration::from_secs(60 * 30);
+
+const GREPTIMEDB_TELEMETRY_CLIENT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
+const GREPTIMEDB_TELEMETRY_CLIENT_TIMEOUT: Duration = Duration::from_secs(10);
+
+pub enum GreptimeDBTelemetryTask {
+    Enable(RepeatedTask<Error>),
+    Disable,
+}
+
+impl GreptimeDBTelemetryTask {
+    pub fn enable(interval: Duration, task_fn: BoxedTaskFunction<Error>) -> Self {
+        GreptimeDBTelemetryTask::Enable(RepeatedTask::new(interval, task_fn))
+    }
+
+    pub fn disable() -> Self {
+        GreptimeDBTelemetryTask::Disable
+    }
+
+    pub fn start(&self, runtime: Runtime) -> Result<()> {
+        match self {
+            GreptimeDBTelemetryTask::Enable(task) => task.start(runtime),
+            GreptimeDBTelemetryTask::Disable => Ok(()),
+        }
+    }
+
+    pub async fn stop(&self) -> Result<()> {
+        match self {
+            GreptimeDBTelemetryTask::Enable(task) => task.stop().await,
+            GreptimeDBTelemetryTask::Disable => Ok(()),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct StatisticData {
+    pub os: String,
+    pub version: String,
+    pub arch: String,
+    pub mode: Mode,
+    pub git_commit: String,
+    pub nodes: Option<i32>,
+    pub uuid: String,
+}
+
+#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum Mode {
+    Distributed,
+    Standalone,
+}
+
+#[async_trait::async_trait]
+pub trait Collector {
+    fn get_version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn get_git_hash(&self) -> String {
+        env!("GIT_COMMIT").to_string()
+    }
+
+    fn get_os(&self) -> String {
+        env::consts::OS.to_string()
+    }
+
+    fn get_arch(&self) -> String {
+        env::consts::ARCH.to_string()
+    }
+
+    fn get_mode(&self) -> Mode;
+
+    fn get_retry(&self) -> i32;
+
+    fn inc_retry(&mut self);
+
+    fn set_uuid_cache(&mut self, uuid: String);
+
+    fn get_uuid_cache(&self) -> Option<String>;
+
+    async fn get_nodes(&self) -> Option<i32>;
+
+    fn get_uuid(&mut self) -> Option<String> {
+        match self.get_uuid_cache() {
+            Some(uuid) => Some(uuid),
+            None => {
+                if self.get_retry() > 3 {
+                    return None;
+                }
+                match default_get_uuid() {
+                    Some(uuid) => {
+                        self.set_uuid_cache(uuid.clone());
+                        Some(uuid)
+                    }
+                    None => {
+                        self.inc_retry();
+                        None
+                    }
+                }
+            }
+        }
+    }
+}
+
+pub fn default_get_uuid() -> Option<String> {
+    let path = (*TELEMETRY_UUID_FILE_NAME).as_path();
+    match std::fs::read(path) {
+        Ok(bytes) => Some(String::from_utf8_lossy(&bytes).to_string()),
+        Err(e) => {
+            if e.kind() == ErrorKind::NotFound {
+                let uuid = uuid::Uuid::new_v4().to_string();
+                let _ = std::fs::write(path, uuid.as_bytes());
+                Some(uuid)
+            } else {
+                None
+            }
+        }
+    }
+}
+
+/// Report version info to GreptimeDB.
+/// We do not collect any identity-sensitive information.
+/// This task is scheduled to run every 30 minutes.
+/// The task will be disabled default. It can be enabled by setting the build feature `greptimedb-telemetry`
+/// Collector is used to collect the version info. It can be implemented by different components.
+/// client is used to send the HTTP request to GreptimeDB.
+/// telemetry_url is the GreptimeDB url.
+pub struct GreptimeDBTelemetry {
+    statistics: Box<dyn Collector + Send + Sync>,
+    client: Option<Client>,
+    telemetry_url: &'static str,
+}
+
+#[async_trait::async_trait]
+impl TaskFunction<Error> for GreptimeDBTelemetry {
+    fn name(&self) -> &str {
+        "Greptimedb-telemetry-task"
+    }
+
+    async fn call(&mut self) -> Result<()> {
+        self.report_telemetry_info().await;
+        Ok(())
+    }
+}
+
+impl GreptimeDBTelemetry {
+    pub fn new(statistics: Box<dyn Collector + Send + Sync>) -> Self {
+        let client = Client::builder()
+            .connect_timeout(GREPTIMEDB_TELEMETRY_CLIENT_CONNECT_TIMEOUT)
+            .timeout(GREPTIMEDB_TELEMETRY_CLIENT_TIMEOUT)
+            .build();
+        Self {
+            statistics,
+            client: client.ok(),
+            telemetry_url: TELEMETRY_URL,
+        }
+    }
+
+    pub async fn report_telemetry_info(&mut self) -> Option<Response> {
+        match self.statistics.get_uuid() {
+            Some(uuid) => {
+                let data = StatisticData {
+                    os: self.statistics.get_os(),
+                    version: self.statistics.get_version(),
+                    git_commit: self.statistics.get_git_hash(),
+                    arch: self.statistics.get_arch(),
+                    mode: self.statistics.get_mode(),
+                    nodes: self.statistics.get_nodes().await,
+                    uuid,
+                };
+
+                if let Some(client) = self.client.as_ref() {
+                    debug!("report version: {:?}", data);
+                    let result = client.post(self.telemetry_url).json(&data).send().await;
+                    debug!("report version result: {:?}", result);
+                    result.ok()
+                } else {
+                    None
+                }
+            }
+            None => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::convert::Infallible;
+    use std::env;
+    use std::sync::atomic::AtomicUsize;
+    use std::time::Duration;
+
+    use common_test_util::ports;
+    use hyper::service::{make_service_fn, service_fn};
+    use hyper::Server;
+    use reqwest::Client;
+    use tokio::spawn;
+
+    use crate::{Collector, GreptimeDBTelemetry, Mode, StatisticData};
+
+    static COUNT: AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
+
+    async fn echo(req: hyper::Request<hyper::Body>) -> hyper::Result<hyper::Response<hyper::Body>> {
+        let path = req.uri().path();
+        if path == "/req-cnt" {
+            let body = hyper::Body::from(format!(
+                "{}",
+                COUNT.load(std::sync::atomic::Ordering::SeqCst)
+            ));
+            Ok(hyper::Response::new(body))
+        } else {
+            COUNT.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+            Ok(hyper::Response::new(req.into_body()))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_gretimedb_telemetry() {
+        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
+        let port: u16 = ports::get_port() as u16;
+        spawn(async move {
+            let make_svc = make_service_fn(|_conn| {
+                // This is the `Service` that will handle the connection.
+                // `service_fn` is a helper to convert a function that
+                // returns a Response into a `Service`.
+                async { Ok::<_, Infallible>(service_fn(echo)) }
+            });
+            let addr = ([127, 0, 0, 1], port).into();
+
+            let server = Server::bind(&addr).serve(make_svc);
+            let graceful = server.with_graceful_shutdown(async {
+                rx.await.ok();
+            });
+            let _ = graceful.await;
+            Ok::<_, Infallible>(())
+        });
+        struct TestStatistic;
+
+        struct FailedStatistic;
+
+        #[async_trait::async_trait]
+        impl Collector for TestStatistic {
+            fn get_mode(&self) -> Mode {
+                Mode::Standalone
+            }
+
+            async fn get_nodes(&self) -> Option<i32> {
+                Some(1)
+            }
+
+            fn get_retry(&self) -> i32 {
+                unimplemented!()
+            }
+
+            fn inc_retry(&mut self) {
+                unimplemented!()
+            }
+
+            fn set_uuid_cache(&mut self, _: String) {
+                unimplemented!()
+            }
+
+            fn get_uuid_cache(&self) -> Option<String> {
+                unimplemented!()
+            }
+
+            fn get_uuid(&mut self) -> Option<String> {
+                Some("test".to_string())
+            }
+        }
+
+        #[async_trait::async_trait]
+        impl Collector for FailedStatistic {
+            fn get_mode(&self) -> Mode {
+                Mode::Standalone
+            }
+
+            async fn get_nodes(&self) -> Option<i32> {
+                None
+            }
+
+            fn get_retry(&self) -> i32 {
+                unimplemented!()
+            }
+
+            fn inc_retry(&mut self) {
+                unimplemented!()
+            }
+
+            fn set_uuid_cache(&mut self, _: String) {
+                unimplemented!()
+            }
+
+            fn get_uuid_cache(&self) -> Option<String> {
+                unimplemented!()
+            }
+
+            fn get_uuid(&mut self) -> Option<String> {
+                None
+            }
+        }
+
+        let test_statistic = Box::new(TestStatistic);
+        let mut test_report = GreptimeDBTelemetry::new(test_statistic);
+        let url = Box::leak(format!("{}:{}", "http://localhost", port).into_boxed_str());
+        test_report.telemetry_url = url;
+        let response = test_report.report_telemetry_info().await.unwrap();
+
+        let body = response.json::<StatisticData>().await.unwrap();
+        assert_eq!(env::consts::ARCH, body.arch);
+        assert_eq!(env::consts::OS, body.os);
+        assert_eq!(env!("CARGO_PKG_VERSION"), body.version);
+        assert_eq!(env!("GIT_COMMIT"), body.git_commit);
+        assert_eq!(Mode::Standalone, body.mode);
+        assert_eq!(1, body.nodes.unwrap());
+
+        let failed_statistic = Box::new(FailedStatistic);
+        let mut failed_report = GreptimeDBTelemetry::new(failed_statistic);
+        failed_report.telemetry_url = url;
+        let response = failed_report.report_telemetry_info().await;
+        assert!(response.is_none());
+
+        let client = Client::builder()
+            .connect_timeout(Duration::from_secs(3))
+            .timeout(Duration::from_secs(3))
+            .build()
+            .unwrap();
+
+        let cnt_url = format!("{}/req-cnt", url);
+        let response = client.get(cnt_url).send().await.unwrap();
+        let body = response.text().await.unwrap();
+        assert_eq!("1", body);
+        tx.send(()).unwrap();
+    }
+}

src/common/grpc-expr/Cargo.toml

@@ -10,7 +10,6 @@ async-trait.workspace = true
 common-base = { path = "../base" }
 common-catalog = { path = "../catalog" }
 common-error = { path = "../error" }
-common-grpc = { path = "../grpc" }
 common-query = { path = "../query" }
 common-telemetry = { path = "../telemetry" }
 common-time = { path = "../time" }

src/common/grpc-expr/src/alter.rs

@@ -38,7 +38,7 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterExpr) -> Result<Alter
     let catalog_name = expr.catalog_name;
     let schema_name = expr.schema_name;
     let kind = expr.kind.context(MissingFieldSnafu { field: "kind" })?;
-    match kind {
+    let alter_kind = match kind {
         Kind::AddColumns(add_columns) => {
             let add_column_requests = add_columns
                 .add_columns
@@ -61,45 +61,27 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterExpr) -> Result<Alter
                 })
                 .collect::<Result<Vec<_>>>()?;
 
-            let alter_kind = AlterKind::AddColumns {
+            AlterKind::AddColumns {
                 columns: add_column_requests,
-            };
-
-            let request = AlterTableRequest {
-                catalog_name,
-                schema_name,
-                table_name: expr.table_name,
-                table_id,
-                alter_kind,
-            };
-            Ok(request)
-        }
-        Kind::DropColumns(DropColumns { drop_columns }) => {
-            let alter_kind = AlterKind::DropColumns {
-                names: drop_columns.into_iter().map(|c| c.name).collect(),
-            };
-
-            let request = AlterTableRequest {
-                catalog_name,
-                schema_name,
-                table_name: expr.table_name,
-                table_id,
-                alter_kind,
-            };
-            Ok(request)
+            }
         }
+        Kind::DropColumns(DropColumns { drop_columns }) => AlterKind::DropColumns {
+            names: drop_columns.into_iter().map(|c| c.name).collect(),
+        },
         Kind::RenameTable(RenameTable { new_table_name }) => {
-            let alter_kind = AlterKind::RenameTable { new_table_name };
-            let request = AlterTableRequest {
-                catalog_name,
-                schema_name,
-                table_name: expr.table_name,
-                table_id,
-                alter_kind,
-            };
-            Ok(request)
+            AlterKind::RenameTable { new_table_name }
         }
-    }
+    };
+
+    let request = AlterTableRequest {
+        catalog_name,
+        schema_name,
+        table_name: expr.table_name,
+        table_id,
+        alter_kind,
+        table_version: Some(expr.table_version),
+    };
+    Ok(request)
 }
 
 pub fn create_table_schema(expr: &CreateTableExpr, require_time_index: bool) -> Result<RawSchema> {

src/common/grpc-expr/src/error.rs

@@ -15,8 +15,8 @@
 use std::any::Any;
 
 use common_error::ext::ErrorExt;
-use common_error::prelude::{Snafu, StatusCode};
-use snafu::Location;
+use common_error::status_code::StatusCode;
+use snafu::{Location, Snafu};
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]

src/common/grpc-expr/src/insert.rs

@@ -16,23 +16,25 @@ use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 
 use api::helper::ColumnDataTypeWrapper;
-use api::v1::column::{SemanticType, Values};
+use api::v1::column::Values;
 use api::v1::{
     AddColumn, AddColumns, Column, ColumnDataType, ColumnDef, CreateTableExpr,
-    InsertRequest as GrpcInsertRequest,
+    InsertRequest as GrpcInsertRequest, SemanticType,
 };
 use common_base::BitVec;
+use common_time::time::Time;
 use common_time::timestamp::Timestamp;
 use common_time::{Date, DateTime};
 use datatypes::data_type::{ConcreteDataType, DataType};
 use datatypes::prelude::{ValueRef, VectorRef};
 use datatypes::scalars::ScalarVector;
 use datatypes::schema::SchemaRef;
-use datatypes::types::{Int16Type, Int8Type, TimestampType, UInt16Type, UInt8Type};
+use datatypes::types::{Int16Type, Int8Type, TimeType, TimestampType, UInt16Type, UInt8Type};
 use datatypes::value::Value;
 use datatypes::vectors::{
     BinaryVector, BooleanVector, DateTimeVector, DateVector, Float32Vector, Float64Vector,
-    Int32Vector, Int64Vector, PrimitiveVector, StringVector, TimestampMicrosecondVector,
+    Int32Vector, Int64Vector, PrimitiveVector, StringVector, TimeMicrosecondVector,
+    TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector, TimestampMicrosecondVector,
     TimestampMillisecondVector, TimestampNanosecondVector, TimestampSecondVector, UInt32Vector,
     UInt64Vector,
 };
@@ -194,6 +196,26 @@ fn collect_column_values(column_datatype: ColumnDataType, values: &Values) -> Ve
                 Timestamp::new_nanosecond(*v)
             ))
         }
+        ColumnDataType::TimeSecond => {
+            collect_values!(values.time_second_values, |v| ValueRef::Time(
+                Time::new_second(*v)
+            ))
+        }
+        ColumnDataType::TimeMillisecond => {
+            collect_values!(values.time_millisecond_values, |v| ValueRef::Time(
+                Time::new_millisecond(*v)
+            ))
+        }
+        ColumnDataType::TimeMicrosecond => {
+            collect_values!(values.time_millisecond_values, |v| ValueRef::Time(
+                Time::new_microsecond(*v)
+            ))
+        }
+        ColumnDataType::TimeNanosecond => {
+            collect_values!(values.time_millisecond_values, |v| ValueRef::Time(
+                Time::new_nanosecond(*v)
+            ))
+        }
     }
 }
 
@@ -387,7 +409,25 @@ fn values_to_vector(data_type: &ConcreteDataType, values: Values) -> VectorRef {
                 values.ts_nanosecond_values,
             )),
         },
-        ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
+        ConcreteDataType::Time(unit) => match unit {
+            TimeType::Second(_) => Arc::new(TimeSecondVector::from_iter_values(
+                values.time_second_values.iter().map(|x| *x as i32),
+            )),
+            TimeType::Millisecond(_) => Arc::new(TimeMillisecondVector::from_iter_values(
+                values.time_millisecond_values.iter().map(|x| *x as i32),
+            )),
+            TimeType::Microsecond(_) => Arc::new(TimeMicrosecondVector::from_vec(
+                values.time_microsecond_values,
+            )),
+            TimeType::Nanosecond(_) => Arc::new(TimeNanosecondVector::from_vec(
+                values.time_nanosecond_values,
+            )),
+        },
+
+        ConcreteDataType::Interval(_)
+        | ConcreteDataType::Null(_)
+        | ConcreteDataType::List(_)
+        | ConcreteDataType::Dictionary(_) => {
             unreachable!()
         }
     }
@@ -495,7 +535,31 @@ fn convert_values(data_type: &ConcreteDataType, values: Values) -> Vec<Value> {
             .into_iter()
             .map(|v| Value::Timestamp(Timestamp::new_nanosecond(v)))
             .collect(),
-        ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
+        ConcreteDataType::Time(TimeType::Second(_)) => values
+            .time_second_values
+            .into_iter()
+            .map(|v| Value::Time(Time::new_second(v)))
+            .collect(),
+        ConcreteDataType::Time(TimeType::Millisecond(_)) => values
+            .time_millisecond_values
+            .into_iter()
+            .map(|v| Value::Time(Time::new_millisecond(v)))
+            .collect(),
+        ConcreteDataType::Time(TimeType::Microsecond(_)) => values
+            .time_microsecond_values
+            .into_iter()
+            .map(|v| Value::Time(Time::new_microsecond(v)))
+            .collect(),
+        ConcreteDataType::Time(TimeType::Nanosecond(_)) => values
+            .time_nanosecond_values
+            .into_iter()
+            .map(|v| Value::Time(Time::new_nanosecond(v)))
+            .collect(),
+
+        ConcreteDataType::Interval(_)
+        | ConcreteDataType::Null(_)
+        | ConcreteDataType::List(_)
+        | ConcreteDataType::Dictionary(_) => {
             unreachable!()
         }
     }
@@ -511,14 +575,17 @@ mod tests {
     use std::{assert_eq, vec};
 
     use api::helper::ColumnDataTypeWrapper;
-    use api::v1::column::{self, SemanticType, Values};
-    use api::v1::{Column, ColumnDataType};
+    use api::v1::column::Values;
+    use api::v1::{Column, ColumnDataType, SemanticType};
     use common_base::BitVec;
     use common_catalog::consts::MITO_ENGINE;
-    use common_time::timestamp::Timestamp;
+    use common_time::timestamp::{TimeUnit, Timestamp};
     use datatypes::data_type::ConcreteDataType;
     use datatypes::schema::{ColumnSchema, SchemaBuilder};
-    use datatypes::types::{TimestampMillisecondType, TimestampSecondType, TimestampType};
+    use datatypes::types::{
+        TimeMillisecondType, TimeSecondType, TimeType, TimestampMillisecondType,
+        TimestampSecondType, TimestampType,
+    };
     use datatypes::value::Value;
     use paste::paste;
     use snafu::ResultExt;
@@ -575,8 +642,8 @@ mod tests {
         );
 
         let column_defs = create_expr.column_defs;
-        assert_eq!(column_defs[3].name, create_expr.time_index);
-        assert_eq!(4, column_defs.len());
+        assert_eq!(column_defs[4].name, create_expr.time_index);
+        assert_eq!(5, column_defs.len());
 
         assert_eq!(
             ConcreteDataType::string_datatype(),
@@ -620,6 +687,20 @@ mod tests {
             )
         );
 
+        assert_eq!(
+            ConcreteDataType::time_datatype(TimeUnit::Millisecond),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    column_defs
+                        .iter()
+                        .find(|c| c.name == "time")
+                        .unwrap()
+                        .datatype
+                )
+                .unwrap()
+            )
+        );
+
         assert_eq!(
             ConcreteDataType::timestamp_millisecond_datatype(),
             ConcreteDataType::from(
@@ -653,7 +734,7 @@ mod tests {
 
         let add_columns = find_new_columns(&schema, &insert_batch.0).unwrap().unwrap();
 
-        assert_eq!(2, add_columns.add_columns.len());
+        assert_eq!(3, add_columns.add_columns.len());
         let host_column = &add_columns.add_columns[0];
         assert!(host_column.is_key);
 
@@ -675,6 +756,17 @@ mod tests {
                     .unwrap()
             )
         );
+
+        let time_column = &add_columns.add_columns[2];
+        assert!(!time_column.is_key);
+
+        assert_eq!(
+            ConcreteDataType::time_datatype(TimeUnit::Millisecond),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(time_column.column_def.as_ref().unwrap().datatype)
+                    .unwrap()
+            )
+        );
     }
 
     #[test]
@@ -886,6 +978,39 @@ mod tests {
         assert_eq!(expect, actual);
     }
 
+    #[test]
+    fn test_convert_time_values() {
+        // second
+        let actual = convert_values(
+            &ConcreteDataType::Time(TimeType::Second(TimeSecondType)),
+            Values {
+                time_second_values: vec![1_i64, 2_i64, 3_i64],
+                ..Default::default()
+            },
+        );
+        let expect = vec![
+            Value::Time(Time::new_second(1_i64)),
+            Value::Time(Time::new_second(2_i64)),
+            Value::Time(Time::new_second(3_i64)),
+        ];
+        assert_eq!(expect, actual);
+
+        // millisecond
+        let actual = convert_values(
+            &ConcreteDataType::Time(TimeType::Millisecond(TimeMillisecondType)),
+            Values {
+                time_millisecond_values: vec![1_i64, 2_i64, 3_i64],
+                ..Default::default()
+            },
+        );
+        let expect = vec![
+            Value::Time(Time::new_millisecond(1_i64)),
+            Value::Time(Time::new_millisecond(2_i64)),
+            Value::Time(Time::new_millisecond(3_i64)),
+        ];
+        assert_eq!(expect, actual);
+    }
+
     #[test]
     fn test_is_null() {
         let null_mask = BitVec::from_slice(&[0b0000_0001, 0b0000_1000]);
@@ -903,7 +1028,7 @@ mod tests {
     fn mock_insert_batch() -> (Vec<Column>, u32) {
         let row_count = 2;
 
-        let host_vals = column::Values {
+        let host_vals = Values {
             string_values: vec!["host1".to_string(), "host2".to_string()],
             ..Default::default()
         };
@@ -915,7 +1040,7 @@ mod tests {
             datatype: ColumnDataType::String as i32,
         };
 
-        let cpu_vals = column::Values {
+        let cpu_vals = Values {
             f64_values: vec![0.31],
             ..Default::default()
         };
@@ -927,7 +1052,7 @@ mod tests {
             datatype: ColumnDataType::Float64 as i32,
         };
 
-        let mem_vals = column::Values {
+        let mem_vals = Values {
             f64_values: vec![0.1],
             ..Default::default()
         };
@@ -939,7 +1064,19 @@ mod tests {
             datatype: ColumnDataType::Float64 as i32,
         };
 
-        let ts_vals = column::Values {
+        let time_vals = Values {
+            time_millisecond_values: vec![100, 101],
+            ..Default::default()
+        };
+        let time_column = Column {
+            column_name: "time".to_string(),
+            semantic_type: SemanticType::Field as i32,
+            values: Some(time_vals),
+            null_mask: vec![0],
+            datatype: ColumnDataType::TimeMillisecond as i32,
+        };
+
+        let ts_vals = Values {
             ts_millisecond_values: vec![100, 101],
             ..Default::default()
         };
@@ -952,7 +1089,7 @@ mod tests {
         };
 
         (
-            vec![host_column, cpu_column, mem_column, ts_column],
+            vec![host_column, cpu_column, mem_column, time_column, ts_column],
             row_count,
         )
     }

src/common/grpc/Cargo.toml

@@ -11,9 +11,6 @@ async-trait = "0.1"
 backtrace = "0.3"
 common-base = { path = "../base" }
 common-error = { path = "../error" }
-common-function-macro = { path = "../function-macro" }
-common-query = { path = "../query" }
-common-meta = { path = "../meta" }
 common-recordbatch = { path = "../recordbatch" }
 common-runtime = { path = "../runtime" }
 common-telemetry = { path = "../telemetry" }
@@ -22,6 +19,7 @@ datafusion.workspace = true
 datatypes = { path = "../../datatypes" }
 flatbuffers = "23.1"
 futures = "0.3"
+lazy_static.workspace = true
 prost.workspace = true
 snafu = { version = "0.7", features = ["backtraces"] }
 tokio.workspace = true

src/common/grpc/src/channel_manager.rs

@@ -12,13 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::atomic::{AtomicUsize, Ordering};
-use std::sync::{Arc, Mutex};
+use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
+use std::sync::Arc;
 use std::time::Duration;
 
 use common_telemetry::info;
 use dashmap::mapref::entry::Entry;
 use dashmap::DashMap;
+use lazy_static::lazy_static;
 use snafu::{OptionExt, ResultExt};
 use tonic::transport::{
     Certificate, Channel as InnerChannel, ClientTlsConfig, Endpoint, Identity, Uri,
@@ -28,14 +29,20 @@ use tower::make::MakeConnection;
 use crate::error::{CreateChannelSnafu, InvalidConfigFilePathSnafu, InvalidTlsConfigSnafu, Result};
 
 const RECYCLE_CHANNEL_INTERVAL_SECS: u64 = 60;
-const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 2;
+pub const DEFAULT_GRPC_REQUEST_TIMEOUT_SECS: u64 = 10;
+pub const DEFAULT_GRPC_CONNECT_TIMEOUT_SECS: u64 = 10;
+
+lazy_static! {
+    static ref ID: AtomicU64 = AtomicU64::new(0);
+}
 
 #[derive(Clone, Debug)]
 pub struct ChannelManager {
+    id: u64,
     config: ChannelConfig,
     client_tls_config: Option<ClientTlsConfig>,
     pool: Arc<Pool>,
-    channel_recycle_started: Arc<Mutex<bool>>,
+    channel_recycle_started: Arc<AtomicBool>,
 }
 
 impl Default for ChannelManager {
@@ -50,30 +57,17 @@ impl ChannelManager {
     }
 
     pub fn with_config(config: ChannelConfig) -> Self {
+        let id = ID.fetch_add(1, Ordering::Relaxed);
         let pool = Arc::new(Pool::default());
         Self {
+            id,
             config,
             client_tls_config: None,
             pool,
-            channel_recycle_started: Arc::new(Mutex::new(false)),
+            channel_recycle_started: Arc::new(AtomicBool::new(false)),
         }
     }
 
-    pub fn start_channel_recycle(&self) {
-        let mut started = self.channel_recycle_started.lock().unwrap();
-        if *started {
-            return;
-        }
-
-        let pool = self.pool.clone();
-        let _handle = common_runtime::spawn_bg(async {
-            recycle_channel_in_loop(pool, RECYCLE_CHANNEL_INTERVAL_SECS).await;
-        });
-        info!("Channel recycle is started, running in the background!");
-
-        *started = true;
-    }
-
     pub fn with_tls_config(config: ChannelConfig) -> Result<Self> {
         let mut cm = Self::with_config(config.clone());
 
@@ -105,6 +99,8 @@ impl ChannelManager {
     }
 
     pub fn get(&self, addr: impl AsRef<str>) -> Result<InnerChannel> {
+        self.trigger_channel_recycling();
+
         let addr = addr.as_ref();
         // It will acquire the read lock.
         if let Some(inner_ch) = self.pool.get(addr) {
@@ -208,6 +204,25 @@ impl ChannelManager {
 
         Ok(endpoint)
     }
+
+    fn trigger_channel_recycling(&self) {
+        if self
+            .channel_recycle_started
+            .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed)
+            .is_err()
+        {
+            return;
+        }
+
+        let pool = self.pool.clone();
+        let _handle = common_runtime::spawn_bg(async {
+            recycle_channel_in_loop(pool, RECYCLE_CHANNEL_INTERVAL_SECS).await;
+        });
+        info!(
+            "ChannelManager: {}, channel recycle is started, running in the background!",
+            self.id
+        );
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -237,8 +252,8 @@ pub struct ChannelConfig {
 impl Default for ChannelConfig {
     fn default() -> Self {
         Self {
-            timeout: Some(Duration::from_secs(DEFAULT_REQUEST_TIMEOUT_SECS)),
-            connect_timeout: Some(Duration::from_secs(4)),
+            timeout: Some(Duration::from_secs(DEFAULT_GRPC_REQUEST_TIMEOUT_SECS)),
+            connect_timeout: Some(Duration::from_secs(DEFAULT_GRPC_CONNECT_TIMEOUT_SECS)),
             concurrency_limit: None,
             rate_limit: None,
             initial_stream_window_size: None,
@@ -468,7 +483,10 @@ mod tests {
 
     #[tokio::test]
     async fn test_access_count() {
-        let mgr = Arc::new(ChannelManager::new());
+        let mgr = ChannelManager::new();
+        // Do not start recycle
+        mgr.channel_recycle_started.store(true, Ordering::Relaxed);
+        let mgr = Arc::new(mgr);
         let addr = "test_uri";
 
         let mut joins = Vec::with_capacity(10);
@@ -498,8 +516,8 @@ mod tests {
         let default_cfg = ChannelConfig::new();
         assert_eq!(
             ChannelConfig {
-                timeout: Some(Duration::from_secs(DEFAULT_REQUEST_TIMEOUT_SECS)),
-                connect_timeout: Some(Duration::from_secs(4)),
+                timeout: Some(Duration::from_secs(DEFAULT_GRPC_REQUEST_TIMEOUT_SECS)),
+                connect_timeout: Some(Duration::from_secs(DEFAULT_GRPC_CONNECT_TIMEOUT_SECS)),
                 concurrency_limit: None,
                 rate_limit: None,
                 initial_stream_window_size: None,

src/common/grpc/src/error.rs

@@ -15,7 +15,8 @@
 use std::any::Any;
 use std::io;
 
-use common_error::prelude::{ErrorExt, StatusCode};
+use common_error::ext::ErrorExt;
+use common_error::status_code::StatusCode;
 use snafu::{Location, Snafu};
 
 pub type Result<T> = std::result::Result<T, Error>;