fix: clippy errors

* fix: Fix several tests compile errors

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix: some compile errors in tests

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix: compile errors in frontend tests

* fix: compile errors in frontend tests

* test: Fix tests in api and common-query

* test: Fix test in sql crate

* fix: resolve substrait error

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* chore: add more test

* test: Fix tests in servers

* fix instance_test

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* test: Fix tests in tests-integration

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: Lei, HUANG <mrsatangel@gmail.com>
Co-authored-by: evenyag <realevenyag@gmail.com>

.cargo/config.toml

@@ -0,0 +1,2 @@
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"

.config/nextest.toml

@@ -0,0 +1,2 @@
+[profile.default]
+slow-timeout = { period = "60s", terminate-after = 3, grace-period = "30s" }

.env.example

@@ -0,0 +1,4 @@
+# Settings for s3 test
+GT_S3_BUCKET=S3 bucket
+GT_S3_ACCESS_KEY_ID=S3 access key id
+GT_S3_ACCESS_KEY=S3 secret access key

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,86 @@
+---
+name: Bug report
+description: Is something not working? Help us fix it!
+labels: [ "bug" ]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Take some time to fill out this bug report. Thank you!
+
+  - type: dropdown
+    id: type
+    attributes:
+      label: What type of bug is this?
+      multiple: true
+      options:
+        - Configuration
+        - Crash
+        - Data corruption
+        - Incorrect result
+        - Locking issue
+        - Performance issue
+        - Unexpected error
+        - Other
+    validations:
+      required: true
+
+  - type: dropdown
+    id: subsystem
+    attributes:
+      label: What subsystems are affected?
+      description: You can pick multiple subsystems.
+      multiple: true
+      options:
+        - Standalone mode
+        - Frontend
+        - Datanode
+        - Meta
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: |
+        Tell us what happened and also what you would have expected to
+        happen instead.
+      placeholder: "Describe the bug"
+    validations:
+      required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: What operating system did you use?
+      description: |
+        Please provide OS, version, and architecture. For example:
+        Windows 10 x64, Ubuntu 21.04 x64, Mac OS X 10.5 ARM, Rasperry
+        Pi i386, etc.
+      placeholder: "Ubuntu 21.04 x64"
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output and stack trace
+      description: |
+        Please copy and paste any relevant log output or a stack
+        trace. This will be automatically formatted into code, so no
+        need for backticks.
+      render: bash
+
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: How can we reproduce the bug?
+      description: |
+        Please walk us through and provide steps and details on how
+        to reproduce the issue. If possible, provide scripts that we
+        can run to trigger the bug.
+      render: bash
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Greptime Community Slack
+    url: https://greptime.com/slack
+    about: Get free help from the Greptime community
+  - name: Greptime Community Discussion
+    url: https://github.com/greptimeTeam/greptimedb/discussions
+    about: Get free help from the Greptime community

.github/ISSUE_TEMPLATE/enhancement.yml

@@ -0,0 +1,39 @@
+---
+name: Enhancement
+description: Suggest an enhancement to existing functionality
+labels: [ "enhancement" ]
+body:
+  - type: dropdown
+    id: type
+    attributes:
+      label: What type of enhancement is this?
+      multiple: true
+      options:
+        - API improvement
+        - Configuration
+        - Performance
+        - Refactor
+        - Tech debt reduction
+        - User experience
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: what
+    attributes:
+      label: What does the enhancement do?
+      description: |
+        Give a high-level overview of how you
+        suggest improving an existing feature or functionality.
+    validations:
+     required: true
+
+  - type: textarea
+    id: implementation
+    attributes:
+      label: Implementation challenges
+      description: |
+        Share any ideas of how to implement the enhancement.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,42 @@
+---
+name: Feature request
+description: Suggest a new feature for GreptimeDB
+labels: [ "feature request" ]
+body:
+  - type: markdown
+    id: info
+    attributes:
+      value: |
+        Only use this template to suggest a new feature that doesn't already exist in GreptimeDB.
+        For enhancements to existing features, use the "Enhancement" issue template. For bugs,
+        use the bug report template.
+
+  - type: textarea
+    id: what
+    attributes:
+      label: What problem does the new feature solve?
+      description: |
+        Describe the problem and why it is important to solve. Did you consider alternative
+        solutions, perhaps outside the database? Why is it better to add the feature to
+        GreptimeDB?
+    validations:
+      required: true
+
+  - type: textarea
+    id: how
+    attributes:
+      label: What does the feature do?
+      description: |
+        Give a high-level overview of what the feature does and how it would work.
+    validations:
+     required: true
+
+  - type: textarea
+    id: implementation
+    attributes:
+      label: Implementation challenges
+      description: |
+        If you have ideas of how to implement the feature, and any particularly
+        challenging issues to overcome, then provide them here.
+    validations:
+      required: false

.github/pr-title-checker-config.json

@@ -4,7 +4,7 @@
     "color": "B60205"
   },
   "CHECKS": {
-    "regexp": "^(feat|fix|test|refactor|chore|style|doc|perf|build|ci|revert)(\\(.*\\))?:.*",
+    "regexp": "^(feat|fix|test|refactor|chore|style|docs|perf|build|ci|revert)(\\(.*\\))?:.*",
     "ignoreLabels" : ["ignore-title"]
   }
 }

.github/pull_request_template.md

@@ -0,0 +1,19 @@
+I hereby agree to the terms of the [GreptimeDB CLA](https://gist.github.com/xtang/6378857777706e568c1949c7578592cc)
+
+## What's changed and what's your intention?
+
+_PLEASE DO NOT LEAVE THIS EMPTY !!!_
+
+Please explain IN DETAIL what the changes are in this PR and why they are needed:
+
+- Summarize your change (**mandatory**)
+- How does this PR work? Need a brief introduction for the changed logic (optional)
+- Describe clearly one logical change and avoid lazy messages (optional)
+- Describe any limitations of the current code (optional)
+
+## Checklist
+
+- [ ]  I have written the necessary rustdoc comments.
+- [ ]  I have added the necessary unit tests and integration tests.
+
+## Refer to a related PR or issue link (optional)

.github/workflows/coverage.yml

@@ -1,10 +1,25 @@
 on:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
   push:
     branches:
       - "main"
       - "develop"
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
+  workflow_dispatch:
 
 name: Code coverage
 
@@ -12,33 +27,33 @@ env:
   RUST_TOOLCHAIN: nightly-2022-07-14
 
 jobs:
-  grcov:
+  coverage:
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-8-cores
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
       - name: Install toolchain
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
-          profile: minimal
-      - name: Execute tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace
+          components: llvm-tools-preview
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install latest nextest release
+        uses: taiki-e/install-action@nextest
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - name: Collect coverage data
+        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info
         env:
           RUST_BACKTRACE: 1
           CARGO_INCREMENTAL: 0
-          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
           GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
           GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
           GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-      - name: Gather coverage data
-        id: coverage
-        uses: actions-rs/grcov@v0.1
+          UNITTEST_LOG_DIR: "__unittest_logs"
       - name: Codecov upload
         uses: codecov/codecov-action@v2
         with:

.github/workflows/develop.yml

@@ -1,84 +1,141 @@
 on:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+  push:
+    branches:
+      - develop
+      - main
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
+  workflow_dispatch:
 
-name: Continuous integration for developing
+name: CI
 
 env:
   RUST_TOOLCHAIN: nightly-2022-07-14
 
 jobs:
+  typos:
+    name: Spell Check with Typos
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: crate-ci/typos@v1.0.4
+
   check:
     name: Check
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@master
         with:
-          profile: minimal
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --workspace --all-targets
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Run cargo check
+        run: cargo check --workspace --all-targets
 
-  test:
-    name: Test Suite
+  toml:
+    name: Toml Check
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
-      - uses: arduino/setup-protoc@v1
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@master
         with:
-          profile: minimal
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace
-        env:
-          RUST_BACKTRACE: 1
-          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install taplo
+        run: cargo install taplo-cli --version ^0.8 --locked
+      - name: Run taplo
+        run: taplo format --check --option "indent_string=    "
+
+  # Use coverage to run test.
+  # test:
+  #   name: Test Suite
+  #   if: github.event.pull_request.draft == false
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 60
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - name: Cache LLVM and Clang
+  #       id: cache-llvm
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: ./llvm
+  #         key: llvm
+  #     - uses: arduino/setup-protoc@v1
+  #     - uses: KyleMayes/install-llvm-action@v1
+  #       with:
+  #         version: "14.0"
+  #         cached: ${{ steps.cache-llvm.outputs.cache-hit }}
+  #     - uses: dtolnay/rust-toolchain@master
+  #       with:
+  #         toolchain: ${{ env.RUST_TOOLCHAIN }}
+  #     - name: Rust Cache
+  #       uses: Swatinem/rust-cache@v2
+  #     - name: Cleanup disk
+  #       uses: curoky/cleanup-disk-action@v2.0
+  #       with:
+  #         retain: 'rust,llvm'
+  #     - name: Install latest nextest release
+  #       uses: taiki-e/install-action@nextest
+  #     - name: Run tests
+  #       run: cargo nextest run
+  #       env:
+  #         CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
+  #         RUST_BACKTRACE: 1
+  #         GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+  #         GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+  #         GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+  #         UNITTEST_LOG_DIR: "__unittest_logs"
 
   fmt:
     name: Rustfmt
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@master
         with:
-          profile: minimal
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
+          components: rustfmt
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Run cargo fmt
+        run: cargo fmt --all -- --check
 
   clippy:
     name: Clippy
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@master
         with:
-          profile: minimal
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
-      - run: rustup component add clippy
-      - uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr
+          components: clippy
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Run cargo clippy
+        run: cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr

.github/workflows/doc-issue.yml

@@ -0,0 +1,25 @@
+name: Create Issue in docs repo on doc related changes
+
+on:
+  issues:
+    types:
+      - labeled
+  pull_request_target:
+    types:
+      - labeled
+
+jobs:
+  doc_issue:
+    if: github.event.label.name == 'doc update required'
+    runs-on: ubuntu-latest
+    steps:
+      - name: create an issue in doc repo
+        uses: dacbd/create-issue-action@main
+        with:
+          owner: GreptimeTeam
+          repo: docs
+          token: ${{ secrets.DOCS_REPO_TOKEN }}
+          title: Update docs for ${{ github.event.issue.title || github.event.pull_request.title }}
+          body: |
+            A document change request is generated from
+            ${{ github.event.issue.html_url || github.event.pull_request.html_url }}

.github/workflows/license.yaml

@@ -0,0 +1,16 @@
+name: License checker
+
+on:
+  push:
+    branches:
+    - develop
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+jobs:
+  license-header-check:
+    runs-on: ubuntu-latest
+    name: license-header-check
+    steps:
+    - uses: actions/checkout@v2
+    - name: Check License Header
+      uses: apache/skywalking-eyes/header@main

.github/workflows/pr-title-checker.yml

@@ -11,6 +11,7 @@ on:
 jobs:
   check:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - uses: thehanimo/pr-title-checker@v1.3.4
         with:

.github/workflows/release.yml

@@ -0,0 +1,226 @@
+on:
+  push:
+    tags:
+      - "v*.*.*"
+  schedule:
+    # At 00:00 on Monday.
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
+
+name: Release
+
+env:
+  RUST_TOOLCHAIN: nightly-2022-07-14
+
+  # FIXME(zyy17): Would be better to use `gh release list -L 1 | cut -f 3` to get the latest release version tag, but for a long time, we will stay at 'v0.1.0-alpha-*'.
+  SCHEDULED_BUILD_VERSION_PREFIX: v0.1.0-alpha
+
+  # In the future, we can change SCHEDULED_PERIOD to nightly.
+  SCHEDULED_PERIOD: weekly
+
+jobs:
+  build:
+    name: Build binary
+    strategy:
+      matrix:
+        # The file format is greptime-<os>-<arch>
+        include:
+          - arch: x86_64-unknown-linux-gnu
+            os: ubuntu-latest-16-cores
+            file: greptime-linux-amd64
+          - arch: aarch64-unknown-linux-gnu
+            os: ubuntu-latest-16-cores
+            file: greptime-linux-arm64
+          - arch: aarch64-apple-darwin
+            os: macos-latest
+            file: greptime-darwin-arm64
+          - arch: x86_64-apple-darwin
+            os: macos-latest
+            file: greptime-darwin-amd64
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Cache cargo assets
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ matrix.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install Protoc for linux
+        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
+        run: | # Make sure the protoc is >= 3.15
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip
+          unzip protoc-21.9-linux-x86_64.zip -d protoc
+          sudo cp protoc/bin/protoc /usr/local/bin/
+          sudo cp -r protoc/include/google /usr/local/include/
+
+      - name: Install Protoc for macos
+        if: contains(matrix.arch, 'darwin')
+        run: |
+          brew install protobuf
+
+      - name: Install dependencies for linux
+        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
+        run: |
+          sudo apt-get -y update
+          sudo apt-get -y install libssl-dev pkg-config g++-aarch64-linux-gnu gcc-aarch64-linux-gnu
+
+      - name: Install rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          targets: ${{ matrix.arch }}
+
+      - name: Output package versions
+        run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version
+
+      - name: Run cargo build
+        run: cargo build ${{ matrix.opts }} --release --locked --target ${{ matrix.arch }}
+
+      - name: Calculate checksum and rename binary
+        shell: bash
+        run: |
+          cd target/${{ matrix.arch }}/release
+          chmod +x greptime
+          tar -zcvf ${{ matrix.file }}.tgz greptime
+          echo $(shasum -a 256 ${{ matrix.file }}.tgz | cut -f1 -d' ') > ${{ matrix.file }}.sha256sum
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.file }}
+          path: target/${{ matrix.arch }}/release/${{ matrix.file }}.tgz
+
+      - name: Upload checksum of artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.file }}.sha256sum
+          path: target/${{ matrix.arch }}/release/${{ matrix.file }}.sha256sum
+  release:
+    name: Release artifacts
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Configure scheduled build version # the version would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}, like v0.1.0-alpha-20221119-weekly.
+        shell: bash
+        if: github.event_name == 'schedule'
+        run: |
+          buildTime=`date "+%Y%m%d"`
+          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
+          echo "SCHEDULED_BUILD_VERSION=${SCHEDULED_BUILD_VERSION}" >> $GITHUB_ENV
+
+      - name: Create scheduled build git tag
+        if: github.event_name == 'schedule'
+        run: |
+          git tag ${{ env.SCHEDULED_BUILD_VERSION }}
+
+      - name: Publish scheduled release # configure the different release title and tags.
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'schedule'
+        with:
+          name: "Release ${{ env.SCHEDULED_BUILD_VERSION }}"
+          tag_name: ${{ env.SCHEDULED_BUILD_VERSION }}
+          generate_release_notes: true
+          files: |
+            **/greptime-*
+
+      - name: Publish release
+        uses: softprops/action-gh-release@v1
+        if: github.event_name != 'schedule'
+        with:
+          name: "Release ${{ github.ref_name }}"
+          files: |
+            **/greptime-*
+
+  docker:
+    name: Build docker image
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Download amd64 binary
+        uses: actions/download-artifact@v3
+        with:
+          name: greptime-linux-amd64
+          path: amd64
+
+      - name: Unzip the amd64 artifacts
+        run: |
+          cd amd64
+          tar xvf greptime-linux-amd64.tgz
+          rm greptime-linux-amd64.tgz
+
+      - name: Download arm64 binary
+        uses: actions/download-artifact@v3
+        with:
+          name: greptime-linux-arm64
+          path: arm64
+
+      - name: Unzip the arm64 artifacts
+        run: |
+          cd arm64
+          tar xvf greptime-linux-arm64.tgz
+          rm greptime-linux-arm64.tgz
+
+      - name: Login to UCloud Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: uhub.service.ucloud.cn
+          username: ${{ secrets.UCLOUD_USERNAME }}
+          password: ${{ secrets.UCLOUD_PASSWORD }}
+
+      - name: Login to Dockerhub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
+        shell: bash
+        if: github.event_name == 'schedule'
+        run: |
+          buildTime=`date "+%Y%m%d"`
+          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
+          echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV
+
+      - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
+        shell: bash
+        if: github.event_name != 'schedule'
+        run: |
+          VERSION=${{ github.ref_name }}
+          echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./docker/ci/Dockerfile
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            greptime/greptimedb:latest
+            greptime/greptimedb:${{ env.IMAGE_TAG }}
+            uhub.service.ucloud.cn/greptime/greptimedb:latest
+            uhub.service.ucloud.cn/greptime/greptimedb:${{ env.IMAGE_TAG }}

.gitignore

@@ -18,13 +18,20 @@ debug/
 
 # JetBrains IDE config directory
 .idea/
+*.iml
+
+# VSCode IDE config directory
+.vscode/
 
 # Logs
 **/__unittest_logs
 logs/
 
-.DS_store
-.gitignore
-
 # cpython's generated python byte code
 **/__pycache__/
+
+# Benchmark dataset
+benchmarks/data
+
+# dotenv
+.env

.licenserc.yaml

@@ -0,0 +1,14 @@
+header:
+  license:
+    spdx-id: Apache-2.0
+    copyright-owner: Greptime Team
+
+  paths:
+    - "**/*.rs"
+    - "**/*.py"
+
+  comment: on-failure
+
+dependency:
+  files:
+    - Cargo.toml

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+-   repo: https://github.com/compilerla/conventional-pre-commit
+    rev: 47923ce11be4a936cd216d427d985dd342adb751
+    hooks:
+    -    id: conventional-pre-commit
+         stages: [commit-msg]
+
+-   repo: https://github.com/DevinR528/cargo-sort
+    rev: e6a795bc6b2c0958f9ef52af4863bbd7cc17238f
+    hooks:
+    -    id: cargo-sort
+         args: ["--workspace"]
+
+-   repo: https://github.com/doublify/pre-commit-rust
+    rev: v1.0
+    hooks:
+    -    id: fmt
+    -    id: clippy
+         args: ["--workspace", "--all-targets", "--", "-D", "warnings", "-D", "clippy::print_stdout", "-D", "clippy::print_stderr"]
+         stages: [push]
+    -    id: cargo-check

CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+info@greptime.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -1,23 +1,92 @@
-# Contributing to GreptimeDB
+# Welcome!
 
-Much appreciate for your interest in contributing to GreptimeDB! This document list some guidelines for contributing to our code base.
+Thanks a lot for considering contributing to GreptimeDB. We believe people like you would make GreptimeDB a great product. We intend to build a community where individuals can have open talks, show respect for one another, and speak with true ❤️. Meanwhile, we are to keep transparency and make your effort count here.
 
-To learn about the design of GreptimeDB, please refer to the [design docs](https://github.com/GrepTimeTeam/docs).
+Read the guidelines, and they can help you get started. Communicate with respect to developers maintaining and developing the project. In return, they should reciprocate that respect by addressing your issue, reviewing changes, as well as helping finalize and merge your pull requests.
 
-## Pull Requests
+Follow our [README](https://github.com/GreptimeTeam/greptimedb#readme) to get the whole picture of the project. To learn about the design of GreptimeDB, please refer to the [design docs](https://github.com/GrepTimeTeam/docs).
+
+## Your First Contribution
+
+It can feel intimidating to contribute to a complex project, but it can also be exciting and fun. These general notes will help everyone participate in this communal activity.
+
+- Follow the [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/develop/CODE_OF_CONDUCT.md)
+- Small changes make huge differences. We will happily accept a PR making a single character change if it helps move forward. Don't wait to have everything working.
+- Check the closed issues before opening your issue.
+- Try to follow the existing style of the code.
+- More importantly, when in doubt, ask away.
+
+Pull requests are great, but we accept all kinds of other help if you like. Such as
+
+- Write tutorials or blog posts. Blog, speak about, or create tutorials about one of GreptimeDB's many features. Mention [@greptime](https://twitter.com/greptime) on Twitter and email info@greptime.com so we can give pointers and tips and help you spread the word by promoting your content on Greptime communication channels.
+- Improve the documentation. [Submit documentation](http://github.com/greptimeTeam/docs/) updates, enhancements, designs, or bug fixes, and fixing any spelling or grammar errors will be very much appreciated.
+- Present at meetups and conferences about your GreptimeDB projects. Your unique challenges and successes in building things with GreptimeDB can provide great speaking material. We'd love to review your talk abstract, so get in touch with us if you'd like some help!
+- Submit bug reports. To report a bug or a security issue, you can [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new).
+- Speak up feature requests. Send feedback is a great way for us to understand your different use cases of GreptimeDB better. If you want to share your experience with GreptimeDB, or if you want to discuss any ideas, you can start a discussion on [GitHub discussions](https://github.com/GreptimeTeam/greptimedb/discussions), chat with the Greptime team on [Slack](https://greptime.com/slack), or you can tweet [@greptime](https://twitter.com/greptime) on Twitter.
+
+## Code of Conduct
+
+Also, there are things that we are not looking for because they don't match the goals of the product or benefit the community. Please read [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/develop/CODE_OF_CONDUCT.md); we hope everyone can keep good manners and become an honored member.
+
+## License
+
+GreptimeDB uses the [Apache 2.0 license](https://github.com/GreptimeTeam/greptimedb/blob/master/LICENSE) to strike a balance between open contributions and allowing you to use the software however you want.
+
+## Getting Started
+
+### Submitting Issues
+
+- Check if an issue already exists. Before filing an issue report, see whether it's already covered. Use the search bar and check out existing issues.
+- File an issue:
+  - To report a bug, a security issue, or anything that you think is a problem and that isn't under the radar, go ahead and [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new).
+  - In the given templates, look for the one that suits you.
+  - If you bump into anything, reach out to our [Slack](https://greptime.com/slack) for a wider audience and ask for help.
+- What happens after:
+  - Once we spot a new issue, we identify and categorize it as soon as possible.
+  - Usually, it gets assigned to other developers. Follow up and see what folks are talking about and how they take care of it.
+  - Please be patient and offer as much information as you can to help reach a solution or a consensus. You are not alone and embrace team power.
 
 ### Before PR
+
+- To ensure that community is free and confident in its ability to use your contributions, please sign the Contributor License Agreement (CLA) which will be incorporated in the pull request process.
+- Make sure all your codes are formatted and follow the [coding style](https://pingcap.github.io/style-guide/rust/).
 - Make sure all unit tests are passed.
 - Make sure all clippy warnings are fixed (you can check it locally by running `cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr`).
 
+#### `pre-commit` Hooks
+You could setup the [`pre-commit`](https://pre-commit.com/#plugins) hooks to run these checks on every commit automatically.
+
+1. Install `pre-commit`
+```
+$ pip install pre-commit
+```
+or
+```
+$ brew install pre-commit
+```
+
+2. Install the `pre-commit` hooks
+```
+$ pre-commit install
+pre-commit installed at .git/hooks/pre-commit
+
+$ pre-commit install --hook-type commit-msg
+pre-commit installed at .git/hooks/commit-msg
+
+$ pre-commit install --hook-type pre-push
+pre-commit installed at .git/hooks/pre-pus
+```
+
+now `pre-commit` will run automatically on `git commit`.
+
 ### Title
 
-The titles of pull requests should be prefixed with category name listed in [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0) 
-like `feat`/`fix`/`doc`, with a concise summary of code change follows. DO NOT use last commit message as pull request title.
+The titles of pull requests should be prefixed with category names listed in [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0)
+like `feat`/`fix`/`docs`, with a concise summary of code change following. DO NOT use last commit message as pull request title.
 
 ### Description
 
-- If your pull request is small, like a typo fix, feel free to go brief.
+- Feel free to go brief if your pull request is small, like a typo fix.
 - But if it contains large code change, make sure to state the motivation/design details of this PR so that reviewers can understand what you're trying to do.
 - If the PR contains any breaking change or API change, make sure that is clearly listed in your description.
 
@@ -25,11 +94,18 @@ like `feat`/`fix`/`doc`, with a concise summary of code change follows. DO NOT u
 
 All commit messages SHOULD adhere to the [Conventional Commits specification](https://conventionalcommits.org/).
 
-## Getting help
+## Getting Help
 
 There are many ways to get help when you're stuck. It is recommended to ask for help by opening an issue, with a detailed description
-of what you were trying to do and what went wrong. You can also reach for help in our Slack channel.
+of what you were trying to do and what went wrong. You can also reach for help in our [Slack channel](https://greptime.com/slack).
 
+## Community
 
-## Bug report
-To report a bug or a security issue, you can [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new). 
+The core team will be thrilled if you participate in any way you like. When you are stuck, try ask for help by filing an issue, with a detailed description of what you were trying to do and what went wrong. If you have any questions or if you would like to get involved in our community, please check out:
+- [GreptimeDB Community Slack](https://greptime.com/slack)
+- [GreptimeDB Github Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)
+
+Also, see some extra GreptimeDB content:
+- [GreptimeDB Docs](https://greptime.com/docs)
+- [Learn GreptimeDB](https://greptime.com/products/db)
+- [Greptime Inc. Website](https://greptime.com)

Cargo.toml

@@ -1,30 +1,43 @@
 [workspace]
 members = [
+    "benchmarks",
     "src/api",
     "src/catalog",
     "src/client",
+    "src/cmd",
     "src/common/base",
+    "src/common/catalog",
     "src/common/error",
     "src/common/function",
+    "src/common/function-macro",
     "src/common/grpc",
+    "src/common/grpc-expr",
     "src/common/query",
     "src/common/recordbatch",
     "src/common/runtime",
+    "src/common/substrait",
     "src/common/telemetry",
     "src/common/time",
-    "src/cmd",
     "src/datanode",
     "src/datatypes",
+    "src/frontend",
     "src/log-store",
-    "src/logical-plans",
+    "src/meta-client",
+    "src/meta-srv",
+    "src/mito",
     "src/object-store",
+    "src/promql",
     "src/query",
     "src/script",
     "src/servers",
+    "src/session",
     "src/sql",
     "src/storage",
     "src/store-api",
     "src/table",
-    "src/table-engine",
-    "test-util",
+    "tests-integration",
+    "tests/runner",
 ]
+
+[profile.release]
+debug = true

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2022 Greptime Team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,67 @@
+IMAGE_REGISTRY ?= greptimedb
+IMAGE_TAG ?= latest
+
+##@ Build
+
+.PHONY: build
+build: ## Build debug version greptime.
+	cargo build
+
+.PHONY: release
+release:  ## Build release version greptime.
+	cargo build --release
+
+.PHONY: clean
+clean: ## Clean the project.
+	cargo clean
+
+.PHONY: fmt
+fmt: ## Format all the Rust code.
+	cargo fmt --all
+
+.PHONY: docker-image
+docker-image: ## Build docker image.
+	docker build --network host -f docker/Dockerfile -t ${IMAGE_REGISTRY}:${IMAGE_TAG} .
+
+##@ Test
+
+.PHONY: unit-test
+unit-test: ## Run unit test.
+	cargo test --workspace
+
+.PHONY: integration-test
+integration-test: ## Run integation test.
+	cargo test integration
+
+.PHONY: sqlness-test
+sqlness-test: ## Run sqlness test.
+	cargo run --bin sqlness-runner
+
+.PHONY: check
+check: ## Cargo check all the targets.
+	cargo check --workspace --all-targets
+
+.PHONY: clippy
+clippy: ## Check clippy rules.
+	cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr
+
+.PHONY: fmt-check
+fmt-check: ## Check code format.
+	cargo fmt --all -- --check
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# https://linuxcommand.org/lc3_adv_awk.php
+
+.PHONY: help
+help: ## Display help messages.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

README.md

@@ -1,92 +1,100 @@
-# GreptimeDB
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: light)" srcset="/docs/logo-text-padding.png">
+    <source media="(prefers-color-scheme: dark)" srcset="/docs/logo-text-padding-dark.png">
+    <img alt="GreptimeDB Logo" src="/docs/logo-text-padding.png" width="400px">
+  </picture>
+</p>
 
-[![codecov](https://codecov.io/gh/GrepTimeTeam/greptimedb/branch/develop/graph/badge.svg?token=FITFDI3J3C)](https://codecov.io/gh/GrepTimeTeam/greptimedb)
 
-GreptimeDB: the next-generation hybrid timeseries/analytics processing database in the cloud.
+<h3 align="center">
+    The next-generation hybrid timeseries/analytics processing database in the cloud
+</h3>
 
-## Getting Started
+<p align="center">
+    <a href="https://codecov.io/gh/GrepTimeTeam/greptimedb"><img src="https://codecov.io/gh/GrepTimeTeam/greptimedb/branch/develop/graph/badge.svg?token=FITFDI3J3C"></img></a>
+    &nbsp;
+    <a href="https://github.com/GreptimeTeam/greptimedb/actions/workflows/develop.yml"><img src="https://github.com/GreptimeTeam/greptimedb/actions/workflows/develop.yml/badge.svg" alt="CI"></img></a>
+    &nbsp;
+    <a href="https://github.com/greptimeTeam/greptimedb/blob/develop/LICENSE"><img src="https://img.shields.io/github/license/greptimeTeam/greptimedb"></a>
+</p>
 
-### Prerequisites
+<p align="center">
+    <a href="https://twitter.com/greptime"><img src="https://img.shields.io/badge/twitter-follow_us-1d9bf0.svg"></a>
+    &nbsp;
+    <a href="https://www.linkedin.com/company/greptime/"><img src="https://img.shields.io/badge/linkedin-connect_with_us-0a66c2.svg"></a>
+</p>
 
-To compile GreptimeDB from source, you'll need the following:
-- Rust
-- Protobuf
-- OpenSSL
+## What is GreptimeDB
 
-#### Rust
+GreptimeDB is an open-source time-series database with a special focus on
+scalability, analytical capabilities and efficiency. It's designed to work on
+infrastructure of the cloud era, and users benefit from its elasticity and commodity
+storage.
 
-The easiest way to install Rust is to use [`rustup`](https://rustup.rs/), which will check our `rust-toolchain` file and install correct Rust version for you.
+Our core developers have been building time-series data platform
+for years. Based on their best-practices, GreptimeDB is born to give you:
 
-#### Protobuf
+- A standalone binary that scales to highly-available distributed cluster, providing a transparent experience for cluster users
+- Optimized columnar layout for handling time-series data; compacted, compressed, stored on various storage backends
+- Flexible index options, tackling high cardinality issues down
+- Distributed, parallel query execution, leveraging elastic computing resource
+- Native SQL, and Python scripting for advanced analytical scenarios
+- Widely adopted database protocols and APIs
+- Extensible table engine architecture for extensive workloads
 
-`protoc` is required for compiling `.proto` files. `protobuf` is available from
-major package manager on macos and linux distributions. You can find an
-installation instructions [here](https://grpc.io/docs/protoc-installation/).
+## Quick Start
 
-#### OpenSSL
+### Build
 
-For Ubuntu:
-```bash
-sudo apt install libssl-dev
-```
+#### Build from Source
 
-For RedHat-based: Fedora, Oracle Linux, etc:
-```bash
-sudo dnf install openssl-devel
-```
+To compile GreptimeDB from source, you'll need:
 
-For macOS:
-```bash
-brew install openssl
-```
+- C/C++ Toolchain: provides basic tools for compiling and linking. This is
+  available as `build-essential` on ubuntu and similar name on other platforms.
+- Rust: the easiest way to install Rust is to use
+  [`rustup`](https://rustup.rs/), which will check our `rust-toolchain` file and
+  install correct Rust version for you.
+- Protobuf: `protoc` is required for compiling `.proto` files. `protobuf` is
+  available from major package manager on macos and linux distributions. You can
+  find an installation instructions [here](https://grpc.io/docs/protoc-installation/).
+  **Note that `protoc` version needs to be >= 3.15** because we have used the `optional`
+  keyword. You can check it with `protoc --version`.
 
-### Build the Docker Image
+#### Build with Docker
+
+A docker image with necessary dependencies is provided:
 
 ```
 docker build --network host -f docker/Dockerfile -t greptimedb .
 ```
 
-## Usage
+### Run
 
-### Start Datanode
+Start GreptimeDB from source code, in standalone mode:
 
 ```
-// Start datanode with default options.
-cargo run -- datanode start
-
-OR
-
-// Start datanode with `http-addr` option.
-cargo run -- datanode start --http-addr=0.0.0.0:9999
-
-OR
-
-// Start datanode with `log-dir` and `log-level` options.
-cargo run -- --log-dir=logs --log-level=debug datanode start
+cargo run -- standalone start
 ```
 
-Start datanode with config file:
+Or if you built from docker:
 
 ```
-cargo run -- --log-dir=logs --log-level=debug datanode start -c ./config/datanode.example.toml
+docker run -p 4002:4002 -v "$(pwd):/tmp/greptimedb" greptime/greptimedb standalone start
 ```
 
-Start datanode by runing docker container:
+For more startup options, greptimedb's **distributed mode** and information
+about Kubernetes deployment, check our [docs](https://docs.greptime.com/).
 
-```
-docker run -p 3000:3000 \
--p 3001:3001 \
--p 3306:3306 \
-greptimedb
-```
+### Connect
 
-### SQL Operations
-
-1. Connecting DB by [mysql client](https://dev.mysql.com/downloads/mysql/):
+1. Connect to GreptimeDB via standard [MySQL
+   client](https://dev.mysql.com/downloads/mysql/):
 
    ```
-   # The datanode listen on port 3306 by default.
-   mysql -h 127.0.0.1 -P 3306
+   # The standalone instance listen on port 4002 by default.
+   mysql -h 127.0.0.1 -P 4002
    ```
 
 2. Create table:
@@ -94,32 +102,102 @@ greptimedb
    ```SQL
    CREATE TABLE monitor (
      host STRING,
-     ts BIGINT,
+     ts TIMESTAMP,
      cpu DOUBLE DEFAULT 0,
      memory DOUBLE,
      TIME INDEX (ts),
-     PRIMARY KEY(ts,host)) ENGINE=mito WITH(regions=1);
+     PRIMARY KEY(host)) ENGINE=mito WITH(regions=1);
    ```
 
-3. Insert data:
+3. Insert some data:
 
    ```SQL
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host1', 66.6, 1024, 1660897955);
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host2', 77.7, 2048, 1660897956);
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host3', 88.8, 4096, 1660897957);
+   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host1', 66.6, 1024, 1660897955000);
+   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host2', 77.7, 2048, 1660897956000);
+   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host3', 88.8, 4096, 1660897957000);
    ```
 
-4. Query data:
+4. Query the data:
 
    ```SQL
-   mysql> SELECT * FROM monitor;
-   +-------+------------+------+--------+
-   | host  | ts         | cpu  | memory |
-   +-------+------------+------+--------+
-   | host1 | 1660897955 | 66.6 |   1024 |
-   | host2 | 1660897956 | 77.7 |   2048 |
-   | host3 | 1660897957 | 88.8 |   4096 |
-   +-------+------------+------+--------+
+   SELECT * FROM monitor;
+   ```
+
+   ```TEXT
+   +-------+---------------------+------+--------+
+   | host  | ts                  | cpu  | memory |
+   +-------+---------------------+------+--------+
+   | host1 | 2022-08-19 08:32:35 | 66.6 |   1024 |
+   | host2 | 2022-08-19 08:32:36 | 77.7 |   2048 |
+   | host3 | 2022-08-19 08:32:37 | 88.8 |   4096 |
+   +-------+---------------------+------+--------+
    3 rows in set (0.01 sec)
    ```
-   You can delete your data by removing `/tmp/greptimedb`.
+
+You can always cleanup test database by removing `/tmp/greptimedb`.
+
+## Resources
+
+### Installation
+
+- [Pre-built Binaries](https://github.com/GreptimeTeam/greptimedb/releases):
+  downloadable pre-built binaries for Linux and MacOS
+- [Docker Images](https://hub.docker.com/r/greptime/greptimedb): pre-built
+  Docker images
+- [`gtctl`](https://github.com/GreptimeTeam/gtctl): the command-line tool for
+  Kubernetes deployment
+
+### Documentation
+
+- GreptimeDB [User Guide](https://docs.greptime.com/user-guide/concepts.html)
+- GreptimeDB [Developer
+  Guide](https://docs.greptime.com/developer-guide/overview.html)
+
+### SDK
+
+- [GreptimeDB Java
+  Client](https://github.com/GreptimeTeam/greptimedb-client-java)
+
+## Project Status
+
+This project is in its early stage and under heavy development. We move fast and
+break things. Benchmark on development branch may not represent its potential
+performance. We release pre-built binaries constantly for functional
+evaluation. Do not use it in production at the moment.
+
+For future plans, check out [GreptimeDB roadmap](https://github.com/GreptimeTeam/greptimedb/issues/669).
+
+## Community
+
+Our core team is thrilled too see you participate in any ways you like. When you are stuck, try to
+ask for help by filling an issue with a detailed description of what you were trying to do
+and what went wrong. If you have any questions or if you would like to get involved in our
+community, please check out:
+
+- GreptimeDB Community on [Slack](https://greptime.com/slack)
+- GreptimeDB GitHub [Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)
+- Greptime official [Website](https://greptime.com)
+
+In addition, you may:
+
+- View our official [Blog](https://greptime.com/blogs/index)
+- Connect us with [Linkedin](https://www.linkedin.com/company/greptime/)
+- Follow us on [Twitter](https://twitter.com/greptime)
+
+## License
+
+GreptimeDB uses the [Apache 2.0 license][1] to strike a balance between
+open contributions and allowing you to use the software however you want.
+
+[1]: <https://github.com/greptimeTeam/greptimedb/blob/develop/LICENSE>
+
+## Contributing
+
+Please refer to [contribution guidelines](CONTRIBUTING.md) for more information.
+
+## Acknowledgement
+- GreptimeDB uses [Apache Arrow](https://arrow.apache.org/) as the memory model and [Apache Parquet](https://parquet.apache.org/) as the persistent file format.
+- GreptimeDB's query engine is powered by [Apache Arrow DataFusion](https://github.com/apache/arrow-datafusion).
+- [OpenDAL](https://github.com/datafuselabs/opendal) from [Datafuse Labs](https://github.com/datafuselabs) gives GreptimeDB a very general and elegant data access abstraction layer.
+- GreptimeDB’s meta service is based on [etcd](https://etcd.io/).
+- GreptimeDB uses [RustPython](https://github.com/RustPython/RustPython) for experimental embedded python scripting.

benchmarks/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "benchmarks"
+version = "0.1.0"
+edition = "2021"
+license = "Apache-2.0"
+
+[dependencies]
+arrow = "26.0.0"
+clap = { version = "4.0", features = ["derive"] }
+client = { path = "../src/client" }
+indicatif = "0.17.1"
+itertools = "0.10.5"
+parquet = "26.0.0"
+tokio = { version = "1.21", features = ["full"] }

benchmarks/src/bin/nyc-taxi.rs

@@ -0,0 +1,444 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Use the taxi trip records from New York City dataset to bench. You can download the dataset from
+//! [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page).
+
+#![feature(once_cell)]
+#![allow(clippy::print_stdout)]
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::time::Instant;
+
+use arrow::array::{ArrayRef, PrimitiveArray, StringArray, TimestampNanosecondArray};
+use arrow::datatypes::{DataType, Float64Type, Int64Type};
+use arrow::record_batch::RecordBatch;
+use clap::Parser;
+use client::admin::Admin;
+use client::api::v1::column::Values;
+use client::api::v1::{Column, ColumnDataType, ColumnDef, CreateExpr, InsertExpr};
+use client::{Client, Database, Select};
+use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+use tokio::task::JoinSet;
+
+const DATABASE_NAME: &str = "greptime";
+const CATALOG_NAME: &str = "greptime";
+const SCHEMA_NAME: &str = "public";
+const TABLE_NAME: &str = "nyc_taxi";
+
+#[derive(Parser)]
+#[command(name = "NYC benchmark runner")]
+struct Args {
+    /// Path to the dataset
+    #[arg(short, long)]
+    path: Option<String>,
+
+    /// Batch size of insert request.
+    #[arg(short = 's', long = "batch-size", default_value_t = 4096)]
+    batch_size: usize,
+
+    /// Number of client threads on write (parallel on file level)
+    #[arg(short = 't', long = "thread-num", default_value_t = 4)]
+    thread_num: usize,
+
+    /// Number of query iteration
+    #[arg(short = 'i', long = "iter-num", default_value_t = 3)]
+    iter_num: usize,
+
+    #[arg(long = "skip-write")]
+    skip_write: bool,
+
+    #[arg(long = "skip-read")]
+    skip_read: bool,
+
+    #[arg(short, long, default_value_t = String::from("127.0.0.1:3001"))]
+    endpoint: String,
+}
+
+fn get_file_list<P: AsRef<Path>>(path: P) -> Vec<PathBuf> {
+    std::fs::read_dir(path)
+        .unwrap()
+        .map(|dir| dir.unwrap().path().canonicalize().unwrap())
+        .collect()
+}
+
+async fn write_data(
+    batch_size: usize,
+    db: &Database,
+    path: PathBuf,
+    mpb: MultiProgress,
+    pb_style: ProgressStyle,
+) -> u128 {
+    let file = std::fs::File::open(&path).unwrap();
+    let record_batch_reader_builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
+    let row_num = record_batch_reader_builder
+        .metadata()
+        .file_metadata()
+        .num_rows();
+    let record_batch_reader = record_batch_reader_builder
+        .with_batch_size(batch_size)
+        .build()
+        .unwrap();
+    let progress_bar = mpb.add(ProgressBar::new(row_num as _));
+    progress_bar.set_style(pb_style);
+    progress_bar.set_message(format!("{:?}", path));
+
+    let mut total_rpc_elapsed_ms = 0;
+
+    for record_batch in record_batch_reader {
+        let record_batch = record_batch.unwrap();
+        let (columns, row_count) = convert_record_batch(record_batch);
+        let insert_expr = InsertExpr {
+            schema_name: "public".to_string(),
+            table_name: TABLE_NAME.to_string(),
+            region_number: 0,
+            columns,
+            row_count,
+        };
+        let now = Instant::now();
+        db.insert(insert_expr).await.unwrap();
+        let elapsed = now.elapsed();
+        total_rpc_elapsed_ms += elapsed.as_millis();
+        progress_bar.inc(row_count as _);
+    }
+
+    progress_bar.finish_with_message(format!(
+        "file {:?} done in {}ms",
+        path, total_rpc_elapsed_ms
+    ));
+    total_rpc_elapsed_ms
+}
+
+fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
+    let schema = record_batch.schema();
+    let fields = schema.fields();
+    let row_count = record_batch.num_rows();
+    let mut columns = vec![];
+
+    for (array, field) in record_batch.columns().iter().zip(fields.iter()) {
+        let values = build_values(array);
+        let column = Column {
+            column_name: field.name().to_owned(),
+            values: Some(values),
+            null_mask: vec![],
+            // datatype and semantic_type are set to default
+            ..Default::default()
+        };
+        columns.push(column);
+    }
+
+    (columns, row_count as _)
+}
+
+fn build_values(column: &ArrayRef) -> Values {
+    match column.data_type() {
+        DataType::Int64 => {
+            let array = column
+                .as_any()
+                .downcast_ref::<PrimitiveArray<Int64Type>>()
+                .unwrap();
+            let values = array.values();
+            Values {
+                i64_values: values.to_vec(),
+                ..Default::default()
+            }
+        }
+        DataType::Float64 => {
+            let array = column
+                .as_any()
+                .downcast_ref::<PrimitiveArray<Float64Type>>()
+                .unwrap();
+            let values = array.values();
+            Values {
+                f64_values: values.to_vec(),
+                ..Default::default()
+            }
+        }
+        DataType::Timestamp(_, _) => {
+            let array = column
+                .as_any()
+                .downcast_ref::<TimestampNanosecondArray>()
+                .unwrap();
+            let values = array.values();
+            Values {
+                i64_values: values.to_vec(),
+                ..Default::default()
+            }
+        }
+        DataType::Utf8 => {
+            let array = column.as_any().downcast_ref::<StringArray>().unwrap();
+            let values = array.iter().filter_map(|s| s.map(String::from)).collect();
+            Values {
+                string_values: values,
+                ..Default::default()
+            }
+        }
+        DataType::Null
+        | DataType::Boolean
+        | DataType::Int8
+        | DataType::Int16
+        | DataType::Int32
+        | DataType::UInt8
+        | DataType::UInt16
+        | DataType::UInt32
+        | DataType::UInt64
+        | DataType::Float16
+        | DataType::Float32
+        | DataType::Date32
+        | DataType::Date64
+        | DataType::Time32(_)
+        | DataType::Time64(_)
+        | DataType::Duration(_)
+        | DataType::Interval(_)
+        | DataType::Binary
+        | DataType::FixedSizeBinary(_)
+        | DataType::LargeBinary
+        | DataType::LargeUtf8
+        | DataType::List(_)
+        | DataType::FixedSizeList(_, _)
+        | DataType::LargeList(_)
+        | DataType::Struct(_)
+        | DataType::Union(_, _, _)
+        | DataType::Dictionary(_, _)
+        | DataType::Decimal128(_, _)
+        | DataType::Decimal256(_, _)
+        | DataType::Map(_, _) => todo!(),
+    }
+}
+
+fn create_table_expr() -> CreateExpr {
+    CreateExpr {
+        catalog_name: Some(CATALOG_NAME.to_string()),
+        schema_name: Some(SCHEMA_NAME.to_string()),
+        table_name: TABLE_NAME.to_string(),
+        desc: None,
+        column_defs: vec![
+            ColumnDef {
+                name: "VendorID".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "tpep_pickup_datetime".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "tpep_dropoff_datetime".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "passenger_count".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "trip_distance".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "RatecodeID".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "store_and_fwd_flag".to_string(),
+                datatype: ColumnDataType::String as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "PULocationID".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "DOLocationID".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "payment_type".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "fare_amount".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "extra".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "mta_tax".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "tip_amount".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "tolls_amount".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "improvement_surcharge".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "total_amount".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "congestion_surcharge".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "airport_fee".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: None,
+            },
+        ],
+        time_index: "tpep_pickup_datetime".to_string(),
+        primary_keys: vec!["VendorID".to_string()],
+        create_if_not_exists: false,
+        table_options: Default::default(),
+        region_ids: vec![0],
+        table_id: Some(0),
+    }
+}
+
+fn query_set() -> HashMap<String, String> {
+    let mut ret = HashMap::new();
+
+    ret.insert(
+        "count_all".to_string(),
+        format!("SELECT COUNT(*) FROM {};", TABLE_NAME),
+    );
+
+    ret.insert(
+        "fare_amt_by_passenger".to_string(),
+        format!("SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM {} GROUP BY passenger_count",TABLE_NAME)
+    );
+
+    ret
+}
+
+async fn do_write(args: &Args, client: &Client) {
+    let admin = Admin::new("admin", client.clone());
+
+    let mut file_list = get_file_list(args.path.clone().expect("Specify data path in argument"));
+    let mut write_jobs = JoinSet::new();
+
+    let create_table_result = admin.create(create_table_expr()).await;
+    println!("Create table result: {:?}", create_table_result);
+
+    let progress_bar_style = ProgressStyle::with_template(
+        "[{elapsed_precise}] {bar:60.cyan/blue} {pos:>7}/{len:7} {msg}",
+    )
+    .unwrap()
+    .progress_chars("##-");
+    let multi_progress_bar = MultiProgress::new();
+    let file_progress = multi_progress_bar.add(ProgressBar::new(file_list.len() as _));
+    file_progress.inc(0);
+
+    let batch_size = args.batch_size;
+    for _ in 0..args.thread_num {
+        if let Some(path) = file_list.pop() {
+            let db = Database::new(DATABASE_NAME, client.clone());
+            let mpb = multi_progress_bar.clone();
+            let pb_style = progress_bar_style.clone();
+            write_jobs.spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
+        }
+    }
+    while write_jobs.join_next().await.is_some() {
+        file_progress.inc(1);
+        if let Some(path) = file_list.pop() {
+            let db = Database::new(DATABASE_NAME, client.clone());
+            let mpb = multi_progress_bar.clone();
+            let pb_style = progress_bar_style.clone();
+            write_jobs.spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
+        }
+    }
+}
+
+async fn do_query(num_iter: usize, db: &Database) {
+    for (query_name, query) in query_set() {
+        println!("Running query: {}", query);
+        for i in 0..num_iter {
+            let now = Instant::now();
+            let _res = db.select(Select::Sql(query.clone())).await.unwrap();
+            let elapsed = now.elapsed();
+            println!(
+                "query {}, iteration {}: {}ms",
+                query_name,
+                i,
+                elapsed.as_millis()
+            );
+        }
+    }
+}
+
+fn main() {
+    let args = Args::parse();
+
+    tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(args.thread_num)
+        .enable_all()
+        .build()
+        .unwrap()
+        .block_on(async {
+            let client = Client::with_urls(vec![&args.endpoint]);
+
+            if !args.skip_write {
+                do_write(&args, &client).await;
+            }
+
+            if !args.skip_read {
+                let db = Database::new(DATABASE_NAME, client.clone());
+                do_query(args.iter_num, &db).await;
+            }
+        })
+}

codecov.yml

@@ -1,10 +1,10 @@
 # codecov config
 coverage:
   status:
-    patch: off # disable patch status
     project:
       default:
-        enable: yes
         threshold: 1%
+    patch: off
 ignore:
   - "**/error*.rs" # ignore all error.rs files
+  - "tests/runner/*.rs" # ignore integration test runner

component/script/python/example/calc_rv.py

@@ -1,69 +0,0 @@
-import sys
-# for annoying releative import beyond top-level package
-sys.path.insert(0, "../")
-from greptime import mock_tester, coprocessor, greptime as gt_builtin
-from greptime.greptime import interval, vector, log, prev, sqrt, datetime
-import greptime.greptime as greptime
-import json
-import numpy as np
-
-
-def data_sample(k_lines, symbol, density=5 * 30 * 86400):
-    """
-    Only return close data for simplicty for now
-    """
-    k_lines = k_lines["result"] if k_lines["ret_msg"] == "OK" else None
-    if k_lines is None:
-        raise Exception("Expect a `OK`ed message")
-    close = [float(i["close"]) for i in k_lines]
-
-    return interval(close, density, "prev")
-
-
-def as_table(kline: list):
-    col_len = len(kline)
-    ret = {
-        k: vector([fn(row[k]) for row in kline], str(ty))
-        for k, fn, ty in
-        [
-            ("symbol", str, "str"),
-            ("period", str, "str"),
-            ("open_time", int, "int"),
-            ("open", float, "float"),
-            ("high", float, "float"),
-            ("low", float, "float"),
-            ("close", float, "float")
-        ]
-    }
-    return ret
-
-@coprocessor(args=["open_time", "close"], returns=[
-    "rv_7d",
-    "rv_15d",
-    "rv_30d",
-    "rv_60d",
-    "rv_90d",
-    "rv_180d"
-],
-sql="select open_time, close from k_line")
-def calc_rvs(open_time, close):
-    from greptime import vector, log, prev, sqrt, datetime, pow, sum
-    def calc_rv(close, open_time, time, interval):
-        mask = (open_time < time) & (open_time > time - interval)
-        close = close[mask]
-
-        avg_time_interval = (open_time[-1] - open_time[0])/(len(open_time)-1)
-        ref = log(close/prev(close))
-        var = sum(pow(ref, 2)/(len(ref)-1))
-        return sqrt(var/avg_time_interval)
-
-    # how to get env var,
-    # maybe through accessing scope and serde then send to remote?
-    timepoint = open_time[-1]
-    rv_7d = calc_rv(close, open_time, timepoint, datetime("7d"))
-    rv_15d = calc_rv(close, open_time, timepoint, datetime("15d"))
-    rv_30d = calc_rv(close, open_time, timepoint, datetime("30d"))
-    rv_60d = calc_rv(close, open_time, timepoint, datetime("60d"))
-    rv_90d = calc_rv(close, open_time, timepoint, datetime("90d"))
-    rv_180d = calc_rv(close, open_time, timepoint, datetime("180d"))
-    return rv_7d, rv_15d, rv_30d, rv_60d, rv_90d, rv_180d

component/script/python/example/fetch_kline.sh

@@ -1 +0,0 @@
-curl "https://api.bybit.com/v2/public/index-price-kline?symbol=BTCUSD&interval=1&limit=$1&from=1581231260" > kline.json

component/script/python/example/kline.json

@@ -1,108 +0,0 @@
-{
-    "ret_code": 0,
-    "ret_msg": "OK",
-    "ext_code": "",
-    "ext_info": "",
-    "result": [
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231300,
-            "open": "10107",
-            "high": "10109.34",
-            "low": "10106.71",
-            "close": "10106.79"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231360,
-            "open": "10106.79",
-            "high": "10109.27",
-            "low": "10105.92",
-            "close": "10106.09"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231420,
-            "open": "10106.09",
-            "high": "10108.75",
-            "low": "10104.66",
-            "close": "10108.73"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231480,
-            "open": "10108.73",
-            "high": "10109.52",
-            "low": "10106.07",
-            "close": "10106.38"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231540,
-            "open": "10106.38",
-            "high": "10109.48",
-            "low": "10104.81",
-            "close": "10106.95"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231600,
-            "open": "10106.95",
-            "high": "10109.48",
-            "low": "10106.6",
-            "close": "10107.55"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231660,
-            "open": "10107.55",
-            "high": "10109.28",
-            "low": "10104.68",
-            "close": "10104.68"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231720,
-            "open": "10104.68",
-            "high": "10109.18",
-            "low": "10104.14",
-            "close": "10108.8"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231780,
-            "open": "10108.8",
-            "high": "10117.36",
-            "low": "10108.8",
-            "close": "10115.96"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231840,
-            "open": "10115.96",
-            "high": "10119.19",
-            "low": "10115.96",
-            "close": "10117.08"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1581231900,
-            "open": "10117.08",
-            "high": "10120.73",
-            "low": "10116.96",
-            "close": "10120.43"
-        }
-    ],
-    "time_now": "1661225351.158190"
-}

component/script/python/greptime/__init__.py

@@ -1,4 +0,0 @@
-from .greptime import coprocessor, copr
-from .greptime import vector, log, prev, sqrt, pow, datetime, sum
-from .mock import mock_tester
-from .cfg import set_conn_addr, get_conn_addr

component/script/python/greptime/cfg.py

@@ -1,11 +0,0 @@
-GREPTIME_DB_CONN_ADDRESS = "localhost:3000"
-"""The Global Variable for address for conntect to database"""
-
-def set_conn_addr(addr: str):
-    """set database address to given `addr`"""
-    global GREPTIME_DB_CONN_ADDRESS
-    GREPTIME_DB_CONN_ADDRESS = addr
-
-def get_conn_addr()->str:
-    global GREPTIME_DB_CONN_ADDRESS
-    return GREPTIME_DB_CONN_ADDRESS

component/script/python/greptime/greptime.py

@@ -1,215 +0,0 @@
-"""
-Be note that this is a mock library, if not connected to database,
-it can only run on mock data and mock function which is supported by numpy
-"""
-import functools
-import numpy as np
-import json
-from urllib import request
-import inspect
-import requests
-
-from .cfg import set_conn_addr, get_conn_addr
-
-log = np.log
-sum = np.nansum
-sqrt = np.sqrt
-pow = np.power
-nan = np.nan
-
-
-class TimeStamp(str):
-    """
-    TODO: impl date time
-    """
-    pass
-
-
-class i32(int):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A signed 32-bit integer.
-    """
-
-    def __repr__(self) -> str:
-        return "i32"
-
-
-class i64(int):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A signed 64-bit integer.
-    """
-
-    def __repr__(self) -> str:
-        return "i64"
-
-
-class f32(float):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A 32-bit floating point number.
-    """
-
-    def __repr__(self) -> str:
-        return "f32"
-
-
-class f64(float):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A 64-bit floating point number.
-    """
-
-    def __repr__(self) -> str:
-        return "f64"
-
-
-class vector(np.ndarray):
-    """
-    A compact Vector with all elements of same Data type.
-    """
-    _datatype: str | None = None
-
-    def __new__(
-        cls,
-        lst,
-        dtype=None
-    ) -> ...:
-        self = np.asarray(lst).view(cls)
-        self._datatype = dtype
-        return self
-
-    def __str__(self) -> str:
-        return "vector({}, \"{}\")".format(super().__str__(), self.datatype())
-
-    def datatype(self):
-        return self._datatype
-
-    def filter(self, lst_bool):
-        return self[lst_bool]
-
-
-def prev(lst):
-    ret = np.zeros(len(lst))
-    ret[1:] = lst[0:-1]
-    ret[0] = nan
-    return ret
-
-
-def query(sql: str):
-    pass
-
-
-def interval(arr: list, duration: int, fill, step: None | int = None, explicitOffset=False):
-    """
-    Note that this is a mock function with same functionailty to the actual Python Coprocessor
-    `arr` is a vector of integral or temporal type.
-
-    `duration` is the length of sliding window
-
-    `step` being the length when sliding window take a step
-
-    `fill` indicate how to fill missing value:
-    - "prev": use previous
-    - "post": next
-    - "linear": linear interpolation, if not possible to interpolate certain types, fallback to prev
-    - "null": use null
-    - "none": do not interpolate
-    """
-    if step is None:
-        step = duration
-
-    tot_len = int(np.ceil(len(arr) // step))
-    slices = np.zeros((tot_len, int(duration)))
-    for idx, start in enumerate(range(0, len(arr), step)):
-        slices[idx] = arr[start:(start + duration)]
-    return slices
-
-
-def factor(unit: str) -> int:
-    if unit == "d":
-        return 24 * 60 * 60
-    elif unit == "h":
-        return 60 * 60
-    elif unit == "m":
-        return 60
-    elif unit == "s":
-        return 1
-    else:
-        raise Exception("Only d,h,m,s, found{}".format(unit))
-
-
-def datetime(input_time: str) -> int:
-    """
-    support `d`(day) `h`(hour) `m`(minute) `s`(second)
-
-    support format:
-    `12s` `7d` `12d2h7m`
-    """
-
-    prev = 0
-    cur = 0
-    state = "Num"
-    parse_res = []
-    for idx, ch in enumerate(input_time):
-        if ch.isdigit():
-            cur = idx
-
-            if state != "Num":
-                parse_res.append((state, input_time[prev:cur], (prev, cur)))
-                prev = idx
-                state = "Num"
-        else:
-            cur = idx
-            if state != "Symbol":
-                parse_res.append((state, input_time[prev:cur], (prev, cur)))
-                prev = idx
-                state = "Symbol"
-    parse_res.append((state, input_time[prev:cur+1], (prev, cur+1)))
-
-    cur_idx = 0
-    res_time = 0
-    while cur_idx < len(parse_res):
-        pair = parse_res[cur_idx]
-        if pair[0] == "Num":
-            val = int(pair[1])
-            nxt = parse_res[cur_idx+1]
-            res_time += val * factor(nxt[1])
-            cur_idx += 2
-        else:
-            raise Exception("Two symbol in a row is impossible")
-
-    return res_time
-
-
-def coprocessor(args=None, returns=None, sql=None):
-    """
-    The actual coprocessor, which will connect to database and update
-    whatever function decorated with `@coprocessor(args=[...], returns=[...], sql=...)`
-    """
-    def decorator_copr(func):
-        @functools.wraps(func)
-        def wrapper_do_actual(*args, **kwargs):
-            if len(args)!=0 or len(kwargs)!=0:
-                raise Exception("Expect call with no arguements(for all args are given by coprocessor itself)")
-            source = inspect.getsource(func)
-            url = "http://{}/v1/scripts".format(get_conn_addr())
-            print("Posting to {}".format(url))
-            data = {
-                    "script": source,
-                    "engine": None,
-                }
-
-            res = requests.post(
-                url,
-                headers={"Content-Type": "application/json"},
-                json=data
-            )
-            return res
-        return wrapper_do_actual
-    return decorator_copr
-
-
-# make a alias for short
-copr = coprocessor

component/script/python/greptime/mock.py

@@ -1,82 +0,0 @@
-"""
-Note this is a mock library, if not connected to database, 
-it can only run on mock data and support by numpy
-"""
-from typing import Any
-import numpy as np
-from .greptime import i32,i64,f32,f64, vector, interval, query, prev, datetime, log, sum, sqrt, pow, nan, copr, coprocessor
-
-import inspect
-import functools
-import ast
-
-
-
-def mock_tester(
-    func,
-    env:dict, 
-    table=None
-):
-    """
-    Mock tester helper function,
-    What it does is replace `@coprocessor` with `@mock_cpor` and add a keyword `env=env`
-    like `@mock_copr(args=...,returns=...,env=env)`
-    """
-    code = inspect.getsource(func)
-    tree = ast.parse(code)
-    tree = HackyReplaceDecorator("env").visit(tree)
-    new_func = tree.body[0]
-    fn_name = new_func.name
-
-    code_obj = compile(tree, "<embedded>", "exec")
-    exec(code_obj)
-
-    ret = eval("{}()".format(fn_name))
-    return ret
-
-def mock_copr(args, returns, sql=None, env:None|dict=None):
-    """
-    This should not be used directly by user
-    """
-    def decorator_copr(func):
-        @functools.wraps(func)
-        def wrapper_do_actual(*fn_args, **fn_kwargs):
-
-            real_args = [env[name] for name in args]
-            ret = func(*real_args)
-            return ret
-        
-        return wrapper_do_actual
-    return decorator_copr
-
-class HackyReplaceDecorator(ast.NodeTransformer):
-    """
-    This class accept a `env` dict for environment to extract args from,
-    and put `env` dict in the param list of `mock_copr` decorator, i.e:
-
-    a `@copr(args=["a", "b"], returns=["c"])` with call like mock_helper(abc, env={"a":2, "b":3})
-    
-    will be transform into `@mock_copr(args=["a", "b"], returns=["c"], env={"a":2, "b":3})`
-    """
-    def __init__(self, env: str) -> None:
-        # just for add `env` keyword
-        self.env = env
-
-    def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
-        new_node = node
-        decorator_list = new_node.decorator_list
-        if len(decorator_list)!=1:
-            return node
-
-        deco = decorator_list[0]
-        if deco.func.id!="coprocessor" and deco.func.id !="copr":
-            raise Exception("Expect a @copr or @coprocessor, found {}.".format(deco.func.id))
-        deco.func = ast.Name(id="mock_copr", ctx=ast.Load())
-        new_kw = ast.keyword(arg="env", value=ast.Name(id=self.env, ctx=ast.Load()))
-        deco.keywords.append(new_kw)
-
-        # Tie up loose ends in the AST.
-        ast.copy_location(new_node, node)
-        ast.fix_missing_locations(new_node)
-        self.generic_visit(node)
-        return new_node

component/script/python/test.py

@@ -1,55 +0,0 @@
-from example.calc_rv import as_table, calc_rvs
-from greptime import coprocessor, set_conn_addr, get_conn_addr, mock_tester
-import sys
-import json
-import requests
-'''
-To run this script, you need to first start a http server of greptime, and 
-`
-python3 component/script/python/test.py 地址:端口
-`
-
-'''
-@coprocessor(sql='select number from numbers limit 10', args=['number'], returns=['n'])
-def test(n):
-    return n+2
-
-def init_table(close, open_time):
-    req_init = "/v1/sql?sql=create table k_line (close double, open_time bigint, TIME INDEX (open_time))"
-    print(get_db(req_init).text)
-    for c1, c2 in zip(close, open_time):
-        req = "/v1/sql?sql=INSERT INTO k_line(close, open_time) VALUES ({}, {})".format(c1, c2)
-        print(get_db(req).text)
-    print(get_db("/v1/sql?sql=select * from k_line").text)
-
-def get_db(req:str):
-    return requests.get("http://{}{}".format(get_conn_addr(), req))
-
-if __name__ == "__main__":
-    if len(sys.argv)!=2:
-        raise Exception("Expect only one address as cmd's args")
-    set_conn_addr(sys.argv[1])
-    res = test()
-    print(res.headers)
-    print(res.text)
-    with open("component/script/python/example/kline.json", "r") as kline_file:
-        kline = json.load(kline_file)
-        # vec = vector([1,2,3], int)
-        # print(vec, vec.datatype())
-        table = as_table(kline["result"])
-        # print(table)
-        close = table["close"]
-        open_time = table["open_time"]
-        init_table(close, open_time)
-
-        # print(repr(close), repr(open_time))
-        # print("calc_rv:", calc_rv(close, open_time, open_time[-1]+datetime("10m"), datetime("7d")))
-        env = {"close":close, "open_time": open_time}
-        # print("env:", env)
-        print("Mock result:", mock_tester(calc_rvs, env=env))
-        real = calc_rvs()
-        print(real)
-        try:
-            print(real.text["error"])
-        except:
-            print(real.text)

config/datanode.example.toml

@@ -1,10 +1,18 @@
-http_addr = '0.0.0.0:3000'
-rpc_addr = '0.0.0.0:3001'
+node_id = 42
+mode = 'distributed'
+rpc_addr = '127.0.0.1:3001'
 wal_dir = '/tmp/greptimedb/wal'
-
-mysql_addr = '0.0.0.0:3306'
+rpc_runtime_size = 8
+mysql_addr = '127.0.0.1:4406'
 mysql_runtime_size = 4
+enable_memory_catalog = false
 
 [storage]
 type = 'File'
 data_dir = '/tmp/greptimedb/data/'
+
+[meta_client_opts]
+metasrv_addrs = ['127.0.0.1:3002']
+timeout_millis = 3000
+connect_timeout_millis = 5000
+tcp_nodelay = false

config/frontend.example.toml

@@ -0,0 +1,12 @@
+mode = 'distributed'
+datanode_rpc_addr = '127.0.0.1:3001'
+
+[http_options]
+addr = '127.0.0.1:4000'
+timeout = "30s"
+
+[meta_client_opts]
+metasrv_addrs = ['127.0.0.1:3002']
+timeout_millis = 3000
+connect_timeout_millis = 5000
+tcp_nodelay = false

config/metasrv.example.toml

@@ -0,0 +1,4 @@
+bind_addr = '127.0.0.1:3002'
+server_addr = '127.0.0.1:3002'
+store_addr = '127.0.0.1:2379'
+datanode_lease_secs = 15

config/standalone.example.toml

@@ -0,0 +1,36 @@
+node_id = 0
+mode = 'standalone'
+wal_dir = '/tmp/greptimedb/wal/'
+enable_memory_catalog = false
+
+[http_options]
+addr = '127.0.0.1:4000'
+timeout = "30s"
+
+[storage]
+type = 'File'
+data_dir = '/tmp/greptimedb/data/'
+
+[grpc_options]
+addr = '127.0.0.1:4001'
+runtime_size = 8
+
+[mysql_options]
+addr = '127.0.0.1:4002'
+runtime_size = 2
+
+[influxdb_options]
+enable = true
+
+[opentsdb_options]
+addr = '127.0.0.1:4242'
+enable = true
+runtime_size = 2
+
+[prometheus_options]
+enable = true
+
+[postgres_options]
+addr = '127.0.0.1:4003'
+runtime_size = 2
+check_pwd = false

docker/Dockerfile

@@ -24,9 +24,8 @@ RUN cargo build --release
 # TODO(zyy17): Maybe should use the more secure container image.
 FROM ubuntu:22.04 as base
 
-WORKDIR /greptimedb
-COPY --from=builder /greptimedb/target/release/greptime /greptimedb/bin/
-ENV PATH /greptimedb/bin/:$PATH
+WORKDIR /greptime
+COPY --from=builder /greptimedb/target/release/greptime /greptime/bin/
+ENV PATH /greptime/bin/:$PATH
 
-ENTRYPOINT [ "greptime" ]
-CMD [ "datanode", "start"]
+ENTRYPOINT ["greptime"]

docker/ci/Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:22.04
+
+ARG TARGETARCH
+
+ADD $TARGETARCH/greptime /greptime/bin/
+
+ENV PATH /greptime/bin/:$PATH
+
+ENTRYPOINT ["greptime"]

docs/how-to/how-to-write-aggregate-function.md

@@ -8,27 +8,29 @@ So is there a way we can make an aggregate function that automatically match the
 
 # 1. Impl `AggregateFunctionCreator` trait for your accumulator creator.
 
-You must first define a struct that can store the input data's type. For example,
+You must first define a struct that will be used to create your accumulator. For example,
 
 ```Rust
-struct MySumAccumulatorCreator {
-    input_types: ArcSwapOption<Vec<ConcreteDataType>>,
-}
+#[as_aggr_func_creator]
+#[derive(Debug, AggrFuncTypeStore)]
+struct MySumAccumulatorCreator {}
 ```
 
+Attribute macro `#[as_aggr_func_creator]` and derive macro `#[derive(Debug, AggrFuncTypeStore)]` must both annotated on the struct. They work together to provide a storage of aggregate function's input data types, which are needed for creating generic accumulator later.  
+
+> Note that the `as_aggr_func_creator` macro will add fields to the struct, so the struct cannot be defined as an empty struct without field like `struct Foo;`, neither as a new type like `struct Foo(bar)`.
+
 Then impl `AggregateFunctionCreator` trait on it. The definition of the trait is:
 
 ```Rust
 pub trait AggregateFunctionCreator: Send + Sync + Debug {
     fn creator(&self) -> AccumulatorCreatorFunction;
-    fn input_types(&self) -> Vec<ConcreteDataType>;
-    fn set_input_types(&self, input_types: Vec<ConcreteDataType>);
     fn output_type(&self) -> ConcreteDataType;
     fn state_types(&self) -> Vec<ConcreteDataType>;
 }
 ```
 
-our query engine will call `set_input_types` the very first, so you can use input data's type in methods that return output type and state types.
+You can use input data's type in methods that return output type and state types (just invoke `input_types()`).
 
 The output type is aggregate function's output data's type. For example, `SUM` aggregate function's output type is `u64` for a `u32` datatype column. The state types are accumulator's internal states' types. Take `AVG` aggregate function on a `i32` column as example, it's state types are `i64` (for sum) and `u64` (for count).
 
@@ -53,7 +55,7 @@ The DataFusion basically execute aggregate like this:
 2. Call `update_batch` on each accumulator with partitioned data, to let you update your aggregate calculation.
 3. Call `state` to get each accumulator's internal state, the medial calculation result.
 4. Call `merge_batch` to merge all accumulator's internal state to one.
-5. Execute `evalute` on the chosen one to get the final calculation result.
+5. Execute `evaluate` on the chosen one to get the final calculation result.
 
 Once you know the meaning of each method, you can easily write your accumulator. You can refer to `Median` accumulator or `SUM` accumulator defined in  file `my_sum_udaf_example.rs` for more details.
 
@@ -61,7 +63,7 @@ Once you know the meaning of each method, you can easily write your accumulator.
 
 You can call `register_aggregate_function` method in query engine to register your aggregate function. To do that, you have to new an instance of struct `AggregateFunctionMeta`. The struct has three fields, first is the name of your aggregate function's name. The function name is case-sensitive due to DataFusion's restriction. We strongly recommend using lowercase for your name. If you have to use uppercase name, wrap your aggregate function with quotation marks. For example, if you define an aggregate function named "my_aggr", you can use "`SELECT MY_AGGR(x)`"; if you define "my_AGGR", you have to use "`SELECT "my_AGGR"(x)`".
 
-The second field is arg_counts ,the count of the arguments. Like accumulator `percentile`, caculating the p_number of the column. We need to input the value of column and the value of p to cacalate, and so the count of the arguments is two.
+The second field is arg_counts ,the count of the arguments. Like accumulator `percentile`, calculating the p_number of the column. We need to input the value of column and the value of p to cacalate, and so the count of the arguments is two.
 
 The third field is a function about how to create your accumulator creator that you defined in step 1 above. Create creator, that's a bit intertwined, but it is how we make DataFusion use a newly created aggregate function each time it executes a SQL, preventing the stored input types from affecting each other. The key detail can be starting looking at our `DfContextProviderAdapter` struct's `get_aggregate_meta` method.
 

rustfmt.toml

@@ -1,3 +1,2 @@
 group_imports = "StdExternalCrate"
-
-
+imports_granularity = "Module"

scripts/install.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+
+set -ue
+
+OS_TYPE=
+ARCH_TYPE=
+VERSION=${1:-latest}
+GITHUB_ORG=GreptimeTeam
+GITHUB_REPO=greptimedb
+BIN=greptime
+
+get_os_type() {
+    os_type="$(uname -s)"
+
+    case "$os_type" in
+    Darwin)
+        OS_TYPE=darwin
+        ;;
+    Linux)
+        OS_TYPE=linux
+        ;;
+    *)
+        echo "Error: Unknown OS type: $os_type"
+        exit 1
+    esac
+}
+
+get_arch_type() {
+    arch_type="$(uname -m)"
+
+    case "$arch_type" in
+    arm64)
+        ARCH_TYPE=arm64
+        ;;
+    aarch64)
+        ARCH_TYPE=arm64
+        ;;
+    x86_64)
+        ARCH_TYPE=amd64
+        ;;
+    amd64)
+        ARCH_TYPE=amd64
+        ;;
+    *)
+        echo "Error: Unknown CPU type: $arch_type"
+        exit 1
+    esac
+}
+
+get_os_type
+get_arch_type
+
+if [ -n "${OS_TYPE}" ] && [ -n "${ARCH_TYPE}" ]; then
+    echo "Downloading ${BIN}, OS: ${OS_TYPE}, Arch: ${ARCH_TYPE}, Version: ${VERSION}"
+
+    if [ "${VERSION}" = "latest" ]; then
+        wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/latest/download/${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz"
+    else
+        wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz"
+    fi
+
+    tar xvf ${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz && rm ${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz && echo "Run '${BIN} --help' to get started"
+fi

src/api/Cargo.toml

@@ -2,11 +2,16 @@
 name = "api"
 version = "0.1.0"
 edition = "2021"
-
+license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+common-base = { path = "../common/base" }
+common-error = { path = "../common/error" }
+common-time = { path = "../common/time" }
+datatypes = { path = "../datatypes" }
 prost = "0.11"
+snafu = { version = "0.7", features = ["backtraces"] }
 tonic = "0.8"
 
 [build-dependencies]

src/api/build.rs

@@ -1,11 +1,32 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::PathBuf;
+
 fn main() {
+    let default_out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
     tonic_build::configure()
+        .file_descriptor_set_path(default_out_dir.join("greptime_fd.bin"))
         .compile(
             &[
-                "greptime/v1/insert.proto",
                 "greptime/v1/select.proto",
-                "greptime/v1/physical_plan.proto",
                 "greptime/v1/greptime.proto",
+                "greptime/v1/meta/common.proto",
+                "greptime/v1/meta/heartbeat.proto",
+                "greptime/v1/meta/route.proto",
+                "greptime/v1/meta/store.proto",
+                "prometheus/remote/remote.proto",
             ],
             &["."],
         )

src/api/greptime/v1/admin.proto

@@ -2,8 +2,84 @@ syntax = "proto3";
 
 package greptime.v1;
 
-// TODO(jiachun)
-message AdminRequest {}
+import "greptime/v1/column.proto";
+import "greptime/v1/common.proto";
 
-// TODO(jiachun)
-message AdminResponse {}
+message AdminRequest {
+  string name = 1;
+  repeated AdminExpr exprs = 2;
+}
+
+message AdminResponse {
+  repeated AdminResult results = 1;
+}
+
+message AdminExpr {
+  ExprHeader header = 1;
+  oneof expr {
+    CreateExpr create = 2;
+    AlterExpr alter = 3;
+    CreateDatabaseExpr create_database = 4;
+    DropTableExpr drop_table = 5;
+  }
+}
+
+message AdminResult {
+  ResultHeader header = 1;
+  oneof result {
+    MutateResult mutate = 2;
+  }
+}
+
+// TODO(hl): rename to CreateTableExpr
+message CreateExpr {
+  optional string catalog_name = 1;
+  optional string schema_name = 2;
+  string table_name = 3;
+  optional string desc = 4;
+  repeated ColumnDef column_defs = 5;
+  string time_index = 6;
+  repeated string primary_keys = 7;
+  bool create_if_not_exists = 8;
+  map<string, string> table_options = 9;
+  optional uint32 table_id = 10;
+  repeated uint32 region_ids = 11;
+}
+
+message AlterExpr {
+  optional string catalog_name = 1;
+  optional string schema_name = 2;
+  string table_name = 3;
+  oneof kind {
+    AddColumns add_columns = 4;
+    DropColumns drop_columns = 5;
+  }
+}
+
+message DropTableExpr {
+  string catalog_name = 1;
+  string schema_name = 2;
+  string table_name = 3;
+}
+
+message AddColumns {
+  repeated AddColumn add_columns = 1;
+}
+
+message DropColumns {
+  repeated DropColumn drop_columns = 1;
+}
+
+message AddColumn {
+  ColumnDef column_def = 1;
+  bool is_key = 2;
+}
+
+message DropColumn {
+  string name = 1;
+}
+
+message CreateDatabaseExpr {
+  //TODO(hl): maybe rename to schema_name?
+  string database_name = 1;
+}

src/api/greptime/v1/column.proto

@@ -29,6 +29,13 @@ message Column {
     repeated bool bool_values = 11;
     repeated bytes binary_values = 12;
     repeated string string_values = 13;
+
+    repeated int32 date_values = 14;
+    repeated int64 datetime_values = 15;
+    repeated int64 ts_second_values = 16;
+    repeated int64 ts_millisecond_values = 17;
+    repeated int64 ts_microsecond_values = 18;
+    repeated int64 ts_nanosecond_values = 19;
   }
   // The array of non-null values in this column.
   //
@@ -43,4 +50,36 @@ message Column {
   // Mask maps the positions of null values.
   // If a bit in null_mask is 1, it indicates that the column value at that position is null.
   bytes null_mask = 4;
+
+  // Helpful in creating vector from column.
+  ColumnDataType datatype = 5;
+}
+
+message ColumnDef {
+  string name = 1;
+  ColumnDataType datatype = 2;
+  bool is_nullable = 3;
+  optional bytes default_constraint = 4;
+}
+
+enum ColumnDataType {
+  BOOLEAN = 0;
+  INT8 = 1;
+  INT16 = 2;
+  INT32 = 3;
+  INT64 = 4;
+  UINT8 = 5;
+  UINT16 = 6;
+  UINT32 = 7;
+  UINT64 = 8;
+  FLOAT32 = 9;
+  FLOAT64 = 10;
+  BINARY = 11;
+  STRING = 12;
+  DATE = 13;
+  DATETIME = 14;
+  TIMESTAMP_SECOND = 15;
+  TIMESTAMP_MILLISECOND = 16;
+  TIMESTAMP_MICROSECOND = 17;
+  TIMESTAMP_NANOSECOND = 18;
 }

src/api/greptime/v1/common.proto

@@ -0,0 +1,22 @@
+syntax = "proto3";
+
+package greptime.v1;
+
+message RequestHeader {
+  string tenant = 1;
+}
+
+message ExprHeader {
+  uint32 version = 1;
+}
+
+message ResultHeader {
+  uint32 version = 1;
+  uint32 code = 2;
+  string err_msg = 3;
+}
+
+message MutateResult {
+  uint32 success = 1;
+  uint32 failure = 2;
+}

src/api/greptime/v1/database.proto

@@ -2,6 +2,9 @@ syntax = "proto3";
 
 package greptime.v1;
 
+import "greptime/v1/column.proto";
+import "greptime/v1/common.proto";
+
 message DatabaseRequest {
   string name = 1;
   repeated ObjectExpr exprs = 2;
@@ -21,26 +24,28 @@ message ObjectExpr {
   }
 }
 
-message ExprHeader {
-  uint32 version = 1;
-}
-
 // TODO(fys): Only support sql now, and will support promql etc in the future
 message SelectExpr {
   oneof expr {
     string sql = 1;
-    PhysicalPlan physical_plan = 15;
+    bytes logical_plan = 2;
   }
 }
 
-message PhysicalPlan {
-  bytes original_ql = 1;
-  bytes plan = 2;
-}
-
 message InsertExpr {
-  string table_name = 1;
-  repeated bytes values = 2;
+  string schema_name = 1;
+  string table_name = 2;
+
+  // Data is represented here.
+  repeated Column columns = 3;
+
+  // The row_count of all columns, which include null and non-null values.
+  //
+  // Note: the row_count of all columns in a InsertExpr must be same.
+  uint32 row_count = 4;
+
+  // The region number of current insert request.
+  uint32 region_number = 5;
 }
 
 // TODO(jiachun)
@@ -59,14 +64,3 @@ message ObjectResult {
 message SelectResult {
   bytes raw_data = 1;
 }
-
-message ResultHeader {
-  uint32 version = 1;
-  uint32 code = 2;
-  string err_msg = 3;
-}
-
-message MutateResult {
-  uint32 success = 1;
-  uint32 failure = 2;
-}

src/api/greptime/v1/greptime.proto

@@ -3,6 +3,7 @@ syntax = "proto3";
 package greptime.v1;
 
 import "greptime/v1/admin.proto";
+import "greptime/v1/common.proto";
 import "greptime/v1/database.proto";
 
 service Greptime {
@@ -10,8 +11,9 @@ service Greptime {
 }
 
 message BatchRequest {
-  repeated AdminRequest admins = 1;
-  repeated DatabaseRequest databases = 2;
+  RequestHeader header = 1;
+  repeated AdminRequest admins = 2;
+  repeated DatabaseRequest databases = 3;
 }
 
 message BatchResponse {

src/api/greptime/v1/insert.proto

@@ -1,10 +0,0 @@
-syntax = "proto3";
-
-package greptime.v1.codec;
-
-import "greptime/v1/column.proto";
-
-message InsertBatch {
-  repeated Column columns = 1;
-  uint32 row_count = 2;
-}

src/api/greptime/v1/meta/common.proto

@@ -0,0 +1,48 @@
+syntax = "proto3";
+
+package greptime.v1.meta;
+
+message RequestHeader {
+  uint64 protocol_version = 1;
+  // cluster_id is the ID of the cluster which be sent to.
+  uint64 cluster_id = 2;
+  // member_id is the ID of the sender server.
+  uint64 member_id = 3;
+}
+
+message ResponseHeader {
+  uint64 protocol_version = 1;
+  // cluster_id is the ID of the cluster which sent the response.
+  uint64 cluster_id = 2;
+  Error error = 3;
+}
+
+message Error {
+  int32 code = 1;
+  string err_msg = 2;
+}
+
+message Peer {
+  uint64 id = 1;
+  string addr = 2;
+}
+
+message TableName {
+  string catalog_name = 1;
+  string schema_name = 2;
+  string table_name = 3;
+}
+
+message TimeInterval {
+  // The unix timestamp in millis of the start of this period.
+  uint64 start_timestamp_millis = 1;
+  // The unix timestamp in millis of the end of this period.
+  uint64 end_timestamp_millis = 2;
+}
+
+message KeyValue {
+  // key is the key in bytes. An empty key is not allowed.
+  bytes key = 1;
+  // value is the value held by the key, in bytes.
+  bytes value = 2;
+}

src/api/greptime/v1/meta/heartbeat.proto

@@ -0,0 +1,92 @@
+syntax = "proto3";
+
+package greptime.v1.meta;
+
+import "greptime/v1/meta/common.proto";
+
+service Heartbeat {
+  // Heartbeat, there may be many contents of the heartbeat, such as:
+  // 1. Metadata to be registered to meta server and discoverable by other nodes.
+  // 2. Some performance metrics, such as Load, CPU usage, etc.
+  // 3. The number of computing tasks being executed.
+  rpc Heartbeat(stream HeartbeatRequest) returns (stream HeartbeatResponse) {}
+
+  // Ask leader's endpoint.
+  rpc AskLeader(AskLeaderRequest) returns (AskLeaderResponse) {}
+}
+
+message HeartbeatRequest {
+  RequestHeader header = 1;
+
+  // Self peer
+  Peer peer = 2;
+  // Leader node
+  bool is_leader = 3;
+  // Actually reported time interval
+  TimeInterval report_interval = 4;
+  // Node stat
+  NodeStat node_stat = 5;
+  // Region stats in this node
+  repeated RegionStat region_stats = 6;
+  // Follower nodes and stats, empty on follower nodes
+  repeated ReplicaStat replica_stats = 7;
+}
+
+message NodeStat {
+  // The read capacity units during this period
+  uint64 rcus = 1;
+  // The write capacity units during this period
+  uint64 wcus = 2;
+  // Table number in this node
+  uint64 table_num = 3;
+  // Region number in this node
+  uint64 region_num = 4;
+
+  double cpu_usage = 5;
+  double load = 6;
+  // Read disk I/O in the node
+  double read_io_rate = 7;
+  // Write disk I/O in the node
+  double write_io_rate = 8;
+
+  // Others
+  map<string, string> attrs = 100;
+}
+
+message RegionStat {
+  uint64 region_id = 1;
+  TableName table_name = 2;
+  // The read capacity units during this period
+  uint64 rcus = 3;
+  // The write capacity units during this period
+  uint64 wcus = 4;
+  // Approximate region size
+  uint64 approximate_size = 5;
+  // Approximate number of rows
+  uint64 approximate_rows = 6;
+
+  // Others
+  map<string, string> attrs = 100;
+}
+
+message ReplicaStat {
+  Peer peer = 1;
+  bool in_sync = 2;
+  bool is_learner = 3;
+}
+
+message HeartbeatResponse {
+  ResponseHeader header = 1;
+
+  repeated bytes payload = 2;
+}
+
+message AskLeaderRequest {
+  RequestHeader header = 1;
+}
+
+message AskLeaderResponse {
+  ResponseHeader header = 1;
+
+  Peer leader = 2;
+}

src/api/greptime/v1/meta/route.proto

@@ -0,0 +1,98 @@
+syntax = "proto3";
+
+package greptime.v1.meta;
+
+import "greptime/v1/meta/common.proto";
+
+service Router {
+  rpc Create(CreateRequest) returns (RouteResponse) {}
+
+  // Fetch routing information for tables. The smallest unit is the complete
+  // routing information(all regions) of a table.
+  // 
+  // ```text
+  // table_1
+  //    table_name
+  //    table_schema
+  //    regions
+  //      region_1
+  //        leader_peer
+  //        follower_peer_1, follower_peer_2
+  //      region_2
+  //        leader_peer
+  //        follower_peer_1, follower_peer_2, follower_peer_3
+  //      region_xxx
+  // table_2
+  //    ...
+  // ```
+  //
+  rpc Route(RouteRequest) returns (RouteResponse) {}
+
+  rpc Delete(DeleteRequest) returns (RouteResponse) {}
+}
+
+message CreateRequest {
+  RequestHeader header = 1;
+
+  TableName table_name = 2;
+  repeated Partition partitions = 3;
+}
+
+message RouteRequest {
+  RequestHeader header = 1;
+
+  repeated TableName table_names = 2;
+}
+
+message DeleteRequest {
+  RequestHeader header = 1;
+
+  TableName table_name = 2; 
+}
+
+message RouteResponse {
+  ResponseHeader header = 1;
+
+  repeated Peer peers = 2;
+  repeated TableRoute table_routes = 3;
+}
+
+message TableRoute {
+  Table table = 1;
+  repeated RegionRoute region_routes = 2;
+}
+
+message RegionRoute {
+  Region region = 1;
+  // single leader node for write task
+  uint64 leader_peer_index = 2;
+  // multiple follower nodes for read task
+  repeated uint64 follower_peer_indexes = 3;
+}
+
+message Table {
+  uint64 id = 1;
+  TableName table_name = 2;
+  bytes table_schema = 3;
+}
+
+message Region {
+  // TODO(LFC): Maybe use message RegionNumber?
+  uint64 id = 1;
+  string name = 2;
+  Partition partition = 3;
+  
+  map<string, string> attrs = 100;
+}
+
+// PARTITION `region_name` VALUES LESS THAN (value_list)
+message Partition {
+  repeated bytes column_list = 1;
+  repeated bytes value_list = 2;
+}
+
+// This message is only for saving into store.
+message TableRouteValue {
+  repeated Peer peers = 1; 
+  TableRoute table_route = 2;
+}

src/api/greptime/v1/meta/store.proto

@@ -0,0 +1,159 @@
+syntax = "proto3";
+
+package greptime.v1.meta;
+
+import "greptime/v1/meta/common.proto";
+
+service Store {
+  // Range gets the keys in the range from the key-value store.
+  rpc Range(RangeRequest) returns (RangeResponse);
+
+  // Put puts the given key into the key-value store.
+  rpc Put(PutRequest) returns (PutResponse);
+
+  // BatchPut atomically puts the given keys into the key-value store.
+  rpc BatchPut(BatchPutRequest) returns (BatchPutResponse);
+
+  // CompareAndPut atomically puts the value to the given updated
+  // value if the current value == the expected value.
+  rpc CompareAndPut(CompareAndPutRequest) returns (CompareAndPutResponse);
+
+  // DeleteRange deletes the given range from the key-value store.
+  rpc DeleteRange(DeleteRangeRequest) returns (DeleteRangeResponse);
+
+  // MoveValue atomically renames the key to the given updated key.
+  rpc MoveValue(MoveValueRequest) returns (MoveValueResponse);
+}
+
+message RangeRequest {
+  RequestHeader header = 1;
+
+  // key is the first key for the range, If range_end is not given, the
+  // request only looks up key.
+  bytes key = 2;
+  // range_end is the upper bound on the requested range [key, range_end).
+  // If range_end is '\0', the range is all keys >= key.
+  // If range_end is key plus one (e.g., "aa"+1 == "ab", "a\xff"+1 == "b"),
+  // then the range request gets all keys prefixed with key.
+  // If both key and range_end are '\0', then the range request returns all
+  // keys.
+  bytes range_end = 3;
+  // limit is a limit on the number of keys returned for the request. When
+  // limit is set to 0, it is treated as no limit.
+  int64 limit = 4;
+  // keys_only when set returns only the keys and not the values.
+  bool keys_only = 5;
+}
+
+message RangeResponse {
+  ResponseHeader header = 1;
+  
+  // kvs is the list of key-value pairs matched by the range request.
+  repeated KeyValue kvs = 2;
+  // more indicates if there are more keys to return in the requested range.
+  bool more = 3;
+}
+
+message PutRequest {
+  RequestHeader header = 1;
+
+  // key is the key, in bytes, to put into the key-value store.
+  bytes key = 2;
+  // value is the value, in bytes, to associate with the key in the
+  // key-value store.
+  bytes value = 3;
+  // If prev_kv is set, gets the previous key-value pair before changing it.
+  // The previous key-value pair will be returned in the put response.
+  bool prev_kv = 4;
+}
+
+message PutResponse {
+  ResponseHeader header = 1;
+
+  // If prev_kv is set in the request, the previous key-value pair will be
+  // returned.
+  KeyValue prev_kv = 2;
+}
+
+message BatchPutRequest {
+  RequestHeader header = 1;
+
+  repeated KeyValue kvs = 2;
+  // If prev_kv is set, gets the previous key-value pairs before changing it.
+  // The previous key-value pairs will be returned in the batch put response.
+  bool prev_kv = 3;
+}
+
+message BatchPutResponse {
+  ResponseHeader header = 1;
+
+  // If prev_kv is set in the request, the previous key-value pairs will be
+  // returned.
+  repeated KeyValue prev_kvs = 2;
+}
+
+message CompareAndPutRequest {
+  RequestHeader header = 1;
+
+  // key is the key, in bytes, to put into the key-value store.
+  bytes key = 2;
+  // expect is the previous value, in bytes
+  bytes expect = 3;
+  // value is the value, in bytes, to associate with the key in the
+  // key-value store.
+  bytes value = 4; 
+}
+
+message CompareAndPutResponse {
+  ResponseHeader header = 1;
+
+  bool success = 2;
+  KeyValue prev_kv = 3;
+}
+
+message DeleteRangeRequest {
+  RequestHeader header = 1;
+
+  // key is the first key to delete in the range.
+  bytes key = 2;
+  // range_end is the key following the last key to delete for the range
+  // [key, range_end).
+  // If range_end is not given, the range is defined to contain only the key
+  // argument.
+  // If range_end is one bit larger than the given key, then the range is all
+  // the keys with the prefix (the given key).
+  // If range_end is '\0', the range is all keys greater than or equal to the
+  // key argument.
+  bytes range_end = 3;
+  // If prev_kv is set, gets the previous key-value pairs before deleting it.
+  // The previous key-value pairs will be returned in the delete response.
+  bool prev_kv = 4;
+}
+
+message DeleteRangeResponse {
+  ResponseHeader header = 1;
+
+  // deleted is the number of keys deleted by the delete range request.
+  int64 deleted = 2;
+  // If prev_kv is set in the request, the previous key-value pairs will be
+  // returned.
+  repeated KeyValue prev_kvs = 3;
+}
+
+message MoveValueRequest {
+  RequestHeader header = 1;
+
+  // If from_key dose not exist, return the value of to_key (if it exists).
+  // If from_key exists, move the value of from_key to to_key (i.e. rename),
+  // and return the value.
+  bytes from_key = 2;
+  bytes to_key = 3;
+}
+
+message MoveValueResponse {
+  ResponseHeader header = 1;
+
+  // If from_key dose not exist, return the value of to_key (if it exists).
+  // If from_key exists, return the value of from_key.
+  KeyValue kv = 2;
+}

src/api/greptime/v1/physical_plan.proto

@@ -1,33 +0,0 @@
-syntax = "proto3";
-
-package greptime.v1.codec;
-
-message PhysicalPlanNode {
-  oneof PhysicalPlanType {
-    ProjectionExecNode projection = 1;
-    MockInputExecNode mock = 99;
-    // TODO(fys): impl other physical plan node
-  }
-}
-
-message ProjectionExecNode {
-  PhysicalPlanNode input = 1;
-  repeated PhysicalExprNode expr = 2;
-  repeated string expr_name = 3;
-}
-
-message PhysicalExprNode {
-  oneof ExprType {
-    PhysicalColumn column = 1;
-    // TODO(fys): impl other physical expr node
-  }
-}
-
-message PhysicalColumn {
-  string name = 1;
-  uint64 index = 2;
-}
-
-message MockInputExecNode {
-  string name = 1;
-}

src/api/prometheus/remote/remote.proto

@@ -0,0 +1,85 @@
+// Copyright 2016 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+package prometheus;
+
+option go_package = "prompb";
+
+import "prometheus/remote/types.proto";
+
+message WriteRequest {
+  repeated prometheus.TimeSeries timeseries = 1;
+  // Cortex uses this field to determine the source of the write request.
+  // We reserve it to avoid any compatibility issues.
+  reserved  2;
+  repeated prometheus.MetricMetadata metadata = 3;
+}
+
+// ReadRequest represents a remote read request.
+message ReadRequest {
+  repeated Query queries = 1;
+
+  enum ResponseType {
+    // Server will return a single ReadResponse message with matched series that includes list of raw samples.
+    // It's recommended to use streamed response types instead.
+    //
+    // Response headers:
+    // Content-Type: "application/x-protobuf"
+    // Content-Encoding: "snappy"
+    SAMPLES = 0;
+    // Server will stream a delimited ChunkedReadResponse message that contains XOR encoded chunks for a single series.
+    // Each message is following varint size and fixed size bigendian uint32 for CRC32 Castagnoli checksum.
+    //
+    // Response headers:
+    // Content-Type: "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse"
+    // Content-Encoding: ""
+    STREAMED_XOR_CHUNKS = 1;
+  }
+
+  // accepted_response_types allows negotiating the content type of the response.
+  //
+  // Response types are taken from the list in the FIFO order. If no response type in `accepted_response_types` is
+  // implemented by server, error is returned.
+  // For request that do not contain `accepted_response_types` field the SAMPLES response type will be used.
+  repeated ResponseType accepted_response_types = 2;
+}
+
+// ReadResponse is a response when response_type equals SAMPLES.
+message ReadResponse {
+  // In same order as the request's queries.
+  repeated QueryResult results = 1;
+}
+
+message Query {
+  int64 start_timestamp_ms = 1;
+  int64 end_timestamp_ms = 2;
+  repeated prometheus.LabelMatcher matchers = 3;
+  prometheus.ReadHints hints = 4;
+}
+
+message QueryResult {
+  // Samples within a time series must be ordered by time.
+  repeated prometheus.TimeSeries timeseries = 1;
+}
+
+// ChunkedReadResponse is a response when response_type equals STREAMED_XOR_CHUNKS.
+// We strictly stream full series after series, optionally split by time. This means that a single frame can contain
+// partition of the single series, but once a new series is started to be streamed it means that no more chunks will
+// be sent for previous one. Series are returned sorted in the same way TSDB block are internally.
+message ChunkedReadResponse {
+  repeated prometheus.ChunkedSeries chunked_series = 1;
+
+  // query_index represents an index of the query from ReadRequest.queries these chunks relates to.
+  int64 query_index = 2;
+}

src/api/prometheus/remote/types.proto

@@ -0,0 +1,117 @@
+// Copyright 2017 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+package prometheus;
+
+option go_package = "prompb";
+
+message MetricMetadata {
+  enum MetricType {
+    UNKNOWN        = 0;
+    COUNTER        = 1;
+    GAUGE          = 2;
+    HISTOGRAM      = 3;
+    GAUGEHISTOGRAM = 4;
+    SUMMARY        = 5;
+    INFO           = 6;
+    STATESET       = 7;
+  }
+
+  // Represents the metric type, these match the set from Prometheus.
+  // Refer to model/textparse/interface.go for details.
+  MetricType type = 1;
+  string metric_family_name = 2;
+  string help = 4;
+  string unit = 5;
+}
+
+message Sample {
+  double value    = 1;
+  // timestamp is in ms format, see model/timestamp/timestamp.go for
+  // conversion from time.Time to Prometheus timestamp.
+  int64 timestamp = 2;
+}
+
+message Exemplar {
+  // Optional, can be empty.
+  repeated Label labels = 1;
+  double value = 2;
+  // timestamp is in ms format, see model/timestamp/timestamp.go for
+  // conversion from time.Time to Prometheus timestamp.
+  int64 timestamp = 3;
+}
+
+// TimeSeries represents samples and labels for a single time series.
+message TimeSeries {
+  // For a timeseries to be valid, and for the samples and exemplars
+  // to be ingested by the remote system properly, the labels field is required.
+  repeated Label labels   = 1;
+  repeated Sample samples = 2;
+  repeated Exemplar exemplars = 3;
+}
+
+message Label {
+  string name  = 1;
+  string value = 2;
+}
+
+message Labels {
+  repeated Label labels = 1;
+}
+
+// Matcher specifies a rule, which can match or set of labels or not.
+message LabelMatcher {
+  enum Type {
+    EQ  = 0;
+    NEQ = 1;
+    RE  = 2;
+    NRE = 3;
+  }
+  Type type    = 1;
+  string name  = 2;
+  string value = 3;
+}
+
+message ReadHints {
+  int64 step_ms = 1;  // Query step size in milliseconds.
+  string func = 2;    // String representation of surrounding function or aggregation.
+  int64 start_ms = 3; // Start time in milliseconds.
+  int64 end_ms = 4;   // End time in milliseconds.
+  repeated string grouping = 5; // List of label names used in aggregation.
+  bool by = 6; // Indicate whether it is without or by.
+  int64 range_ms = 7; // Range vector selector range in milliseconds.
+}
+
+// Chunk represents a TSDB chunk.
+// Time range [min, max] is inclusive.
+message Chunk {
+  int64 min_time_ms = 1;
+  int64 max_time_ms = 2;
+
+  // We require this to match chunkenc.Encoding.
+  enum Encoding {
+    UNKNOWN = 0;
+    XOR     = 1;
+  }
+  Encoding type  = 3;
+  bytes data     = 4;
+}
+
+// ChunkedSeries represents single, encoded time series.
+message ChunkedSeries {
+  // Labels should be sorted.
+  repeated Label labels = 1;
+  // Chunks will be in start time order and may overlap.
+  repeated Chunk chunks = 2;
+}

src/api/src/error.rs

@@ -0,0 +1,76 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+
+use common_error::ext::ErrorExt;
+use common_error::prelude::StatusCode;
+use datatypes::prelude::ConcreteDataType;
+use snafu::prelude::*;
+use snafu::{Backtrace, ErrorCompat};
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug, Snafu)]
+#[snafu(visibility(pub))]
+pub enum Error {
+    #[snafu(display("Unknown proto column datatype: {}", datatype))]
+    UnknownColumnDataType { datatype: i32, backtrace: Backtrace },
+
+    #[snafu(display("Failed to create column datatype from {:?}", from))]
+    IntoColumnDataType {
+        from: ConcreteDataType,
+        backtrace: Backtrace,
+    },
+
+    #[snafu(display(
+        "Failed to convert column default constraint, column: {}, source: {}",
+        column,
+        source
+    ))]
+    ConvertColumnDefaultConstraint {
+        column: String,
+        #[snafu(backtrace)]
+        source: datatypes::error::Error,
+    },
+
+    #[snafu(display(
+        "Invalid column default constraint, column: {}, source: {}",
+        column,
+        source
+    ))]
+    InvalidColumnDefaultConstraint {
+        column: String,
+        #[snafu(backtrace)]
+        source: datatypes::error::Error,
+    },
+}
+
+impl ErrorExt for Error {
+    fn status_code(&self) -> StatusCode {
+        match self {
+            Error::UnknownColumnDataType { .. } => StatusCode::InvalidArguments,
+            Error::IntoColumnDataType { .. } => StatusCode::Unexpected,
+            Error::ConvertColumnDefaultConstraint { source, .. }
+            | Error::InvalidColumnDefaultConstraint { source, .. } => source.status_code(),
+        }
+    }
+    fn backtrace_opt(&self) -> Option<&Backtrace> {
+        ErrorCompat::backtrace(self)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/api/src/helper.rs

@@ -0,0 +1,517 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_base::BitVec;
+use common_time::timestamp::TimeUnit;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::types::TimestampType;
+use datatypes::value::Value;
+use datatypes::vectors::VectorRef;
+use snafu::prelude::*;
+
+use crate::error::{self, Result};
+use crate::v1::column::Values;
+use crate::v1::{Column, ColumnDataType};
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct ColumnDataTypeWrapper(ColumnDataType);
+
+impl ColumnDataTypeWrapper {
+    pub fn try_new(datatype: i32) -> Result<Self> {
+        let datatype = ColumnDataType::from_i32(datatype)
+            .context(error::UnknownColumnDataTypeSnafu { datatype })?;
+        Ok(Self(datatype))
+    }
+
+    pub fn datatype(&self) -> ColumnDataType {
+        self.0
+    }
+}
+
+impl From<ColumnDataTypeWrapper> for ConcreteDataType {
+    fn from(datatype: ColumnDataTypeWrapper) -> Self {
+        match datatype.0 {
+            ColumnDataType::Boolean => ConcreteDataType::boolean_datatype(),
+            ColumnDataType::Int8 => ConcreteDataType::int8_datatype(),
+            ColumnDataType::Int16 => ConcreteDataType::int16_datatype(),
+            ColumnDataType::Int32 => ConcreteDataType::int32_datatype(),
+            ColumnDataType::Int64 => ConcreteDataType::int64_datatype(),
+            ColumnDataType::Uint8 => ConcreteDataType::uint8_datatype(),
+            ColumnDataType::Uint16 => ConcreteDataType::uint16_datatype(),
+            ColumnDataType::Uint32 => ConcreteDataType::uint32_datatype(),
+            ColumnDataType::Uint64 => ConcreteDataType::uint64_datatype(),
+            ColumnDataType::Float32 => ConcreteDataType::float32_datatype(),
+            ColumnDataType::Float64 => ConcreteDataType::float64_datatype(),
+            ColumnDataType::Binary => ConcreteDataType::binary_datatype(),
+            ColumnDataType::String => ConcreteDataType::string_datatype(),
+            ColumnDataType::Date => ConcreteDataType::date_datatype(),
+            ColumnDataType::Datetime => ConcreteDataType::datetime_datatype(),
+            ColumnDataType::TimestampSecond => ConcreteDataType::timestamp_second_datatype(),
+            ColumnDataType::TimestampMillisecond => {
+                ConcreteDataType::timestamp_millisecond_datatype()
+            }
+            ColumnDataType::TimestampMicrosecond => {
+                ConcreteDataType::timestamp_microsecond_datatype()
+            }
+            ColumnDataType::TimestampNanosecond => {
+                ConcreteDataType::timestamp_nanosecond_datatype()
+            }
+        }
+    }
+}
+
+impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
+    type Error = error::Error;
+
+    fn try_from(datatype: ConcreteDataType) -> Result<Self> {
+        let datatype = ColumnDataTypeWrapper(match datatype {
+            ConcreteDataType::Boolean(_) => ColumnDataType::Boolean,
+            ConcreteDataType::Int8(_) => ColumnDataType::Int8,
+            ConcreteDataType::Int16(_) => ColumnDataType::Int16,
+            ConcreteDataType::Int32(_) => ColumnDataType::Int32,
+            ConcreteDataType::Int64(_) => ColumnDataType::Int64,
+            ConcreteDataType::UInt8(_) => ColumnDataType::Uint8,
+            ConcreteDataType::UInt16(_) => ColumnDataType::Uint16,
+            ConcreteDataType::UInt32(_) => ColumnDataType::Uint32,
+            ConcreteDataType::UInt64(_) => ColumnDataType::Uint64,
+            ConcreteDataType::Float32(_) => ColumnDataType::Float32,
+            ConcreteDataType::Float64(_) => ColumnDataType::Float64,
+            ConcreteDataType::Binary(_) => ColumnDataType::Binary,
+            ConcreteDataType::String(_) => ColumnDataType::String,
+            ConcreteDataType::Date(_) => ColumnDataType::Date,
+            ConcreteDataType::DateTime(_) => ColumnDataType::Datetime,
+            ConcreteDataType::Timestamp(unit) => match unit {
+                TimestampType::Second(_) => ColumnDataType::TimestampSecond,
+                TimestampType::Millisecond(_) => ColumnDataType::TimestampMillisecond,
+                TimestampType::Microsecond(_) => ColumnDataType::TimestampMicrosecond,
+                TimestampType::Nanosecond(_) => ColumnDataType::TimestampNanosecond,
+            },
+            ConcreteDataType::Null(_) | ConcreteDataType::List(_) => {
+                return error::IntoColumnDataTypeSnafu { from: datatype }.fail()
+            }
+        });
+        Ok(datatype)
+    }
+}
+
+impl Values {
+    pub fn with_capacity(datatype: ColumnDataType, capacity: usize) -> Self {
+        match datatype {
+            ColumnDataType::Boolean => Values {
+                bool_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Int8 => Values {
+                i8_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Int16 => Values {
+                i16_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Int32 => Values {
+                i32_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Int64 => Values {
+                i64_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Uint8 => Values {
+                u8_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Uint16 => Values {
+                u16_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Uint32 => Values {
+                u32_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Uint64 => Values {
+                u64_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Float32 => Values {
+                f32_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Float64 => Values {
+                f64_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Binary => Values {
+                binary_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::String => Values {
+                string_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Date => Values {
+                date_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::Datetime => Values {
+                datetime_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::TimestampSecond => Values {
+                ts_second_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::TimestampMillisecond => Values {
+                ts_millisecond_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::TimestampMicrosecond => Values {
+                ts_microsecond_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+            ColumnDataType::TimestampNanosecond => Values {
+                ts_nanosecond_values: Vec::with_capacity(capacity),
+                ..Default::default()
+            },
+        }
+    }
+}
+
+impl Column {
+    // The type of vals must be same.
+    pub fn push_vals(&mut self, origin_count: usize, vector: VectorRef) {
+        let values = self.values.get_or_insert_with(Values::default);
+        let mut null_mask = BitVec::from_slice(&self.null_mask);
+        let len = vector.len();
+        null_mask.reserve_exact(origin_count + len);
+        null_mask.extend(BitVec::repeat(false, len));
+
+        (0..len).into_iter().for_each(|idx| match vector.get(idx) {
+            Value::Null => null_mask.set(idx + origin_count, true),
+            Value::Boolean(val) => values.bool_values.push(val),
+            Value::UInt8(val) => values.u8_values.push(val.into()),
+            Value::UInt16(val) => values.u16_values.push(val.into()),
+            Value::UInt32(val) => values.u32_values.push(val),
+            Value::UInt64(val) => values.u64_values.push(val),
+            Value::Int8(val) => values.i8_values.push(val.into()),
+            Value::Int16(val) => values.i16_values.push(val.into()),
+            Value::Int32(val) => values.i32_values.push(val),
+            Value::Int64(val) => values.i64_values.push(val),
+            Value::Float32(val) => values.f32_values.push(*val),
+            Value::Float64(val) => values.f64_values.push(*val),
+            Value::String(val) => values.string_values.push(val.as_utf8().to_string()),
+            Value::Binary(val) => values.binary_values.push(val.to_vec()),
+            Value::Date(val) => values.date_values.push(val.val()),
+            Value::DateTime(val) => values.datetime_values.push(val.val()),
+            Value::Timestamp(val) => match val.unit() {
+                TimeUnit::Second => values.ts_second_values.push(val.value()),
+                TimeUnit::Millisecond => values.ts_millisecond_values.push(val.value()),
+                TimeUnit::Microsecond => values.ts_microsecond_values.push(val.value()),
+                TimeUnit::Nanosecond => values.ts_nanosecond_values.push(val.value()),
+            },
+            Value::List(_) => unreachable!(),
+        });
+        self.null_mask = null_mask.into_vec();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use datatypes::vectors::{
+        BooleanVector, TimestampMicrosecondVector, TimestampMillisecondVector,
+        TimestampNanosecondVector, TimestampSecondVector,
+    };
+
+    use super::*;
+
+    #[test]
+    fn test_values_with_capacity() {
+        let values = Values::with_capacity(ColumnDataType::Int8, 2);
+        let values = values.i8_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Int32, 2);
+        let values = values.i32_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Int64, 2);
+        let values = values.i64_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Uint8, 2);
+        let values = values.u8_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Uint32, 2);
+        let values = values.u32_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Uint64, 2);
+        let values = values.u64_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Float32, 2);
+        let values = values.f32_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Float64, 2);
+        let values = values.f64_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Binary, 2);
+        let values = values.binary_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Boolean, 2);
+        let values = values.bool_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::String, 2);
+        let values = values.string_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Date, 2);
+        let values = values.date_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::Datetime, 2);
+        let values = values.datetime_values;
+        assert_eq!(2, values.capacity());
+
+        let values = Values::with_capacity(ColumnDataType::TimestampMillisecond, 2);
+        let values = values.ts_millisecond_values;
+        assert_eq!(2, values.capacity());
+    }
+
+    #[test]
+    fn test_concrete_datatype_from_column_datatype() {
+        assert_eq!(
+            ConcreteDataType::boolean_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Boolean).into()
+        );
+        assert_eq!(
+            ConcreteDataType::int8_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Int8).into()
+        );
+        assert_eq!(
+            ConcreteDataType::int16_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Int16).into()
+        );
+        assert_eq!(
+            ConcreteDataType::int32_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Int32).into()
+        );
+        assert_eq!(
+            ConcreteDataType::int64_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Int64).into()
+        );
+        assert_eq!(
+            ConcreteDataType::uint8_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Uint8).into()
+        );
+        assert_eq!(
+            ConcreteDataType::uint16_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Uint16).into()
+        );
+        assert_eq!(
+            ConcreteDataType::uint32_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Uint32).into()
+        );
+        assert_eq!(
+            ConcreteDataType::uint64_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Uint64).into()
+        );
+        assert_eq!(
+            ConcreteDataType::float32_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Float32).into()
+        );
+        assert_eq!(
+            ConcreteDataType::float64_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Float64).into()
+        );
+        assert_eq!(
+            ConcreteDataType::binary_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Binary).into()
+        );
+        assert_eq!(
+            ConcreteDataType::string_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::String).into()
+        );
+        assert_eq!(
+            ConcreteDataType::date_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Date).into()
+        );
+        assert_eq!(
+            ConcreteDataType::datetime_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Datetime).into()
+        );
+        assert_eq!(
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::TimestampMillisecond).into()
+        );
+    }
+
+    #[test]
+    fn test_column_datatype_from_concrete_datatype() {
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Boolean),
+            ConcreteDataType::boolean_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Int8),
+            ConcreteDataType::int8_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Int16),
+            ConcreteDataType::int16_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Int32),
+            ConcreteDataType::int32_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Int64),
+            ConcreteDataType::int64_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Uint8),
+            ConcreteDataType::uint8_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Uint16),
+            ConcreteDataType::uint16_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Uint32),
+            ConcreteDataType::uint32_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Uint64),
+            ConcreteDataType::uint64_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Float32),
+            ConcreteDataType::float32_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Float64),
+            ConcreteDataType::float64_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Binary),
+            ConcreteDataType::binary_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::String),
+            ConcreteDataType::string_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Date),
+            ConcreteDataType::date_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Datetime),
+            ConcreteDataType::datetime_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::TimestampMillisecond),
+            ConcreteDataType::timestamp_millisecond_datatype()
+                .try_into()
+                .unwrap()
+        );
+
+        let result: Result<ColumnDataTypeWrapper> = ConcreteDataType::null_datatype().try_into();
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Failed to create column datatype from Null(NullType)"
+        );
+
+        let result: Result<ColumnDataTypeWrapper> =
+            ConcreteDataType::list_datatype(ConcreteDataType::boolean_datatype()).try_into();
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Failed to create column datatype from List(ListType { item_type: Boolean(BooleanType) })"
+        );
+    }
+
+    #[test]
+    fn test_column_put_timestamp_values() {
+        let mut column = Column {
+            column_name: "test".to_string(),
+            semantic_type: 0,
+            values: Some(Values {
+                ..Default::default()
+            }),
+            null_mask: vec![],
+            datatype: 0,
+        };
+
+        let vector = Arc::new(TimestampNanosecondVector::from_vec(vec![1, 2, 3]));
+        column.push_vals(3, vector);
+        assert_eq!(
+            vec![1, 2, 3],
+            column.values.as_ref().unwrap().ts_nanosecond_values
+        );
+
+        let vector = Arc::new(TimestampMillisecondVector::from_vec(vec![4, 5, 6]));
+        column.push_vals(3, vector);
+        assert_eq!(
+            vec![4, 5, 6],
+            column.values.as_ref().unwrap().ts_millisecond_values
+        );
+
+        let vector = Arc::new(TimestampMicrosecondVector::from_vec(vec![7, 8, 9]));
+        column.push_vals(3, vector);
+        assert_eq!(
+            vec![7, 8, 9],
+            column.values.as_ref().unwrap().ts_microsecond_values
+        );
+
+        let vector = Arc::new(TimestampSecondVector::from_vec(vec![10, 11, 12]));
+        column.push_vals(3, vector);
+        assert_eq!(
+            vec![10, 11, 12],
+            column.values.as_ref().unwrap().ts_second_values
+        );
+    }
+
+    #[test]
+    fn test_column_put_vector() {
+        use crate::v1::column::SemanticType;
+        // Some(false), None, Some(true), Some(true)
+        let mut column = Column {
+            column_name: "test".to_string(),
+            semantic_type: SemanticType::Field as i32,
+            values: Some(Values {
+                bool_values: vec![false, true, true],
+                ..Default::default()
+            }),
+            null_mask: vec![2],
+            datatype: ColumnDataType::Boolean as i32,
+        };
+        let row_count = 4;
+
+        let vector = Arc::new(BooleanVector::from(vec![Some(true), None, Some(false)]));
+        column.push_vals(row_count, vector);
+        // Some(false), None, Some(true), Some(true), Some(true), None, Some(false)
+        let bool_values = column.values.unwrap().bool_values;
+        assert_eq!(vec![false, true, true, true, false], bool_values);
+        let null_mask = column.null_mask;
+        assert_eq!(34, null_mask[0]);
+    }
+}

src/api/src/lib.rs

@@ -1,3 +1,21 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod error;
+pub mod helper;
+pub mod prometheus;
+pub mod result;
 pub mod serde;
 pub mod v1;
 

src/api/src/prometheus.rs

@@ -0,0 +1,19 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(clippy::derive_partial_eq_without_eq)]
+
+pub mod remote {
+    tonic::include_proto!("prometheus");
+}

src/api/src/result.rs

@@ -1,8 +1,24 @@
-use api::v1::{
-    codec::SelectResult, object_result, MutateResult, ObjectResult, ResultHeader,
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_error::prelude::ErrorExt;
+
+use crate::v1::codec::SelectResult;
+use crate::v1::{
+    admin_result, object_result, AdminResult, MutateResult, ObjectResult, ResultHeader,
     SelectResult as SelectResultRaw,
 };
-use common_error::prelude::ErrorExt;
 
 pub const PROTOCOL_VERSION: u32 = 1;
 
@@ -10,14 +26,14 @@ pub type Success = u32;
 pub type Failure = u32;
 
 #[derive(Default)]
-pub(crate) struct ObjectResultBuilder {
+pub struct ObjectResultBuilder {
     version: u32,
     code: u32,
     err_msg: Option<String>,
     result: Option<Body>,
 }
 
-pub(crate) enum Body {
+pub enum Body {
     Mutate((Success, Failure)),
     Select(SelectResult),
 }
@@ -80,20 +96,75 @@ impl ObjectResultBuilder {
     }
 }
 
-pub(crate) fn build_err_result(err: &impl ErrorExt) -> ObjectResult {
+pub fn build_err_result(err: &impl ErrorExt) -> ObjectResult {
     ObjectResultBuilder::new()
         .status_code(err.status_code() as u32)
         .err_msg(err.to_string())
         .build()
 }
 
+#[derive(Debug)]
+pub struct AdminResultBuilder {
+    version: u32,
+    code: u32,
+    err_msg: Option<String>,
+    mutate: Option<(Success, Failure)>,
+}
+
+impl AdminResultBuilder {
+    pub fn status_code(mut self, code: u32) -> Self {
+        self.code = code;
+        self
+    }
+
+    pub fn err_msg(mut self, err_msg: String) -> Self {
+        self.err_msg = Some(err_msg);
+        self
+    }
+
+    pub fn mutate_result(mut self, success: u32, failure: u32) -> Self {
+        self.mutate = Some((success, failure));
+        self
+    }
+
+    pub fn build(self) -> AdminResult {
+        let header = Some(ResultHeader {
+            version: self.version,
+            code: self.code,
+            err_msg: self.err_msg.unwrap_or_default(),
+        });
+
+        let result = if let Some((success, failure)) = self.mutate {
+            Some(admin_result::Result::Mutate(MutateResult {
+                success,
+                failure,
+            }))
+        } else {
+            None
+        };
+
+        AdminResult { header, result }
+    }
+}
+
+impl Default for AdminResultBuilder {
+    fn default() -> Self {
+        Self {
+            version: PROTOCOL_VERSION,
+            code: 0,
+            err_msg: None,
+            mutate: None,
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use api::v1::{object_result, MutateResult};
     use common_error::status_code::StatusCode;
 
     use super::*;
-    use crate::error::UnsupportedExprSnafu;
+    use crate::error::UnknownColumnDataTypeSnafu;
+    use crate::v1::{object_result, MutateResult};
 
     #[test]
     fn test_object_result_builder() {
@@ -120,14 +191,13 @@ mod tests {
 
     #[test]
     fn test_build_err_result() {
-        let err = UnsupportedExprSnafu { name: "select" }.build();
+        let err = UnknownColumnDataTypeSnafu { datatype: 1 }.build();
         let err_result = build_err_result(&err);
         let header = err_result.header.unwrap();
         let result = err_result.result;
 
         assert_eq!(PROTOCOL_VERSION, header.version);
-        assert_eq!(StatusCode::Internal as u32, header.code);
-        assert_eq!("Unsupported expr type: select", header.err_msg);
+        assert_eq!(StatusCode::InvalidArguments as u32, header.code);
         assert!(result.is_none());
     }
 }

src/api/src/serde.rs

@@ -1,7 +1,22 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 pub use prost::DecodeError;
 use prost::Message;
 
-use crate::v1::codec::{InsertBatch, PhysicalPlanNode, SelectResult};
+use crate::v1::codec::SelectResult;
+use crate::v1::meta::TableRouteValue;
 
 macro_rules! impl_convert_with_bytes {
     ($data_type: ty) => {
@@ -21,66 +36,18 @@ macro_rules! impl_convert_with_bytes {
     };
 }
 
-impl_convert_with_bytes!(InsertBatch);
 impl_convert_with_bytes!(SelectResult);
-impl_convert_with_bytes!(PhysicalPlanNode);
+impl_convert_with_bytes!(TableRouteValue);
 
 #[cfg(test)]
 mod tests {
     use std::ops::Deref;
 
     use crate::v1::codec::*;
-    use crate::v1::column;
-    use crate::v1::Column;
+    use crate::v1::{column, Column};
 
     const SEMANTIC_TAG: i32 = 0;
 
-    #[test]
-    fn test_convert_insert_batch() {
-        let insert_batch = mock_insert_batch();
-
-        let bytes: Vec<u8> = insert_batch.into();
-        let insert: InsertBatch = bytes.deref().try_into().unwrap();
-
-        assert_eq!(8, insert.row_count);
-        assert_eq!(1, insert.columns.len());
-
-        let column = &insert.columns[0];
-        assert_eq!("foo", column.column_name);
-        assert_eq!(SEMANTIC_TAG, column.semantic_type);
-        assert_eq!(vec![1], column.null_mask);
-        assert_eq!(
-            vec![2, 3, 4, 5, 6, 7, 8],
-            column.values.as_ref().unwrap().i32_values
-        );
-    }
-
-    #[should_panic]
-    #[test]
-    fn test_convert_insert_batch_wrong() {
-        let insert_batch = mock_insert_batch();
-
-        let mut bytes: Vec<u8> = insert_batch.into();
-
-        // modify some bytes
-        bytes[0] = 0b1;
-        bytes[1] = 0b1;
-
-        let insert: InsertBatch = bytes.deref().try_into().unwrap();
-
-        assert_eq!(8, insert.row_count);
-        assert_eq!(1, insert.columns.len());
-
-        let column = &insert.columns[0];
-        assert_eq!("foo", column.column_name);
-        assert_eq!(SEMANTIC_TAG, column.semantic_type);
-        assert_eq!(vec![1], column.null_mask);
-        assert_eq!(
-            vec![2, 3, 4, 5, 6, 7, 8],
-            column.values.as_ref().unwrap().i32_values
-        );
-    }
-
     #[test]
     fn test_convert_select_result() {
         let select_result = mock_select_result();
@@ -127,24 +94,6 @@ mod tests {
         );
     }
 
-    fn mock_insert_batch() -> InsertBatch {
-        let values = column::Values {
-            i32_values: vec![2, 3, 4, 5, 6, 7, 8],
-            ..Default::default()
-        };
-        let null_mask = vec![1];
-        let column = Column {
-            column_name: "foo".to_string(),
-            semantic_type: SEMANTIC_TAG,
-            values: Some(values),
-            null_mask,
-        };
-        InsertBatch {
-            columns: vec![column],
-            row_count: 8,
-        }
-    }
-
     fn mock_select_result() -> SelectResult {
         let values = column::Values {
             i32_values: vec![2, 3, 4, 5, 6, 7, 8],
@@ -156,6 +105,7 @@ mod tests {
             semantic_type: SEMANTIC_TAG,
             values: Some(values),
             null_mask,
+            ..Default::default()
         };
         SelectResult {
             columns: vec![column],

src/api/src/v1.rs

@@ -1,6 +1,25 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #![allow(clippy::derive_partial_eq_without_eq)]
 tonic::include_proto!("greptime.v1");
 
+pub const GREPTIME_FD_SET: &[u8] = tonic::include_file_descriptor_set!("greptime_fd");
+
 pub mod codec {
     tonic::include_proto!("greptime.v1.codec");
 }
+
+mod column_def;
+pub mod meta;

src/api/src/v1/column_def.rs

@@ -0,0 +1,38 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema};
+use snafu::ResultExt;
+
+use crate::error::{self, Result};
+use crate::helper::ColumnDataTypeWrapper;
+use crate::v1::ColumnDef;
+
+impl ColumnDef {
+    pub fn try_as_column_schema(&self) -> Result<ColumnSchema> {
+        let data_type = ColumnDataTypeWrapper::try_new(self.datatype)?;
+
+        let constraint = match &self.default_constraint {
+            None => None,
+            Some(v) => Some(
+                ColumnDefaultConstraint::try_from(&v[..])
+                    .context(error::ConvertColumnDefaultConstraintSnafu { column: &self.name })?,
+            ),
+        };
+
+        ColumnSchema::new(&self.name, data_type.into(), self.is_nullable)
+            .with_default_constraint(constraint)
+            .context(error::InvalidColumnDefaultConstraintSnafu { column: &self.name })
+    }
+}

src/api/src/v1/meta.rs

@@ -0,0 +1,209 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+tonic::include_proto!("greptime.v1.meta");
+
+use std::collections::HashMap;
+use std::hash::{Hash, Hasher};
+
+pub const PROTOCOL_VERSION: u64 = 1;
+
+#[derive(Default)]
+pub struct PeerDict {
+    peers: HashMap<Peer, usize>,
+    index: usize,
+}
+
+impl PeerDict {
+    pub fn get_or_insert(&mut self, peer: Peer) -> usize {
+        let index = self.peers.entry(peer).or_insert_with(|| {
+            let v = self.index;
+            self.index += 1;
+            v
+        });
+
+        *index
+    }
+
+    pub fn into_peers(self) -> Vec<Peer> {
+        let mut array = vec![Peer::default(); self.index];
+        for (p, i) in self.peers {
+            array[i] = p;
+        }
+        array
+    }
+}
+
+#[allow(clippy::derive_hash_xor_eq)]
+impl Hash for Peer {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.id.hash(state);
+        self.addr.hash(state);
+    }
+}
+
+impl Eq for Peer {}
+
+impl RequestHeader {
+    #[inline]
+    pub fn new((cluster_id, member_id): (u64, u64)) -> Self {
+        Self {
+            protocol_version: PROTOCOL_VERSION,
+            cluster_id,
+            member_id,
+        }
+    }
+}
+
+impl ResponseHeader {
+    #[inline]
+    pub fn success(cluster_id: u64) -> Self {
+        Self {
+            protocol_version: PROTOCOL_VERSION,
+            cluster_id,
+            ..Default::default()
+        }
+    }
+
+    #[inline]
+    pub fn failed(cluster_id: u64, error: Error) -> Self {
+        Self {
+            protocol_version: PROTOCOL_VERSION,
+            cluster_id,
+            error: Some(error),
+        }
+    }
+
+    #[inline]
+    pub fn is_not_leader(&self) -> bool {
+        if let Some(error) = &self.error {
+            if error.code == ErrorCode::NotLeader as i32 {
+                return true;
+            }
+        }
+        false
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ErrorCode {
+    NoActiveDatanodes = 1,
+    NotLeader = 2,
+}
+
+impl Error {
+    #[inline]
+    pub fn no_active_datanodes() -> Self {
+        Self {
+            code: ErrorCode::NoActiveDatanodes as i32,
+            err_msg: "No active datanodes".to_string(),
+        }
+    }
+
+    #[inline]
+    pub fn is_not_leader() -> Self {
+        Self {
+            code: ErrorCode::NotLeader as i32,
+            err_msg: "Current server is not leader".to_string(),
+        }
+    }
+}
+
+impl HeartbeatResponse {
+    #[inline]
+    pub fn is_not_leader(&self) -> bool {
+        if let Some(header) = &self.header {
+            return header.is_not_leader();
+        }
+        false
+    }
+}
+
+macro_rules! gen_set_header {
+    ($req: ty) => {
+        impl $req {
+            #[inline]
+            pub fn set_header(&mut self, (cluster_id, member_id): (u64, u64)) {
+                self.header = Some(RequestHeader::new((cluster_id, member_id)));
+            }
+        }
+    };
+}
+
+gen_set_header!(HeartbeatRequest);
+gen_set_header!(RouteRequest);
+gen_set_header!(CreateRequest);
+gen_set_header!(RangeRequest);
+gen_set_header!(DeleteRequest);
+gen_set_header!(PutRequest);
+gen_set_header!(BatchPutRequest);
+gen_set_header!(CompareAndPutRequest);
+gen_set_header!(DeleteRangeRequest);
+gen_set_header!(MoveValueRequest);
+
+#[cfg(test)]
+mod tests {
+    use std::vec;
+
+    use super::*;
+
+    #[test]
+    fn test_peer_dict() {
+        let mut dict = PeerDict::default();
+
+        dict.get_or_insert(Peer {
+            id: 1,
+            addr: "111".to_string(),
+        });
+        dict.get_or_insert(Peer {
+            id: 2,
+            addr: "222".to_string(),
+        });
+        dict.get_or_insert(Peer {
+            id: 1,
+            addr: "111".to_string(),
+        });
+        dict.get_or_insert(Peer {
+            id: 1,
+            addr: "111".to_string(),
+        });
+        dict.get_or_insert(Peer {
+            id: 1,
+            addr: "111".to_string(),
+        });
+        dict.get_or_insert(Peer {
+            id: 1,
+            addr: "111".to_string(),
+        });
+        dict.get_or_insert(Peer {
+            id: 2,
+            addr: "222".to_string(),
+        });
+
+        assert_eq!(2, dict.index);
+        assert_eq!(
+            vec![
+                Peer {
+                    id: 1,
+                    addr: "111".to_string(),
+                },
+                Peer {
+                    id: 2,
+                    addr: "222".to_string(),
+                }
+            ],
+            dict.into_peers()
+        );
+    }
+}

src/catalog/Cargo.toml

@@ -2,25 +2,42 @@
 name = "catalog"
 version = "0.1.0"
 edition = "2021"
-
+license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1"
+api = { path = "../api" }
+arc-swap = "1.0"
 async-stream = "0.3"
+async-trait = "0.1"
+backoff = { version = "0.4", features = ["tokio"] }
+common-catalog = { path = "../common/catalog" }
 common-error = { path = "../common/error" }
+common-grpc = { path = "../common/grpc" }
 common-query = { path = "../common/query" }
 common-recordbatch = { path = "../common/recordbatch" }
+common-runtime = { path = "../common/runtime" }
 common-telemetry = { path = "../common/telemetry" }
 common-time = { path = "../common/time" }
-datafusion = { git = "https://github.com/apache/arrow-datafusion.git", branch = "arrow2", features = ["simd"] }
+datafusion = "14.0.0"
 datatypes = { path = "../datatypes" }
 futures = "0.3"
 futures-util = "0.3"
+lazy_static = "1.4"
+meta-client = { path = "../meta-client" }
+regex = "1.6"
 serde = "1.0"
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }
+storage = { path = "../storage" }
 table = { path = "../table" }
+tokio = { version = "1.18", features = ["full"] }
 
 [dev-dependencies]
+chrono = "0.4"
+log-store = { path = "../log-store" }
+mito = { path = "../mito", features = ["test"] }
+object-store = { path = "../object-store" }
+storage = { path = "../storage" }
+tempdir = "0.3"
 tokio = { version = "1.0", features = ["full"] }

src/catalog/src/consts.rs

@@ -1,6 +0,0 @@
-pub const SYSTEM_CATALOG_NAME: &str = "system";
-pub const INFORMATION_SCHEMA_NAME: &str = "information_schema";
-pub const SYSTEM_CATALOG_TABLE_ID: u32 = 0;
-pub const SYSTEM_CATALOG_TABLE_NAME: &str = "system_catalog";
-pub const DEFAULT_CATALOG_NAME: &str = "greptime";
-pub const DEFAULT_SCHEMA_NAME: &str = "public";

src/catalog/src/error.rs

@@ -1,9 +1,24 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::prelude::{Snafu, StatusCode};
 use datafusion::error::DataFusionError;
-use datatypes::arrow;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::schema::RawSchema;
 use snafu::{Backtrace, ErrorCompat};
 
 #[derive(Debug, Snafu)]
@@ -21,28 +36,43 @@ pub enum Error {
         source: table::error::Error,
     },
 
+    #[snafu(display(
+        "Failed to create table, table info: {}, source: {}",
+        table_info,
+        source
+    ))]
+    CreateTable {
+        table_info: String,
+        #[snafu(backtrace)]
+        source: table::error::Error,
+    },
+
     #[snafu(display("System catalog is not valid: {}", msg))]
     SystemCatalog { msg: String, backtrace: Backtrace },
 
     #[snafu(display(
-        "System catalog table type mismatch, expected: binary, found: {:?} source: {}",
+        "System catalog table type mismatch, expected: binary, found: {:?}",
         data_type,
-        source
     ))]
     SystemCatalogTypeMismatch {
-        data_type: arrow::datatypes::DataType,
-        #[snafu(backtrace)]
-        source: datatypes::error::Error,
+        data_type: ConcreteDataType,
+        backtrace: Backtrace,
     },
 
     #[snafu(display("Invalid system catalog entry type: {:?}", entry_type))]
-    InvalidEntryType { entry_type: Option<u8> },
+    InvalidEntryType {
+        entry_type: Option<u8>,
+        backtrace: Backtrace,
+    },
 
     #[snafu(display("Invalid system catalog key: {:?}", key))]
-    InvalidKey { key: Option<String> },
+    InvalidKey {
+        key: Option<String>,
+        backtrace: Backtrace,
+    },
 
     #[snafu(display("Catalog value is not present"))]
-    EmptyValue,
+    EmptyValue { backtrace: Backtrace },
 
     #[snafu(display("Failed to deserialize value, source: {}", source))]
     ValueDeserialize {
@@ -51,20 +81,38 @@ pub enum Error {
     },
 
     #[snafu(display("Cannot find catalog by name: {}", catalog_name))]
-    CatalogNotFound { catalog_name: String },
+    CatalogNotFound {
+        catalog_name: String,
+        backtrace: Backtrace,
+    },
 
     #[snafu(display("Cannot find schema, schema info: {}", schema_info))]
-    SchemaNotFound { schema_info: String },
+    SchemaNotFound {
+        schema_info: String,
+        backtrace: Backtrace,
+    },
 
-    #[snafu(display("Table {} already exists", table))]
+    #[snafu(display("Table `{}` already exists", table))]
     TableExists { table: String, backtrace: Backtrace },
 
+    #[snafu(display("Schema {} already exists", schema))]
+    SchemaExists {
+        schema: String,
+        backtrace: Backtrace,
+    },
+
     #[snafu(display("Failed to register table"))]
     RegisterTable {
         #[snafu(backtrace)]
         source: BoxedError,
     },
 
+    #[snafu(display("Operation {} not implemented yet", operation))]
+    Unimplemented {
+        operation: String,
+        backtrace: Backtrace,
+    },
+
     #[snafu(display("Failed to open table, table info: {}, source: {}", table_info, source))]
     OpenTable {
         table_info: String,
@@ -73,7 +121,10 @@ pub enum Error {
     },
 
     #[snafu(display("Table not found while opening table, table info: {}", table_info))]
-    TableNotFound { table_info: String },
+    TableNotFound {
+        table_info: String,
+        backtrace: Backtrace,
+    },
 
     #[snafu(display("Failed to read system catalog table records"))]
     ReadSystemCatalog {
@@ -85,10 +136,70 @@ pub enum Error {
         "Failed to insert table creation record to system catalog, source: {}",
         source
     ))]
-    InsertTableRecord {
+    InsertCatalogRecord {
         #[snafu(backtrace)]
         source: table::error::Error,
     },
+
+    #[snafu(display("Illegal catalog manager state: {}", msg))]
+    IllegalManagerState { backtrace: Backtrace, msg: String },
+
+    #[snafu(display("Failed to scan system catalog table, source: {}", source))]
+    SystemCatalogTableScan {
+        #[snafu(backtrace)]
+        source: table::error::Error,
+    },
+
+    #[snafu(display(
+        "Invalid table schema in catalog entry, table:{}, schema: {:?}, source: {}",
+        table_info,
+        schema,
+        source
+    ))]
+    InvalidTableSchema {
+        table_info: String,
+        schema: RawSchema,
+        #[snafu(backtrace)]
+        source: datatypes::error::Error,
+    },
+
+    #[snafu(display("Failed to execute system catalog table scan, source: {}", source))]
+    SystemCatalogTableScanExec {
+        #[snafu(backtrace)]
+        source: common_query::error::Error,
+    },
+    #[snafu(display("Cannot parse catalog value, source: {}", source))]
+    InvalidCatalogValue {
+        #[snafu(backtrace)]
+        source: common_catalog::error::Error,
+    },
+
+    #[snafu(display("IO error occurred while fetching catalog info, source: {}", source))]
+    Io {
+        backtrace: Backtrace,
+        source: std::io::Error,
+    },
+
+    #[snafu(display("Local and remote catalog data are inconsistent, msg: {}", msg))]
+    CatalogStateInconsistent { msg: String, backtrace: Backtrace },
+
+    #[snafu(display("Failed to perform metasrv operation, source: {}", source))]
+    MetaSrv {
+        #[snafu(backtrace)]
+        source: meta_client::error::Error,
+    },
+
+    #[snafu(display("Invalid table info in catalog, source: {}", source))]
+    InvalidTableInfoInCatalog {
+        #[snafu(backtrace)]
+        source: datatypes::error::Error,
+    },
+
+    #[snafu(display("Catalog internal error: {}", source))]
+    Internal {
+        #[snafu(backtrace)]
+        source: BoxedError,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -97,21 +208,41 @@ impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         match self {
             Error::InvalidKey { .. }
-            | Error::OpenSystemCatalog { .. }
-            | Error::CreateSystemCatalog { .. }
             | Error::SchemaNotFound { .. }
             | Error::TableNotFound { .. }
-            | Error::InvalidEntryType { .. } => StatusCode::Unexpected,
-            Error::SystemCatalog { .. }
-            | Error::SystemCatalogTypeMismatch { .. }
-            | Error::EmptyValue
-            | Error::ValueDeserialize { .. }
+            | Error::IllegalManagerState { .. }
             | Error::CatalogNotFound { .. }
-            | Error::OpenTable { .. }
-            | Error::ReadSystemCatalog { .. }
-            | Error::InsertTableRecord { .. } => StatusCode::StorageUnavailable,
-            Error::RegisterTable { .. } => StatusCode::Internal,
+            | Error::InvalidEntryType { .. }
+            | Error::CatalogStateInconsistent { .. } => StatusCode::Unexpected,
+
+            Error::SystemCatalog { .. }
+            | Error::EmptyValue { .. }
+            | Error::ValueDeserialize { .. }
+            | Error::Io { .. } => StatusCode::StorageUnavailable,
+
+            Error::RegisterTable { .. } | Error::SystemCatalogTypeMismatch { .. } => {
+                StatusCode::Internal
+            }
+
+            Error::ReadSystemCatalog { source, .. } => source.status_code(),
+            Error::InvalidCatalogValue { source, .. } => source.status_code(),
+
             Error::TableExists { .. } => StatusCode::TableAlreadyExists,
+            Error::SchemaExists { .. } => StatusCode::InvalidArguments,
+
+            Error::OpenSystemCatalog { source, .. }
+            | Error::CreateSystemCatalog { source, .. }
+            | Error::InsertCatalogRecord { source, .. }
+            | Error::OpenTable { source, .. }
+            | Error::CreateTable { source, .. } => source.status_code(),
+            Error::MetaSrv { source, .. } => source.status_code(),
+            Error::SystemCatalogTableScan { source } => source.status_code(),
+            Error::SystemCatalogTableScanExec { source } => source.status_code(),
+            Error::InvalidTableSchema { source, .. } => source.status_code(),
+            Error::InvalidTableInfoInCatalog { .. } => StatusCode::Unexpected,
+            Error::Internal { source, .. } => source.status_code(),
+
+            Error::Unimplemented { .. } => StatusCode::Unsupported,
         }
     }
 
@@ -133,7 +264,6 @@ impl From<Error> for DataFusionError {
 #[cfg(test)]
 mod tests {
     use common_error::mock::MockError;
-    use datatypes::arrow::datatypes::DataType;
     use snafu::GenerateImplicitData;
 
     use super::*;
@@ -151,11 +281,11 @@ mod tests {
 
         assert_eq!(
             StatusCode::Unexpected,
-            Error::InvalidKey { key: None }.status_code()
+            InvalidKeySnafu { key: None }.build().status_code()
         );
 
         assert_eq!(
-            StatusCode::Unexpected,
+            StatusCode::StorageUnavailable,
             Error::OpenSystemCatalog {
                 source: table::error::Error::new(MockError::new(StatusCode::StorageUnavailable))
             }
@@ -163,7 +293,7 @@ mod tests {
         );
 
         assert_eq!(
-            StatusCode::Unexpected,
+            StatusCode::StorageUnavailable,
             Error::CreateSystemCatalog {
                 source: table::error::Error::new(MockError::new(StatusCode::StorageUnavailable))
             }
@@ -180,19 +310,16 @@ mod tests {
         );
 
         assert_eq!(
-            StatusCode::StorageUnavailable,
+            StatusCode::Internal,
             Error::SystemCatalogTypeMismatch {
-                data_type: DataType::Boolean,
-                source: datatypes::error::Error::UnsupportedArrowType {
-                    arrow_type: DataType::Boolean,
-                    backtrace: Backtrace::generate()
-                }
+                data_type: ConcreteDataType::binary_datatype(),
+                backtrace: Backtrace::generate(),
             }
             .status_code()
         );
         assert_eq!(
             StatusCode::StorageUnavailable,
-            Error::EmptyValue.status_code()
+            EmptyValueSnafu {}.build().status_code()
         );
     }
 

src/catalog/src/helper.rs

@@ -0,0 +1,384 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::fmt::{Display, Formatter};
+
+use common_catalog::error::{
+    DeserializeCatalogEntryValueSnafu, Error, InvalidCatalogSnafu, SerializeCatalogEntryValueSnafu,
+};
+use lazy_static::lazy_static;
+use regex::Regex;
+use serde::{Deserialize, Serialize, Serializer};
+use snafu::{ensure, OptionExt, ResultExt};
+use table::metadata::{RawTableInfo, TableId, TableVersion};
+
+const CATALOG_KEY_PREFIX: &str = "__c";
+const SCHEMA_KEY_PREFIX: &str = "__s";
+const TABLE_GLOBAL_KEY_PREFIX: &str = "__tg";
+const TABLE_REGIONAL_KEY_PREFIX: &str = "__tr";
+
+const ALPHANUMERICS_NAME_PATTERN: &str = "[a-zA-Z_][a-zA-Z0-9_]*";
+
+lazy_static! {
+    static ref CATALOG_KEY_PATTERN: Regex = Regex::new(&format!(
+        "^{}-({})$",
+        CATALOG_KEY_PREFIX, ALPHANUMERICS_NAME_PATTERN
+    ))
+    .unwrap();
+}
+
+lazy_static! {
+    static ref SCHEMA_KEY_PATTERN: Regex = Regex::new(&format!(
+        "^{}-({})-({})$",
+        SCHEMA_KEY_PREFIX, ALPHANUMERICS_NAME_PATTERN, ALPHANUMERICS_NAME_PATTERN
+    ))
+    .unwrap();
+}
+
+lazy_static! {
+    static ref TABLE_GLOBAL_KEY_PATTERN: Regex = Regex::new(&format!(
+        "^{}-({})-({})-({})$",
+        TABLE_GLOBAL_KEY_PREFIX,
+        ALPHANUMERICS_NAME_PATTERN,
+        ALPHANUMERICS_NAME_PATTERN,
+        ALPHANUMERICS_NAME_PATTERN
+    ))
+    .unwrap();
+}
+
+lazy_static! {
+    static ref TABLE_REGIONAL_KEY_PATTERN: Regex = Regex::new(&format!(
+        "^{}-({})-({})-({})-([0-9]+)$",
+        TABLE_REGIONAL_KEY_PREFIX,
+        ALPHANUMERICS_NAME_PATTERN,
+        ALPHANUMERICS_NAME_PATTERN,
+        ALPHANUMERICS_NAME_PATTERN
+    ))
+    .unwrap();
+}
+
+pub fn build_catalog_prefix() -> String {
+    format!("{}-", CATALOG_KEY_PREFIX)
+}
+
+pub fn build_schema_prefix(catalog_name: impl AsRef<str>) -> String {
+    format!("{}-{}-", SCHEMA_KEY_PREFIX, catalog_name.as_ref())
+}
+
+pub fn build_table_global_prefix(
+    catalog_name: impl AsRef<str>,
+    schema_name: impl AsRef<str>,
+) -> String {
+    format!(
+        "{}-{}-{}-",
+        TABLE_GLOBAL_KEY_PREFIX,
+        catalog_name.as_ref(),
+        schema_name.as_ref()
+    )
+}
+
+pub fn build_table_regional_prefix(
+    catalog_name: impl AsRef<str>,
+    schema_name: impl AsRef<str>,
+) -> String {
+    format!(
+        "{}-{}-{}-",
+        TABLE_REGIONAL_KEY_PREFIX,
+        catalog_name.as_ref(),
+        schema_name.as_ref()
+    )
+}
+
+/// Table global info has only one key across all datanodes so it does not have `node_id` field.
+pub struct TableGlobalKey {
+    pub catalog_name: String,
+    pub schema_name: String,
+    pub table_name: String,
+}
+
+impl Display for TableGlobalKey {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(TABLE_GLOBAL_KEY_PREFIX)?;
+        f.write_str("-")?;
+        f.write_str(&self.catalog_name)?;
+        f.write_str("-")?;
+        f.write_str(&self.schema_name)?;
+        f.write_str("-")?;
+        f.write_str(&self.table_name)
+    }
+}
+
+impl TableGlobalKey {
+    pub fn parse<S: AsRef<str>>(s: S) -> Result<Self, Error> {
+        let key = s.as_ref();
+        let captures = TABLE_GLOBAL_KEY_PATTERN
+            .captures(key)
+            .context(InvalidCatalogSnafu { key })?;
+        ensure!(captures.len() == 4, InvalidCatalogSnafu { key });
+
+        Ok(Self {
+            catalog_name: captures[1].to_string(),
+            schema_name: captures[2].to_string(),
+            table_name: captures[3].to_string(),
+        })
+    }
+}
+
+/// Table global info contains necessary info for a datanode to create table regions, including
+/// table id, table meta(schema...), region id allocation across datanodes.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct TableGlobalValue {
+    /// Id of datanode that created the global table info kv. only for debugging.
+    pub node_id: u64,
+    // TODO(LFC): Maybe remove it?
+    /// Allocation of region ids across all datanodes.
+    pub regions_id_map: HashMap<u64, Vec<u32>>,
+    pub table_info: RawTableInfo,
+}
+
+impl TableGlobalValue {
+    pub fn table_id(&self) -> TableId {
+        self.table_info.ident.table_id
+    }
+}
+
+/// Table regional info that varies between datanode, so it contains a `node_id` field.
+pub struct TableRegionalKey {
+    pub catalog_name: String,
+    pub schema_name: String,
+    pub table_name: String,
+    pub node_id: u64,
+}
+
+impl Display for TableRegionalKey {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(TABLE_REGIONAL_KEY_PREFIX)?;
+        f.write_str("-")?;
+        f.write_str(&self.catalog_name)?;
+        f.write_str("-")?;
+        f.write_str(&self.schema_name)?;
+        f.write_str("-")?;
+        f.write_str(&self.table_name)?;
+        f.write_str("-")?;
+        f.serialize_u64(self.node_id)
+    }
+}
+
+impl TableRegionalKey {
+    pub fn parse<S: AsRef<str>>(s: S) -> Result<Self, Error> {
+        let key = s.as_ref();
+        let captures = TABLE_REGIONAL_KEY_PATTERN
+            .captures(key)
+            .context(InvalidCatalogSnafu { key })?;
+        ensure!(captures.len() == 5, InvalidCatalogSnafu { key });
+        let node_id = captures[4]
+            .to_string()
+            .parse()
+            .map_err(|_| InvalidCatalogSnafu { key }.build())?;
+        Ok(Self {
+            catalog_name: captures[1].to_string(),
+            schema_name: captures[2].to_string(),
+            table_name: captures[3].to_string(),
+            node_id,
+        })
+    }
+}
+
+/// Regional table info of specific datanode, including table version on that datanode and
+/// region ids allocated by metasrv.
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct TableRegionalValue {
+    pub version: TableVersion,
+    pub regions_ids: Vec<u32>,
+}
+
+pub struct CatalogKey {
+    pub catalog_name: String,
+}
+
+impl Display for CatalogKey {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(CATALOG_KEY_PREFIX)?;
+        f.write_str("-")?;
+        f.write_str(&self.catalog_name)
+    }
+}
+
+impl CatalogKey {
+    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
+        let key = s.as_ref();
+        let captures = CATALOG_KEY_PATTERN
+            .captures(key)
+            .context(InvalidCatalogSnafu { key })?;
+        ensure!(captures.len() == 2, InvalidCatalogSnafu { key });
+        Ok(Self {
+            catalog_name: captures[1].to_string(),
+        })
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct CatalogValue;
+
+pub struct SchemaKey {
+    pub catalog_name: String,
+    pub schema_name: String,
+}
+
+impl Display for SchemaKey {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(SCHEMA_KEY_PREFIX)?;
+        f.write_str("-")?;
+        f.write_str(&self.catalog_name)?;
+        f.write_str("-")?;
+        f.write_str(&self.schema_name)
+    }
+}
+
+impl SchemaKey {
+    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
+        let key = s.as_ref();
+        let captures = SCHEMA_KEY_PATTERN
+            .captures(key)
+            .context(InvalidCatalogSnafu { key })?;
+        ensure!(captures.len() == 3, InvalidCatalogSnafu { key });
+        Ok(Self {
+            catalog_name: captures[1].to_string(),
+            schema_name: captures[2].to_string(),
+        })
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SchemaValue;
+
+macro_rules! define_catalog_value {
+    ( $($val_ty: ty), *) => {
+            $(
+                impl $val_ty {
+                    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
+                        serde_json::from_str(s.as_ref())
+                            .context(DeserializeCatalogEntryValueSnafu { raw: s.as_ref() })
+                    }
+
+                    pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result<Self, Error> {
+                         Self::parse(&String::from_utf8_lossy(bytes.as_ref()))
+                    }
+
+                    pub fn as_bytes(&self) -> Result<Vec<u8>, Error> {
+                        Ok(serde_json::to_string(self)
+                            .context(SerializeCatalogEntryValueSnafu)?
+                            .into_bytes())
+                    }
+                }
+            )*
+        }
+}
+
+define_catalog_value!(
+    TableRegionalValue,
+    TableGlobalValue,
+    CatalogValue,
+    SchemaValue
+);
+
+#[cfg(test)]
+mod tests {
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::schema::{ColumnSchema, RawSchema, Schema};
+    use table::metadata::{RawTableMeta, TableIdent, TableType};
+
+    use super::*;
+
+    #[test]
+    fn test_parse_catalog_key() {
+        let key = "__c-C";
+        let catalog_key = CatalogKey::parse(key).unwrap();
+        assert_eq!("C", catalog_key.catalog_name);
+        assert_eq!(key, catalog_key.to_string());
+    }
+
+    #[test]
+    fn test_parse_schema_key() {
+        let key = "__s-C-S";
+        let schema_key = SchemaKey::parse(key).unwrap();
+        assert_eq!("C", schema_key.catalog_name);
+        assert_eq!("S", schema_key.schema_name);
+        assert_eq!(key, schema_key.to_string());
+    }
+
+    #[test]
+    fn test_parse_table_key() {
+        let key = "__tg-C-S-T";
+        let entry = TableGlobalKey::parse(key).unwrap();
+        assert_eq!("C", entry.catalog_name);
+        assert_eq!("S", entry.schema_name);
+        assert_eq!("T", entry.table_name);
+        assert_eq!(key, &entry.to_string());
+    }
+
+    #[test]
+    fn test_build_prefix() {
+        assert_eq!("__c-", build_catalog_prefix());
+        assert_eq!("__s-CATALOG-", build_schema_prefix("CATALOG"));
+        assert_eq!(
+            "__tg-CATALOG-SCHEMA-",
+            build_table_global_prefix("CATALOG", "SCHEMA")
+        );
+    }
+
+    #[test]
+    fn test_serialize_schema() {
+        let schema = Schema::new(vec![ColumnSchema::new(
+            "name",
+            ConcreteDataType::string_datatype(),
+            true,
+        )]);
+
+        let meta = RawTableMeta {
+            schema: RawSchema::from(&schema),
+            engine: "mito".to_string(),
+            created_on: chrono::DateTime::default(),
+            primary_key_indices: vec![0, 1],
+            next_column_id: 3,
+            engine_options: Default::default(),
+            value_indices: vec![2, 3],
+            options: Default::default(),
+            region_numbers: vec![1],
+        };
+
+        let table_info = RawTableInfo {
+            ident: TableIdent {
+                table_id: 42,
+                version: 1,
+            },
+            name: "table_1".to_string(),
+            desc: Some("blah".to_string()),
+            catalog_name: "catalog_1".to_string(),
+            schema_name: "schema_1".to_string(),
+            meta,
+            table_type: TableType::Base,
+        };
+
+        let value = TableGlobalValue {
+            node_id: 0,
+            regions_id_map: HashMap::from([(0, vec![1, 2, 3])]),
+            table_info,
+        };
+        let serialized = serde_json::to_string(&value).unwrap();
+        let deserialized = TableGlobalValue::parse(&serialized).unwrap();
+        assert_eq!(value, deserialized);
+    }
+}

src/catalog/src/lib.rs

@@ -1,22 +1,40 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #![feature(assert_matches)]
 
 use std::any::Any;
+use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
+use common_telemetry::info;
+use snafu::ResultExt;
+use table::engine::{EngineContext, TableEngineRef};
 use table::metadata::TableId;
+use table::requests::CreateTableRequest;
 use table::TableRef;
 
-pub use crate::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-pub use crate::manager::LocalCatalogManager;
+use crate::error::{CreateTableSnafu, Result};
 pub use crate::schema::{SchemaProvider, SchemaProviderRef};
 
-mod consts;
 pub mod error;
-mod manager;
-pub mod memory;
+pub mod helper;
+pub mod local;
+pub mod remote;
 pub mod schema;
-mod system;
-mod tables;
+pub mod system;
+pub mod tables;
 
 /// Represent a list of named catalogs
 pub trait CatalogList: Sync + Send {
@@ -30,13 +48,13 @@ pub trait CatalogList: Sync + Send {
         &self,
         name: String,
         catalog: CatalogProviderRef,
-    ) -> Option<CatalogProviderRef>;
+    ) -> Result<Option<CatalogProviderRef>>;
 
     /// Retrieves the list of available catalog names
-    fn catalog_names(&self) -> Vec<String>;
+    fn catalog_names(&self) -> Result<Vec<String>>;
 
     /// Retrieves a specific catalog by name, provided it exists.
-    fn catalog(&self, name: &str) -> Option<CatalogProviderRef>;
+    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>>;
 }
 
 /// Represents a catalog, comprising a number of named schemas.
@@ -46,14 +64,17 @@ pub trait CatalogProvider: Sync + Send {
     fn as_any(&self) -> &dyn Any;
 
     /// Retrieves the list of available schema names in this catalog.
-    fn schema_names(&self) -> Vec<String>;
+    fn schema_names(&self) -> Result<Vec<String>>;
 
     /// Registers schema to this catalog.
-    fn register_schema(&self, name: String, schema: SchemaProviderRef)
-        -> Option<SchemaProviderRef>;
+    fn register_schema(
+        &self,
+        name: String,
+        schema: SchemaProviderRef,
+    ) -> Result<Option<SchemaProviderRef>>;
 
     /// Retrieves a specific schema from the catalog by name, provided it exists.
-    fn schema(&self, name: &str) -> Option<SchemaProviderRef>;
+    fn schema(&self, name: &str) -> Result<Option<SchemaProviderRef>>;
 }
 
 pub type CatalogListRef = Arc<dyn CatalogList>;
@@ -62,27 +83,129 @@ pub type CatalogProviderRef = Arc<dyn CatalogProvider>;
 #[async_trait::async_trait]
 pub trait CatalogManager: CatalogList {
     /// Starts a catalog manager.
-    async fn start(&self) -> error::Result<()>;
+    async fn start(&self) -> Result<()>;
 
-    /// Returns next available table id.
-    fn next_table_id(&self) -> TableId;
+    /// Registers a table within given catalog/schema to catalog manager,
+    /// returns whether the table registered.
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool>;
 
-    /// Registers a table given given catalog/schema to catalog manager,
-    /// returns table registered.
-    async fn register_table(&self, request: RegisterTableRequest) -> error::Result<usize>;
+    /// Deregisters a table within given catalog/schema to catalog manager,
+    /// returns whether the table deregistered.
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool>;
+
+    /// Register a schema with catalog name and schema name. Retuens whether the
+    /// schema registered.
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool>;
+
+    /// Register a system table, should be called before starting the manager.
+    async fn register_system_table(&self, request: RegisterSystemTableRequest)
+        -> error::Result<()>;
+
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>>;
+
+    /// Returns the table by catalog, schema and table name.
+    fn table(&self, catalog: &str, schema: &str, table_name: &str) -> Result<Option<TableRef>>;
 }
 
 pub type CatalogManagerRef = Arc<dyn CatalogManager>;
 
+/// Hook called after system table opening.
+pub type OpenSystemTableHook = Arc<dyn Fn(TableRef) -> Result<()> + Send + Sync>;
+
+/// Register system table request:
+/// - When system table is already created and registered, the hook will be called
+///     with table ref after opening the system table
+/// - When system table is not exists, create and register the table by create_table_request and calls open_hook with the created table.
+pub struct RegisterSystemTableRequest {
+    pub create_table_request: CreateTableRequest,
+    pub open_hook: Option<OpenSystemTableHook>,
+}
+
+#[derive(Clone)]
 pub struct RegisterTableRequest {
-    pub catalog: Option<String>,
-    pub schema: Option<String>,
+    pub catalog: String,
+    pub schema: String,
     pub table_name: String,
     pub table_id: TableId,
     pub table: TableRef,
 }
 
+impl Debug for RegisterTableRequest {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RegisterTableRequest")
+            .field("catalog", &self.catalog)
+            .field("schema", &self.schema)
+            .field("table_name", &self.table_name)
+            .field("table_id", &self.table_id)
+            .field("table", &self.table.table_info())
+            .finish()
+    }
+}
+
+#[derive(Clone)]
+pub struct DeregisterTableRequest {
+    pub catalog: String,
+    pub schema: String,
+    pub table_name: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct RegisterSchemaRequest {
+    pub catalog: String,
+    pub schema: String,
+}
+
 /// Formats table fully-qualified name
 pub fn format_full_table_name(catalog: &str, schema: &str, table: &str) -> String {
     format!("{}.{}.{}", catalog, schema, table)
 }
+
+pub trait CatalogProviderFactory {
+    fn create(&self, catalog_name: String) -> CatalogProviderRef;
+}
+
+pub trait SchemaProviderFactory {
+    fn create(&self, catalog_name: String, schema_name: String) -> SchemaProviderRef;
+}
+
+pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
+    manager: &'a M,
+    engine: TableEngineRef,
+    sys_table_requests: &'a mut Vec<RegisterSystemTableRequest>,
+) -> Result<()> {
+    for req in sys_table_requests.drain(..) {
+        let catalog_name = &req.create_table_request.catalog_name;
+        let schema_name = &req.create_table_request.schema_name;
+        let table_name = &req.create_table_request.table_name;
+        let table_id = req.create_table_request.id;
+
+        let table = if let Some(table) = manager.table(catalog_name, schema_name, table_name)? {
+            table
+        } else {
+            let table = engine
+                .create_table(&EngineContext::default(), req.create_table_request.clone())
+                .await
+                .with_context(|_| CreateTableSnafu {
+                    table_info: format!(
+                        "{}.{}.{}, id: {}",
+                        catalog_name, schema_name, table_name, table_id,
+                    ),
+                })?;
+            manager
+                .register_table(RegisterTableRequest {
+                    catalog: catalog_name.clone(),
+                    schema: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    table_id,
+                    table: table.clone(),
+                })
+                .await?;
+            info!("Created and registered system table: {}", table_name);
+            table
+        };
+        if let Some(hook) = req.open_hook {
+            (hook)(table)?;
+        }
+    }
+    Ok(())
+}

src/catalog/src/local.rs

@@ -0,0 +1,21 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod manager;
+pub mod memory;
+
+pub use manager::LocalCatalogManager;
+pub use memory::{
+    new_memory_catalog_list, MemoryCatalogManager, MemoryCatalogProvider, MemorySchemaProvider,
+};

src/catalog/src/local/manager.rs

@@ -0,0 +1,506 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
+    SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_NAME,
+};
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use common_telemetry::{error, info};
+use datatypes::prelude::ScalarVector;
+use datatypes::vectors::{BinaryVector, UInt8Vector};
+use futures_util::lock::Mutex;
+use snafu::{ensure, OptionExt, ResultExt};
+use table::engine::{EngineContext, TableEngineRef};
+use table::metadata::TableId;
+use table::requests::OpenTableRequest;
+use table::table::numbers::NumbersTable;
+use table::table::TableIdProvider;
+use table::TableRef;
+
+use crate::error::{
+    CatalogNotFoundSnafu, IllegalManagerStateSnafu, OpenTableSnafu, ReadSystemCatalogSnafu, Result,
+    SchemaExistsSnafu, SchemaNotFoundSnafu, SystemCatalogSnafu, SystemCatalogTypeMismatchSnafu,
+    TableExistsSnafu, TableNotFoundSnafu, UnimplementedSnafu,
+};
+use crate::local::memory::{MemoryCatalogManager, MemoryCatalogProvider, MemorySchemaProvider};
+use crate::system::{
+    decode_system_catalog, Entry, SystemCatalogTable, TableEntry, ENTRY_TYPE_INDEX, KEY_INDEX,
+    VALUE_INDEX,
+};
+use crate::tables::SystemCatalog;
+use crate::{
+    format_full_table_name, handle_system_table_request, CatalogList, CatalogManager,
+    CatalogProvider, CatalogProviderRef, DeregisterTableRequest, RegisterSchemaRequest,
+    RegisterSystemTableRequest, RegisterTableRequest, SchemaProvider, SchemaProviderRef,
+};
+
+/// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
+pub struct LocalCatalogManager {
+    system: Arc<SystemCatalog>,
+    catalogs: Arc<MemoryCatalogManager>,
+    engine: TableEngineRef,
+    next_table_id: AtomicU32,
+    init_lock: Mutex<bool>,
+    register_lock: Mutex<()>,
+    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
+}
+
+impl LocalCatalogManager {
+    /// Create a new [CatalogManager] with given user catalogs and table engine
+    pub async fn try_new(engine: TableEngineRef) -> Result<Self> {
+        let table = SystemCatalogTable::new(engine.clone()).await?;
+        let memory_catalog_list = crate::local::memory::new_memory_catalog_list()?;
+        let system_catalog = Arc::new(SystemCatalog::new(
+            table,
+            memory_catalog_list.clone(),
+            engine.clone(),
+        ));
+        Ok(Self {
+            system: system_catalog,
+            catalogs: memory_catalog_list,
+            engine,
+            next_table_id: AtomicU32::new(MIN_USER_TABLE_ID),
+            init_lock: Mutex::new(false),
+            register_lock: Mutex::new(()),
+            system_table_requests: Mutex::new(Vec::default()),
+        })
+    }
+
+    /// Scan all entries from system catalog table
+    pub async fn init(&self) -> Result<()> {
+        self.init_system_catalog()?;
+        let system_records = self.system.information_schema.system.records().await?;
+        let entries = self.collect_system_catalog_entries(system_records).await?;
+        let max_table_id = self.handle_system_catalog_entries(entries).await?;
+
+        info!(
+            "All system catalog entries processed, max table id: {}",
+            max_table_id
+        );
+        self.next_table_id
+            .store((max_table_id + 1).max(MIN_USER_TABLE_ID), Ordering::Relaxed);
+        *self.init_lock.lock().await = true;
+
+        // Processing system table hooks
+        let mut sys_table_requests = self.system_table_requests.lock().await;
+        handle_system_table_request(self, self.engine.clone(), &mut sys_table_requests).await?;
+        Ok(())
+    }
+
+    fn init_system_catalog(&self) -> Result<()> {
+        let system_schema = Arc::new(MemorySchemaProvider::new());
+        system_schema.register_table(
+            SYSTEM_CATALOG_TABLE_NAME.to_string(),
+            self.system.information_schema.system.clone(),
+        )?;
+        let system_catalog = Arc::new(MemoryCatalogProvider::new());
+        system_catalog.register_schema(INFORMATION_SCHEMA_NAME.to_string(), system_schema)?;
+        self.catalogs
+            .register_catalog(SYSTEM_CATALOG_NAME.to_string(), system_catalog)?;
+
+        let default_catalog = Arc::new(MemoryCatalogProvider::new());
+        let default_schema = Arc::new(MemorySchemaProvider::new());
+
+        // Add numbers table for test
+        let table = Arc::new(NumbersTable::default());
+        default_schema.register_table("numbers".to_string(), table)?;
+
+        default_catalog.register_schema(DEFAULT_SCHEMA_NAME.to_string(), default_schema)?;
+        self.catalogs
+            .register_catalog(DEFAULT_CATALOG_NAME.to_string(), default_catalog)?;
+        Ok(())
+    }
+
+    /// Collect stream of system catalog entries to `Vec<Entry>`
+    async fn collect_system_catalog_entries(
+        &self,
+        stream: SendableRecordBatchStream,
+    ) -> Result<Vec<Entry>> {
+        let record_batch = common_recordbatch::util::collect(stream)
+            .await
+            .context(ReadSystemCatalogSnafu)?;
+        let rbs = record_batch
+            .into_iter()
+            .map(Self::record_batch_to_entry)
+            .collect::<Result<Vec<_>>>()?;
+        Ok(rbs.into_iter().flat_map(Vec::into_iter).collect::<_>())
+    }
+
+    /// Convert `RecordBatch` to a vector of `Entry`.
+    fn record_batch_to_entry(rb: RecordBatch) -> Result<Vec<Entry>> {
+        ensure!(
+            rb.num_columns() >= 6,
+            SystemCatalogSnafu {
+                msg: format!("Length mismatch: {}", rb.num_columns())
+            }
+        );
+
+        let entry_type = rb
+            .column(ENTRY_TYPE_INDEX)
+            .as_any()
+            .downcast_ref::<UInt8Vector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(ENTRY_TYPE_INDEX).data_type(),
+            })?;
+
+        let key = rb
+            .column(KEY_INDEX)
+            .as_any()
+            .downcast_ref::<BinaryVector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(KEY_INDEX).data_type(),
+            })?;
+
+        let value = rb
+            .column(VALUE_INDEX)
+            .as_any()
+            .downcast_ref::<BinaryVector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(VALUE_INDEX).data_type(),
+            })?;
+
+        let mut res = Vec::with_capacity(rb.num_rows());
+        for ((t, k), v) in entry_type
+            .iter_data()
+            .zip(key.iter_data())
+            .zip(value.iter_data())
+        {
+            let entry = decode_system_catalog(t, k, v)?;
+            res.push(entry);
+        }
+        Ok(res)
+    }
+
+    /// Processes records from system catalog table and returns the max table id persisted
+    /// in system catalog table.
+    async fn handle_system_catalog_entries(&self, entries: Vec<Entry>) -> Result<TableId> {
+        let entries = Self::sort_entries(entries);
+        let mut max_table_id = 0;
+        for entry in entries {
+            match entry {
+                Entry::Catalog(c) => {
+                    self.catalogs.register_catalog_if_absent(
+                        c.catalog_name.clone(),
+                        Arc::new(MemoryCatalogProvider::new()),
+                    );
+                    info!("Register catalog: {}", c.catalog_name);
+                }
+                Entry::Schema(s) => {
+                    let catalog =
+                        self.catalogs
+                            .catalog(&s.catalog_name)?
+                            .context(CatalogNotFoundSnafu {
+                                catalog_name: &s.catalog_name,
+                            })?;
+                    catalog.register_schema(
+                        s.schema_name.clone(),
+                        Arc::new(MemorySchemaProvider::new()),
+                    )?;
+                    info!("Registered schema: {:?}", s);
+                }
+                Entry::Table(t) => {
+                    self.open_and_register_table(&t).await?;
+                    info!("Registered table: {:?}", t);
+                    max_table_id = max_table_id.max(t.table_id);
+                }
+            }
+        }
+        Ok(max_table_id)
+    }
+
+    /// Sort catalog entries to ensure catalog entries comes first, then schema entries,
+    /// and table entries is the last.
+    fn sort_entries(mut entries: Vec<Entry>) -> Vec<Entry> {
+        entries.sort();
+        entries
+    }
+
+    async fn open_and_register_table(&self, t: &TableEntry) -> Result<()> {
+        let catalog = self
+            .catalogs
+            .catalog(&t.catalog_name)?
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &t.catalog_name,
+            })?;
+        let schema = catalog
+            .schema(&t.schema_name)?
+            .context(SchemaNotFoundSnafu {
+                schema_info: format!("{}.{}", &t.catalog_name, &t.schema_name),
+            })?;
+
+        let context = EngineContext {};
+        let request = OpenTableRequest {
+            catalog_name: t.catalog_name.clone(),
+            schema_name: t.schema_name.clone(),
+            table_name: t.table_name.clone(),
+            table_id: t.table_id,
+            region_numbers: vec![0],
+        };
+
+        let option = self
+            .engine
+            .open_table(&context, request)
+            .await
+            .with_context(|_| OpenTableSnafu {
+                table_info: format!(
+                    "{}.{}.{}, id: {}",
+                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
+                ),
+            })?
+            .with_context(|| TableNotFoundSnafu {
+                table_info: format!(
+                    "{}.{}.{}, id: {}",
+                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
+                ),
+            })?;
+
+        schema.register_table(t.table_name.clone(), option)?;
+        Ok(())
+    }
+}
+
+impl CatalogList for LocalCatalogManager {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn register_catalog(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Result<Option<CatalogProviderRef>> {
+        self.catalogs.register_catalog(name, catalog)
+    }
+
+    fn catalog_names(&self) -> Result<Vec<String>> {
+        let mut res = self.catalogs.catalog_names()?;
+        res.push(SYSTEM_CATALOG_NAME.to_string());
+        Ok(res)
+    }
+
+    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>> {
+        if name.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
+            Ok(Some(self.system.clone()))
+        } else {
+            self.catalogs.catalog(name)
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl TableIdProvider for LocalCatalogManager {
+    async fn next_table_id(&self) -> table::Result<TableId> {
+        Ok(self.next_table_id.fetch_add(1, Ordering::Relaxed))
+    }
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for LocalCatalogManager {
+    /// Start [LocalCatalogManager] to load all information from system catalog table.
+    /// Make sure table engine is initialized before starting [MemoryCatalogManager].
+    async fn start(&self) -> Result<()> {
+        self.init().await
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        let started = self.init_lock.lock().await;
+
+        ensure!(
+            *started,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager not started",
+            }
+        );
+
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+
+        let catalog = self
+            .catalogs
+            .catalog(catalog_name)?
+            .context(CatalogNotFoundSnafu { catalog_name })?;
+        let schema = catalog
+            .schema(schema_name)?
+            .with_context(|| SchemaNotFoundSnafu {
+                schema_info: format!("{}.{}", catalog_name, schema_name),
+            })?;
+
+        {
+            let _lock = self.register_lock.lock().await;
+            if let Some(existing) = schema.table(&request.table_name)? {
+                if existing.table_info().ident.table_id != request.table_id {
+                    error!(
+                        "Unexpected table register request: {:?}, existing: {:?}",
+                        request,
+                        existing.table_info()
+                    );
+                    return TableExistsSnafu {
+                        table: format_full_table_name(
+                            catalog_name,
+                            schema_name,
+                            &request.table_name,
+                        ),
+                    }
+                    .fail();
+                }
+                // Try to register table with same table id, just ignore.
+                Ok(false)
+            } else {
+                // table does not exist
+                self.system
+                    .register_table(
+                        catalog_name.clone(),
+                        schema_name.clone(),
+                        request.table_name.clone(),
+                        request.table_id,
+                    )
+                    .await?;
+                schema.register_table(request.table_name, request.table)?;
+                Ok(true)
+            }
+        }
+    }
+
+    async fn deregister_table(&self, _request: DeregisterTableRequest) -> Result<bool> {
+        UnimplementedSnafu {
+            operation: "deregister table",
+        }
+        .fail()
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        let started = self.init_lock.lock().await;
+        ensure!(
+            *started,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager not started",
+            }
+        );
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+
+        let catalog = self
+            .catalogs
+            .catalog(catalog_name)?
+            .context(CatalogNotFoundSnafu { catalog_name })?;
+
+        {
+            let _lock = self.register_lock.lock().await;
+            ensure!(
+                catalog.schema(schema_name)?.is_none(),
+                SchemaExistsSnafu {
+                    schema: schema_name,
+                }
+            );
+            self.system
+                .register_schema(request.catalog, schema_name.clone())
+                .await?;
+            catalog.register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))?;
+            Ok(true)
+        }
+    }
+
+    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
+        ensure!(
+            !*self.init_lock.lock().await,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager already started",
+            }
+        );
+
+        let mut sys_table_requests = self.system_table_requests.lock().await;
+        sys_table_requests.push(request);
+
+        Ok(())
+    }
+
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+        self.catalogs
+            .catalog(catalog)?
+            .context(CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .schema(schema)
+    }
+
+    fn table(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        let catalog = self
+            .catalogs
+            .catalog(catalog_name)?
+            .context(CatalogNotFoundSnafu { catalog_name })?;
+        let schema = catalog
+            .schema(schema_name)?
+            .with_context(|| SchemaNotFoundSnafu {
+                schema_info: format!("{}.{}", catalog_name, schema_name),
+            })?;
+        schema.table(table_name)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+
+    use super::*;
+    use crate::system::{CatalogEntry, SchemaEntry};
+
+    #[test]
+    fn test_sort_entry() {
+        let vec = vec![
+            Entry::Table(TableEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+                table_name: "T1".to_string(),
+                table_id: 1,
+            }),
+            Entry::Catalog(CatalogEntry {
+                catalog_name: "C2".to_string(),
+            }),
+            Entry::Schema(SchemaEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+            }),
+            Entry::Schema(SchemaEntry {
+                catalog_name: "C2".to_string(),
+                schema_name: "S2".to_string(),
+            }),
+            Entry::Catalog(CatalogEntry {
+                catalog_name: "".to_string(),
+            }),
+            Entry::Table(TableEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+                table_name: "T2".to_string(),
+                table_id: 2,
+            }),
+        ];
+        let res = LocalCatalogManager::sort_entries(vec);
+        assert_matches!(res[0], Entry::Catalog(..));
+        assert_matches!(res[1], Entry::Catalog(..));
+        assert_matches!(res[2], Entry::Schema(..));
+        assert_matches!(res[3], Entry::Schema(..));
+        assert_matches!(res[4], Entry::Table(..));
+        assert_matches!(res[5], Entry::Table(..));
+    }
+}

src/catalog/src/local/memory.rs

@@ -0,0 +1,403 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::collections::hash_map::Entry;
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::{Arc, RwLock};
+
+use common_catalog::consts::MIN_USER_TABLE_ID;
+use common_telemetry::error;
+use snafu::OptionExt;
+use table::metadata::TableId;
+use table::table::TableIdProvider;
+use table::TableRef;
+
+use crate::error::{CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu};
+use crate::schema::SchemaProvider;
+use crate::{
+    CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef, DeregisterTableRequest,
+    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, SchemaProviderRef,
+};
+
+/// Simple in-memory list of catalogs
+pub struct MemoryCatalogManager {
+    /// Collection of catalogs containing schemas and ultimately Tables
+    pub catalogs: RwLock<HashMap<String, CatalogProviderRef>>,
+    pub table_id: AtomicU32,
+}
+
+impl Default for MemoryCatalogManager {
+    fn default() -> Self {
+        let manager = Self {
+            table_id: AtomicU32::new(MIN_USER_TABLE_ID),
+            catalogs: Default::default(),
+        };
+        let default_catalog = Arc::new(MemoryCatalogProvider::new());
+        manager
+            .register_catalog("greptime".to_string(), default_catalog.clone())
+            .unwrap();
+        default_catalog
+            .register_schema("public".to_string(), Arc::new(MemorySchemaProvider::new()))
+            .unwrap();
+        manager
+    }
+}
+
+#[async_trait::async_trait]
+impl TableIdProvider for MemoryCatalogManager {
+    async fn next_table_id(&self) -> table::error::Result<TableId> {
+        Ok(self.table_id.fetch_add(1, Ordering::Relaxed))
+    }
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for MemoryCatalogManager {
+    async fn start(&self) -> Result<()> {
+        self.table_id.store(MIN_USER_TABLE_ID, Ordering::Relaxed);
+        Ok(())
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        let catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get(&request.catalog)
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .clone();
+        let schema = catalog
+            .schema(&request.schema)?
+            .with_context(|| SchemaNotFoundSnafu {
+                schema_info: format!("{}.{}", &request.catalog, &request.schema),
+            })?;
+        schema
+            .register_table(request.table_name, request.table)
+            .map(|v| v.is_none())
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool> {
+        let catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get(&request.catalog)
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .clone();
+        let schema = catalog
+            .schema(&request.schema)?
+            .with_context(|| SchemaNotFoundSnafu {
+                schema_info: format!("{}.{}", &request.catalog, &request.schema),
+            })?;
+        schema
+            .deregister_table(&request.table_name)
+            .map(|v| v.is_some())
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        let catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get(&request.catalog)
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?;
+        catalog.register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))?;
+        Ok(true)
+    }
+
+    async fn register_system_table(&self, _request: RegisterSystemTableRequest) -> Result<()> {
+        // TODO(ruihang): support register system table request
+        Ok(())
+    }
+
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+        let catalogs = self.catalogs.read().unwrap();
+        if let Some(c) = catalogs.get(catalog) {
+            c.schema(schema)
+        } else {
+            Ok(None)
+        }
+    }
+
+    fn table(&self, catalog: &str, schema: &str, table_name: &str) -> Result<Option<TableRef>> {
+        let c = self.catalogs.read().unwrap();
+        let catalog = if let Some(c) = c.get(catalog) {
+            c.clone()
+        } else {
+            return Ok(None);
+        };
+        match catalog.schema(schema)? {
+            None => Ok(None),
+            Some(s) => s.table(table_name),
+        }
+    }
+}
+
+impl MemoryCatalogManager {
+    /// Registers a catalog and return `None` if no catalog with the same name was already
+    /// registered, or `Some` with the previously registered catalog.
+    pub fn register_catalog_if_absent(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Option<CatalogProviderRef> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let entry = catalogs.entry(name);
+        match entry {
+            Entry::Occupied(v) => Some(v.get().clone()),
+            Entry::Vacant(v) => {
+                v.insert(catalog);
+                None
+            }
+        }
+    }
+}
+
+impl CatalogList for MemoryCatalogManager {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn register_catalog(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Result<Option<CatalogProviderRef>> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        Ok(catalogs.insert(name, catalog))
+    }
+
+    fn catalog_names(&self) -> Result<Vec<String>> {
+        let catalogs = self.catalogs.read().unwrap();
+        Ok(catalogs.keys().map(|s| s.to_string()).collect())
+    }
+
+    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>> {
+        let catalogs = self.catalogs.read().unwrap();
+        Ok(catalogs.get(name).cloned())
+    }
+}
+
+impl Default for MemoryCatalogProvider {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Simple in-memory implementation of a catalog.
+pub struct MemoryCatalogProvider {
+    schemas: RwLock<HashMap<String, Arc<dyn SchemaProvider>>>,
+}
+
+impl MemoryCatalogProvider {
+    /// Instantiates a new MemoryCatalogProvider with an empty collection of schemas.
+    pub fn new() -> Self {
+        Self {
+            schemas: RwLock::new(HashMap::new()),
+        }
+    }
+}
+
+impl CatalogProvider for MemoryCatalogProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema_names(&self) -> Result<Vec<String>> {
+        let schemas = self.schemas.read().unwrap();
+        Ok(schemas.keys().cloned().collect())
+    }
+
+    fn register_schema(
+        &self,
+        name: String,
+        schema: SchemaProviderRef,
+    ) -> Result<Option<SchemaProviderRef>> {
+        let mut schemas = self.schemas.write().unwrap();
+        Ok(schemas.insert(name, schema))
+    }
+
+    fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>> {
+        let schemas = self.schemas.read().unwrap();
+        Ok(schemas.get(name).cloned())
+    }
+}
+
+/// Simple in-memory implementation of a schema.
+pub struct MemorySchemaProvider {
+    tables: RwLock<HashMap<String, TableRef>>,
+}
+
+impl MemorySchemaProvider {
+    /// Instantiates a new MemorySchemaProvider with an empty collection of tables.
+    pub fn new() -> Self {
+        Self {
+            tables: RwLock::new(HashMap::new()),
+        }
+    }
+}
+
+impl Default for MemorySchemaProvider {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SchemaProvider for MemorySchemaProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn table_names(&self) -> Result<Vec<String>> {
+        let tables = self.tables.read().unwrap();
+        Ok(tables.keys().cloned().collect())
+    }
+
+    fn table(&self, name: &str) -> Result<Option<TableRef>> {
+        let tables = self.tables.read().unwrap();
+        Ok(tables.get(name).cloned())
+    }
+
+    fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
+        let mut tables = self.tables.write().unwrap();
+        if let Some(existing) = tables.get(name.as_str()) {
+            // if table with the same name but different table id exists, then it's a fatal bug
+            if existing.table_info().ident.table_id != table.table_info().ident.table_id {
+                error!(
+                    "Unexpected table register: {:?}, existing: {:?}",
+                    table.table_info(),
+                    existing.table_info()
+                );
+                return TableExistsSnafu { table: name }.fail()?;
+            }
+            Ok(Some(existing.clone()))
+        } else {
+            Ok(tables.insert(name, table))
+        }
+    }
+
+    fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
+        let mut tables = self.tables.write().unwrap();
+        Ok(tables.remove(name))
+    }
+
+    fn table_exist(&self, name: &str) -> Result<bool> {
+        let tables = self.tables.read().unwrap();
+        Ok(tables.contains_key(name))
+    }
+}
+
+/// Create a memory catalog list contains a numbers table for test
+pub fn new_memory_catalog_list() -> Result<Arc<MemoryCatalogManager>> {
+    Ok(Arc::new(MemoryCatalogManager::default()))
+}
+
+#[cfg(test)]
+mod tests {
+    use common_catalog::consts::*;
+    use common_error::ext::ErrorExt;
+    use common_error::prelude::StatusCode;
+    use table::table::numbers::NumbersTable;
+
+    use super::*;
+
+    #[test]
+    fn test_new_memory_catalog_list() {
+        let catalog_list = new_memory_catalog_list().unwrap();
+        let default_catalog = catalog_list.catalog(DEFAULT_CATALOG_NAME).unwrap().unwrap();
+
+        let default_schema = default_catalog
+            .schema(DEFAULT_SCHEMA_NAME)
+            .unwrap()
+            .unwrap();
+
+        default_schema
+            .register_table("numbers".to_string(), Arc::new(NumbersTable::default()))
+            .unwrap();
+
+        let table = default_schema.table("numbers").unwrap();
+        assert!(table.is_some());
+        assert!(default_schema.table("not_exists").unwrap().is_none());
+    }
+
+    #[tokio::test]
+    async fn test_mem_provider() {
+        let provider = MemorySchemaProvider::new();
+        let table_name = "numbers";
+        assert!(!provider.table_exist(table_name).unwrap());
+        assert!(provider.deregister_table(table_name).unwrap().is_none());
+        let test_table = NumbersTable::default();
+        // register table successfully
+        assert!(provider
+            .register_table(table_name.to_string(), Arc::new(test_table))
+            .unwrap()
+            .is_none());
+        assert!(provider.table_exist(table_name).unwrap());
+        let other_table = NumbersTable::new(12);
+        let result = provider.register_table(table_name.to_string(), Arc::new(other_table));
+        let err = result.err().unwrap();
+        assert!(err.backtrace_opt().is_some());
+        assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
+    }
+
+    #[test]
+    pub fn test_register_if_absent() {
+        let list = MemoryCatalogManager::default();
+        assert!(list
+            .register_catalog_if_absent(
+                "test_catalog".to_string(),
+                Arc::new(MemoryCatalogProvider::new())
+            )
+            .is_none());
+        list.register_catalog_if_absent(
+            "test_catalog".to_string(),
+            Arc::new(MemoryCatalogProvider::new()),
+        )
+        .unwrap();
+        list.as_any()
+            .downcast_ref::<MemoryCatalogManager>()
+            .unwrap();
+    }
+
+    #[tokio::test]
+    pub async fn test_catalog_deregister_table() {
+        let catalog = MemoryCatalogManager::default();
+        let schema = catalog
+            .schema(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
+            .unwrap()
+            .unwrap();
+
+        let register_table_req = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "numbers".to_string(),
+            table_id: 2333,
+            table: Arc::new(NumbersTable::default()),
+        };
+        catalog.register_table(register_table_req).await.unwrap();
+        assert!(schema.table_exist("numbers").unwrap());
+
+        let deregister_table_req = DeregisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "numbers".to_string(),
+        };
+        catalog
+            .deregister_table(deregister_table_req)
+            .await
+            .unwrap();
+        assert!(!schema.table_exist("numbers").unwrap());
+    }
+}

src/catalog/src/manager.rs

@@ -1,307 +0,0 @@
-use std::any::Any;
-use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::Arc;
-
-use common_recordbatch::RecordBatch;
-use common_telemetry::{debug, info};
-use datatypes::prelude::ScalarVector;
-use datatypes::vectors::{BinaryVector, UInt8Vector};
-use futures_util::lock::Mutex;
-use futures_util::StreamExt;
-use snafu::{ensure, OptionExt, ResultExt};
-use table::engine::{EngineContext, TableEngineRef};
-use table::metadata::TableId;
-use table::requests::OpenTableRequest;
-use table::table::numbers::NumbersTable;
-
-use super::error::Result;
-use crate::consts::{INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_NAME};
-use crate::error::{
-    CatalogNotFoundSnafu, OpenTableSnafu, ReadSystemCatalogSnafu, SchemaNotFoundSnafu,
-    SystemCatalogSnafu, SystemCatalogTypeMismatchSnafu, TableExistsSnafu, TableNotFoundSnafu,
-};
-use crate::memory::{MemoryCatalogList, MemoryCatalogProvider, MemorySchemaProvider};
-use crate::system::{
-    decode_system_catalog, Entry, SystemCatalogTable, TableEntry, ENTRY_TYPE_INDEX, KEY_INDEX,
-    VALUE_INDEX,
-};
-use crate::tables::SystemCatalog;
-use crate::{
-    format_full_table_name, CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef,
-    RegisterTableRequest, SchemaProvider, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME,
-};
-
-/// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
-pub struct LocalCatalogManager {
-    system: Arc<SystemCatalog>,
-    catalogs: Arc<MemoryCatalogList>,
-    engine: TableEngineRef,
-    next_table_id: AtomicU32,
-    lock: Mutex<()>,
-}
-
-impl LocalCatalogManager {
-    /// Create a new [CatalogManager] with given user catalogs and table engine
-    pub async fn try_new(engine: TableEngineRef) -> Result<Self> {
-        let table = SystemCatalogTable::new(engine.clone()).await?;
-        let memory_catalog_list = crate::memory::new_memory_catalog_list()?;
-        let system_catalog = Arc::new(SystemCatalog::new(
-            table,
-            memory_catalog_list.clone(),
-            engine.clone(),
-        ));
-        Ok(Self {
-            system: system_catalog,
-            catalogs: memory_catalog_list,
-            engine,
-            next_table_id: AtomicU32::new(0),
-            lock: Mutex::new(()),
-        })
-    }
-
-    /// Scan all entries from system catalog table
-    pub async fn init(&self) -> Result<()> {
-        self.init_system_catalog()?;
-        let mut system_records = self.system.information_schema.system.records().await?;
-        let mut max_table_id = 0;
-        while let Some(records) = system_records
-            .next()
-            .await
-            .transpose()
-            .context(ReadSystemCatalogSnafu)?
-        {
-            let table_id = self.handle_system_catalog_entries(records).await?;
-            max_table_id = max_table_id.max(table_id);
-        }
-        info!(
-            "All system catalog entries processed, max table id: {}",
-            max_table_id
-        );
-        self.next_table_id
-            .store(max_table_id + 1, Ordering::Relaxed);
-        Ok(())
-    }
-
-    fn init_system_catalog(&self) -> Result<()> {
-        let system_schema = Arc::new(MemorySchemaProvider::new());
-        system_schema.register_table(
-            SYSTEM_CATALOG_TABLE_NAME.to_string(),
-            self.system.information_schema.system.clone(),
-        )?;
-        let system_catalog = Arc::new(MemoryCatalogProvider::new());
-        system_catalog.register_schema(INFORMATION_SCHEMA_NAME.to_string(), system_schema);
-        self.catalogs
-            .register_catalog(SYSTEM_CATALOG_NAME.to_string(), system_catalog);
-
-        let default_catalog = Arc::new(MemoryCatalogProvider::new());
-        let default_schema = Arc::new(MemorySchemaProvider::new());
-
-        // Add numbers table for test
-        // TODO(hl): remove this registration
-        let table = Arc::new(NumbersTable::default());
-        default_schema.register_table("numbers".to_string(), table)?;
-
-        default_catalog.register_schema(DEFAULT_SCHEMA_NAME.to_string(), default_schema);
-        self.catalogs
-            .register_catalog(DEFAULT_CATALOG_NAME.to_string(), default_catalog);
-        Ok(())
-    }
-
-    /// Processes records from system catalog table and returns the max table id persisted
-    /// in system catalog table.
-    async fn handle_system_catalog_entries(&self, records: RecordBatch) -> Result<TableId> {
-        ensure!(
-            records.df_recordbatch.columns().len() >= 6,
-            SystemCatalogSnafu {
-                msg: format!(
-                    "Length mismatch: {}",
-                    records.df_recordbatch.columns().len()
-                )
-            }
-        );
-
-        let entry_type = UInt8Vector::try_from_arrow_array(&records.df_recordbatch.columns()[0])
-            .with_context(|_| SystemCatalogTypeMismatchSnafu {
-                data_type: records.df_recordbatch.columns()[ENTRY_TYPE_INDEX]
-                    .data_type()
-                    .clone(),
-            })?;
-
-        let key = BinaryVector::try_from_arrow_array(&records.df_recordbatch.columns()[1])
-            .with_context(|_| SystemCatalogTypeMismatchSnafu {
-                data_type: records.df_recordbatch.columns()[KEY_INDEX]
-                    .data_type()
-                    .clone(),
-            })?;
-
-        let value = BinaryVector::try_from_arrow_array(&records.df_recordbatch.columns()[3])
-            .with_context(|_| SystemCatalogTypeMismatchSnafu {
-                data_type: records.df_recordbatch.columns()[VALUE_INDEX]
-                    .data_type()
-                    .clone(),
-            })?;
-
-        let mut max_table_id = 0;
-        for ((t, k), v) in entry_type
-            .iter_data()
-            .zip(key.iter_data())
-            .zip(value.iter_data())
-        {
-            let entry = decode_system_catalog(t, k, v)?;
-            match entry {
-                Entry::Catalog(c) => {
-                    self.catalogs.register_catalog_if_absent(
-                        c.catalog_name.clone(),
-                        Arc::new(MemoryCatalogProvider::new()),
-                    );
-                    info!("Register catalog: {}", c.catalog_name);
-                }
-                Entry::Schema(s) => {
-                    let catalog =
-                        self.catalogs
-                            .catalog(&s.catalog_name)
-                            .context(CatalogNotFoundSnafu {
-                                catalog_name: &s.catalog_name,
-                            })?;
-                    catalog.register_schema(
-                        s.schema_name.clone(),
-                        Arc::new(MemorySchemaProvider::new()),
-                    );
-                    info!("Registered schema: {:?}", s);
-                }
-                Entry::Table(t) => {
-                    debug!("t: {:?}", t);
-                    self.open_and_register_table(&t).await?;
-                    info!("Registered table: {:?}", t);
-                    max_table_id = max_table_id.max(t.table_id);
-                }
-            }
-        }
-        Ok(max_table_id)
-    }
-
-    async fn open_and_register_table(&self, t: &TableEntry) -> Result<()> {
-        let catalog = self
-            .catalogs
-            .catalog(&t.catalog_name)
-            .context(CatalogNotFoundSnafu {
-                catalog_name: &t.catalog_name,
-            })?;
-        let schema = catalog
-            .schema(&t.schema_name)
-            .context(SchemaNotFoundSnafu {
-                schema_info: format!("{}.{}", &t.catalog_name, &t.schema_name),
-            })?;
-
-        let context = EngineContext {};
-        let request = OpenTableRequest {
-            catalog_name: t.catalog_name.clone(),
-            schema_name: t.schema_name.clone(),
-            table_name: t.table_name.clone(),
-            table_id: t.table_id,
-        };
-
-        let option = self
-            .engine
-            .open_table(&context, request)
-            .await
-            .with_context(|_| OpenTableSnafu {
-                table_info: format!(
-                    "{}.{}.{}, id: {}",
-                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
-                ),
-            })?
-            .with_context(|| TableNotFoundSnafu {
-                table_info: format!(
-                    "{}.{}.{}, id: {}",
-                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
-                ),
-            })?;
-
-        schema.register_table(t.table_name.clone(), option)?;
-        Ok(())
-    }
-}
-
-impl CatalogList for LocalCatalogManager {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn register_catalog(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Option<Arc<dyn CatalogProvider>> {
-        self.catalogs.register_catalog(name, catalog)
-    }
-
-    fn catalog_names(&self) -> Vec<String> {
-        let mut res = self.catalogs.catalog_names();
-        res.push(SYSTEM_CATALOG_NAME.to_string());
-        res
-    }
-
-    fn catalog(&self, name: &str) -> Option<Arc<dyn CatalogProvider>> {
-        if name.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
-            Some(self.system.clone())
-        } else {
-            self.catalogs.catalog(name)
-        }
-    }
-}
-
-#[async_trait::async_trait]
-impl CatalogManager for LocalCatalogManager {
-    /// Start [MemoryCatalogManager] to load all information from system catalog table.
-    /// Make sure table engine is initialized before starting [MemoryCatalogManager].
-    async fn start(&self) -> Result<()> {
-        self.init().await
-    }
-
-    #[inline]
-    fn next_table_id(&self) -> TableId {
-        self.next_table_id.fetch_add(1, Ordering::Relaxed)
-    }
-
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<usize> {
-        let _lock = self.lock.lock().await;
-        let catalog_name = request
-            .catalog
-            .unwrap_or_else(|| DEFAULT_CATALOG_NAME.to_string());
-        let schema_name = request
-            .schema
-            .unwrap_or_else(|| DEFAULT_SCHEMA_NAME.to_string());
-
-        let catalog = self
-            .catalogs
-            .catalog(&catalog_name)
-            .context(CatalogNotFoundSnafu {
-                catalog_name: &catalog_name,
-            })?;
-        let schema = catalog
-            .schema(&schema_name)
-            .with_context(|| SchemaNotFoundSnafu {
-                schema_info: format!("{}.{}", catalog_name, schema_name),
-            })?;
-
-        if schema.table_exist(&request.table_name) {
-            return TableExistsSnafu {
-                table: format_full_table_name(&catalog_name, &schema_name, &request.table_name),
-            }
-            .fail();
-        }
-
-        self.system
-            .register_table(
-                catalog_name,
-                schema_name,
-                request.table_name.clone(),
-                request.table_id,
-            )
-            .await?;
-
-        schema.register_table(request.table_name, request.table)?;
-        Ok(1)
-    }
-}

src/catalog/src/memory.rs

@@ -1,236 +0,0 @@
-use std::any::Any;
-use std::collections::hash_map::Entry;
-use std::collections::HashMap;
-use std::sync::Arc;
-use std::sync::RwLock;
-
-use table::TableRef;
-
-use crate::error::{Result, TableExistsSnafu};
-use crate::schema::SchemaProvider;
-use crate::{CatalogList, CatalogProvider, CatalogProviderRef, SchemaProviderRef};
-
-/// Simple in-memory list of catalogs
-#[derive(Default)]
-pub struct MemoryCatalogList {
-    /// Collection of catalogs containing schemas and ultimately Tables
-    pub catalogs: RwLock<HashMap<String, CatalogProviderRef>>,
-}
-
-impl MemoryCatalogList {
-    /// Registers a catalog and return `None` if no catalog with the same name was already
-    /// registered, or `Some` with the previously registered catalog.
-    pub fn register_catalog_if_absent(
-        &self,
-        name: String,
-        catalog: Arc<dyn CatalogProvider>,
-    ) -> Option<CatalogProviderRef> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let entry = catalogs.entry(name);
-        match entry {
-            Entry::Occupied(v) => Some(v.get().clone()),
-            Entry::Vacant(v) => {
-                v.insert(catalog);
-                None
-            }
-        }
-    }
-}
-
-impl CatalogList for MemoryCatalogList {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn register_catalog(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Option<CatalogProviderRef> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        catalogs.insert(name, catalog)
-    }
-
-    fn catalog_names(&self) -> Vec<String> {
-        let catalogs = self.catalogs.read().unwrap();
-        catalogs.keys().map(|s| s.to_string()).collect()
-    }
-
-    fn catalog(&self, name: &str) -> Option<CatalogProviderRef> {
-        let catalogs = self.catalogs.read().unwrap();
-        catalogs.get(name).cloned()
-    }
-}
-
-impl Default for MemoryCatalogProvider {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-/// Simple in-memory implementation of a catalog.
-pub struct MemoryCatalogProvider {
-    schemas: RwLock<HashMap<String, Arc<dyn SchemaProvider>>>,
-}
-
-impl MemoryCatalogProvider {
-    /// Instantiates a new MemoryCatalogProvider with an empty collection of schemas.
-    pub fn new() -> Self {
-        Self {
-            schemas: RwLock::new(HashMap::new()),
-        }
-    }
-}
-
-impl CatalogProvider for MemoryCatalogProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema_names(&self) -> Vec<String> {
-        let schemas = self.schemas.read().unwrap();
-        schemas.keys().cloned().collect()
-    }
-
-    fn register_schema(
-        &self,
-        name: String,
-        schema: SchemaProviderRef,
-    ) -> Option<SchemaProviderRef> {
-        let mut schemas = self.schemas.write().unwrap();
-        schemas.insert(name, schema)
-    }
-
-    fn schema(&self, name: &str) -> Option<Arc<dyn SchemaProvider>> {
-        let schemas = self.schemas.read().unwrap();
-        schemas.get(name).cloned()
-    }
-}
-
-/// Simple in-memory implementation of a schema.
-pub struct MemorySchemaProvider {
-    tables: RwLock<HashMap<String, TableRef>>,
-}
-
-impl MemorySchemaProvider {
-    /// Instantiates a new MemorySchemaProvider with an empty collection of tables.
-    pub fn new() -> Self {
-        Self {
-            tables: RwLock::new(HashMap::new()),
-        }
-    }
-}
-
-impl Default for MemorySchemaProvider {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl SchemaProvider for MemorySchemaProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn table_names(&self) -> Vec<String> {
-        let tables = self.tables.read().unwrap();
-        tables.keys().cloned().collect()
-    }
-
-    fn table(&self, name: &str) -> Option<TableRef> {
-        let tables = self.tables.read().unwrap();
-        tables.get(name).cloned()
-    }
-
-    fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
-        if self.table_exist(name.as_str()) {
-            return TableExistsSnafu { table: name }.fail()?;
-        }
-        let mut tables = self.tables.write().unwrap();
-        Ok(tables.insert(name, table))
-    }
-
-    fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
-        let mut tables = self.tables.write().unwrap();
-        Ok(tables.remove(name))
-    }
-
-    fn table_exist(&self, name: &str) -> bool {
-        let tables = self.tables.read().unwrap();
-        tables.contains_key(name)
-    }
-}
-
-/// Create a memory catalog list contains a numbers table for test
-pub fn new_memory_catalog_list() -> Result<Arc<MemoryCatalogList>> {
-    Ok(Arc::new(MemoryCatalogList::default()))
-}
-
-#[cfg(test)]
-mod tests {
-    use common_error::ext::ErrorExt;
-    use common_error::prelude::StatusCode;
-    use table::table::numbers::NumbersTable;
-
-    use super::*;
-    use crate::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-
-    #[test]
-    fn test_new_memory_catalog_list() {
-        let catalog_list = new_memory_catalog_list().unwrap();
-
-        assert!(catalog_list.catalog(DEFAULT_CATALOG_NAME).is_none());
-        let default_catalog = Arc::new(MemoryCatalogProvider::default());
-        catalog_list.register_catalog(DEFAULT_CATALOG_NAME.to_string(), default_catalog.clone());
-
-        assert!(default_catalog.schema(DEFAULT_SCHEMA_NAME).is_none());
-        let default_schema = Arc::new(MemorySchemaProvider::default());
-        default_catalog.register_schema(DEFAULT_SCHEMA_NAME.to_string(), default_schema.clone());
-
-        default_schema
-            .register_table("numbers".to_string(), Arc::new(NumbersTable::default()))
-            .unwrap();
-
-        let table = default_schema.table("numbers");
-        assert!(table.is_some());
-
-        assert!(default_schema.table("not_exists").is_none());
-    }
-
-    #[tokio::test]
-    async fn test_mem_provider() {
-        let provider = MemorySchemaProvider::new();
-        let table_name = "numbers";
-        assert!(!provider.table_exist(table_name));
-        assert!(provider.deregister_table(table_name).unwrap().is_none());
-        let test_table = NumbersTable::default();
-        // register table successfully
-        assert!(provider
-            .register_table(table_name.to_string(), Arc::new(test_table))
-            .unwrap()
-            .is_none());
-        assert!(provider.table_exist(table_name));
-        let other_table = NumbersTable::default();
-        let result = provider.register_table(table_name.to_string(), Arc::new(other_table));
-        let err = result.err().unwrap();
-        assert!(err.backtrace_opt().is_some());
-        assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
-    }
-
-    #[test]
-    pub fn test_register_if_absent() {
-        let list = MemoryCatalogList::default();
-        assert!(list
-            .register_catalog_if_absent(
-                "test_catalog".to_string(),
-                Arc::new(MemoryCatalogProvider::new())
-            )
-            .is_none());
-        list.register_catalog_if_absent(
-            "test_catalog".to_string(),
-            Arc::new(MemoryCatalogProvider::new()),
-        )
-        .unwrap();
-        list.as_any().downcast_ref::<MemoryCatalogList>().unwrap();
-    }
-}

src/catalog/src/remote.rs

@@ -0,0 +1,131 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Debug;
+use std::pin::Pin;
+use std::sync::Arc;
+
+pub use client::MetaKvBackend;
+use futures::Stream;
+use futures_util::StreamExt;
+pub use manager::{RemoteCatalogManager, RemoteCatalogProvider, RemoteSchemaProvider};
+
+use crate::error::Error;
+
+mod client;
+mod manager;
+
+#[derive(Debug, Clone)]
+pub struct Kv(pub Vec<u8>, pub Vec<u8>);
+
+pub type ValueIter<'a, E> = Pin<Box<dyn Stream<Item = Result<Kv, E>> + Send + 'a>>;
+
+#[async_trait::async_trait]
+pub trait KvBackend: Send + Sync {
+    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
+    where
+        'a: 'b;
+
+    async fn set(&self, key: &[u8], val: &[u8]) -> Result<(), Error>;
+
+    /// Compare and set value of key. `expect` is the expected value, if backend's current value associated
+    /// with key is the same as `expect`, the value will be updated to `val`.
+    ///
+    /// - If the compare-and-set operation successfully updated value, this method will return an `Ok(Ok())`
+    /// - If associated value is not the same as `expect`, no value will be updated and an `Ok(Err(Vec<u8>))`
+    /// will be returned, the `Err(Vec<u8>)` indicates the current associated value of key.
+    /// - If any error happens during operation, an `Err(Error)` will be returned.
+    async fn compare_and_set(
+        &self,
+        key: &[u8],
+        expect: &[u8],
+        val: &[u8],
+    ) -> Result<Result<(), Option<Vec<u8>>>, Error>;
+
+    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Error>;
+
+    async fn delete(&self, key: &[u8]) -> Result<(), Error> {
+        self.delete_range(key, &[]).await
+    }
+
+    /// Default get is implemented based on `range` method.
+    async fn get(&self, key: &[u8]) -> Result<Option<Kv>, Error> {
+        let mut iter = self.range(key);
+        while let Some(r) = iter.next().await {
+            let kv = r?;
+            if kv.0 == key {
+                return Ok(Some(kv));
+            }
+        }
+        return Ok(None);
+    }
+}
+
+pub type KvBackendRef = Arc<dyn KvBackend>;
+
+#[cfg(test)]
+mod tests {
+    use async_stream::stream;
+
+    use super::*;
+
+    struct MockKvBackend {}
+
+    #[async_trait::async_trait]
+    impl KvBackend for MockKvBackend {
+        fn range<'a, 'b>(&'a self, _key: &[u8]) -> ValueIter<'b, Error>
+        where
+            'a: 'b,
+        {
+            Box::pin(stream!({
+                for i in 0..3 {
+                    yield Ok(Kv(
+                        i.to_string().as_bytes().to_vec(),
+                        i.to_string().as_bytes().to_vec(),
+                    ))
+                }
+            }))
+        }
+
+        async fn set(&self, _key: &[u8], _val: &[u8]) -> Result<(), Error> {
+            unimplemented!()
+        }
+
+        async fn compare_and_set(
+            &self,
+            _key: &[u8],
+            _expect: &[u8],
+            _val: &[u8],
+        ) -> Result<Result<(), Option<Vec<u8>>>, Error> {
+            unimplemented!()
+        }
+
+        async fn delete_range(&self, _key: &[u8], _end: &[u8]) -> Result<(), Error> {
+            unimplemented!()
+        }
+    }
+
+    #[tokio::test]
+    async fn test_get() {
+        let backend = MockKvBackend {};
+        let result = backend.get(0.to_string().as_bytes()).await;
+        assert_eq!(0.to_string().as_bytes(), result.unwrap().unwrap().0);
+        let result = backend.get(1.to_string().as_bytes()).await;
+        assert_eq!(1.to_string().as_bytes(), result.unwrap().unwrap().0);
+        let result = backend.get(2.to_string().as_bytes()).await;
+        assert_eq!(2.to_string().as_bytes(), result.unwrap().unwrap().0);
+        let result = backend.get(3.to_string().as_bytes()).await;
+        assert!(result.unwrap().is_none());
+    }
+}

src/catalog/src/remote/client.rs

@@ -0,0 +1,108 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use async_stream::stream;
+use common_telemetry::info;
+use meta_client::client::MetaClient;
+use meta_client::rpc::{CompareAndPutRequest, DeleteRangeRequest, PutRequest, RangeRequest};
+use snafu::ResultExt;
+
+use crate::error::{Error, MetaSrvSnafu};
+use crate::remote::{Kv, KvBackend, ValueIter};
+#[derive(Debug)]
+pub struct MetaKvBackend {
+    pub client: Arc<MetaClient>,
+}
+
+/// Implement `KvBackend` trait for `MetaKvBackend` instead of opendal's `Accessor` since
+/// `MetaClient`'s range method can return both keys and values, which can reduce IO overhead
+/// comparing to `Accessor`'s list and get method.
+#[async_trait::async_trait]
+impl KvBackend for MetaKvBackend {
+    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
+    where
+        'a: 'b,
+    {
+        let key = key.to_vec();
+        Box::pin(stream!({
+            let mut resp = self
+                .client
+                .range(RangeRequest::new().with_prefix(key))
+                .await
+                .context(MetaSrvSnafu)?;
+            let kvs = resp.take_kvs();
+            for mut kv in kvs.into_iter() {
+                yield Ok(Kv(kv.take_key(), kv.take_value()))
+            }
+        }))
+    }
+
+    async fn get(&self, key: &[u8]) -> Result<Option<Kv>, Error> {
+        let mut response = self
+            .client
+            .range(RangeRequest::new().with_key(key))
+            .await
+            .context(MetaSrvSnafu)?;
+        Ok(response
+            .take_kvs()
+            .get_mut(0)
+            .map(|kv| Kv(kv.take_key(), kv.take_value())))
+    }
+
+    async fn set(&self, key: &[u8], val: &[u8]) -> Result<(), Error> {
+        let req = PutRequest::new()
+            .with_key(key.to_vec())
+            .with_value(val.to_vec());
+        let _ = self.client.put(req).await.context(MetaSrvSnafu)?;
+        Ok(())
+    }
+
+    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Error> {
+        let req = DeleteRangeRequest::new().with_range(key.to_vec(), end.to_vec());
+        let resp = self.client.delete_range(req).await.context(MetaSrvSnafu)?;
+        info!(
+            "Delete range, key: {}, end: {}, deleted: {}",
+            String::from_utf8_lossy(key),
+            String::from_utf8_lossy(end),
+            resp.deleted()
+        );
+
+        Ok(())
+    }
+
+    async fn compare_and_set(
+        &self,
+        key: &[u8],
+        expect: &[u8],
+        val: &[u8],
+    ) -> Result<Result<(), Option<Vec<u8>>>, Error> {
+        let request = CompareAndPutRequest::new()
+            .with_key(key.to_vec())
+            .with_expect(expect.to_vec())
+            .with_value(val.to_vec());
+        let mut response = self
+            .client
+            .compare_and_put(request)
+            .await
+            .context(MetaSrvSnafu)?;
+        if response.is_success() {
+            Ok(Ok(()))
+        } else {
+            Ok(Err(response.take_prev_kv().map(|v| v.value().to_vec())))
+        }
+    }
+}

src/catalog/src/remote/manager.rs

@@ -0,0 +1,775 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::collections::HashMap;
+use std::pin::Pin;
+use std::sync::Arc;
+
+use arc_swap::ArcSwap;
+use async_stream::stream;
+use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID};
+use common_telemetry::{debug, info};
+use futures::Stream;
+use futures_util::StreamExt;
+use snafu::{OptionExt, ResultExt};
+use table::engine::{EngineContext, TableEngineRef};
+use table::metadata::TableId;
+use table::requests::{CreateTableRequest, OpenTableRequest};
+use table::table::numbers::NumbersTable;
+use table::TableRef;
+use tokio::sync::Mutex;
+
+use crate::error::{
+    CatalogNotFoundSnafu, CreateTableSnafu, InvalidCatalogValueSnafu, InvalidTableSchemaSnafu,
+    OpenTableSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu, UnimplementedSnafu,
+};
+use crate::helper::{
+    build_catalog_prefix, build_schema_prefix, build_table_global_prefix, CatalogKey, CatalogValue,
+    SchemaKey, SchemaValue, TableGlobalKey, TableGlobalValue, TableRegionalKey, TableRegionalValue,
+};
+use crate::remote::{Kv, KvBackendRef};
+use crate::{
+    handle_system_table_request, CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef,
+    DeregisterTableRequest, RegisterSchemaRequest, RegisterSystemTableRequest,
+    RegisterTableRequest, SchemaProvider, SchemaProviderRef,
+};
+
+/// Catalog manager based on metasrv.
+pub struct RemoteCatalogManager {
+    node_id: u64,
+    backend: KvBackendRef,
+    catalogs: Arc<ArcSwap<HashMap<String, CatalogProviderRef>>>,
+    engine: TableEngineRef,
+    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
+    mutex: Arc<Mutex<()>>,
+}
+
+impl RemoteCatalogManager {
+    pub fn new(engine: TableEngineRef, node_id: u64, backend: KvBackendRef) -> Self {
+        Self {
+            engine,
+            node_id,
+            backend,
+            catalogs: Default::default(),
+            system_table_requests: Default::default(),
+            mutex: Default::default(),
+        }
+    }
+
+    fn build_catalog_key(&self, catalog_name: impl AsRef<str>) -> CatalogKey {
+        CatalogKey {
+            catalog_name: catalog_name.as_ref().to_string(),
+        }
+    }
+
+    fn new_catalog_provider(&self, catalog_name: &str) -> CatalogProviderRef {
+        Arc::new(RemoteCatalogProvider {
+            node_id: self.node_id,
+            catalog_name: catalog_name.to_string(),
+            backend: self.backend.clone(),
+            schemas: Default::default(),
+            mutex: Default::default(),
+        }) as _
+    }
+
+    fn new_schema_provider(&self, catalog_name: &str, schema_name: &str) -> SchemaProviderRef {
+        Arc::new(RemoteSchemaProvider {
+            catalog_name: catalog_name.to_string(),
+            schema_name: schema_name.to_string(),
+            tables: Default::default(),
+            node_id: self.node_id,
+            backend: self.backend.clone(),
+            mutex: Default::default(),
+        }) as _
+    }
+
+    async fn iter_remote_catalogs(
+        &self,
+    ) -> Pin<Box<dyn Stream<Item = Result<CatalogKey>> + Send + '_>> {
+        let catalog_range_prefix = build_catalog_prefix();
+        info!("catalog_range_prefix: {}", catalog_range_prefix);
+        let mut catalogs = self.backend.range(catalog_range_prefix.as_bytes());
+        Box::pin(stream!({
+            while let Some(r) = catalogs.next().await {
+                let Kv(k, _) = r?;
+                if !k.starts_with(catalog_range_prefix.as_bytes()) {
+                    debug!("Ignoring non-catalog key: {}", String::from_utf8_lossy(&k));
+                    continue;
+                }
+                let key = CatalogKey::parse(&String::from_utf8_lossy(&k))
+                    .context(InvalidCatalogValueSnafu)?;
+                yield Ok(key)
+            }
+        }))
+    }
+
+    async fn iter_remote_schemas(
+        &self,
+        catalog_name: &str,
+    ) -> Pin<Box<dyn Stream<Item = Result<SchemaKey>> + Send + '_>> {
+        let schema_prefix = build_schema_prefix(catalog_name);
+        let mut schemas = self.backend.range(schema_prefix.as_bytes());
+
+        Box::pin(stream!({
+            while let Some(r) = schemas.next().await {
+                let Kv(k, _) = r?;
+                if !k.starts_with(schema_prefix.as_bytes()) {
+                    debug!("Ignoring non-schema key: {}", String::from_utf8_lossy(&k));
+                    continue;
+                }
+
+                let schema_key = SchemaKey::parse(&String::from_utf8_lossy(&k))
+                    .context(InvalidCatalogValueSnafu)?;
+                yield Ok(schema_key)
+            }
+        }))
+    }
+
+    /// Iterate over all table entries on metasrv
+    async fn iter_remote_tables(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+    ) -> Pin<Box<dyn Stream<Item = Result<(TableGlobalKey, TableGlobalValue)>> + Send + '_>> {
+        let table_prefix = build_table_global_prefix(catalog_name, schema_name);
+        let mut tables = self.backend.range(table_prefix.as_bytes());
+        Box::pin(stream!({
+            while let Some(r) = tables.next().await {
+                let Kv(k, v) = r?;
+                if !k.starts_with(table_prefix.as_bytes()) {
+                    debug!("Ignoring non-table prefix: {}", String::from_utf8_lossy(&k));
+                    continue;
+                }
+                let table_key = TableGlobalKey::parse(&String::from_utf8_lossy(&k))
+                    .context(InvalidCatalogValueSnafu)?;
+                let table_value =
+                    TableGlobalValue::from_bytes(&v).context(InvalidCatalogValueSnafu)?;
+
+                info!(
+                    "Found catalog table entry, key: {}, value: {:?}",
+                    table_key, table_value
+                );
+                // metasrv has allocated region ids to current datanode
+                if table_value
+                    .regions_id_map
+                    .get(&self.node_id)
+                    .map(|v| !v.is_empty())
+                    .unwrap_or(false)
+                {
+                    yield Ok((table_key, table_value))
+                }
+            }
+        }))
+    }
+
+    /// Fetch catalogs/schemas/tables from remote catalog manager along with max table id allocated.
+    async fn initiate_catalogs(&self) -> Result<(HashMap<String, CatalogProviderRef>, TableId)> {
+        let mut res = HashMap::new();
+        let max_table_id = MIN_USER_TABLE_ID - 1;
+
+        // initiate default catalog and schema
+        let default_catalog = self.initiate_default_catalog().await?;
+        res.insert(DEFAULT_CATALOG_NAME.to_string(), default_catalog);
+        info!("Default catalog and schema registered");
+
+        let mut catalogs = self.iter_remote_catalogs().await;
+        while let Some(r) = catalogs.next().await {
+            let CatalogKey { catalog_name, .. } = r?;
+            info!("Fetch catalog from metasrv: {}", catalog_name);
+            let catalog = res
+                .entry(catalog_name.clone())
+                .or_insert_with(|| self.new_catalog_provider(&catalog_name))
+                .clone();
+
+            self.initiate_schemas(catalog_name, catalog, max_table_id)
+                .await?;
+        }
+
+        Ok((res, max_table_id))
+    }
+
+    async fn initiate_schemas(
+        &self,
+        catalog_name: String,
+        catalog: CatalogProviderRef,
+        max_table_id: TableId,
+    ) -> Result<()> {
+        let mut schemas = self.iter_remote_schemas(&catalog_name).await;
+        while let Some(r) = schemas.next().await {
+            let SchemaKey {
+                catalog_name,
+                schema_name,
+                ..
+            } = r?;
+            info!("Found schema: {}.{}", catalog_name, schema_name);
+            let schema = match catalog.schema(&schema_name)? {
+                None => {
+                    let schema = self.new_schema_provider(&catalog_name, &schema_name);
+                    catalog.register_schema(schema_name.clone(), schema.clone())?;
+                    info!("Registered schema: {}", &schema_name);
+                    schema
+                }
+                Some(schema) => schema,
+            };
+
+            info!(
+                "Fetch schema from metasrv: {}.{}",
+                &catalog_name, &schema_name
+            );
+            self.initiate_tables(&catalog_name, &schema_name, schema, max_table_id)
+                .await?;
+        }
+        Ok(())
+    }
+
+    /// Initiates all tables inside a catalog by fetching data from metasrv.
+    async fn initiate_tables<'a>(
+        &'a self,
+        catalog_name: &'a str,
+        schema_name: &'a str,
+        schema: SchemaProviderRef,
+        mut max_table_id: TableId,
+    ) -> Result<()> {
+        info!("initializing tables in {}.{}", catalog_name, schema_name);
+        let mut table_num = 0;
+        let mut tables = self.iter_remote_tables(catalog_name, schema_name).await;
+        while let Some(r) = tables.next().await {
+            let (table_key, table_value) = r?;
+            let table_ref = self.open_or_create_table(&table_key, &table_value).await?;
+            schema.register_table(table_key.table_name.to_string(), table_ref)?;
+            info!("Registered table {}", &table_key.table_name);
+            max_table_id = max_table_id.max(table_value.table_id());
+            table_num += 1;
+        }
+        info!(
+            "initialized tables in {}.{}, total: {}",
+            catalog_name, schema_name, table_num
+        );
+        Ok(())
+    }
+
+    async fn initiate_default_catalog(&self) -> Result<CatalogProviderRef> {
+        let default_catalog = self.new_catalog_provider(DEFAULT_CATALOG_NAME);
+        let default_schema = self.new_schema_provider(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
+        default_catalog.register_schema(DEFAULT_SCHEMA_NAME.to_string(), default_schema.clone())?;
+        let schema_key = SchemaKey {
+            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        }
+        .to_string();
+        self.backend
+            .set(
+                schema_key.as_bytes(),
+                &SchemaValue {}
+                    .as_bytes()
+                    .context(InvalidCatalogValueSnafu)?,
+            )
+            .await?;
+        info!("Registered default schema");
+
+        let catalog_key = CatalogKey {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        }
+        .to_string();
+        self.backend
+            .set(
+                catalog_key.as_bytes(),
+                &CatalogValue {}
+                    .as_bytes()
+                    .context(InvalidCatalogValueSnafu)?,
+            )
+            .await?;
+        info!("Registered default catalog");
+        Ok(default_catalog)
+    }
+
+    async fn open_or_create_table(
+        &self,
+        table_key: &TableGlobalKey,
+        table_value: &TableGlobalValue,
+    ) -> Result<TableRef> {
+        let context = EngineContext {};
+        let TableGlobalKey {
+            catalog_name,
+            schema_name,
+            table_name,
+            ..
+        } = table_key;
+
+        let table_id = table_value.table_id();
+
+        let TableGlobalValue {
+            table_info,
+            regions_id_map,
+            ..
+        } = table_value;
+
+        // unwrap safety: checked in yielding this table when `iter_remote_tables`
+        let region_numbers = regions_id_map.get(&self.node_id).unwrap();
+
+        let request = OpenTableRequest {
+            catalog_name: catalog_name.clone(),
+            schema_name: schema_name.clone(),
+            table_name: table_name.clone(),
+            table_id,
+            region_numbers: region_numbers.clone(),
+        };
+        match self
+            .engine
+            .open_table(&context, request)
+            .await
+            .with_context(|_| OpenTableSnafu {
+                table_info: format!(
+                    "{}.{}.{}, id:{}",
+                    catalog_name, schema_name, table_name, table_id
+                ),
+            })? {
+            Some(table) => {
+                info!(
+                    "Table opened: {}.{}.{}",
+                    catalog_name, schema_name, table_name
+                );
+                Ok(table)
+            }
+            None => {
+                info!(
+                    "Try create table: {}.{}.{}",
+                    catalog_name, schema_name, table_name
+                );
+
+                let meta = &table_info.meta;
+                let schema = meta
+                    .schema
+                    .clone()
+                    .try_into()
+                    .context(InvalidTableSchemaSnafu {
+                        table_info: format!("{}.{}.{}", catalog_name, schema_name, table_name,),
+                        schema: meta.schema.clone(),
+                    })?;
+                let req = CreateTableRequest {
+                    id: table_id,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    desc: None,
+                    schema: Arc::new(schema),
+                    region_numbers: region_numbers.clone(),
+                    primary_key_indices: meta.primary_key_indices.clone(),
+                    create_if_not_exists: true,
+                    table_options: meta.options.clone(),
+                };
+
+                self.engine
+                    .create_table(&context, req)
+                    .await
+                    .context(CreateTableSnafu {
+                        table_info: format!(
+                            "{}.{}.{}, id:{}",
+                            &catalog_name, &schema_name, &table_name, table_id
+                        ),
+                    })
+            }
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for RemoteCatalogManager {
+    async fn start(&self) -> Result<()> {
+        let (catalogs, max_table_id) = self.initiate_catalogs().await?;
+        info!(
+            "Initialized catalogs: {:?}",
+            catalogs.keys().cloned().collect::<Vec<_>>()
+        );
+        self.catalogs.store(Arc::new(catalogs));
+        info!("Max table id allocated: {}", max_table_id);
+
+        let mut system_table_requests = self.system_table_requests.lock().await;
+        handle_system_table_request(self, self.engine.clone(), &mut system_table_requests).await?;
+        info!("All system table opened");
+
+        self.catalog(DEFAULT_CATALOG_NAME)
+            .unwrap()
+            .unwrap()
+            .schema(DEFAULT_SCHEMA_NAME)
+            .unwrap()
+            .unwrap()
+            .register_table("numbers".to_string(), Arc::new(NumbersTable::default()))
+            .unwrap();
+        Ok(())
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        let catalog_name = request.catalog;
+        let schema_name = request.schema;
+        let catalog_provider = self.catalog(&catalog_name)?.context(CatalogNotFoundSnafu {
+            catalog_name: &catalog_name,
+        })?;
+        let schema_provider =
+            catalog_provider
+                .schema(&schema_name)?
+                .with_context(|| SchemaNotFoundSnafu {
+                    schema_info: format!("{}.{}", &catalog_name, &schema_name),
+                })?;
+        if schema_provider.table_exist(&request.table_name)? {
+            return TableExistsSnafu {
+                table: format!("{}.{}.{}", &catalog_name, &schema_name, &request.table_name),
+            }
+            .fail();
+        }
+        schema_provider.register_table(request.table_name, request.table)?;
+        Ok(true)
+    }
+
+    async fn deregister_table(&self, _request: DeregisterTableRequest) -> Result<bool> {
+        UnimplementedSnafu {
+            operation: "deregister table",
+        }
+        .fail()
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        let catalog_name = request.catalog;
+        let schema_name = request.schema;
+        let catalog_provider = self.catalog(&catalog_name)?.context(CatalogNotFoundSnafu {
+            catalog_name: &catalog_name,
+        })?;
+        let schema_provider = self.new_schema_provider(&catalog_name, &schema_name);
+        catalog_provider.register_schema(schema_name, schema_provider)?;
+        Ok(true)
+    }
+
+    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
+        let mut requests = self.system_table_requests.lock().await;
+        requests.push(request);
+        Ok(())
+    }
+
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+        self.catalog(catalog)?
+            .context(CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .schema(schema)
+    }
+
+    fn table(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        let catalog = self
+            .catalog(catalog_name)?
+            .with_context(|| CatalogNotFoundSnafu { catalog_name })?;
+        let schema = catalog
+            .schema(schema_name)?
+            .with_context(|| SchemaNotFoundSnafu {
+                schema_info: format!("{}.{}", catalog_name, schema_name),
+            })?;
+        schema.table(table_name)
+    }
+}
+
+impl CatalogList for RemoteCatalogManager {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn register_catalog(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Result<Option<CatalogProviderRef>> {
+        let key = self.build_catalog_key(&name).to_string();
+        let backend = self.backend.clone();
+        let mutex = self.mutex.clone();
+        let catalogs = self.catalogs.clone();
+
+        std::thread::spawn(|| {
+            common_runtime::block_on_write(async move {
+                let _guard = mutex.lock().await;
+                backend
+                    .set(
+                        key.as_bytes(),
+                        &CatalogValue {}
+                            .as_bytes()
+                            .context(InvalidCatalogValueSnafu)?,
+                    )
+                    .await?;
+                let prev_catalogs = catalogs.load();
+                let mut new_catalogs = HashMap::with_capacity(prev_catalogs.len() + 1);
+                new_catalogs.clone_from(&prev_catalogs);
+                let prev = new_catalogs.insert(name, catalog);
+                catalogs.store(Arc::new(new_catalogs));
+                Ok(prev)
+            })
+        })
+        .join()
+        .unwrap()
+    }
+
+    /// List all catalogs from metasrv
+    fn catalog_names(&self) -> Result<Vec<String>> {
+        Ok(self.catalogs.load().keys().cloned().collect::<Vec<_>>())
+    }
+
+    /// Read catalog info of given name from metasrv.
+    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>> {
+        Ok(self.catalogs.load().get(name).cloned())
+    }
+}
+
+pub struct RemoteCatalogProvider {
+    node_id: u64,
+    catalog_name: String,
+    backend: KvBackendRef,
+    schemas: Arc<ArcSwap<HashMap<String, SchemaProviderRef>>>,
+    mutex: Arc<Mutex<()>>,
+}
+
+impl RemoteCatalogProvider {
+    pub fn new(catalog_name: String, backend: KvBackendRef, node_id: u64) -> Self {
+        Self {
+            node_id,
+            catalog_name,
+            backend,
+            schemas: Default::default(),
+            mutex: Default::default(),
+        }
+    }
+
+    pub fn refresh_schemas(&self) -> Result<()> {
+        let schemas = self.schemas.clone();
+        let schema_prefix = build_schema_prefix(&self.catalog_name);
+        let catalog_name = self.catalog_name.clone();
+        let mutex = self.mutex.clone();
+        let backend = self.backend.clone();
+        let node_id = self.node_id;
+
+        std::thread::spawn(move || {
+            common_runtime::block_on_write(async move {
+                let _guard = mutex.lock().await;
+                let prev_schemas = schemas.load();
+                let mut new_schemas = HashMap::with_capacity(prev_schemas.len() + 1);
+                new_schemas.clone_from(&prev_schemas);
+
+                let mut remote_schemas = backend.range(schema_prefix.as_bytes());
+                while let Some(r) = remote_schemas.next().await {
+                    let Kv(k, _) = r?;
+                    let schema_key = SchemaKey::parse(&String::from_utf8_lossy(&k))
+                        .context(InvalidCatalogValueSnafu)?;
+                    if !new_schemas.contains_key(&schema_key.schema_name) {
+                        new_schemas.insert(
+                            schema_key.schema_name.clone(),
+                            Arc::new(RemoteSchemaProvider::new(
+                                catalog_name.clone(),
+                                schema_key.schema_name,
+                                node_id,
+                                backend.clone(),
+                            )),
+                        );
+                    }
+                }
+                schemas.store(Arc::new(new_schemas));
+                Ok(())
+            })
+        })
+        .join()
+        .unwrap()?;
+
+        Ok(())
+    }
+
+    fn build_schema_key(&self, schema_name: impl AsRef<str>) -> SchemaKey {
+        SchemaKey {
+            catalog_name: self.catalog_name.clone(),
+            schema_name: schema_name.as_ref().to_string(),
+        }
+    }
+}
+
+impl CatalogProvider for RemoteCatalogProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema_names(&self) -> Result<Vec<String>> {
+        self.refresh_schemas()?;
+        Ok(self.schemas.load().keys().cloned().collect::<Vec<_>>())
+    }
+
+    fn register_schema(
+        &self,
+        name: String,
+        schema: SchemaProviderRef,
+    ) -> Result<Option<SchemaProviderRef>> {
+        let key = self.build_schema_key(&name).to_string();
+        let backend = self.backend.clone();
+        let mutex = self.mutex.clone();
+        let schemas = self.schemas.clone();
+
+        std::thread::spawn(|| {
+            common_runtime::block_on_write(async move {
+                let _guard = mutex.lock().await;
+                backend
+                    .set(
+                        key.as_bytes(),
+                        &SchemaValue {}
+                            .as_bytes()
+                            .context(InvalidCatalogValueSnafu)?,
+                    )
+                    .await?;
+
+                let prev_schemas = schemas.load();
+                let mut new_schemas = HashMap::with_capacity(prev_schemas.len() + 1);
+                new_schemas.clone_from(&prev_schemas);
+                let prev_schema = new_schemas.insert(name, schema);
+                schemas.store(Arc::new(new_schemas));
+                Ok(prev_schema)
+            })
+        })
+        .join()
+        .unwrap()
+    }
+
+    fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>> {
+        // TODO(hl): We should refresh whole catalog before calling datafusion's query engine.
+        self.refresh_schemas()?;
+        Ok(self.schemas.load().get(name).cloned())
+    }
+}
+
+pub struct RemoteSchemaProvider {
+    catalog_name: String,
+    schema_name: String,
+    node_id: u64,
+    backend: KvBackendRef,
+    tables: Arc<ArcSwap<HashMap<String, TableRef>>>,
+    mutex: Arc<Mutex<()>>,
+}
+
+impl RemoteSchemaProvider {
+    pub fn new(
+        catalog_name: String,
+        schema_name: String,
+        node_id: u64,
+        backend: KvBackendRef,
+    ) -> Self {
+        Self {
+            catalog_name,
+            schema_name,
+            node_id,
+            backend,
+            tables: Default::default(),
+            mutex: Default::default(),
+        }
+    }
+
+    fn build_regional_table_key(&self, table_name: impl AsRef<str>) -> TableRegionalKey {
+        TableRegionalKey {
+            catalog_name: self.catalog_name.clone(),
+            schema_name: self.schema_name.clone(),
+            table_name: table_name.as_ref().to_string(),
+            node_id: self.node_id,
+        }
+    }
+}
+
+impl SchemaProvider for RemoteSchemaProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn table_names(&self) -> Result<Vec<String>> {
+        Ok(self.tables.load().keys().cloned().collect::<Vec<_>>())
+    }
+
+    fn table(&self, name: &str) -> Result<Option<TableRef>> {
+        Ok(self.tables.load().get(name).cloned())
+    }
+
+    fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
+        let table_info = table.table_info();
+        let table_version = table_info.ident.version;
+        let table_value = TableRegionalValue {
+            version: table_version,
+            regions_ids: table.table_info().meta.region_numbers.clone(),
+        };
+        let backend = self.backend.clone();
+        let mutex = self.mutex.clone();
+        let tables = self.tables.clone();
+        let table_key = self.build_regional_table_key(&name).to_string();
+
+        let prev = std::thread::spawn(move || {
+            common_runtime::block_on_read(async move {
+                let _guard = mutex.lock().await;
+                backend
+                    .set(
+                        table_key.as_bytes(),
+                        &table_value.as_bytes().context(InvalidCatalogValueSnafu)?,
+                    )
+                    .await?;
+                debug!(
+                    "Successfully set catalog table entry, key: {}, table value: {:?}",
+                    table_key, table_value
+                );
+
+                let prev_tables = tables.load();
+                let mut new_tables = HashMap::with_capacity(prev_tables.len() + 1);
+                new_tables.clone_from(&prev_tables);
+                let prev = new_tables.insert(name, table);
+                tables.store(Arc::new(new_tables));
+                Ok(prev)
+            })
+        })
+        .join()
+        .unwrap();
+        prev
+    }
+
+    fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
+        let table_name = name.to_string();
+        let table_key = self.build_regional_table_key(&table_name).to_string();
+        let backend = self.backend.clone();
+        let mutex = self.mutex.clone();
+        let tables = self.tables.clone();
+        let prev = std::thread::spawn(move || {
+            common_runtime::block_on_read(async move {
+                let _guard = mutex.lock().await;
+                backend.delete(table_key.as_bytes()).await?;
+                debug!(
+                    "Successfully deleted catalog table entry, key: {}",
+                    table_key
+                );
+
+                let prev_tables = tables.load();
+                let mut new_tables = HashMap::with_capacity(prev_tables.len() + 1);
+                new_tables.clone_from(&prev_tables);
+                let prev = new_tables.remove(&table_name);
+                tables.store(Arc::new(new_tables));
+                Ok(prev)
+            })
+        })
+        .join()
+        .unwrap();
+        prev
+    }
+
+    /// Checks if table exists in schema provider based on locally opened table map.
+    fn table_exist(&self, name: &str) -> Result<bool> {
+        Ok(self.tables.load().contains_key(name))
+    }
+}

src/catalog/src/schema.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::sync::Arc;
 
@@ -12,10 +26,10 @@ pub trait SchemaProvider: Sync + Send {
     fn as_any(&self) -> &dyn Any;
 
     /// Retrieves the list of available table names in this schema.
-    fn table_names(&self) -> Vec<String>;
+    fn table_names(&self) -> Result<Vec<String>>;
 
     /// Retrieves a specific table from the schema by name, provided it exists.
-    fn table(&self, name: &str) -> Option<TableRef>;
+    fn table(&self, name: &str) -> Result<Option<TableRef>>;
 
     /// If supported by the implementation, adds a new table to this schema.
     /// If a table of the same name existed before, it returns "Table already exists" error.
@@ -28,7 +42,7 @@ pub trait SchemaProvider: Sync + Send {
     /// If supported by the implementation, checks the table exist in the schema provider or not.
     /// If no matched table in the schema provider, return false.
     /// Otherwise, return true.
-    fn table_exist(&self, name: &str) -> bool;
+    fn table_exist(&self, name: &str) -> Result<bool>;
 }
 
 pub type SchemaProviderRef = Arc<dyn SchemaProvider>;

src/catalog/src/system.rs

@@ -1,37 +1,51 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_NAME,
+    SYSTEM_CATALOG_TABLE_ID, SYSTEM_CATALOG_TABLE_NAME,
+};
 use common_query::logical_plan::Expr;
+use common_query::physical_plan::{PhysicalPlanRef, SessionContext};
 use common_recordbatch::SendableRecordBatchStream;
 use common_telemetry::debug;
 use common_time::util;
 use datatypes::prelude::{ConcreteDataType, ScalarVector};
 use datatypes::schema::{ColumnSchema, Schema, SchemaBuilder, SchemaRef};
-use datatypes::vectors::{BinaryVector, Int64Vector, UInt8Vector};
+use datatypes::vectors::{BinaryVector, TimestampMillisecondVector, UInt8Vector};
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
 use table::engine::{EngineContext, TableEngineRef};
-use table::metadata::TableId;
+use table::metadata::{TableId, TableInfoRef};
 use table::requests::{CreateTableRequest, InsertRequest, OpenTableRequest};
 use table::{Table, TableRef};
 
-use crate::consts::{
-    INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID,
-    SYSTEM_CATALOG_TABLE_NAME,
-};
 use crate::error::{
-    CreateSystemCatalogSnafu, EmptyValueSnafu, Error, InvalidEntryTypeSnafu, InvalidKeySnafu,
+    self, CreateSystemCatalogSnafu, EmptyValueSnafu, Error, InvalidEntryTypeSnafu, InvalidKeySnafu,
     OpenSystemCatalogSnafu, Result, ValueDeserializeSnafu,
 };
 
 pub const ENTRY_TYPE_INDEX: usize = 0;
 pub const KEY_INDEX: usize = 1;
-pub const TIMESTAMP_INDEX: usize = 2;
 pub const VALUE_INDEX: usize = 3;
 
 pub struct SystemCatalogTable {
-    schema: SchemaRef,
+    table_info: TableInfoRef,
     pub table: TableRef,
 }
 
@@ -42,7 +56,7 @@ impl Table for SystemCatalogTable {
     }
 
     fn schema(&self) -> SchemaRef {
-        self.schema.clone()
+        self.table_info.meta.schema.clone()
     }
 
     async fn scan(
@@ -50,7 +64,7 @@ impl Table for SystemCatalogTable {
         _projection: &Option<Vec<usize>>,
         _filters: &[Expr],
         _limit: Option<usize>,
-    ) -> table::Result<SendableRecordBatchStream> {
+    ) -> table::Result<PhysicalPlanRef> {
         panic!("System catalog table does not support scan!")
     }
 
@@ -58,6 +72,10 @@ impl Table for SystemCatalogTable {
     async fn insert(&self, request: InsertRequest) -> table::error::Result<usize> {
         self.table.insert(request).await
     }
+
+    fn table_info(&self) -> TableInfoRef {
+        self.table_info.clone()
+    }
 }
 
 impl SystemCatalogTable {
@@ -67,6 +85,7 @@ impl SystemCatalogTable {
             schema_name: INFORMATION_SCHEMA_NAME.to_string(),
             table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
             table_id: SYSTEM_CATALOG_TABLE_ID,
+            region_numbers: vec![0],
         };
         let schema = Arc::new(build_system_catalog_schema());
         let ctx = EngineContext::default();
@@ -76,32 +95,46 @@ impl SystemCatalogTable {
             .await
             .context(OpenSystemCatalogSnafu)?
         {
-            Ok(Self { table, schema })
+            Ok(Self {
+                table_info: table.table_info(),
+                table,
+            })
         } else {
             // system catalog table is not yet created, try to create
             let request = CreateTableRequest {
                 id: SYSTEM_CATALOG_TABLE_ID,
-                catalog_name: Some(SYSTEM_CATALOG_NAME.to_string()),
-                schema_name: Some(INFORMATION_SCHEMA_NAME.to_string()),
+                catalog_name: SYSTEM_CATALOG_NAME.to_string(),
+                schema_name: INFORMATION_SCHEMA_NAME.to_string(),
                 table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
                 desc: Some("System catalog table".to_string()),
                 schema: schema.clone(),
-                primary_key_indices: vec![ENTRY_TYPE_INDEX, KEY_INDEX, TIMESTAMP_INDEX],
+                region_numbers: vec![0],
+                primary_key_indices: vec![ENTRY_TYPE_INDEX, KEY_INDEX],
                 create_if_not_exists: true,
+                table_options: HashMap::new(),
             };
 
             let table = engine
                 .create_table(&ctx, request)
                 .await
                 .context(CreateSystemCatalogSnafu)?;
-            Ok(Self { table, schema })
+            let table_info = table.table_info();
+            Ok(Self { table, table_info })
         }
     }
 
     /// Create a stream of all entries inside system catalog table
     pub async fn records(&self) -> Result<SendableRecordBatchStream> {
         let full_projection = None;
-        let stream = self.table.scan(&full_projection, &[], None).await.unwrap();
+        let ctx = SessionContext::new();
+        let scan = self
+            .table
+            .scan(&full_projection, &[], None)
+            .await
+            .context(error::SystemCatalogTableScanSnafu)?;
+        let stream = scan
+            .execute(0, ctx.task_ctx())
+            .context(error::SystemCatalogTableScanExecSnafu)?;
         Ok(stream)
     }
 }
@@ -128,9 +161,10 @@ fn build_system_catalog_schema() -> Schema {
         ),
         ColumnSchema::new(
             "timestamp".to_string(),
-            ConcreteDataType::int64_datatype(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
             false,
-        ),
+        )
+        .with_time_index(true),
         ColumnSchema::new(
             "value".to_string(),
             ConcreteDataType::binary_datatype(),
@@ -138,61 +172,78 @@ fn build_system_catalog_schema() -> Schema {
         ),
         ColumnSchema::new(
             "gmt_created".to_string(),
-            ConcreteDataType::int64_datatype(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
             false,
         ),
         ColumnSchema::new(
             "gmt_modified".to_string(),
-            ConcreteDataType::int64_datatype(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
             false,
         ),
     ];
 
     // The schema of this table must be valid.
-    SchemaBuilder::from(cols)
-        .timestamp_index(2)
-        .build()
-        .unwrap()
+    SchemaBuilder::try_from(cols).unwrap().build().unwrap()
 }
 
 pub fn build_table_insert_request(full_table_name: String, table_id: TableId) -> InsertRequest {
+    build_insert_request(
+        EntryType::Table,
+        full_table_name.as_bytes(),
+        serde_json::to_string(&TableEntryValue { table_id })
+            .unwrap()
+            .as_bytes(),
+    )
+}
+
+pub fn build_schema_insert_request(catalog_name: String, schema_name: String) -> InsertRequest {
+    let full_schema_name = format!("{}.{}", catalog_name, schema_name);
+    build_insert_request(
+        EntryType::Schema,
+        full_schema_name.as_bytes(),
+        serde_json::to_string(&SchemaEntryValue {})
+            .unwrap()
+            .as_bytes(),
+    )
+}
+
+pub fn build_insert_request(entry_type: EntryType, key: &[u8], value: &[u8]) -> InsertRequest {
     let mut columns_values = HashMap::with_capacity(6);
     columns_values.insert(
         "entry_type".to_string(),
-        Arc::new(UInt8Vector::from_slice(&[EntryType::Table as u8])) as _,
+        Arc::new(UInt8Vector::from_slice(&[entry_type as u8])) as _,
     );
 
     columns_values.insert(
         "key".to_string(),
-        Arc::new(BinaryVector::from_slice(&[full_table_name.as_bytes()])) as _,
+        Arc::new(BinaryVector::from_slice(&[key])) as _,
     );
 
     // Timestamp in key part is intentionally left to 0
     columns_values.insert(
         "timestamp".to_string(),
-        Arc::new(Int64Vector::from_slice(&[0])) as _,
+        Arc::new(TimestampMillisecondVector::from_slice(&[0])) as _,
     );
 
     columns_values.insert(
         "value".to_string(),
-        Arc::new(BinaryVector::from_slice(&[serde_json::to_string(
-            &TableEntryValue { table_id },
-        )
-        .unwrap()
-        .as_bytes()])) as _,
+        Arc::new(BinaryVector::from_slice(&[value])) as _,
     );
 
+    let now = util::current_time_millis();
     columns_values.insert(
         "gmt_created".to_string(),
-        Arc::new(Int64Vector::from_slice(&[util::current_time_millis()])) as _,
+        Arc::new(TimestampMillisecondVector::from_slice(&[now])) as _,
     );
 
     columns_values.insert(
         "gmt_modified".to_string(),
-        Arc::new(Int64Vector::from_slice(&[util::current_time_millis()])) as _,
+        Arc::new(TimestampMillisecondVector::from_slice(&[now])) as _,
     );
 
     InsertRequest {
+        catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        schema_name: DEFAULT_SCHEMA_NAME.to_string(),
         table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
         columns_values,
     }
@@ -280,25 +331,28 @@ impl TryFrom<u8> for EntryType {
     }
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
 pub enum Entry {
     Catalog(CatalogEntry),
     Schema(SchemaEntry),
     Table(TableEntry),
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
 pub struct CatalogEntry {
     pub catalog_name: String,
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
 pub struct SchemaEntry {
     pub catalog_name: String,
     pub schema_name: String,
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SchemaEntryValue;
+
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
 pub struct TableEntry {
     pub catalog_name: String,
     pub schema_name: String,
@@ -313,10 +367,20 @@ pub struct TableEntryValue {
 
 #[cfg(test)]
 mod tests {
+    use log_store::fs::noop::NoopLogStore;
+    use mito::config::EngineConfig;
+    use mito::engine::MitoEngine;
+    use object_store::ObjectStore;
+    use storage::config::EngineConfig as StorageEngineConfig;
+    use storage::EngineImpl;
+    use table::metadata::TableType;
+    use table::metadata::TableType::Base;
+    use tempdir::TempDir;
+
     use super::*;
 
     #[test]
-    pub fn test_decode_catalog_enrty() {
+    pub fn test_decode_catalog_entry() {
         let entry = decode_system_catalog(
             Some(EntryType::Catalog as u8),
             Some("some_catalog".as_bytes()),
@@ -384,4 +448,43 @@ mod tests {
         assert_eq!(EntryType::Table, EntryType::try_from(3).unwrap());
         assert!(EntryType::try_from(4).is_err());
     }
+
+    pub async fn prepare_table_engine() -> (TempDir, TableEngineRef) {
+        let dir = TempDir::new("system-table-test").unwrap();
+        let store_dir = dir.path().to_string_lossy();
+        let accessor = object_store::backend::fs::Builder::default()
+            .root(&store_dir)
+            .build()
+            .unwrap();
+        let object_store = ObjectStore::new(accessor);
+        let table_engine = Arc::new(MitoEngine::new(
+            EngineConfig::default(),
+            EngineImpl::new(
+                StorageEngineConfig::default(),
+                Arc::new(NoopLogStore::default()),
+                object_store.clone(),
+            ),
+            object_store,
+        ));
+        (dir, table_engine)
+    }
+
+    #[tokio::test]
+    async fn test_system_table_type() {
+        let (_dir, table_engine) = prepare_table_engine().await;
+        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
+        assert_eq!(Base, system_table.table_type());
+    }
+
+    #[tokio::test]
+    async fn test_system_table_info() {
+        let (_dir, table_engine) = prepare_table_engine().await;
+        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
+        let info = system_table.table_info();
+        assert_eq!(TableType::Base, info.table_type);
+        assert_eq!(SYSTEM_CATALOG_TABLE_NAME, info.name);
+        assert_eq!(SYSTEM_CATALOG_TABLE_ID, info.ident.table_id);
+        assert_eq!(SYSTEM_CATALOG_NAME, info.catalog_name);
+        assert_eq!(INFORMATION_SCHEMA_NAME, info.schema_name);
+    }
 }

src/catalog/src/tables.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // The `tables` table in system catalog keeps a record of all tables created by user.
 
 use std::any::Any;
@@ -6,22 +20,26 @@ use std::sync::Arc;
 use std::task::{Context, Poll};
 
 use async_stream::stream;
+use common_catalog::consts::{INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_TABLE_NAME};
+use common_error::ext::BoxedError;
 use common_query::logical_plan::Expr;
+use common_query::physical_plan::PhysicalPlanRef;
 use common_recordbatch::error::Result as RecordBatchResult;
-use common_recordbatch::{RecordBatch, RecordBatchStream, SendableRecordBatchStream};
-use datatypes::prelude::{ConcreteDataType, VectorBuilder};
+use common_recordbatch::{RecordBatch, RecordBatchStream};
+use datatypes::prelude::{ConcreteDataType, DataType};
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
-use datatypes::value::Value;
+use datatypes::value::ValueRef;
 use datatypes::vectors::VectorRef;
 use futures::Stream;
 use snafu::ResultExt;
 use table::engine::TableEngineRef;
-use table::metadata::TableId;
+use table::error::TablesRecordBatchSnafu;
+use table::metadata::{TableId, TableInfoRef};
+use table::table::scan::SimpleTableScan;
 use table::{Table, TableRef};
 
-use crate::consts::{INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_TABLE_NAME};
-use crate::error::InsertTableRecordSnafu;
-use crate::system::{build_table_insert_request, SystemCatalogTable};
+use crate::error::{Error, InsertCatalogRecordSnafu};
+use crate::system::{build_schema_insert_request, build_table_insert_request, SystemCatalogTable};
 use crate::{
     format_full_table_name, CatalogListRef, CatalogProvider, SchemaProvider, SchemaProviderRef,
 };
@@ -53,23 +71,53 @@ impl Table for Tables {
         self.schema.clone()
     }
 
+    fn table_info(&self) -> TableInfoRef {
+        unreachable!("Tables does not support table_info method")
+    }
+
     async fn scan(
         &self,
         _projection: &Option<Vec<usize>>,
         _filters: &[Expr],
         _limit: Option<usize>,
-    ) -> table::error::Result<SendableRecordBatchStream> {
+    ) -> table::error::Result<PhysicalPlanRef> {
         let catalogs = self.catalogs.clone();
         let schema_ref = self.schema.clone();
         let engine_name = self.engine_name.clone();
 
         let stream = stream!({
-            for catalog_name in catalogs.catalog_names() {
-                let catalog = catalogs.catalog(&catalog_name).unwrap();
-                for schema_name in catalog.schema_names() {
-                    let mut tables_in_schema = Vec::with_capacity(catalog.schema_names().len());
-                    let schema = catalog.schema(&schema_name).unwrap();
-                    for table_name in schema.table_names() {
+            for catalog_name in catalogs
+                .catalog_names()
+                .map_err(BoxedError::new)
+                .context(TablesRecordBatchSnafu)?
+            {
+                let catalog = catalogs
+                    .catalog(&catalog_name)
+                    .map_err(BoxedError::new)
+                    .context(TablesRecordBatchSnafu)?
+                    .unwrap();
+                for schema_name in catalog
+                    .schema_names()
+                    .map_err(BoxedError::new)
+                    .context(TablesRecordBatchSnafu)?
+                {
+                    let mut tables_in_schema = Vec::with_capacity(
+                        catalog
+                            .schema_names()
+                            .map_err(BoxedError::new)
+                            .context(TablesRecordBatchSnafu)?
+                            .len(),
+                    );
+                    let schema = catalog
+                        .schema(&schema_name)
+                        .map_err(BoxedError::new)
+                        .context(TablesRecordBatchSnafu)?
+                        .unwrap();
+                    for table_name in schema
+                        .table_names()
+                        .map_err(BoxedError::new)
+                        .context(TablesRecordBatchSnafu)?
+                    {
                         tables_in_schema.push(table_name);
                     }
 
@@ -85,10 +133,11 @@ impl Table for Tables {
             }
         });
 
-        Ok(Box::pin(TablesRecordBatchStream {
+        let stream = Box::pin(TablesRecordBatchStream {
             schema: self.schema.clone(),
             stream: Box::pin(stream),
-        }))
+        });
+        Ok(Arc::new(SimpleTableScan::new(stream)))
     }
 }
 
@@ -100,26 +149,33 @@ fn tables_to_record_batch(
     engine: &str,
 ) -> Vec<VectorRef> {
     let mut catalog_vec =
-        VectorBuilder::with_capacity(ConcreteDataType::string_datatype(), table_names.len());
+        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
     let mut schema_vec =
-        VectorBuilder::with_capacity(ConcreteDataType::string_datatype(), table_names.len());
+        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
     let mut table_name_vec =
-        VectorBuilder::with_capacity(ConcreteDataType::string_datatype(), table_names.len());
+        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
     let mut engine_vec =
-        VectorBuilder::with_capacity(ConcreteDataType::string_datatype(), table_names.len());
+        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
 
     for table_name in table_names {
-        catalog_vec.push(&Value::String(catalog_name.into()));
-        schema_vec.push(&Value::String(schema_name.into()));
-        table_name_vec.push(&Value::String(table_name.into()));
-        engine_vec.push(&Value::String(engine.into()));
+        // Safety: All these vectors are string type.
+        catalog_vec
+            .push_value_ref(ValueRef::String(catalog_name))
+            .unwrap();
+        schema_vec
+            .push_value_ref(ValueRef::String(schema_name))
+            .unwrap();
+        table_name_vec
+            .push_value_ref(ValueRef::String(&table_name))
+            .unwrap();
+        engine_vec.push_value_ref(ValueRef::String(engine)).unwrap();
     }
 
     vec![
-        catalog_vec.finish(),
-        schema_vec.finish(),
-        table_name_vec.finish(),
-        engine_vec.finish(),
+        catalog_vec.to_vector(),
+        schema_vec.to_vector(),
+        table_name_vec.to_vector(),
+        engine_vec.to_vector(),
     ]
 }
 
@@ -152,17 +208,20 @@ impl SchemaProvider for InformationSchema {
         self
     }
 
-    fn table_names(&self) -> Vec<String> {
-        vec!["tables".to_string(), SYSTEM_CATALOG_TABLE_NAME.to_string()]
+    fn table_names(&self) -> Result<Vec<String>, Error> {
+        Ok(vec![
+            "tables".to_string(),
+            SYSTEM_CATALOG_TABLE_NAME.to_string(),
+        ])
     }
 
-    fn table(&self, name: &str) -> Option<TableRef> {
+    fn table(&self, name: &str) -> Result<Option<TableRef>, Error> {
         if name.eq_ignore_ascii_case("tables") {
-            Some(self.tables.clone())
+            Ok(Some(self.tables.clone()))
         } else if name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME) {
-            Some(self.system.clone())
+            Ok(Some(self.system.clone()))
         } else {
-            None
+            Ok(None)
         }
     }
 
@@ -178,8 +237,9 @@ impl SchemaProvider for InformationSchema {
         panic!("System catalog & schema does not support deregister table")
     }
 
-    fn table_exist(&self, name: &str) -> bool {
-        name.eq_ignore_ascii_case("tables") || name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME)
+    fn table_exist(&self, name: &str) -> Result<bool, Error> {
+        Ok(name.eq_ignore_ascii_case("tables")
+            || name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME))
     }
 }
 
@@ -215,7 +275,20 @@ impl SystemCatalog {
             .system
             .insert(request)
             .await
-            .context(InsertTableRecordSnafu)
+            .context(InsertCatalogRecordSnafu)
+    }
+
+    pub async fn register_schema(
+        &self,
+        catalog: String,
+        schema: String,
+    ) -> crate::error::Result<usize> {
+        let request = build_schema_insert_request(catalog, schema);
+        self.information_schema
+            .system
+            .insert(request)
+            .await
+            .context(InsertCatalogRecordSnafu)
     }
 }
 
@@ -224,23 +297,23 @@ impl CatalogProvider for SystemCatalog {
         self
     }
 
-    fn schema_names(&self) -> Vec<String> {
-        vec![INFORMATION_SCHEMA_NAME.to_string()]
+    fn schema_names(&self) -> Result<Vec<String>, Error> {
+        Ok(vec![INFORMATION_SCHEMA_NAME.to_string()])
     }
 
     fn register_schema(
         &self,
         _name: String,
         _schema: SchemaProviderRef,
-    ) -> Option<SchemaProviderRef> {
+    ) -> Result<Option<SchemaProviderRef>, Error> {
         panic!("System catalog does not support registering schema!")
     }
 
-    fn schema(&self, name: &str) -> Option<Arc<dyn SchemaProvider>> {
+    fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>, Error> {
         if name.eq_ignore_ascii_case(INFORMATION_SCHEMA_NAME) {
-            Some(self.information_schema.clone())
+            Ok(Some(self.information_schema.clone()))
         } else {
-            None
+            Ok(None)
         }
     }
 }
@@ -273,74 +346,72 @@ fn build_schema_for_tables() -> Schema {
 
 #[cfg(test)]
 mod tests {
-    use datatypes::arrow::array::Utf8Array;
-    use datatypes::arrow::datatypes::DataType;
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use common_query::physical_plan::SessionContext;
     use futures_util::StreamExt;
     use table::table::numbers::NumbersTable;
 
     use super::*;
-    use crate::memory::{new_memory_catalog_list, MemoryCatalogProvider, MemorySchemaProvider};
+    use crate::local::memory::new_memory_catalog_list;
     use crate::CatalogList;
 
     #[tokio::test]
     async fn test_tables() {
         let catalog_list = new_memory_catalog_list().unwrap();
-        let catalog_provider = Arc::new(MemoryCatalogProvider::default());
-        let schema = Arc::new(MemorySchemaProvider::new());
+        let schema = catalog_list
+            .catalog(DEFAULT_CATALOG_NAME)
+            .unwrap()
+            .unwrap()
+            .schema(DEFAULT_SCHEMA_NAME)
+            .unwrap()
+            .unwrap();
         schema
             .register_table("test_table".to_string(), Arc::new(NumbersTable::default()))
             .unwrap();
-        catalog_provider.register_schema("test_schema".to_string(), schema);
-        catalog_list.register_catalog("test_catalog".to_string(), catalog_provider);
-        let tables = Tables::new(catalog_list, "test_engine".to_string());
 
-        let mut tables_stream = tables.scan(&None, &[], None).await.unwrap();
+        let tables = Tables::new(catalog_list, "test_engine".to_string());
+        let tables_stream = tables.scan(&None, &[], None).await.unwrap();
+        let session_ctx = SessionContext::new();
+        let mut tables_stream = tables_stream.execute(0, session_ctx.task_ctx()).unwrap();
+
         if let Some(t) = tables_stream.next().await {
-            let batch = t.unwrap().df_recordbatch;
+            let batch = t.unwrap();
             assert_eq!(1, batch.num_rows());
             assert_eq!(4, batch.num_columns());
-            assert_eq!(&DataType::LargeUtf8, batch.column(0).data_type());
-            assert_eq!(&DataType::LargeUtf8, batch.column(1).data_type());
-            assert_eq!(&DataType::LargeUtf8, batch.column(2).data_type());
-            assert_eq!(&DataType::LargeUtf8, batch.column(3).data_type());
             assert_eq!(
-                "test_catalog",
-                batch
-                    .column(0)
-                    .as_any()
-                    .downcast_ref::<Utf8Array<i64>>()
-                    .unwrap()
-                    .value(0)
+                ConcreteDataType::string_datatype(),
+                batch.column(0).data_type()
+            );
+            assert_eq!(
+                ConcreteDataType::string_datatype(),
+                batch.column(1).data_type()
+            );
+            assert_eq!(
+                ConcreteDataType::string_datatype(),
+                batch.column(2).data_type()
+            );
+            assert_eq!(
+                ConcreteDataType::string_datatype(),
+                batch.column(3).data_type()
+            );
+            assert_eq!(
+                "greptime",
+                batch.column(0).get_ref(0).as_string().unwrap().unwrap()
             );
 
             assert_eq!(
-                "test_schema",
-                batch
-                    .column(1)
-                    .as_any()
-                    .downcast_ref::<Utf8Array<i64>>()
-                    .unwrap()
-                    .value(0)
+                "public",
+                batch.column(1).get_ref(0).as_string().unwrap().unwrap()
             );
 
             assert_eq!(
                 "test_table",
-                batch
-                    .column(2)
-                    .as_any()
-                    .downcast_ref::<Utf8Array<i64>>()
-                    .unwrap()
-                    .value(0)
+                batch.column(2).get_ref(0).as_string().unwrap().unwrap()
             );
 
             assert_eq!(
                 "test_engine",
-                batch
-                    .column(3)
-                    .as_any()
-                    .downcast_ref::<Utf8Array<i64>>()
-                    .unwrap()
-                    .value(0)
+                batch.column(3).get_ref(0).as_string().unwrap().unwrap()
             );
         } else {
             panic!("Record batch should not be empty!")

src/catalog/tests/local_catalog_tests.rs

@@ -0,0 +1,132 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use catalog::local::LocalCatalogManager;
+    use catalog::{CatalogManager, RegisterTableRequest};
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use common_telemetry::{error, info};
+    use mito::config::EngineConfig;
+    use table::table::numbers::NumbersTable;
+    use table::TableRef;
+    use tokio::sync::Mutex;
+
+    async fn create_local_catalog_manager() -> Result<LocalCatalogManager, catalog::error::Error> {
+        let (_dir, object_store) =
+            mito::table::test_util::new_test_object_store("setup_mock_engine_and_table").await;
+        let mock_engine = Arc::new(mito::table::test_util::MockMitoEngine::new(
+            EngineConfig::default(),
+            mito::table::test_util::MockEngine::default(),
+            object_store,
+        ));
+        let catalog_manager = LocalCatalogManager::try_new(mock_engine).await.unwrap();
+        catalog_manager.start().await?;
+        Ok(catalog_manager)
+    }
+
+    #[tokio::test]
+    async fn test_duplicate_register() {
+        let catalog_manager = create_local_catalog_manager().await.unwrap();
+        let request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "test_table".to_string(),
+            table_id: 42,
+            table: Arc::new(NumbersTable::new(42)),
+        };
+        assert!(catalog_manager
+            .register_table(request.clone())
+            .await
+            .unwrap());
+
+        // register table with same table id will succeed with 0 as return val.
+        assert!(!catalog_manager.register_table(request).await.unwrap());
+
+        let err = catalog_manager
+            .register_table(RegisterTableRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_SCHEMA_NAME.to_string(),
+                table_name: "test_table".to_string(),
+                table_id: 43,
+                table: Arc::new(NumbersTable::new(43)),
+            })
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("Table `greptime.public.test_table` already exists"),
+            "Actual error message: {}",
+            err
+        );
+    }
+
+    #[test]
+    fn test_concurrent_register() {
+        common_telemetry::init_default_ut_logging();
+        let rt = Arc::new(tokio::runtime::Builder::new_multi_thread().build().unwrap());
+        let catalog_manager =
+            Arc::new(rt.block_on(async { create_local_catalog_manager().await.unwrap() }));
+
+        let succeed: Arc<Mutex<Option<TableRef>>> = Arc::new(Mutex::new(None));
+
+        let mut handles = Vec::with_capacity(8);
+        for i in 0..8 {
+            let catalog = catalog_manager.clone();
+            let succeed = succeed.clone();
+            let handle = rt.spawn(async move {
+                let table_id = 42 + i;
+                let table = Arc::new(NumbersTable::new(table_id));
+                let req = RegisterTableRequest {
+                    catalog: DEFAULT_CATALOG_NAME.to_string(),
+                    schema: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "test_table".to_string(),
+                    table_id,
+                    table: table.clone(),
+                };
+                match catalog.register_table(req).await {
+                    Ok(res) => {
+                        if res {
+                            let mut succeed = succeed.lock().await;
+                            info!("Successfully registered table: {}", table_id);
+                            *succeed = Some(table);
+                        }
+                    }
+                    Err(_) => {
+                        error!("Failed to register table {}", table_id);
+                    }
+                }
+            });
+            handles.push(handle);
+        }
+
+        rt.block_on(async move {
+            for handle in handles {
+                handle.await.unwrap();
+            }
+            let guard = succeed.lock().await;
+            let table = guard.as_ref().unwrap();
+            let table_registered = catalog_manager
+                .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "test_table")
+                .unwrap()
+                .unwrap();
+            assert_eq!(
+                table_registered.table_info().ident.table_id,
+                table.table_info().ident.table_id
+            );
+        });
+    }
+}

src/catalog/tests/mock.rs

@@ -0,0 +1,223 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::btree_map::Entry;
+use std::collections::{BTreeMap, HashMap};
+use std::fmt::{Display, Formatter};
+use std::str::FromStr;
+use std::sync::Arc;
+
+use async_stream::stream;
+use catalog::error::Error;
+use catalog::remote::{Kv, KvBackend, ValueIter};
+use common_recordbatch::RecordBatch;
+use common_telemetry::logging::info;
+use datatypes::data_type::ConcreteDataType;
+use datatypes::schema::{ColumnSchema, Schema};
+use datatypes::vectors::StringVector;
+use serde::Serializer;
+use table::engine::{EngineContext, TableEngine, TableReference};
+use table::metadata::TableId;
+use table::requests::{AlterTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest};
+use table::test_util::MemTable;
+use table::TableRef;
+use tokio::sync::RwLock;
+
+#[derive(Default)]
+pub struct MockKvBackend {
+    map: RwLock<BTreeMap<Vec<u8>, Vec<u8>>>,
+}
+
+impl Display for MockKvBackend {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        futures::executor::block_on(async {
+            let map = self.map.read().await;
+            for (k, v) in map.iter() {
+                f.serialize_str(&String::from_utf8_lossy(k))?;
+                f.serialize_str(" -> ")?;
+                f.serialize_str(&String::from_utf8_lossy(v))?;
+                f.serialize_str("\n")?;
+            }
+            Ok(())
+        })
+    }
+}
+
+#[async_trait::async_trait]
+impl KvBackend for MockKvBackend {
+    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
+    where
+        'a: 'b,
+    {
+        let prefix = key.to_vec();
+        let prefix_string = String::from_utf8_lossy(&prefix).to_string();
+        Box::pin(stream!({
+            let maps = self.map.read().await.clone();
+            for (k, v) in maps.range(prefix.clone()..) {
+                let key_string = String::from_utf8_lossy(k).to_string();
+                let matches = key_string.starts_with(&prefix_string);
+                if matches {
+                    yield Ok(Kv(k.clone(), v.clone()))
+                } else {
+                    info!("Stream finished");
+                    return;
+                }
+            }
+        }))
+    }
+
+    async fn set(&self, key: &[u8], val: &[u8]) -> Result<(), Error> {
+        let mut map = self.map.write().await;
+        map.insert(key.to_vec(), val.to_vec());
+        Ok(())
+    }
+
+    async fn compare_and_set(
+        &self,
+        key: &[u8],
+        expect: &[u8],
+        val: &[u8],
+    ) -> Result<Result<(), Option<Vec<u8>>>, Error> {
+        let mut map = self.map.write().await;
+        let existing = map.entry(key.to_vec());
+        match existing {
+            Entry::Vacant(e) => {
+                if expect.is_empty() {
+                    e.insert(val.to_vec());
+                    Ok(Ok(()))
+                } else {
+                    Ok(Err(None))
+                }
+            }
+            Entry::Occupied(mut existing) => {
+                if existing.get() == expect {
+                    existing.insert(val.to_vec());
+                    Ok(Ok(()))
+                } else {
+                    Ok(Err(Some(existing.get().clone())))
+                }
+            }
+        }
+    }
+
+    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Error> {
+        let start = key.to_vec();
+        let end = end.to_vec();
+        let range = start..end;
+
+        let mut map = self.map.write().await;
+        map.retain(|k, _| !range.contains(k));
+        Ok(())
+    }
+}
+
+#[derive(Default)]
+pub struct MockTableEngine {
+    tables: RwLock<HashMap<String, TableRef>>,
+}
+
+#[async_trait::async_trait]
+impl TableEngine for MockTableEngine {
+    fn name(&self) -> &str {
+        "MockTableEngine"
+    }
+
+    /// Create a table with only one column
+    async fn create_table(
+        &self,
+        _ctx: &EngineContext,
+        request: CreateTableRequest,
+    ) -> table::Result<TableRef> {
+        let table_name = request.table_name.clone();
+        let catalog_name = request.catalog_name.clone();
+        let schema_name = request.schema_name.clone();
+
+        let default_table_id = "0".to_owned();
+        let table_id = TableId::from_str(
+            request
+                .table_options
+                .get("table_id")
+                .unwrap_or(&default_table_id),
+        )
+        .unwrap();
+        let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
+            "name",
+            ConcreteDataType::string_datatype(),
+            true,
+        )]));
+
+        let data = vec![Arc::new(StringVector::from(vec!["a", "b", "c"])) as _];
+        let record_batch = RecordBatch::new(schema, data).unwrap();
+        let table: TableRef = Arc::new(MemTable::new_with_catalog(
+            &table_name,
+            record_batch,
+            table_id,
+            catalog_name,
+            schema_name,
+            vec![0],
+        )) as Arc<_>;
+
+        let mut tables = self.tables.write().await;
+        tables.insert(table_name, table.clone() as TableRef);
+        Ok(table)
+    }
+
+    async fn open_table(
+        &self,
+        _ctx: &EngineContext,
+        request: OpenTableRequest,
+    ) -> table::Result<Option<TableRef>> {
+        Ok(self.tables.read().await.get(&request.table_name).cloned())
+    }
+
+    async fn alter_table(
+        &self,
+        _ctx: &EngineContext,
+        _request: AlterTableRequest,
+    ) -> table::Result<TableRef> {
+        unimplemented!()
+    }
+
+    fn get_table<'a>(
+        &self,
+        _ctx: &EngineContext,
+        table_ref: &'a TableReference,
+    ) -> table::Result<Option<TableRef>> {
+        futures::executor::block_on(async {
+            Ok(self
+                .tables
+                .read()
+                .await
+                .get(&table_ref.to_string())
+                .cloned())
+        })
+    }
+
+    fn table_exists<'a>(&self, _ctx: &EngineContext, table_ref: &'a TableReference) -> bool {
+        futures::executor::block_on(async {
+            self.tables
+                .read()
+                .await
+                .contains_key(&table_ref.to_string())
+        })
+    }
+
+    async fn drop_table(
+        &self,
+        _ctx: &EngineContext,
+        _request: DropTableRequest,
+    ) -> table::Result<bool> {
+        unimplemented!()
+    }
+}

src/catalog/tests/remote_catalog_tests.rs

@@ -0,0 +1,297 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(assert_matches)]
+
+mod mock;
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+    use std::collections::HashSet;
+    use std::sync::Arc;
+
+    use catalog::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
+    use catalog::remote::{
+        KvBackend, KvBackendRef, RemoteCatalogManager, RemoteCatalogProvider, RemoteSchemaProvider,
+    };
+    use catalog::{CatalogList, CatalogManager, RegisterTableRequest};
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use datatypes::schema::Schema;
+    use futures_util::StreamExt;
+    use table::engine::{EngineContext, TableEngineRef};
+    use table::requests::CreateTableRequest;
+
+    use crate::mock::{MockKvBackend, MockTableEngine};
+
+    #[tokio::test]
+    async fn test_backend() {
+        common_telemetry::init_default_ut_logging();
+        let backend = MockKvBackend::default();
+
+        let default_catalog_key = CatalogKey {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        }
+        .to_string();
+
+        backend
+            .set(
+                default_catalog_key.as_bytes(),
+                &CatalogValue {}.as_bytes().unwrap(),
+            )
+            .await
+            .unwrap();
+
+        let schema_key = SchemaKey {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+        }
+        .to_string();
+        backend
+            .set(schema_key.as_bytes(), &SchemaValue {}.as_bytes().unwrap())
+            .await
+            .unwrap();
+
+        let mut iter = backend.range("__c-".as_bytes());
+        let mut res = HashSet::new();
+        while let Some(r) = iter.next().await {
+            let kv = r.unwrap();
+            res.insert(String::from_utf8_lossy(&kv.0).to_string());
+        }
+        assert_eq!(
+            vec!["__c-greptime".to_string()],
+            res.into_iter().collect::<Vec<_>>()
+        );
+    }
+
+    async fn prepare_components(
+        node_id: u64,
+    ) -> (KvBackendRef, TableEngineRef, Arc<RemoteCatalogManager>) {
+        let backend = Arc::new(MockKvBackend::default()) as KvBackendRef;
+        let table_engine = Arc::new(MockTableEngine::default());
+        let catalog_manager =
+            RemoteCatalogManager::new(table_engine.clone(), node_id, backend.clone());
+        catalog_manager.start().await.unwrap();
+        (backend, table_engine, Arc::new(catalog_manager))
+    }
+
+    #[tokio::test]
+    async fn test_remote_catalog_default() {
+        common_telemetry::init_default_ut_logging();
+        let node_id = 42;
+        let (_, _, catalog_manager) = prepare_components(node_id).await;
+        assert_eq!(
+            vec![DEFAULT_CATALOG_NAME.to_string()],
+            catalog_manager.catalog_names().unwrap()
+        );
+
+        let default_catalog = catalog_manager
+            .catalog(DEFAULT_CATALOG_NAME)
+            .unwrap()
+            .unwrap();
+        assert_eq!(
+            vec![DEFAULT_SCHEMA_NAME.to_string()],
+            default_catalog.schema_names().unwrap()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_remote_catalog_register_nonexistent() {
+        common_telemetry::init_default_ut_logging();
+        let node_id = 42;
+        let (_, table_engine, catalog_manager) = prepare_components(node_id).await;
+        // register a new table with an nonexistent catalog
+        let catalog_name = "nonexistent_catalog".to_string();
+        let schema_name = "nonexistent_schema".to_string();
+        let table_name = "fail_table".to_string();
+        // this schema has no effect
+        let table_schema = Arc::new(Schema::new(vec![]));
+        let table = table_engine
+            .create_table(
+                &EngineContext {},
+                CreateTableRequest {
+                    id: 1,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    desc: None,
+                    schema: table_schema.clone(),
+                    region_numbers: vec![0],
+                    primary_key_indices: vec![],
+                    create_if_not_exists: false,
+                    table_options: Default::default(),
+                },
+            )
+            .await
+            .unwrap();
+        let reg_req = RegisterTableRequest {
+            catalog: catalog_name,
+            schema: schema_name,
+            table_name,
+            table_id: 1,
+            table,
+        };
+        let res = catalog_manager.register_table(reg_req).await;
+
+        // because nonexistent_catalog does not exist yet.
+        assert_matches!(
+            res.err().unwrap(),
+            catalog::error::Error::CatalogNotFound { .. }
+        );
+    }
+
+    #[tokio::test]
+    async fn test_register_table() {
+        let node_id = 42;
+        let (_, table_engine, catalog_manager) = prepare_components(node_id).await;
+        let default_catalog = catalog_manager
+            .catalog(DEFAULT_CATALOG_NAME)
+            .unwrap()
+            .unwrap();
+        assert_eq!(
+            vec![DEFAULT_SCHEMA_NAME.to_string()],
+            default_catalog.schema_names().unwrap()
+        );
+
+        let default_schema = default_catalog
+            .schema(DEFAULT_SCHEMA_NAME)
+            .unwrap()
+            .unwrap();
+        assert_eq!(vec!["numbers"], default_schema.table_names().unwrap());
+
+        // register a new table with an nonexistent catalog
+        let catalog_name = DEFAULT_CATALOG_NAME.to_string();
+        let schema_name = DEFAULT_SCHEMA_NAME.to_string();
+        let table_name = "test_table".to_string();
+        let table_id = 1;
+        // this schema has no effect
+        let table_schema = Arc::new(Schema::new(vec![]));
+        let table = table_engine
+            .create_table(
+                &EngineContext {},
+                CreateTableRequest {
+                    id: table_id,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    desc: None,
+                    schema: table_schema.clone(),
+                    region_numbers: vec![0],
+                    primary_key_indices: vec![],
+                    create_if_not_exists: false,
+                    table_options: Default::default(),
+                },
+            )
+            .await
+            .unwrap();
+        let reg_req = RegisterTableRequest {
+            catalog: catalog_name,
+            schema: schema_name,
+            table_name: table_name.clone(),
+            table_id,
+            table,
+        };
+        assert!(catalog_manager.register_table(reg_req).await.unwrap());
+        assert_eq!(
+            HashSet::from([table_name, "numbers".to_string()]),
+            default_schema
+                .table_names()
+                .unwrap()
+                .into_iter()
+                .collect::<HashSet<_>>()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_register_catalog_schema_table() {
+        let node_id = 42;
+        let (backend, table_engine, catalog_manager) = prepare_components(node_id).await;
+
+        let catalog_name = "test_catalog".to_string();
+        let schema_name = "nonexistent_schema".to_string();
+        let catalog = Arc::new(RemoteCatalogProvider::new(
+            catalog_name.clone(),
+            backend.clone(),
+            node_id,
+        ));
+
+        // register catalog to catalog manager
+        catalog_manager
+            .register_catalog(catalog_name.clone(), catalog)
+            .unwrap();
+        assert_eq!(
+            HashSet::<String>::from_iter(
+                vec![DEFAULT_CATALOG_NAME.to_string(), catalog_name.clone()].into_iter()
+            ),
+            HashSet::from_iter(catalog_manager.catalog_names().unwrap().into_iter())
+        );
+
+        let table_to_register = table_engine
+            .create_table(
+                &EngineContext {},
+                CreateTableRequest {
+                    id: 2,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: "".to_string(),
+                    desc: None,
+                    schema: Arc::new(Schema::new(vec![])),
+                    region_numbers: vec![0],
+                    primary_key_indices: vec![],
+                    create_if_not_exists: false,
+                    table_options: Default::default(),
+                },
+            )
+            .await
+            .unwrap();
+
+        let reg_req = RegisterTableRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+            table_name: " fail_table".to_string(),
+            table_id: 2,
+            table: table_to_register,
+        };
+        // this register will fail since schema does not exist yet
+        assert_matches!(
+            catalog_manager
+                .register_table(reg_req.clone())
+                .await
+                .unwrap_err(),
+            catalog::error::Error::SchemaNotFound { .. }
+        );
+
+        let new_catalog = catalog_manager
+            .catalog(&catalog_name)
+            .unwrap()
+            .expect("catalog should exist since it's already registered");
+        let schema = Arc::new(RemoteSchemaProvider::new(
+            catalog_name.clone(),
+            schema_name.clone(),
+            node_id,
+            backend.clone(),
+        ));
+
+        let prev = new_catalog
+            .register_schema(schema_name.clone(), schema.clone())
+            .expect("Register schema should not fail");
+        assert!(prev.is_none());
+        assert!(catalog_manager.register_table(reg_req).await.unwrap());
+
+        assert_eq!(
+            HashSet::from([schema_name.clone()]),
+            new_catalog.schema_names().unwrap().into_iter().collect()
+        )
+    }
+}

src/client/Cargo.toml

@@ -2,18 +2,39 @@
 name = "client"
 version = "0.1.0"
 edition = "2021"
-
+license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 api = { path = "../api" }
+async-stream = "0.3"
+common-base = { path = "../common/base" }
 common-error = { path = "../common/error" }
 common-grpc = { path = "../common/grpc" }
-datafusion = { git = "https://github.com/apache/arrow-datafusion.git", branch = "arrow2", features = ["simd"] }
+common-grpc-expr = { path = "../common/grpc-expr" }
+common-query = { path = "../common/query" }
+common-recordbatch = { path = "../common/recordbatch" }
+common-time = { path = "../common/time" }
+datafusion = "14.0.0"
+datatypes = { path = "../datatypes" }
+enum_dispatch = "0.3"
+parking_lot = "0.12"
+rand = "0.8"
 snafu = { version = "0.7", features = ["backtraces"] }
 tonic = "0.8"
 
 [dev-dependencies]
+datanode = { path = "../datanode" }
+substrait = { path = "../common/substrait" }
 tokio = { version = "1.0", features = ["full"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+# TODO(ruihang): upgrade to 0.11 once substrait-rs supports it.
+[dev-dependencies.prost_09]
+package = "prost"
+version = "0.9"
+
+[dev-dependencies.substrait_proto]
+package = "substrait"
+version = "0.2"

src/client/examples/insert.rs

@@ -1,4 +1,18 @@
-use api::v1::{codec::InsertBatch, *};
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::*;
 use client::{Client, Database};
 
 fn main() {
@@ -10,15 +24,24 @@ fn main() {
 
 #[tokio::main]
 async fn run() {
-    let client = Client::connect("http://127.0.0.1:3001").await.unwrap();
+    let client = Client::with_urls(vec!["127.0.0.1:3001"]);
     let db = Database::new("greptime", client);
 
-    db.insert("demo", insert_batches()).await.unwrap();
+    let (columns, row_count) = insert_data();
+
+    let expr = InsertExpr {
+        schema_name: "public".to_string(),
+        table_name: "demo".to_string(),
+        region_number: 0,
+        columns,
+        row_count,
+    };
+    db.insert(expr).await.unwrap();
 }
 
-fn insert_batches() -> Vec<Vec<u8>> {
+fn insert_data() -> (Vec<Column>, u32) {
     const SEMANTIC_TAG: i32 = 0;
-    const SEMANTIC_FEILD: i32 = 1;
+    const SEMANTIC_FIELD: i32 = 1;
     const SEMANTIC_TS: i32 = 2;
 
     let row_count = 4;
@@ -37,6 +60,7 @@ fn insert_batches() -> Vec<Vec<u8>> {
         semantic_type: SEMANTIC_TAG,
         values: Some(host_vals),
         null_mask: vec![0],
+        ..Default::default()
     };
 
     let cpu_vals = column::Values {
@@ -45,9 +69,10 @@ fn insert_batches() -> Vec<Vec<u8>> {
     };
     let cpu_column = Column {
         column_name: "cpu".to_string(),
-        semantic_type: SEMANTIC_FEILD,
+        semantic_type: SEMANTIC_FIELD,
         values: Some(cpu_vals),
         null_mask: vec![2],
+        ..Default::default()
     };
 
     let mem_vals = column::Values {
@@ -56,9 +81,10 @@ fn insert_batches() -> Vec<Vec<u8>> {
     };
     let mem_column = Column {
         column_name: "memory".to_string(),
-        semantic_type: SEMANTIC_FEILD,
+        semantic_type: SEMANTIC_FIELD,
         values: Some(mem_vals),
         null_mask: vec![4],
+        ..Default::default()
     };
 
     let ts_vals = column::Values {
@@ -70,11 +96,11 @@ fn insert_batches() -> Vec<Vec<u8>> {
         semantic_type: SEMANTIC_TS,
         values: Some(ts_vals),
         null_mask: vec![0],
+        ..Default::default()
     };
 
-    let insert_batch = InsertBatch {
-        columns: vec![host_column, cpu_column, mem_column, ts_column],
+    (
+        vec![host_column, cpu_column, mem_column, ts_column],
         row_count,
-    };
-    vec![insert_batch.into()]
+    )
 }

src/client/examples/logical.rs

@@ -0,0 +1,111 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::{ColumnDataType, ColumnDef, CreateExpr};
+use client::admin::Admin;
+use client::{Client, Database};
+use prost_09::Message;
+use substrait_proto::protobuf::plan_rel::RelType as PlanRelType;
+use substrait_proto::protobuf::read_rel::{NamedTable, ReadType};
+use substrait_proto::protobuf::rel::RelType;
+use substrait_proto::protobuf::{PlanRel, ReadRel, Rel};
+use tracing::{event, Level};
+
+fn main() {
+    tracing::subscriber::set_global_default(tracing_subscriber::FmtSubscriber::builder().finish())
+        .unwrap();
+
+    run();
+}
+
+#[tokio::main]
+async fn run() {
+    let client = Client::with_urls(vec!["127.0.0.1:3001"]);
+
+    let create_table_expr = CreateExpr {
+        catalog_name: Some("greptime".to_string()),
+        schema_name: Some("public".to_string()),
+        table_name: "test_logical_dist_exec".to_string(),
+        desc: None,
+        column_defs: vec![
+            ColumnDef {
+                name: "timestamp".to_string(),
+                datatype: ColumnDataType::TimestampMillisecond as i32,
+                is_nullable: false,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "key".to_string(),
+                datatype: ColumnDataType::Uint64 as i32,
+                is_nullable: false,
+                default_constraint: None,
+            },
+            ColumnDef {
+                name: "value".to_string(),
+                datatype: ColumnDataType::Uint64 as i32,
+                is_nullable: false,
+                default_constraint: None,
+            },
+        ],
+        time_index: "timestamp".to_string(),
+        primary_keys: vec!["key".to_string()],
+        create_if_not_exists: false,
+        table_options: Default::default(),
+        table_id: Some(1024),
+        region_ids: vec![0],
+    };
+
+    let admin = Admin::new("create table", client.clone());
+    let result = admin.create(create_table_expr).await.unwrap();
+    event!(Level::INFO, "create table result: {:#?}", result);
+
+    let logical = mock_logical_plan();
+    event!(Level::INFO, "plan size: {:#?}", logical.len());
+    let db = Database::new("greptime", client);
+    let result = db.logical_plan(logical).await.unwrap();
+
+    event!(Level::INFO, "result: {:#?}", result);
+}
+
+fn mock_logical_plan() -> Vec<u8> {
+    let catalog_name = "greptime".to_string();
+    let schema_name = "public".to_string();
+    let table_name = "test_logical_dist_exec".to_string();
+
+    let named_table = NamedTable {
+        names: vec![catalog_name, schema_name, table_name],
+        advanced_extension: None,
+    };
+    let read_type = ReadType::NamedTable(named_table);
+
+    let read_rel = ReadRel {
+        common: None,
+        base_schema: None,
+        filter: None,
+        projection: None,
+        advanced_extension: None,
+        read_type: Some(read_type),
+    };
+
+    let mut buf = vec![];
+    let rel = Rel {
+        rel_type: Some(RelType::Read(Box::new(read_rel))),
+    };
+    let plan_rel = PlanRel {
+        rel_type: Some(PlanRelType::Rel(rel)),
+    };
+    plan_rel.encode(&mut buf).unwrap();
+
+    buf
+}

src/client/examples/physical.rs

@@ -1,37 +0,0 @@
-use std::sync::Arc;
-
-use client::{Client, Database};
-use common_grpc::MockExecution;
-use datafusion::physical_plan::{
-    expressions::Column, projection::ProjectionExec, ExecutionPlan, PhysicalExpr,
-};
-use tracing::{event, Level};
-
-fn main() {
-    tracing::subscriber::set_global_default(tracing_subscriber::FmtSubscriber::builder().finish())
-        .unwrap();
-
-    run();
-}
-
-#[tokio::main]
-async fn run() {
-    let client = Client::connect("http://127.0.0.1:3001").await.unwrap();
-    let db = Database::new("greptime", client);
-
-    let physical = mock_physical_plan();
-    let result = db.physical_plan(physical, None).await;
-
-    event!(Level::INFO, "result: {:#?}", result);
-}
-
-fn mock_physical_plan() -> Arc<dyn ExecutionPlan> {
-    let id_expr = Arc::new(Column::new("id", 0)) as Arc<dyn PhysicalExpr>;
-    let age_expr = Arc::new(Column::new("age", 2)) as Arc<dyn PhysicalExpr>;
-    let expr = vec![(id_expr, "id".to_string()), (age_expr, "age".to_string())];
-
-    let input =
-        Arc::new(MockExecution::new("mock_input_exec".to_string())) as Arc<dyn ExecutionPlan>;
-    let projection = ProjectionExec::try_new(expr, input).unwrap();
-    Arc::new(projection)
-}

src/client/examples/select.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use client::{Client, Database, Select};
 use tracing::{event, Level};
 
@@ -10,7 +24,7 @@ fn main() {
 
 #[tokio::main]
 async fn run() {
-    let client = Client::connect("http://127.0.0.1:3001").await.unwrap();
+    let client = Client::with_urls(vec!["127.0.0.1:3001"]);
     let db = Database::new("greptime", client);
 
     let sql = Select::Sql("select * from demo".to_string());

src/client/src/admin.rs

@@ -1,14 +1,137 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use api::v1::*;
+use common_error::prelude::StatusCode;
+use common_query::Output;
+use snafu::prelude::*;
+
+use crate::database::PROTOCOL_VERSION;
+use crate::{error, Client, Result};
 
 #[derive(Clone, Debug)]
 pub struct Admin {
+    name: String,
     client: Client,
 }
 
 impl Admin {
-    pub fn new(client: Client) -> Self {
-        Self { client }
+    pub fn new(name: impl Into<String>, client: Client) -> Self {
+        Self {
+            name: name.into(),
+            client,
+        }
     }
 
-    // TODO(jiachun): admin api
+    pub async fn create(&self, expr: CreateExpr) -> Result<AdminResult> {
+        let header = ExprHeader {
+            version: PROTOCOL_VERSION,
+        };
+        let expr = AdminExpr {
+            header: Some(header),
+            expr: Some(admin_expr::Expr::Create(expr)),
+        };
+        self.do_request(expr).await
+    }
+
+    pub async fn do_request(&self, expr: AdminExpr) -> Result<AdminResult> {
+        // `remove(0)` is safe because of `do_requests`'s invariants.
+        Ok(self.do_requests(vec![expr]).await?.remove(0))
+    }
+
+    pub async fn alter(&self, expr: AlterExpr) -> Result<AdminResult> {
+        let header = ExprHeader {
+            version: PROTOCOL_VERSION,
+        };
+        let expr = AdminExpr {
+            header: Some(header),
+            expr: Some(admin_expr::Expr::Alter(expr)),
+        };
+        self.do_request(expr).await
+    }
+
+    pub async fn drop_table(&self, expr: DropTableExpr) -> Result<AdminResult> {
+        let header = ExprHeader {
+            version: PROTOCOL_VERSION,
+        };
+        let expr = AdminExpr {
+            header: Some(header),
+            expr: Some(admin_expr::Expr::DropTable(expr)),
+        };
+
+        self.do_request(expr).await
+    }
+
+    /// Invariants: the lengths of input vec (`Vec<AdminExpr>`) and output vec (`Vec<AdminResult>`) are equal.
+    async fn do_requests(&self, exprs: Vec<AdminExpr>) -> Result<Vec<AdminResult>> {
+        let expr_count = exprs.len();
+        let req = AdminRequest {
+            name: self.name.clone(),
+            exprs,
+        };
+
+        let resp = self.client.admin(req).await?;
+
+        let results = resp.results;
+        ensure!(
+            results.len() == expr_count,
+            error::MissingResultSnafu {
+                name: "admin_results",
+                expected: expr_count,
+                actual: results.len(),
+            }
+        );
+        Ok(results)
+    }
+
+    pub async fn create_database(&self, expr: CreateDatabaseExpr) -> Result<AdminResult> {
+        let header = ExprHeader {
+            version: PROTOCOL_VERSION,
+        };
+        let expr = AdminExpr {
+            header: Some(header),
+            expr: Some(admin_expr::Expr::CreateDatabase(expr)),
+        };
+        Ok(self.do_requests(vec![expr]).await?.remove(0))
+    }
+}
+
+pub fn admin_result_to_output(admin_result: AdminResult) -> Result<Output> {
+    let header = admin_result.header.context(error::MissingHeaderSnafu)?;
+    if !StatusCode::is_success(header.code) {
+        return error::DatanodeSnafu {
+            code: header.code,
+            msg: header.err_msg,
+        }
+        .fail();
+    }
+
+    let result = admin_result.result.context(error::MissingResultSnafu {
+        name: "result".to_string(),
+        expected: 1_usize,
+        actual: 0_usize,
+    })?;
+    let output = match result {
+        admin_result::Result::Mutate(mutate) => {
+            if mutate.failure != 0 {
+                return error::MutateFailureSnafu {
+                    failure: mutate.failure,
+                }
+                .fail();
+            }
+            Output::AffectedRows(mutate.success as usize)
+        }
+    };
+    Ok(output)
 }

src/client/src/client.rs

@@ -1,22 +1,107 @@
-use api::v1::{greptime_client::GreptimeClient, *};
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use api::v1::greptime_client::GreptimeClient;
+use api::v1::*;
+use common_grpc::channel_manager::ChannelManager;
+use parking_lot::RwLock;
 use snafu::{OptionExt, ResultExt};
 use tonic::transport::Channel;
 
-use crate::error;
-use crate::Result;
+use crate::load_balance::{LoadBalance, Loadbalancer};
+use crate::{error, Result};
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default)]
 pub struct Client {
-    client: GreptimeClient<Channel>,
+    inner: Arc<Inner>,
+}
+
+#[derive(Debug, Default)]
+struct Inner {
+    channel_manager: ChannelManager,
+    peers: Arc<RwLock<Vec<String>>>,
+    load_balance: Loadbalancer,
+}
+
+impl Inner {
+    fn with_manager(channel_manager: ChannelManager) -> Self {
+        Self {
+            channel_manager,
+            ..Default::default()
+        }
+    }
+
+    fn set_peers(&self, peers: Vec<String>) {
+        let mut guard = self.peers.write();
+        *guard = peers;
+    }
+
+    fn get_peer(&self) -> Option<String> {
+        let guard = self.peers.read();
+        self.load_balance.get_peer(&guard).cloned()
+    }
 }
 
 impl Client {
-    pub async fn connect(url: impl Into<String>) -> Result<Self> {
-        let url = url.into();
-        let client = GreptimeClient::connect(url.clone())
-            .await
-            .context(error::ConnectFailedSnafu { url })?;
-        Ok(Self { client })
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    pub fn with_manager(channel_manager: ChannelManager) -> Self {
+        let inner = Arc::new(Inner::with_manager(channel_manager));
+        Self { inner }
+    }
+
+    pub fn with_urls<U, A>(urls: A) -> Self
+    where
+        U: AsRef<str>,
+        A: AsRef<[U]>,
+    {
+        Self::with_manager_and_urls(ChannelManager::new(), urls)
+    }
+
+    pub fn with_manager_and_urls<U, A>(channel_manager: ChannelManager, urls: A) -> Self
+    where
+        U: AsRef<str>,
+        A: AsRef<[U]>,
+    {
+        let inner = Inner::with_manager(channel_manager);
+        let urls: Vec<String> = urls
+            .as_ref()
+            .iter()
+            .map(|peer| peer.as_ref().to_string())
+            .collect();
+        inner.set_peers(urls);
+        Self {
+            inner: Arc::new(inner),
+        }
+    }
+
+    pub fn start<U, A>(&self, urls: A)
+    where
+        U: AsRef<str>,
+        A: AsRef<[U]>,
+    {
+        let urls: Vec<String> = urls
+            .as_ref()
+            .iter()
+            .map(|peer| peer.as_ref().to_string())
+            .collect();
+
+        self.inner.set_peers(urls);
     }
 
     pub async fn admin(&self, req: AdminRequest) -> Result<AdminResponse> {
@@ -48,12 +133,62 @@ impl Client {
     }
 
     pub async fn batch(&self, req: BatchRequest) -> Result<BatchResponse> {
-        let res = self
-            .client
-            .clone()
+        let peer = self
+            .inner
+            .get_peer()
+            .context(error::IllegalGrpcClientStateSnafu {
+                err_msg: "No available peer found",
+            })?;
+        let mut client = self.make_client(&peer)?;
+        let result = client
             .batch(req)
             .await
-            .context(error::TonicStatusSnafu)?;
-        Ok(res.into_inner())
+            .context(error::TonicStatusSnafu { addr: peer })?;
+        Ok(result.into_inner())
+    }
+
+    fn make_client(&self, addr: impl AsRef<str>) -> Result<GreptimeClient<Channel>> {
+        let addr = addr.as_ref();
+        let channel = self
+            .inner
+            .channel_manager
+            .get(addr)
+            .context(error::CreateChannelSnafu { addr })?;
+        Ok(GreptimeClient::new(channel))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashSet;
+
+    use super::Inner;
+    use crate::load_balance::Loadbalancer;
+
+    fn mock_peers() -> Vec<String> {
+        vec![
+            "127.0.0.1:3001".to_string(),
+            "127.0.0.1:3002".to_string(),
+            "127.0.0.1:3003".to_string(),
+        ]
+    }
+
+    #[test]
+    fn test_inner() {
+        let inner = Inner::default();
+
+        assert!(matches!(
+            inner.load_balance,
+            Loadbalancer::Random(crate::load_balance::Random)
+        ));
+        assert!(inner.get_peer().is_none());
+
+        let peers = mock_peers();
+        inner.set_peers(peers.clone());
+        let all: HashSet<String> = peers.into_iter().collect();
+
+        for _ in 0..20 {
+            assert!(all.contains(&inner.get_peer().unwrap()));
+        }
     }
 }

src/client/src/database.rs

@@ -1,28 +1,38 @@
-use std::ops::Deref;
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::sync::Arc;
 
 use api::v1::codec::SelectResult as GrpcSelectResult;
+use api::v1::column::SemanticType;
 use api::v1::{
     object_expr, object_result, select_expr, DatabaseRequest, ExprHeader, InsertExpr,
-    MutateResult as GrpcMutateResult, ObjectExpr, ObjectResult as GrpcObjectResult, PhysicalPlan,
-    SelectExpr,
+    MutateResult as GrpcMutateResult, ObjectExpr, ObjectResult as GrpcObjectResult, SelectExpr,
 };
 use common_error::status_code::StatusCode;
-use common_grpc::AsExcutionPlan;
-use common_grpc::DefaultAsPlanImpl;
-use datafusion::physical_plan::ExecutionPlan;
+use common_grpc_expr::column_to_vector;
+use common_query::Output;
+use common_recordbatch::{RecordBatch, RecordBatches};
+use datatypes::prelude::*;
+use datatypes::schema::{ColumnSchema, Schema};
 use snafu::{ensure, OptionExt, ResultExt};
 
-use crate::error::{self, MissingResultSnafu};
-use crate::{
-    error::DatanodeSnafu, error::DecodeSelectSnafu, error::EncodePhysicalSnafu,
-    error::MissingHeaderSnafu, Client, Result,
-};
+use crate::error::{ColumnToVectorSnafu, ConvertSchemaSnafu, DatanodeSnafu, DecodeSelectSnafu};
+use crate::{error, Client, Result};
 
 pub const PROTOCOL_VERSION: u32 = 1;
 
-pub type Bytes = Vec<u8>;
-
 #[derive(Clone, Debug)]
 pub struct Database {
     name: String,
@@ -41,22 +51,33 @@ impl Database {
         &self.name
     }
 
-    pub async fn insert(&self, table: impl Into<String>, values: Vec<Bytes>) -> Result<()> {
+    pub async fn insert(&self, insert: InsertExpr) -> Result<ObjectResult> {
         let header = ExprHeader {
             version: PROTOCOL_VERSION,
         };
-        let insert = InsertExpr {
-            table_name: table.into(),
-            values,
-        };
         let expr = ObjectExpr {
             header: Some(header),
             expr: Some(object_expr::Expr::Insert(insert)),
         };
+        self.object(expr).await?.try_into()
+    }
 
-        self.object(expr).await?;
-
-        Ok(())
+    pub async fn batch_insert(&self, insert_exprs: Vec<InsertExpr>) -> Result<Vec<ObjectResult>> {
+        let header = ExprHeader {
+            version: PROTOCOL_VERSION,
+        };
+        let obj_exprs = insert_exprs
+            .into_iter()
+            .map(|expr| ObjectExpr {
+                header: Some(header.clone()),
+                expr: Some(object_expr::Expr::Insert(expr)),
+            })
+            .collect();
+        self.objects(obj_exprs)
+            .await?
+            .into_iter()
+            .map(|result| result.try_into())
+            .collect()
     }
 
     pub async fn select(&self, expr: Select) -> Result<ObjectResult> {
@@ -68,20 +89,9 @@ impl Database {
         self.do_select(select_expr).await
     }
 
-    pub async fn physical_plan(
-        &self,
-        physical: Arc<dyn ExecutionPlan>,
-        original_ql: Option<String>,
-    ) -> Result<ObjectResult> {
-        let plan = DefaultAsPlanImpl::try_from_physical_plan(physical.clone())
-            .context(EncodePhysicalSnafu { physical })?
-            .bytes;
-        let original_ql = original_ql.unwrap_or_default();
+    pub async fn logical_plan(&self, logical_plan: Vec<u8>) -> Result<ObjectResult> {
         let select_expr = SelectExpr {
-            expr: Some(select_expr::Expr::PhysicalPlan(PhysicalPlan {
-                original_ql: original_ql.into_bytes(),
-                plan,
-            })),
+            expr: Some(select_expr::Expr::LogicalPlan(logical_plan)),
         };
         self.do_select(select_expr).await
     }
@@ -97,41 +107,10 @@ impl Database {
         };
 
         let obj_result = self.object(expr).await?;
-
-        let header = obj_result.header.context(MissingHeaderSnafu)?;
-
-        if !StatusCode::is_success(header.code) {
-            return DatanodeSnafu {
-                code: header.code,
-                msg: header.err_msg,
-            }
-            .fail();
-        }
-
-        let obj_result = obj_result.result.context(MissingResultSnafu {
-            name: "select_result".to_string(),
-            expected: 1_usize,
-            actual: 0_usize,
-        })?;
-
-        let result = match obj_result {
-            object_result::Result::Select(select) => {
-                let result = select
-                    .raw_data
-                    .deref()
-                    .try_into()
-                    .context(DecodeSelectSnafu)?;
-                ObjectResult::Select(result)
-            }
-            object_result::Result::Mutate(mutate) => ObjectResult::Mutate(mutate),
-        };
-
-        Ok(result)
+        obj_result.try_into()
     }
 
-    // TODO(jiachun) update/delete
-
-    async fn object(&self, expr: ObjectExpr) -> Result<GrpcObjectResult> {
+    pub async fn object(&self, expr: ObjectExpr) -> Result<GrpcObjectResult> {
         let res = self.objects(vec![expr]).await?.pop().unwrap();
         Ok(res)
     }
@@ -165,6 +144,186 @@ pub enum ObjectResult {
     Mutate(GrpcMutateResult),
 }
 
+impl TryFrom<api::v1::ObjectResult> for ObjectResult {
+    type Error = error::Error;
+
+    fn try_from(object_result: api::v1::ObjectResult) -> std::result::Result<Self, Self::Error> {
+        let header = object_result.header.context(error::MissingHeaderSnafu)?;
+        if !StatusCode::is_success(header.code) {
+            return DatanodeSnafu {
+                code: header.code,
+                msg: header.err_msg,
+            }
+            .fail();
+        }
+
+        let obj_result = object_result.result.context(error::MissingResultSnafu {
+            name: "result".to_string(),
+            expected: 1_usize,
+            actual: 0_usize,
+        })?;
+        Ok(match obj_result {
+            object_result::Result::Select(select) => {
+                let result = (*select.raw_data).try_into().context(DecodeSelectSnafu)?;
+                ObjectResult::Select(result)
+            }
+            object_result::Result::Mutate(mutate) => ObjectResult::Mutate(mutate),
+        })
+    }
+}
+
 pub enum Select {
     Sql(String),
 }
+
+impl TryFrom<ObjectResult> for Output {
+    type Error = error::Error;
+
+    fn try_from(value: ObjectResult) -> Result<Self> {
+        let output = match value {
+            ObjectResult::Select(select) => {
+                let vectors = select
+                    .columns
+                    .iter()
+                    .map(|column| {
+                        column_to_vector(column, select.row_count).context(ColumnToVectorSnafu)
+                    })
+                    .collect::<Result<Vec<VectorRef>>>()?;
+
+                let column_schemas = select
+                    .columns
+                    .iter()
+                    .zip(vectors.iter())
+                    .map(|(column, vector)| {
+                        let datatype = vector.data_type();
+                        // nullable or not, does not affect the output
+                        let mut column_schema =
+                            ColumnSchema::new(&column.column_name, datatype, true);
+                        if column.semantic_type == SemanticType::Timestamp as i32 {
+                            column_schema = column_schema.with_time_index(true);
+                        }
+                        column_schema
+                    })
+                    .collect::<Vec<ColumnSchema>>();
+
+                let schema = Arc::new(Schema::try_new(column_schemas).context(ConvertSchemaSnafu)?);
+                let recordbatches = if vectors.is_empty() {
+                    RecordBatches::try_new(schema, vec![])
+                } else {
+                    RecordBatch::new(schema, vectors)
+                        .and_then(|batch| RecordBatches::try_new(batch.schema.clone(), vec![batch]))
+                }
+                .context(error::CreateRecordBatchesSnafu)?;
+                Output::RecordBatches(recordbatches)
+            }
+            ObjectResult::Mutate(mutate) => {
+                if mutate.failure != 0 {
+                    return error::MutateFailureSnafu {
+                        failure: mutate.failure,
+                    }
+                    .fail();
+                }
+                Output::AffectedRows(mutate.success as usize)
+            }
+        };
+        Ok(output)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use api::helper::ColumnDataTypeWrapper;
+    use api::v1::Column;
+    use common_grpc::select::{null_mask, values};
+    use datatypes::vectors::{
+        BinaryVector, BooleanVector, DateTimeVector, DateVector, Float32Vector, Float64Vector,
+        Int16Vector, Int32Vector, Int64Vector, Int8Vector, StringVector, UInt16Vector,
+        UInt32Vector, UInt64Vector, UInt8Vector,
+    };
+
+    use super::*;
+
+    #[test]
+    fn test_column_to_vector() {
+        let mut column = create_test_column(Arc::new(BooleanVector::from(vec![true])));
+        column.datatype = -100;
+        let result = column_to_vector(&column, 1);
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Column datatype error, source: Unknown proto column datatype: -100"
+        );
+
+        macro_rules! test_with_vector {
+            ($vector: expr) => {
+                let vector = Arc::new($vector);
+                let column = create_test_column(vector.clone());
+                let result = column_to_vector(&column, vector.len() as u32).unwrap();
+                assert_eq!(result, vector as VectorRef);
+            };
+        }
+
+        test_with_vector!(BooleanVector::from(vec![Some(true), None, Some(false)]));
+        test_with_vector!(Int8Vector::from(vec![Some(i8::MIN), None, Some(i8::MAX)]));
+        test_with_vector!(Int16Vector::from(vec![
+            Some(i16::MIN),
+            None,
+            Some(i16::MAX)
+        ]));
+        test_with_vector!(Int32Vector::from(vec![
+            Some(i32::MIN),
+            None,
+            Some(i32::MAX)
+        ]));
+        test_with_vector!(Int64Vector::from(vec![
+            Some(i64::MIN),
+            None,
+            Some(i64::MAX)
+        ]));
+        test_with_vector!(UInt8Vector::from(vec![Some(u8::MIN), None, Some(u8::MAX)]));
+        test_with_vector!(UInt16Vector::from(vec![
+            Some(u16::MIN),
+            None,
+            Some(u16::MAX)
+        ]));
+        test_with_vector!(UInt32Vector::from(vec![
+            Some(u32::MIN),
+            None,
+            Some(u32::MAX)
+        ]));
+        test_with_vector!(UInt64Vector::from(vec![
+            Some(u64::MIN),
+            None,
+            Some(u64::MAX)
+        ]));
+        test_with_vector!(Float32Vector::from(vec![
+            Some(f32::MIN),
+            None,
+            Some(f32::MAX)
+        ]));
+        test_with_vector!(Float64Vector::from(vec![
+            Some(f64::MIN),
+            None,
+            Some(f64::MAX)
+        ]));
+        test_with_vector!(BinaryVector::from(vec![
+            Some(b"".to_vec()),
+            None,
+            Some(b"hello".to_vec())
+        ]));
+        test_with_vector!(StringVector::from(vec![Some(""), None, Some("foo"),]));
+        test_with_vector!(DateVector::from(vec![Some(1), None, Some(3)]));
+        test_with_vector!(DateTimeVector::from(vec![Some(4), None, Some(6)]));
+    }
+
+    fn create_test_column(vector: VectorRef) -> Column {
+        let wrapper: ColumnDataTypeWrapper = vector.data_type().try_into().unwrap();
+        Column {
+            column_name: "test".to_string(),
+            semantic_type: 1,
+            values: Some(values(&[vector.clone()]).unwrap()),
+            null_mask: null_mask(&[vector.clone()], vector.len()),
+            datatype: wrapper.datatype() as i32,
+        }
+    }
+}