chore: add build_table_route_prefix (#1269)

* test: Add compaction test

* test: Test read during compaction

* test: Add s3 object store to test

* test: only run compact test

* feat: Hold file handle in chunk stream

* test: check files still exist after compact

* feat: Revert changes to develop.yaml

* test: Simplify MockPurgeHandler

.cargo/config.toml

@@ -0,0 +1,5 @@
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"
+
+[alias]
+sqlness = "run --bin sqlness-runner --"

.config/nextest.toml

@@ -0,0 +1,2 @@
+[profile.default]
+slow-timeout = { period = "60s", terminate-after = 3, grace-period = "30s" }

.env.example

@@ -0,0 +1,10 @@
+# Settings for s3 test
+GT_S3_BUCKET=S3 bucket
+GT_S3_ACCESS_KEY_ID=S3 access key id
+GT_S3_ACCESS_KEY=S3 secret access key
+GT_S3_ENDPOINT_URL=S3 endpoint url
+# Settings for oss test
+GT_OSS_BUCKET=OSS bucket
+GT_OSS_ACCESS_KEY_ID=OSS access key id
+GT_OSS_ACCESS_KEY=OSS access key
+GT_OSS_ENDPOINT=OSS endpoint

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,86 @@
+---
+name: Bug report
+description: Is something not working? Help us fix it!
+labels: [ "bug" ]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Take some time to fill out this bug report. Thank you!
+
+  - type: dropdown
+    id: type
+    attributes:
+      label: What type of bug is this?
+      multiple: true
+      options:
+        - Configuration
+        - Crash
+        - Data corruption
+        - Incorrect result
+        - Locking issue
+        - Performance issue
+        - Unexpected error
+        - Other
+    validations:
+      required: true
+
+  - type: dropdown
+    id: subsystem
+    attributes:
+      label: What subsystems are affected?
+      description: You can pick multiple subsystems.
+      multiple: true
+      options:
+        - Standalone mode
+        - Frontend
+        - Datanode
+        - Meta
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: |
+        Tell us what happened and also what you would have expected to
+        happen instead.
+      placeholder: "Describe the bug"
+    validations:
+      required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: What operating system did you use?
+      description: |
+        Please provide OS, version, and architecture. For example:
+        Windows 10 x64, Ubuntu 21.04 x64, Mac OS X 10.5 ARM, Rasperry
+        Pi i386, etc.
+      placeholder: "Ubuntu 21.04 x64"
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output and stack trace
+      description: |
+        Please copy and paste any relevant log output or a stack
+        trace. This will be automatically formatted into code, so no
+        need for backticks.
+      render: bash
+
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: How can we reproduce the bug?
+      description: |
+        Please walk us through and provide steps and details on how
+        to reproduce the issue. If possible, provide scripts that we
+        can run to trigger the bug.
+      render: bash
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Greptime Community Slack
+    url: https://greptime.com/slack
+    about: Get free help from the Greptime community
+  - name: Greptime Community Discussion
+    url: https://github.com/greptimeTeam/greptimedb/discussions
+    about: Get free help from the Greptime community

.github/ISSUE_TEMPLATE/enhancement.yml

@@ -0,0 +1,39 @@
+---
+name: Enhancement
+description: Suggest an enhancement to existing functionality
+labels: [ "enhancement" ]
+body:
+  - type: dropdown
+    id: type
+    attributes:
+      label: What type of enhancement is this?
+      multiple: true
+      options:
+        - API improvement
+        - Configuration
+        - Performance
+        - Refactor
+        - Tech debt reduction
+        - User experience
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: what
+    attributes:
+      label: What does the enhancement do?
+      description: |
+        Give a high-level overview of how you
+        suggest improving an existing feature or functionality.
+    validations:
+     required: true
+
+  - type: textarea
+    id: implementation
+    attributes:
+      label: Implementation challenges
+      description: |
+        Share any ideas of how to implement the enhancement.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,42 @@
+---
+name: Feature request
+description: Suggest a new feature for GreptimeDB
+labels: [ "feature request" ]
+body:
+  - type: markdown
+    id: info
+    attributes:
+      value: |
+        Only use this template to suggest a new feature that doesn't already exist in GreptimeDB.
+        For enhancements to existing features, use the "Enhancement" issue template. For bugs,
+        use the bug report template.
+
+  - type: textarea
+    id: what
+    attributes:
+      label: What problem does the new feature solve?
+      description: |
+        Describe the problem and why it is important to solve. Did you consider alternative
+        solutions, perhaps outside the database? Why is it better to add the feature to
+        GreptimeDB?
+    validations:
+      required: true
+
+  - type: textarea
+    id: how
+    attributes:
+      label: What does the feature do?
+      description: |
+        Give a high-level overview of what the feature does and how it would work.
+    validations:
+     required: true
+
+  - type: textarea
+    id: implementation
+    attributes:
+      label: Implementation challenges
+      description: |
+        If you have ideas of how to implement the feature, and any particularly
+        challenging issues to overcome, then provide them here.
+    validations:
+      required: false

.github/pr-title-breaking-change-label-config.json

@@ -0,0 +1,13 @@
+{
+    "LABEL": {
+        "name": "breaking change",
+        "color": "D93F0B"
+    },
+    "CHECKS": {
+        "regexp": "^(?:(?!!:).)*$",
+        "ignoreLabels": [
+            "ignore-title"
+        ],
+        "alwaysPassCI": true
+    }
+}

.github/pr-title-checker-config.json

@@ -1,10 +1,12 @@
 {
-  "LABEL": {
-    "name": "Invalid PR Title",
-    "color": "B60205"
-  },
-  "CHECKS": {
-    "regexp": "^(feat|fix|test|refactor|chore|style|doc|perf|build|ci|revert)(\\(.*\\))?:.*",
-    "ignoreLabels" : ["ignore-title"]
-  }
+    "LABEL": {
+        "name": "Invalid PR Title",
+        "color": "B60205"
+    },
+    "CHECKS": {
+        "regexp": "^(feat|fix|test|refactor|chore|style|docs|perf|build|ci|revert)(\\(.*\\))?\\!?:.*",
+        "ignoreLabels": [
+            "ignore-title"
+        ]
+    }
 }

.github/pull_request_template.md

@@ -0,0 +1,19 @@
+I hereby agree to the terms of the [GreptimeDB CLA](https://gist.github.com/xtang/6378857777706e568c1949c7578592cc)
+
+## What's changed and what's your intention?
+
+_PLEASE DO NOT LEAVE THIS EMPTY !!!_
+
+Please explain IN DETAIL what the changes are in this PR and why they are needed:
+
+- Summarize your change (**mandatory**)
+- How does this PR work? Need a brief introduction for the changed logic (optional)
+- Describe clearly one logical change and avoid lazy messages (optional)
+- Describe any limitations of the current code (optional)
+
+## Checklist
+
+- [ ]  I have written the necessary rustdoc comments.
+- [ ]  I have added the necessary unit tests and integration tests.
+
+## Refer to a related PR or issue link (optional)

.github/workflows/apidoc.yml

@@ -0,0 +1,42 @@
+on:
+  push:
+    branches:
+      - develop
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
+
+name: Build API docs
+
+env:
+  RUST_TOOLCHAIN: nightly-2023-02-26
+
+jobs:
+  apidoc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: arduino/setup-protoc@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ env.RUST_TOOLCHAIN }}
+    - run: cargo doc --workspace --no-deps --document-private-items
+    - run: |
+        cat <<EOF > target/doc/index.html
+        <!DOCTYPE html>
+        <html>
+        <head>
+        <meta http-equiv="refresh" content="0; url='greptime/'" />
+        </head>
+        <body></body></html>
+        EOF
+    - name: Publish dist directory
+      uses: JamesIves/github-pages-deploy-action@v4
+      with:
+        folder: target/doc

.github/workflows/coverage.yml

@@ -1,56 +0,0 @@
-on:
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
-  push:
-    branches:
-      - "main"
-      - "develop"
-
-name: Code coverage
-
-env:
-  RUST_TOOLCHAIN: nightly-2022-07-14
-
-jobs:
-  grcov:
-    if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: arduino/setup-protoc@v1
-      - name: Install toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
-          profile: minimal
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2.0.0
-      - name: Cleanup disk
-        uses: curoky/cleanup-disk-action@v2.0
-        with:
-          retain: 'rust'
-      - name: Execute tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace
-        env:
-          RUST_BACKTRACE: 1
-          CARGO_INCREMENTAL: 0
-          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=unwind -Zpanic_abort_tests"
-          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-          UNITTEST_LOG_DIR: "__unittest_logs"
-      - name: Gather coverage data
-        id: coverage
-        uses: actions-rs/grcov@v0.1
-      - name: Codecov upload
-        uses: codecov/codecov-action@v2
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./lcov.info
-          flags: rust
-          fail_ci_if_error: true
-          verbose: true

.github/workflows/develop.yml

@@ -1,93 +1,235 @@
 on:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
+  push:
+    branches:
+      - develop
+      - main
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
+  workflow_dispatch:
 
-name: Continuous integration for developing
+name: CI
 
 env:
-  RUST_TOOLCHAIN: nightly-2022-07-14
+  RUST_TOOLCHAIN: nightly-2023-02-26
 
 jobs:
+  typos:
+    name: Spell Check with Typos
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: crate-ci/typos@v1.13.10
+
   check:
     name: Check
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
-      - uses: actions-rs/toolchain@v1
         with:
-          profile: minimal
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@master
+        with:
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
       - name: Rust Cache
-        uses: Swatinem/rust-cache@v2.0.0
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --workspace --all-targets
+        uses: Swatinem/rust-cache@v2
+      - name: Run cargo check
+        run: cargo check --workspace --all-targets
 
-  test:
-    name: Test Suite
+  toml:
+    name: Toml Check
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
-      - uses: arduino/setup-protoc@v1
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@master
         with:
-          profile: minimal
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
       - name: Rust Cache
-        uses: Swatinem/rust-cache@v2.0.0
-      - uses: actions-rs/cargo@v1
+        uses: Swatinem/rust-cache@v2
+      - name: Install taplo
+        run: cargo install taplo-cli --version ^0.8 --locked
+      - name: Run taplo
+        run: taplo format --check --option "indent_string=    "
+
+  # Use coverage to run test.
+  # test:
+  #   name: Test Suite
+  #   if: github.event.pull_request.draft == false
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 60
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - name: Cache LLVM and Clang
+  #       id: cache-llvm
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: ./llvm
+  #         key: llvm
+  #     - uses: arduino/setup-protoc@v1
+  #       with:
+  #         repo-token: ${{ secrets.GITHUB_TOKEN }}
+  #     - uses: KyleMayes/install-llvm-action@v1
+  #       with:
+  #         version: "14.0"
+  #         cached: ${{ steps.cache-llvm.outputs.cache-hit }}
+  #     - uses: dtolnay/rust-toolchain@master
+  #       with:
+  #         toolchain: ${{ env.RUST_TOOLCHAIN }}
+  #     - name: Rust Cache
+  #       uses: Swatinem/rust-cache@v2
+  #     - name: Cleanup disk
+  #       uses: curoky/cleanup-disk-action@v2.0
+  #       with:
+  #         retain: 'rust,llvm'
+  #     - name: Install latest nextest release
+  #       uses: taiki-e/install-action@nextest
+  #     - name: Run tests
+  #       run: cargo nextest run
+  #       env:
+  #         CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
+  #         RUST_BACKTRACE: 1
+  #         GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+  #         GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+  #         GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+  #         UNITTEST_LOG_DIR: "__unittest_logs"
+
+  sqlness:
+    name: Sqlness Test
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest-8-cores
+    timeout-minutes: 60
+    needs: [clippy]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: arduino/setup-protoc@v1
         with:
-          command: test
-          args: --workspace
-        env:
-          RUST_BACKTRACE: 1
-          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-          UNITTEST_LOG_DIR: "__unittest_logs"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Run etcd
+        run: |
+          ETCD_VER=v3.5.7
+          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
+          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+          mkdir -p /tmp/etcd-download
+          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1
+          rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+
+          sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/
+          nohup etcd >/tmp/etcd.log 2>&1 &
+      - name: Run sqlness
+        run: cargo sqlness && ls /tmp
+      - name: Upload sqlness logs
+        uses: actions/upload-artifact@v3
+        with:
+          name: sqlness-logs
+          path: /tmp/greptime-*.log
+          retention-days: 3
 
   fmt:
     name: Rustfmt
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
-      - uses: actions-rs/toolchain@v1
         with:
-          profile: minimal
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@master
+        with:
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
+          components: rustfmt
       - name: Rust Cache
-        uses: Swatinem/rust-cache@v2.0.0
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
+        uses: Swatinem/rust-cache@v2
+      - name: Run cargo fmt
+        run: cargo fmt --all -- --check
 
   clippy:
     name: Clippy
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
-      - uses: actions-rs/toolchain@v1
         with:
-          profile: minimal
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@master
+        with:
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
+          components: clippy
       - name: Rust Cache
-        uses: Swatinem/rust-cache@v2.0.0
-      - run: rustup component add clippy
-      - uses: actions-rs/cargo@v1
+        uses: Swatinem/rust-cache@v2
+      - name: Run cargo clippy
+        run: cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr
+
+  coverage:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest-8-cores
+    timeout-minutes: 60
+    needs: [clippy]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: arduino/setup-protoc@v1
         with:
-          command: clippy
-          args: --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: KyleMayes/install-llvm-action@v1
+        with:
+          version: "14.0"
+      - name: Install toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: llvm-tools-preview
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install latest nextest release
+        uses: taiki-e/install-action@nextest
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install PyArrow Package
+        run: pip install pyarrow
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - name: Collect coverage data
+        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend
+        env:
+          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
+          RUST_BACKTRACE: 1
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          UNITTEST_LOG_DIR: "__unittest_logs"
+      - name: Codecov upload
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./lcov.info
+          flags: rust
+          fail_ci_if_error: false
+          verbose: true

.github/workflows/doc-issue.yml

@@ -0,0 +1,39 @@
+name: Create Issue in downstream repos
+
+on:
+  issues:
+    types:
+      - labeled
+  pull_request_target:
+    types:
+      - labeled
+
+jobs:
+  doc_issue:
+    if: github.event.label.name == 'doc update required'
+    runs-on: ubuntu-latest
+    steps:
+      - name: create an issue in doc repo
+        uses: dacbd/create-issue-action@main
+        with:
+          owner: GreptimeTeam
+          repo: docs
+          token: ${{ secrets.DOCS_REPO_TOKEN }}
+          title: Update docs for ${{ github.event.issue.title || github.event.pull_request.title }}
+          body: |
+            A document change request is generated from
+            ${{ github.event.issue.html_url || github.event.pull_request.html_url }}
+  cloud_issue:
+    if: github.event.label.name == 'cloud followup required'
+    runs-on: ubuntu-latest
+    steps:
+      - name: create an issue in cloud repo
+        uses: dacbd/create-issue-action@main
+        with:
+          owner: GreptimeTeam
+          repo: greptimedb-cloud
+          token: ${{ secrets.DOCS_REPO_TOKEN }}
+          title: Followup changes in ${{ github.event.issue.title || github.event.pull_request.title }}
+          body: |
+            A followup request is generated from
+            ${{ github.event.issue.html_url || github.event.pull_request.html_url }}

.github/workflows/docs.yml

@@ -0,0 +1,55 @@
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
+  push:
+    branches:
+      - develop
+      - main
+    paths:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
+  workflow_dispatch:
+
+name: CI
+
+# To pass the required status check, see:
+# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
+
+jobs:
+  check:
+    name: Check
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No action required"'
+
+  fmt:
+    name: Rustfmt
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No action required"'
+
+  clippy:
+    name: Clippy
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No action required"'
+
+  coverage:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No action required"'

.github/workflows/license.yaml

@@ -0,0 +1,16 @@
+name: License checker
+
+on:
+  push:
+    branches:
+    - develop
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+jobs:
+  license-header-check:
+    runs-on: ubuntu-latest
+    name: license-header-check
+    steps:
+    - uses: actions/checkout@v2
+    - name: Check License Header
+      uses: apache/skywalking-eyes/header@df70871af1a8109c9a5b1dc824faaf65246c5236

.github/workflows/pr-title-checker.yml

@@ -11,9 +11,19 @@ on:
 jobs:
   check:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - uses: thehanimo/pr-title-checker@v1.3.4
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           pass_on_octokit_error: false
           configuration_path: ".github/pr-title-checker-config.json"
+  breaking:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: thehanimo/pr-title-checker@v1.3.4
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          pass_on_octokit_error: false
+          configuration_path: ".github/pr-title-breaking-change-label-config.json"

.github/workflows/release.yml

@@ -0,0 +1,384 @@
+on:
+  push:
+    tags:
+      - "v*.*.*"
+  schedule:
+    # At 00:00 on Monday.
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
+
+name: Release
+
+env:
+  RUST_TOOLCHAIN: nightly-2023-02-26
+
+  SCHEDULED_BUILD_VERSION_PREFIX: v0.2.0
+
+  SCHEDULED_PERIOD: nightly
+
+  CARGO_PROFILE: nightly
+
+  ## FIXME(zyy17): Enable it after the tests are stabled.
+  DISABLE_RUN_TESTS: true
+
+jobs:
+  build:
+    name: Build binary
+    strategy:
+      matrix:
+        # The file format is greptime-<os>-<arch>
+        include:
+          - arch: x86_64-unknown-linux-gnu
+            os: ubuntu-2004-16-cores
+            file: greptime-linux-amd64
+            continue-on-error: false
+          - arch: aarch64-unknown-linux-gnu
+            os: ubuntu-2004-16-cores
+            file: greptime-linux-arm64
+            continue-on-error: false
+          - arch: aarch64-apple-darwin
+            os: macos-latest
+            file: greptime-darwin-arm64
+            continue-on-error: false
+          - arch: x86_64-apple-darwin
+            os: macos-latest
+            file: greptime-darwin-amd64
+            continue-on-error: false
+          - arch: x86_64-unknown-linux-gnu
+            os: ubuntu-2004-16-cores
+            file: greptime-linux-amd64-pyo3
+            continue-on-error: false
+            opts: "-F pyo3_backend"
+          - arch: aarch64-unknown-linux-gnu
+            os: ubuntu-2004-16-cores
+            file: greptime-linux-arm64-pyo3
+            continue-on-error: false
+            opts: "-F pyo3_backend"
+          - arch: aarch64-apple-darwin
+            os: macos-latest
+            file: greptime-darwin-arm64-pyo3
+            continue-on-error: false
+            opts: "-F pyo3_backend"
+          - arch: x86_64-apple-darwin
+            os: macos-latest
+            file: greptime-darwin-amd64-pyo3
+            continue-on-error: false
+            opts: "-F pyo3_backend"
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.continue-on-error }}
+    if: github.repository == 'GreptimeTeam/greptimedb'
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Cache cargo assets
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ matrix.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install Protoc for linux
+        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
+        run: | # Make sure the protoc is >= 3.15
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip
+          unzip protoc-21.9-linux-x86_64.zip -d protoc
+          sudo cp protoc/bin/protoc /usr/local/bin/
+          sudo cp -r protoc/include/google /usr/local/include/
+
+      - name: Install Protoc for macos
+        if: contains(matrix.arch, 'darwin')
+        run: |
+          brew install protobuf
+
+      - name: Install etcd for linux
+        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
+        run: |
+          ETCD_VER=v3.5.7
+          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
+          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+          mkdir -p /tmp/etcd-download
+          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1
+          rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+
+          sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/
+          nohup etcd >/tmp/etcd.log 2>&1 &
+
+      - name: Install etcd for macos
+        if: contains(matrix.arch, 'darwin')
+        run: |
+          brew install etcd
+          brew services start etcd
+
+      - name: Install dependencies for linux
+        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
+        run: |
+          sudo apt-get -y update
+          sudo apt-get -y install libssl-dev pkg-config g++-aarch64-linux-gnu gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu wget
+
+      # FIXME(zyy17): Should we specify the version of python when building binary for darwin?
+      - name: Compile Python 3.10.10 from source for linux
+        if: contains(matrix.arch, 'linux') && contains(matrix.opts, 'pyo3_backend')
+        run: |
+          sudo chmod +x ./docker/aarch64/compile-python.sh
+          sudo ./docker/aarch64/compile-python.sh ${{ matrix.arch }}
+
+      - name: Install rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          targets: ${{ matrix.arch }}
+
+      - name: Output package versions
+        run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version
+
+      - name: Run tests
+        if: env.DISABLE_RUN_TESTS == 'false'
+        run: make unit-test integration-test sqlness-test
+
+      - name: Run cargo build with pyo3 for aarch64-linux
+        if: contains(matrix.arch, 'aarch64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend')
+        run: |
+          # TODO(zyy17): We should make PYO3_CROSS_LIB_DIR configurable.
+          export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64
+          export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
+          export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
+          export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
+
+          export PYO3_CROSS_LIB_DIR=${PWD}/python-3.10.10/aarch64
+          echo "PYO3_CROSS_LIB_DIR: $PYO3_CROSS_LIB_DIR"
+          alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3
+          alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip
+
+          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
+
+      - name: Run cargo build with pyo3 for amd64-linux
+        if: contains(matrix.arch, 'x86_64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend')
+        run: |
+          export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64
+          export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
+          export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
+          export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
+          
+          echo "implementation=CPython" >> pyo3.config
+          echo "version=3.10" >> pyo3.config
+          echo "implementation=CPython" >> pyo3.config
+          echo "shared=true" >> pyo3.config
+          echo "abi3=true" >> pyo3.config
+          echo "lib_name=python3.10" >> pyo3.config
+          echo "lib_dir=$PYTHON_INSTALL_PATH_AMD64/lib" >> pyo3.config
+          echo "executable=$PYTHON_INSTALL_PATH_AMD64/bin/python3" >> pyo3.config
+          echo "pointer_width=64" >> pyo3.config
+          echo "build_flags=" >> pyo3.config
+          echo "suppress_build_script_link_lines=false" >> pyo3.config
+
+          cat pyo3.config
+          export PYO3_CONFIG_FILE=${PWD}/pyo3.config
+          alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3
+          alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip
+
+          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
+
+      - name: Run cargo build
+        if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false
+        run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
+
+      - name: Calculate checksum and rename binary
+        shell: bash
+        run: |
+          cd target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}
+          chmod +x greptime
+          tar -zcvf ${{ matrix.file }}.tgz greptime
+          echo $(shasum -a 256 ${{ matrix.file }}.tgz | cut -f1 -d' ') > ${{ matrix.file }}.sha256sum
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.file }}
+          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.tgz
+
+      - name: Upload checksum of artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.file }}.sha256sum
+          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.sha256sum
+
+  docker:
+    name: Build docker image
+    needs: [build]
+    runs-on: ubuntu-latest
+    if: github.repository == 'GreptimeTeam/greptimedb'
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Login to Dockerhub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
+        shell: bash
+        if: github.event_name == 'schedule'
+        run: |
+          buildTime=`date "+%Y%m%d"`
+          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
+          echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV
+
+      - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
+        shell: bash
+        if: github.event_name != 'schedule'
+        run: |
+          VERSION=${{ github.ref_name }}
+          echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Download amd64 binary
+        uses: actions/download-artifact@v3
+        with:
+          name: greptime-linux-amd64-pyo3
+          path: amd64
+
+      - name: Unzip the amd64 artifacts
+        run: |
+          tar xvf amd64/greptime-linux-amd64-pyo3.tgz -C amd64/ && rm amd64/greptime-linux-amd64-pyo3.tgz
+          cp -r amd64 docker/ci
+
+      - name: Download arm64 binary
+        id: download-arm64
+        uses: actions/download-artifact@v3
+        with:
+          name: greptime-linux-arm64-pyo3
+          path: arm64
+
+      - name: Unzip the arm64 artifacts
+        id: unzip-arm64
+        if: success() || steps.download-arm64.conclusion == 'success'
+        run: |
+          tar xvf arm64/greptime-linux-arm64-pyo3.tgz -C arm64/ && rm arm64/greptime-linux-arm64-pyo3.tgz
+          cp -r arm64 docker/ci
+
+      - name: Build and push all
+        uses: docker/build-push-action@v3
+        if: success() || steps.unzip-arm64.conclusion == 'success' # Build and push all platform if unzip-arm64 succeeds
+        with:
+          context: ./docker/ci/
+          file: ./docker/ci/Dockerfile
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            greptime/greptimedb:latest
+            greptime/greptimedb:${{ env.IMAGE_TAG }}
+
+      - name: Build and push amd64 only
+        uses: docker/build-push-action@v3
+        if: success() || steps.download-arm64.conclusion == 'failure' # Only build and push amd64 platform if download-arm64 fails
+        with:
+          context: ./docker/ci/
+          file: ./docker/ci/Dockerfile
+          push: true
+          platforms: linux/amd64
+          tags: |
+            greptime/greptimedb:latest
+            greptime/greptimedb:${{ env.IMAGE_TAG }}
+
+  release:
+    name: Release artifacts
+    # Release artifacts only when all the artifacts are built successfully.
+    needs: [build,docker]
+    runs-on: ubuntu-latest
+    if: github.repository == 'GreptimeTeam/greptimedb'
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Configure scheduled build version # the version would be ${SCHEDULED_BUILD_VERSION_PREFIX}-${SCHEDULED_PERIOD}-YYYYMMDD, like v0.2.0-nigthly-20230313.
+        shell: bash
+        if: github.event_name == 'schedule'
+        run: |
+          buildTime=`date "+%Y%m%d"`
+          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-${{ env.SCHEDULED_PERIOD }}-$buildTime
+          echo "SCHEDULED_BUILD_VERSION=${SCHEDULED_BUILD_VERSION}" >> $GITHUB_ENV
+
+      - name: Create scheduled build git tag
+        if: github.event_name == 'schedule'
+        run: |
+          git tag ${{ env.SCHEDULED_BUILD_VERSION }}
+
+      - name: Publish scheduled release # configure the different release title and tags.
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'schedule'
+        with:
+          name: "Release ${{ env.SCHEDULED_BUILD_VERSION }}"
+          tag_name: ${{ env.SCHEDULED_BUILD_VERSION }}
+          generate_release_notes: true
+          files: |
+            **/greptime-*
+
+      - name: Publish release
+        uses: softprops/action-gh-release@v1
+        if: github.event_name != 'schedule'
+        with:
+          name: "Release ${{ github.ref_name }}"
+          files: |
+            **/greptime-*
+
+  docker-push-uhub:
+    name: Push docker image to UCloud Container Registry
+    needs: [docker]
+    runs-on: ubuntu-latest
+    if: github.repository == 'GreptimeTeam/greptimedb'
+    # Push to uhub may fail(500 error), but we don't want to block the release process. The failed job will be retried manually.
+    continue-on-error: true
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to UCloud Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: uhub.service.ucloud.cn
+          username: ${{ secrets.UCLOUD_USERNAME }}
+          password: ${{ secrets.UCLOUD_PASSWORD }}
+
+      - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
+        shell: bash
+        if: github.event_name == 'schedule'
+        run: |
+          buildTime=`date "+%Y%m%d"`
+          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
+          echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV
+
+      - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
+        shell: bash
+        if: github.event_name != 'schedule'
+        run: |
+          VERSION=${{ github.ref_name }}
+          echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV
+
+      - name: Push image to uhub # Use 'docker buildx imagetools create' to create a new image base on source image.
+        run: |
+          docker buildx imagetools create \
+            --tag uhub.service.ucloud.cn/greptime/greptimedb:latest \
+            --tag uhub.service.ucloud.cn/greptime/greptimedb:${{ env.IMAGE_TAG }} \
+            greptime/greptimedb:${{ env.IMAGE_TAG }}

.gitignore

@@ -18,13 +18,24 @@ debug/
 
 # JetBrains IDE config directory
 .idea/
+*.iml
+
+# VSCode IDE config directory
+.vscode/
 
 # Logs
 **/__unittest_logs
 logs/
 
-.DS_store
-.gitignore
-
 # cpython's generated python byte code
 **/__pycache__/
+
+# Benchmark dataset
+benchmarks/data
+
+# dotenv
+.env
+
+# dashboard files
+!/src/servers/dashboard/VERSION
+/src/servers/dashboard/*

.licenserc.yaml

@@ -0,0 +1,14 @@
+header:
+  license:
+    spdx-id: Apache-2.0
+    copyright-owner: Greptime Team
+
+  paths:
+    - "**/*.rs"
+    - "**/*.py"
+
+  comment: on-failure
+
+dependency:
+  files:
+    - Cargo.toml

.pre-commit-config.yaml

@@ -5,11 +5,11 @@ repos:
     -    id: conventional-pre-commit
          stages: [commit-msg]
 
--   repo: https://github.com/DevinR528/cargo-sort
-    rev: e6a795bc6b2c0958f9ef52af4863bbd7cc17238f
-    hooks:
-    -    id: cargo-sort
-         args: ["--workspace"]
+# -   repo: https://github.com/DevinR528/cargo-sort
+#     rev: e6a795bc6b2c0958f9ef52af4863bbd7cc17238f
+#     hooks:
+#     -    id: cargo-sort
+#          args: ["--workspace"]
 
 -   repo: https://github.com/doublify/pre-commit-rust
     rev: v1.0

CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+info@greptime.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -1,23 +1,91 @@
-# Contributing to GreptimeDB
+# Welcome 👋
 
-Much appreciate for your interest in contributing to GreptimeDB! This document list some guidelines for contributing to our code base.
+Thanks a lot for considering contributing to GreptimeDB. We believe people like you would make GreptimeDB a great product. We intend to build a community where individuals can have open talks, show respect for one another, and speak with true ❤️. Meanwhile, we are to keep transparency and make your effort count here.
 
-To learn about the design of GreptimeDB, please refer to the [design docs](https://github.com/GrepTimeTeam/docs).
+Read the guidelines, and they can help you get started. Communicate with respect to developers maintaining and developing the project. In return, they should reciprocate that respect by addressing your issue, reviewing changes, as well as helping finalize and merge your pull requests.
 
-## Pull Requests
+Follow our [README](https://github.com/GreptimeTeam/greptimedb#readme) to get the whole picture of the project. To learn about the design of GreptimeDB, please refer to the [design docs](https://github.com/GrepTimeTeam/docs).
+
+## Your First Contribution
+
+It can feel intimidating to contribute to a complex project, but it can also be exciting and fun. These general notes will help everyone participate in this communal activity.
+
+- Follow the [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/develop/CODE_OF_CONDUCT.md)
+- Small changes make huge differences. We will happily accept a PR making a single character change if it helps move forward. Don't wait to have everything working.
+- Check the closed issues before opening your issue.
+- Try to follow the existing style of the code.
+- More importantly, when in doubt, ask away.
+
+Pull requests are great, but we accept all kinds of other help if you like. Such as
+
+- Write tutorials or blog posts. Blog, speak about, or create tutorials about one of GreptimeDB's many features. Mention [@greptime](https://twitter.com/greptime) on Twitter and email info@greptime.com so we can give pointers and tips and help you spread the word by promoting your content on Greptime communication channels.
+- Improve the documentation. [Submit documentation](http://github.com/greptimeTeam/docs/) updates, enhancements, designs, or bug fixes, and fixing any spelling or grammar errors will be very much appreciated.
+- Present at meetups and conferences about your GreptimeDB projects. Your unique challenges and successes in building things with GreptimeDB can provide great speaking material. We'd love to review your talk abstract, so get in touch with us if you'd like some help!
+- Submit bug reports. To report a bug or a security issue, you can [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new).
+- Speak up feature requests. Send feedback is a great way for us to understand your different use cases of GreptimeDB better. If you want to share your experience with GreptimeDB, or if you want to discuss any ideas, you can start a discussion on [GitHub discussions](https://github.com/GreptimeTeam/greptimedb/discussions), chat with the Greptime team on [Slack](https://greptime.com/slack), or you can tweet [@greptime](https://twitter.com/greptime) on Twitter.
+
+## Code of Conduct
+
+Also, there are things that we are not looking for because they don't match the goals of the product or benefit the community. Please read [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/develop/CODE_OF_CONDUCT.md); we hope everyone can keep good manners and become an honored member.
+
+## License
+
+GreptimeDB uses the [Apache 2.0 license](https://github.com/GreptimeTeam/greptimedb/blob/master/LICENSE) to strike a balance between open contributions and allowing you to use the software however you want.
+
+## Getting Started
+
+### Submitting Issues
+
+- Check if an issue already exists. Before filing an issue report, see whether it's already covered. Use the search bar and check out existing issues.
+- File an issue:
+  - To report a bug, a security issue, or anything that you think is a problem and that isn't under the radar, go ahead and [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new).
+  - In the given templates, look for the one that suits you.
+  - If you bump into anything, reach out to our [Slack](https://greptime.com/slack) for a wider audience and ask for help.
+- What happens after:
+  - Once we spot a new issue, we identify and categorize it as soon as possible.
+  - Usually, it gets assigned to other developers. Follow up and see what folks are talking about and how they take care of it.
+  - Please be patient and offer as much information as you can to help reach a solution or a consensus. You are not alone and embrace team power.
 
 ### Before PR
-- Make sure all unit tests are passed.
+
+- To ensure that community is free and confident in its ability to use your contributions, please sign the Contributor License Agreement (CLA) which will be incorporated in the pull request process.
+- Make sure all your codes are formatted and follow the [coding style](https://pingcap.github.io/style-guide/rust/).
+- Make sure all unit tests are passed (using `cargo test --workspace` or [nextest](https://nexte.st/index.html) `cargo nextest run`).
 - Make sure all clippy warnings are fixed (you can check it locally by running `cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr`).
 
+#### `pre-commit` Hooks
+
+You could setup the [`pre-commit`](https://pre-commit.com/#plugins) hooks to run these checks on every commit automatically.
+
+1. Install `pre-commit`
+
+        pip install pre-commit
+
+    or
+
+        brew install pre-commit
+
+2. Install the `pre-commit` hooks
+
+        $ pre-commit install
+        pre-commit installed at .git/hooks/pre-commit
+
+        $ pre-commit install --hook-type commit-msg
+        pre-commit installed at .git/hooks/commit-msg
+
+        $ pre-commit install --hook-type pre-push
+        pre-commit installed at .git/hooks/pre-push
+
+Now, `pre-commit` will run automatically on `git commit`.
+
 ### Title
 
-The titles of pull requests should be prefixed with category name listed in [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0) 
-like `feat`/`fix`/`doc`, with a concise summary of code change follows. DO NOT use last commit message as pull request title.
+The titles of pull requests should be prefixed with category names listed in [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0)
+like `feat`/`fix`/`docs`, with a concise summary of code change following. DO NOT use last commit message as pull request title.
 
 ### Description
 
-- If your pull request is small, like a typo fix, feel free to go brief.
+- Feel free to go brief if your pull request is small, like a typo fix.
 - But if it contains large code change, make sure to state the motivation/design details of this PR so that reviewers can understand what you're trying to do.
 - If the PR contains any breaking change or API change, make sure that is clearly listed in your description.
 
@@ -25,11 +93,20 @@ like `feat`/`fix`/`doc`, with a concise summary of code change follows. DO NOT u
 
 All commit messages SHOULD adhere to the [Conventional Commits specification](https://conventionalcommits.org/).
 
-## Getting help
+## Getting Help
 
 There are many ways to get help when you're stuck. It is recommended to ask for help by opening an issue, with a detailed description
-of what you were trying to do and what went wrong. You can also reach for help in our Slack channel.
+of what you were trying to do and what went wrong. You can also reach for help in our [Slack channel](https://greptime.com/slack).
 
+## Community
 
-## Bug report
-To report a bug or a security issue, you can [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new). 
+The core team will be thrilled if you participate in any way you like. When you are stuck, try ask for help by filing an issue, with a detailed description of what you were trying to do and what went wrong. If you have any questions or if you would like to get involved in our community, please check out:
+
+- [GreptimeDB Community Slack](https://greptime.com/slack)
+- [GreptimeDB Github Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)
+
+Also, see some extra GreptimeDB content:
+
+- [GreptimeDB Docs](https://greptime.com/docs)
+- [Learn GreptimeDB](https://greptime.com/products/db)
+- [Greptime Inc. Website](https://greptime.com)

Cargo.toml

@@ -1,32 +1,91 @@
 [workspace]
 members = [
+    "benchmarks",
     "src/api",
     "src/catalog",
     "src/client",
     "src/cmd",
     "src/common/base",
+    "src/common/catalog",
+    "src/common/datasource",
     "src/common/error",
     "src/common/function",
     "src/common/function-macro",
     "src/common/grpc",
+    "src/common/grpc-expr",
+    "src/common/mem-prof",
+    "src/common/procedure",
     "src/common/query",
     "src/common/recordbatch",
     "src/common/runtime",
+    "src/common/substrait",
     "src/common/telemetry",
+    "src/common/test-util",
     "src/common/time",
     "src/datanode",
     "src/datatypes",
     "src/frontend",
     "src/log-store",
-    "src/logical-plans",
+    "src/meta-client",
+    "src/meta-srv",
+    "src/mito",
     "src/object-store",
+    "src/partition",
+    "src/promql",
     "src/query",
     "src/script",
     "src/servers",
+    "src/session",
     "src/sql",
     "src/storage",
     "src/store-api",
     "src/table",
-    "src/table-engine",
-    "test-util",
+    "src/table-procedure",
+    "tests-integration",
+    "tests/runner",
 ]
+
+[workspace.package]
+version = "0.1.1"
+edition = "2021"
+license = "Apache-2.0"
+
+[workspace.dependencies]
+arrow = { version = "34.0" }
+arrow-array = "34.0"
+arrow-flight = "34.0"
+arrow-schema = { version = "34.0", features = ["serde"] }
+async-stream = "0.3"
+async-trait = "0.1"
+chrono = { version = "0.4", features = ["serde"] }
+datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
+datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
+datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
+futures = "0.3"
+futures-util = "0.3"
+parquet = "34.0"
+paste = "1.0"
+prost = "0.11"
+rand = "0.8"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+snafu = { version = "0.7", features = ["backtraces"] }
+sqlparser = "0.32"
+tempfile = "3"
+tokio = { version = "1.24.2", features = ["full"] }
+tokio-util = "0.7"
+tonic = { version = "0.8", features = ["tls"] }
+uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
+
+[profile.release]
+debug = true
+
+[profile.nightly]
+inherits = "release"
+strip = true
+lto = "thin"
+debug = false
+incremental = false

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2022 Greptime Team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,71 @@
+IMAGE_REGISTRY ?= greptimedb
+IMAGE_TAG ?= latest
+
+##@ Build
+
+.PHONY: build
+build: ## Build debug version greptime.
+	cargo build
+
+.PHONY: release
+release:  ## Build release version greptime.
+	cargo build --release
+
+.PHONY: clean
+clean: ## Clean the project.
+	cargo clean
+
+.PHONY: fmt
+fmt: ## Format all the Rust code.
+	cargo fmt --all
+
+.PHONY: fmt-toml
+fmt-toml: ## Format all TOML files.
+	taplo format --check --option "indent_string=    "
+
+.PHONY: docker-image
+docker-image: ## Build docker image.
+	docker build --network host -f docker/Dockerfile -t ${IMAGE_REGISTRY}:${IMAGE_TAG} .
+
+##@ Test
+
+.PHONY: unit-test
+unit-test: ## Run unit test.
+	cargo test --workspace
+
+.PHONY: integration-test
+integration-test: ## Run integation test.
+	cargo test integration
+
+.PHONY: sqlness-test
+sqlness-test: ## Run sqlness test.
+	cargo sqlness
+
+.PHONY: check
+check: ## Cargo check all the targets.
+	cargo check --workspace --all-targets
+
+.PHONY: clippy
+clippy: ## Check clippy rules.
+	cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr
+
+.PHONY: fmt-check
+fmt-check: ## Check code format.
+	cargo fmt --all -- --check
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# https://linuxcommand.org/lc3_adv_awk.php
+
+.PHONY: help
+help: ## Display help messages.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

README.md

@@ -1,119 +1,106 @@
-# GreptimeDB
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: light)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding.png">
+    <source media="(prefers-color-scheme: dark)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding-dark.png">
+    <img alt="GreptimeDB Logo" src="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding.png" width="400px">
+  </picture>
+</p>
 
-[![codecov](https://codecov.io/gh/GrepTimeTeam/greptimedb/branch/develop/graph/badge.svg?token=FITFDI3J3C)](https://codecov.io/gh/GrepTimeTeam/greptimedb)
 
-GreptimeDB: the next-generation hybrid timeseries/analytics processing database in the cloud.
+<h3 align="center">
+    The next-generation hybrid timeseries/analytics processing database in the cloud
+</h3>
 
-## Getting Started
+<p align="center">
+    <a href="https://codecov.io/gh/GrepTimeTeam/greptimedb"><img src="https://codecov.io/gh/GrepTimeTeam/greptimedb/branch/develop/graph/badge.svg?token=FITFDI3J3C"></img></a>
+    &nbsp;
+    <a href="https://github.com/GreptimeTeam/greptimedb/actions/workflows/develop.yml"><img src="https://github.com/GreptimeTeam/greptimedb/actions/workflows/develop.yml/badge.svg" alt="CI"></img></a>
+    &nbsp;
+    <a href="https://github.com/greptimeTeam/greptimedb/blob/develop/LICENSE"><img src="https://img.shields.io/github/license/greptimeTeam/greptimedb"></a>
+</p>
 
-### Prerequisites
+<p align="center">
+    <a href="https://twitter.com/greptime"><img src="https://img.shields.io/badge/twitter-follow_us-1d9bf0.svg"></a>
+    &nbsp;
+    <a href="https://www.linkedin.com/company/greptime/"><img src="https://img.shields.io/badge/linkedin-connect_with_us-0a66c2.svg"></a>
+</p>
 
-To compile GreptimeDB from source, you'll need the following:
-- Rust
-- Protobuf
-- OpenSSL
+## What is GreptimeDB
 
-#### Rust
+GreptimeDB is an open-source time-series database with a special focus on
+scalability, analytical capabilities and efficiency. It's designed to work on
+infrastructure of the cloud era, and users benefit from its elasticity and commodity
+storage.
 
-The easiest way to install Rust is to use [`rustup`](https://rustup.rs/), which will check our `rust-toolchain` file and install correct Rust version for you.
+Our core developers have been building time-series data platform
+for years. Based on their best-practices, GreptimeDB is born to give you:
 
-#### Protobuf
+- A standalone binary that scales to highly-available distributed cluster, providing a transparent experience for cluster users
+- Optimized columnar layout for handling time-series data; compacted, compressed, stored on various storage backends
+- Flexible index options, tackling high cardinality issues down
+- Distributed, parallel query execution, leveraging elastic computing resource
+- Native SQL, and Python scripting for advanced analytical scenarios
+- Widely adopted database protocols and APIs
+- Extensible table engine architecture for extensive workloads
 
-`protoc` is required for compiling `.proto` files. `protobuf` is available from
-major package manager on macos and linux distributions. You can find an
-installation instructions [here](https://grpc.io/docs/protoc-installation/).
+## Quick Start
 
-#### OpenSSL
+### Build
 
-For Ubuntu:
-```bash
-sudo apt install libssl-dev
-```
+#### Build from Source
 
-For RedHat-based: Fedora, Oracle Linux, etc:
-```bash
-sudo dnf install openssl-devel
-```
+To compile GreptimeDB from source, you'll need:
 
-For macOS:
-```bash
-brew install openssl
-```
+- C/C++ Toolchain: provides basic tools for compiling and linking. This is
+  available as `build-essential` on ubuntu and similar name on other platforms.
+- Rust: the easiest way to install Rust is to use
+  [`rustup`](https://rustup.rs/), which will check our `rust-toolchain` file and
+  install correct Rust version for you.
+- Protobuf: `protoc` is required for compiling `.proto` files. `protobuf` is
+  available from major package manager on macos and linux distributions. You can
+  find an installation instructions [here](https://grpc.io/docs/protoc-installation/).
+  **Note that `protoc` version needs to be >= 3.15** because we have used the `optional`
+  keyword. You can check it with `protoc --version`.
+- python3-dev or python3-devel(Optional feature, only needed if you want to run scripts
+  in CPython, and also need to enable `pyo3_backend` feature when compiling(by `cargo run -F pyo3_backend` or add `pyo3_backend` to src/script/Cargo.toml 's `features.default` like `default = ["python", "pyo3_backend]`)): this install a Python shared library required for running Python
+  scripting engine(In CPython Mode). This is available as `python3-dev` on
+  ubuntu, you can install it with `sudo apt install python3-dev`, or
+  `python3-devel` on RPM based distributions (e.g. Fedora, Red Hat, SuSE). Mac's
+  `Python3` package should have this shared library by default. More detail for compiling with PyO3 can be found in [PyO3](https://pyo3.rs/v0.18.1/building_and_distribution#configuring-the-python-version)'s documentation.
 
-### Build the Docker Image
+#### Build with Docker
+
+A docker image with necessary dependencies is provided:
 
 ```
 docker build --network host -f docker/Dockerfile -t greptimedb .
 ```
 
-## Usage
+### Run
 
-### Start Datanode
+Start GreptimeDB from source code, in standalone mode:
 
 ```
-// Start datanode with default options.
-cargo run -- datanode start
-
-OR
-
-// Start datanode with `http-addr` option.
-cargo run -- datanode start --http-addr=0.0.0.0:9999
-
-OR
-
-// Start datanode with `log-dir` and `log-level` options.
-cargo run -- --log-dir=logs --log-level=debug datanode start
+cargo run -- standalone start
 ```
 
-Start datanode with config file:
+Or if you built from docker:
 
 ```
-cargo run -- --log-dir=logs --log-level=debug datanode start -c ./config/datanode.example.toml
+docker run -p 4002:4002 -v "$(pwd):/tmp/greptimedb" greptime/greptimedb standalone start
 ```
 
-Start datanode by runing docker container:
+For more startup options, greptimedb's **distributed mode** and information
+about Kubernetes deployment, check our [docs](https://docs.greptime.com/).
 
-```
-docker run -p 3000:3000 \
--p 3001:3001 \
--p 3306:3306 \
-greptimedb
-```
+### Connect
 
-### Start Frontend
-
-Frontend should connect to Datanode, so **Datanode must have been started** at first!
-
-```
-// Connects to local Datanode at its default GRPC port: 3001
-
-// Start Frontend with default options.
-cargo run -- frontend start
-
-OR
-
-// Start Frontend with `mysql-addr` option.
-cargo run -- frontend start --mysql-addr=0.0.0.0:9999
-
-OR
-
-// Start datanode with `log-dir` and `log-level` options.
-cargo run -- --log-dir=logs --log-level=debug frontend start
-```
-
-Start datanode with config file:
-
-```
-cargo run -- --log-dir=logs --log-level=debug frontend start -c ./config/frontend.example.toml
-```
-
-### SQL Operations
-
-1. Connecting DB by [mysql client](https://dev.mysql.com/downloads/mysql/):
+1. Connect to GreptimeDB via standard [MySQL
+   client](https://dev.mysql.com/downloads/mysql/):
 
    ```
-   # The datanode listen on port 3306 by default.
-   mysql -h 127.0.0.1 -P 3306
+   # The standalone instance listen on port 4002 by default.
+   mysql -h 127.0.0.1 -P 4002
    ```
 
 2. Create table:
@@ -125,59 +112,102 @@ cargo run -- --log-dir=logs --log-level=debug frontend start -c ./config/fronten
      cpu DOUBLE DEFAULT 0,
      memory DOUBLE,
      TIME INDEX (ts),
-     PRIMARY KEY(ts,host)) ENGINE=mito WITH(regions=1);
+     PRIMARY KEY(host)) ENGINE=mito WITH(regions=1);
    ```
 
-3. Insert data:
+3. Insert some data:
 
    ```SQL
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host1', 66.6, 1024, 1660897955);
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host2', 77.7, 2048, 1660897956);
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host3', 88.8, 4096, 1660897957);
+   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host1', 66.6, 1024, 1660897955000);
+   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host2', 77.7, 2048, 1660897956000);
+   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host3', 88.8, 4096, 1660897957000);
    ```
 
-4. Query data:
+4. Query the data:
 
    ```SQL
-   mysql> SELECT * FROM monitor;
-   +-------+------------+------+--------+
-   | host  | ts         | cpu  | memory |
-   +-------+------------+------+--------+
-   | host1 | 1660897955 | 66.6 |   1024 |
-   | host2 | 1660897956 | 77.7 |   2048 |
-   | host3 | 1660897957 | 88.8 |   4096 |
-   +-------+------------+------+--------+
+   SELECT * FROM monitor;
+   ```
+
+   ```TEXT
+   +-------+---------------------+------+--------+
+   | host  | ts                  | cpu  | memory |
+   +-------+---------------------+------+--------+
+   | host1 | 2022-08-19 08:32:35 | 66.6 |   1024 |
+   | host2 | 2022-08-19 08:32:36 | 77.7 |   2048 |
+   | host3 | 2022-08-19 08:32:37 | 88.8 |   4096 |
+   +-------+---------------------+------+--------+
    3 rows in set (0.01 sec)
    ```
-   You can delete your data by removing `/tmp/greptimedb`.
 
-## Contribute
+You can always cleanup test database by removing `/tmp/greptimedb`.
 
-1. [Install rust](https://www.rust-lang.org/tools/install)
-2. [Install `pre-commit`](https://pre-commit.com/#plugins) for run hooks on every commit automatically such as `cargo fmt` etc.
+## Resources
 
-```
-$ pip install pre-commit
+### Installation
 
-or
+- [Pre-built Binaries](https://github.com/GreptimeTeam/greptimedb/releases):
+  For Linux and macOS, you can easily download pre-built binaries that are ready to use. In most cases, downloading the version without PyO3 is sufficient. However, if you plan to run scripts in CPython (and use Python packages like NumPy and Pandas), you will need to download the version with PyO3 and install a Python with the same version as the Python in the PyO3 version. We recommend using virtualenv for the installation process to manage multiple Python versions.
+- [Docker Images](https://hub.docker.com/r/greptime/greptimedb)(**recommended**): pre-built
+  Docker images, this is the easiest way to try GreptimeDB. By default it runs CPython script with `pyo3_backend` enabled.
+- [`gtctl`](https://github.com/GreptimeTeam/gtctl): the command-line tool for
+  Kubernetes deployment
 
-$ brew install pre-commit
-$
-```
+### Documentation
 
-3. Install the git hook scripts:
+- GreptimeDB [User Guide](https://docs.greptime.com/user-guide/concepts.html)
+- GreptimeDB [Developer
+  Guide](https://docs.greptime.com/developer-guide/overview.html)
+- GreptimeDB [internal code document](https://greptimedb.rs)
 
-```
-$ pre-commit install
-pre-commit installed at .git/hooks/pre-commit
+### Dashboard
+- [The dashboard UI for GreptimeDB](https://github.com/GreptimeTeam/dashboard)
 
-$ pre-commit install --hook-type commit-msg
-pre-commit installed at .git/hooks/commit-msg
+### SDK
 
-$ pre-commit install --hook-type pre-push
-pre-commit installed at .git/hooks/pre-pus
-```
+- [GreptimeDB Java
+  Client](https://github.com/GreptimeTeam/greptimedb-client-java)
 
-now `pre-commit` will run automatically on `git commit`.
+## Project Status
 
-4. Check out branch from `develop` and make your contribution. Follow the [style guide](https://github.com/GreptimeTeam/docs/blob/main/style-guide/zh.md). Create a PR when you are ready, feel free and have fun!
+This project is in its early stage and under heavy development. We move fast and
+break things. Benchmark on development branch may not represent its potential
+performance. We release pre-built binaries constantly for functional
+evaluation. Do not use it in production at the moment.
+
+For future plans, check out [GreptimeDB roadmap](https://github.com/GreptimeTeam/greptimedb/issues/669).
+
+## Community
+
+Our core team is thrilled to see you participate in any ways you like. When you are stuck, try to
+ask for help by filling an issue with a detailed description of what you were trying to do
+and what went wrong. If you have any questions or if you would like to get involved in our
+community, please check out:
+
+- GreptimeDB Community on [Slack](https://greptime.com/slack)
+- GreptimeDB GitHub [Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)
+- Greptime official [Website](https://greptime.com)
+
+In addition, you may:
+
+- View our official [Blog](https://greptime.com/blogs/index)
+- Connect us with [Linkedin](https://www.linkedin.com/company/greptime/)
+- Follow us on [Twitter](https://twitter.com/greptime)
+
+## License
+
+GreptimeDB uses the [Apache 2.0 license][1] to strike a balance between
+open contributions and allowing you to use the software however you want.
+
+[1]: <https://github.com/greptimeTeam/greptimedb/blob/develop/LICENSE>
+
+## Contributing
+
+Please refer to [contribution guidelines](CONTRIBUTING.md) for more information.
+
+## Acknowledgement
+- GreptimeDB uses [Apache Arrow](https://arrow.apache.org/) as the memory model and [Apache Parquet](https://parquet.apache.org/) as the persistent file format.
+- GreptimeDB's query engine is powered by [Apache Arrow DataFusion](https://github.com/apache/arrow-datafusion).
+- [OpenDAL](https://github.com/datafuselabs/opendal) from [Datafuse Labs](https://github.com/datafuselabs) gives GreptimeDB a very general and elegant data access abstraction layer.
+- GreptimeDB’s meta service is based on [etcd](https://etcd.io/).
+- GreptimeDB uses [RustPython](https://github.com/RustPython/RustPython) for experimental embedded python scripting.

benchmarks/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "benchmarks"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+arrow.workspace = true
+clap = { version = "4.0", features = ["derive"] }
+client = { path = "../src/client" }
+indicatif = "0.17.1"
+itertools = "0.10.5"
+parquet.workspace = true
+tokio.workspace = true

benchmarks/src/bin/nyc-taxi.rs

@@ -0,0 +1,460 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Use the taxi trip records from New York City dataset to bench. You can download the dataset from
+//! [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page).
+
+#![allow(clippy::print_stdout)]
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::time::Instant;
+
+use arrow::array::{ArrayRef, PrimitiveArray, StringArray, TimestampMicrosecondArray};
+use arrow::datatypes::{DataType, Float64Type, Int64Type};
+use arrow::record_batch::RecordBatch;
+use clap::Parser;
+use client::api::v1::column::Values;
+use client::api::v1::{Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest};
+use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+use tokio::task::JoinSet;
+
+const CATALOG_NAME: &str = "greptime";
+const SCHEMA_NAME: &str = "public";
+const TABLE_NAME: &str = "nyc_taxi";
+
+#[derive(Parser)]
+#[command(name = "NYC benchmark runner")]
+struct Args {
+    /// Path to the dataset
+    #[arg(short, long)]
+    path: Option<String>,
+
+    /// Batch size of insert request.
+    #[arg(short = 's', long = "batch-size", default_value_t = 4096)]
+    batch_size: usize,
+
+    /// Number of client threads on write (parallel on file level)
+    #[arg(short = 't', long = "thread-num", default_value_t = 4)]
+    thread_num: usize,
+
+    /// Number of query iteration
+    #[arg(short = 'i', long = "iter-num", default_value_t = 3)]
+    iter_num: usize,
+
+    #[arg(long = "skip-write")]
+    skip_write: bool,
+
+    #[arg(long = "skip-read")]
+    skip_read: bool,
+
+    #[arg(short, long, default_value_t = String::from("127.0.0.1:4001"))]
+    endpoint: String,
+}
+
+fn get_file_list<P: AsRef<Path>>(path: P) -> Vec<PathBuf> {
+    std::fs::read_dir(path)
+        .unwrap()
+        .map(|dir| dir.unwrap().path().canonicalize().unwrap())
+        .collect()
+}
+
+async fn write_data(
+    batch_size: usize,
+    db: &Database,
+    path: PathBuf,
+    mpb: MultiProgress,
+    pb_style: ProgressStyle,
+) -> u128 {
+    let file = std::fs::File::open(&path).unwrap();
+    let record_batch_reader_builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
+    let row_num = record_batch_reader_builder
+        .metadata()
+        .file_metadata()
+        .num_rows();
+    let record_batch_reader = record_batch_reader_builder
+        .with_batch_size(batch_size)
+        .build()
+        .unwrap();
+    let progress_bar = mpb.add(ProgressBar::new(row_num as _));
+    progress_bar.set_style(pb_style);
+    progress_bar.set_message(format!("{path:?}"));
+
+    let mut total_rpc_elapsed_ms = 0;
+
+    for record_batch in record_batch_reader {
+        let record_batch = record_batch.unwrap();
+        if !is_record_batch_full(&record_batch) {
+            continue;
+        }
+        let (columns, row_count) = convert_record_batch(record_batch);
+        let request = InsertRequest {
+            table_name: TABLE_NAME.to_string(),
+            region_number: 0,
+            columns,
+            row_count,
+        };
+        let now = Instant::now();
+        db.insert(request).await.unwrap();
+        let elapsed = now.elapsed();
+        total_rpc_elapsed_ms += elapsed.as_millis();
+        progress_bar.inc(row_count as _);
+    }
+
+    progress_bar.finish_with_message(format!("file {path:?} done in {total_rpc_elapsed_ms}ms",));
+    total_rpc_elapsed_ms
+}
+
+fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
+    let schema = record_batch.schema();
+    let fields = schema.fields();
+    let row_count = record_batch.num_rows();
+    let mut columns = vec![];
+
+    for (array, field) in record_batch.columns().iter().zip(fields.iter()) {
+        let (values, datatype) = build_values(array);
+        let column = Column {
+            column_name: field.name().to_owned(),
+            values: Some(values),
+            null_mask: array
+                .data()
+                .null_bitmap()
+                .map(|bitmap| bitmap.buffer().as_slice().to_vec())
+                .unwrap_or_default(),
+            datatype: datatype.into(),
+            // datatype and semantic_type are set to default
+            ..Default::default()
+        };
+        columns.push(column);
+    }
+
+    (columns, row_count as _)
+}
+
+fn build_values(column: &ArrayRef) -> (Values, ColumnDataType) {
+    match column.data_type() {
+        DataType::Int64 => {
+            let array = column
+                .as_any()
+                .downcast_ref::<PrimitiveArray<Int64Type>>()
+                .unwrap();
+            let values = array.values();
+            (
+                Values {
+                    i64_values: values.to_vec(),
+                    ..Default::default()
+                },
+                ColumnDataType::Int64,
+            )
+        }
+        DataType::Float64 => {
+            let array = column
+                .as_any()
+                .downcast_ref::<PrimitiveArray<Float64Type>>()
+                .unwrap();
+            let values = array.values();
+            (
+                Values {
+                    f64_values: values.to_vec(),
+                    ..Default::default()
+                },
+                ColumnDataType::Float64,
+            )
+        }
+        DataType::Timestamp(_, _) => {
+            let array = column
+                .as_any()
+                .downcast_ref::<TimestampMicrosecondArray>()
+                .unwrap();
+            let values = array.values();
+            (
+                Values {
+                    i64_values: values.to_vec(),
+                    ..Default::default()
+                },
+                ColumnDataType::Int64,
+            )
+        }
+        DataType::Utf8 => {
+            let array = column.as_any().downcast_ref::<StringArray>().unwrap();
+            let values = array.iter().filter_map(|s| s.map(String::from)).collect();
+            (
+                Values {
+                    string_values: values,
+                    ..Default::default()
+                },
+                ColumnDataType::String,
+            )
+        }
+        DataType::Null
+        | DataType::Boolean
+        | DataType::Int8
+        | DataType::Int16
+        | DataType::Int32
+        | DataType::UInt8
+        | DataType::UInt16
+        | DataType::UInt32
+        | DataType::UInt64
+        | DataType::Float16
+        | DataType::Float32
+        | DataType::Date32
+        | DataType::Date64
+        | DataType::Time32(_)
+        | DataType::Time64(_)
+        | DataType::Duration(_)
+        | DataType::Interval(_)
+        | DataType::Binary
+        | DataType::FixedSizeBinary(_)
+        | DataType::LargeBinary
+        | DataType::LargeUtf8
+        | DataType::List(_)
+        | DataType::FixedSizeList(_, _)
+        | DataType::LargeList(_)
+        | DataType::Struct(_)
+        | DataType::Union(_, _, _)
+        | DataType::Dictionary(_, _)
+        | DataType::Decimal128(_, _)
+        | DataType::Decimal256(_, _)
+        | DataType::RunEndEncoded(_, _)
+        | DataType::Map(_, _) => todo!(),
+    }
+}
+
+fn is_record_batch_full(batch: &RecordBatch) -> bool {
+    batch.columns().iter().all(|col| col.null_count() == 0)
+}
+
+fn create_table_expr() -> CreateTableExpr {
+    CreateTableExpr {
+        catalog_name: CATALOG_NAME.to_string(),
+        schema_name: SCHEMA_NAME.to_string(),
+        table_name: TABLE_NAME.to_string(),
+        desc: "".to_string(),
+        column_defs: vec![
+            ColumnDef {
+                name: "VendorID".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "tpep_pickup_datetime".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "tpep_dropoff_datetime".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "passenger_count".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "trip_distance".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "RatecodeID".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "store_and_fwd_flag".to_string(),
+                datatype: ColumnDataType::String as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "PULocationID".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "DOLocationID".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "payment_type".to_string(),
+                datatype: ColumnDataType::Int64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "fare_amount".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "extra".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "mta_tax".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "tip_amount".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "tolls_amount".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "improvement_surcharge".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "total_amount".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "congestion_surcharge".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+            ColumnDef {
+                name: "airport_fee".to_string(),
+                datatype: ColumnDataType::Float64 as i32,
+                is_nullable: true,
+                default_constraint: vec![],
+            },
+        ],
+        time_index: "tpep_pickup_datetime".to_string(),
+        primary_keys: vec!["VendorID".to_string()],
+        create_if_not_exists: false,
+        table_options: Default::default(),
+        region_ids: vec![0],
+        table_id: None,
+    }
+}
+
+fn query_set() -> HashMap<String, String> {
+    let mut ret = HashMap::new();
+
+    ret.insert(
+        "count_all".to_string(),
+        format!("SELECT COUNT(*) FROM {TABLE_NAME};"),
+    );
+
+    ret.insert(
+        "fare_amt_by_passenger".to_string(),
+        format!("SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM {TABLE_NAME} GROUP BY passenger_count")
+    );
+
+    ret
+}
+
+async fn do_write(args: &Args, db: &Database) {
+    let mut file_list = get_file_list(args.path.clone().expect("Specify data path in argument"));
+    let mut write_jobs = JoinSet::new();
+
+    let create_table_result = db.create(create_table_expr()).await;
+    println!("Create table result: {create_table_result:?}");
+
+    let progress_bar_style = ProgressStyle::with_template(
+        "[{elapsed_precise}] {bar:60.cyan/blue} {pos:>7}/{len:7} {msg}",
+    )
+    .unwrap()
+    .progress_chars("##-");
+    let multi_progress_bar = MultiProgress::new();
+    let file_progress = multi_progress_bar.add(ProgressBar::new(file_list.len() as _));
+    file_progress.inc(0);
+
+    let batch_size = args.batch_size;
+    for _ in 0..args.thread_num {
+        if let Some(path) = file_list.pop() {
+            let db = db.clone();
+            let mpb = multi_progress_bar.clone();
+            let pb_style = progress_bar_style.clone();
+            write_jobs.spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
+        }
+    }
+    while write_jobs.join_next().await.is_some() {
+        file_progress.inc(1);
+        if let Some(path) = file_list.pop() {
+            let db = db.clone();
+            let mpb = multi_progress_bar.clone();
+            let pb_style = progress_bar_style.clone();
+            write_jobs.spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
+        }
+    }
+}
+
+async fn do_query(num_iter: usize, db: &Database) {
+    for (query_name, query) in query_set() {
+        println!("Running query: {query}");
+        for i in 0..num_iter {
+            let now = Instant::now();
+            let _res = db.sql(&query).await.unwrap();
+            let elapsed = now.elapsed();
+            println!(
+                "query {}, iteration {}: {}ms",
+                query_name,
+                i,
+                elapsed.as_millis()
+            );
+        }
+    }
+}
+
+fn main() {
+    let args = Args::parse();
+
+    tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(args.thread_num)
+        .enable_all()
+        .build()
+        .unwrap()
+        .block_on(async {
+            let client = Client::with_urls(vec![&args.endpoint]);
+            let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
+
+            if !args.skip_write {
+                do_write(&args, &db).await;
+            }
+
+            if !args.skip_read {
+                do_query(args.iter_num, &db).await;
+            }
+        })
+}

codecov.yml

@@ -1,10 +1,12 @@
 # codecov config
 coverage:
   status:
-    patch: off # disable patch status
     project:
       default:
-        enable: yes
         threshold: 1%
+    patch: off
 ignore:
   - "**/error*.rs" # ignore all error.rs files
+  - "tests/runner/*.rs" # ignore integration test runner
+comment:                  # this is a top-level key
+  layout: "diff"

component/script/python/example/calc_rv.py

@@ -1,71 +0,0 @@
-import sys
-# for annoying releative import beyond top-level package
-sys.path.insert(0, "../")
-from greptime import mock_tester, coprocessor, greptime as gt_builtin
-from greptime.greptime import interval, vector, log, prev, sqrt, datetime
-import greptime.greptime as greptime
-import json
-import numpy as np
-
-
-def data_sample(k_lines, symbol, density=5 * 30 * 86400):
-    """
-    Only return close data for simplicty for now
-    """
-    k_lines = k_lines["result"] if k_lines["ret_msg"] == "OK" else None
-    if k_lines is None:
-        raise Exception("Expect a `OK`ed message")
-    close = [float(i["close"]) for i in k_lines]
-
-    return interval(close, density, "prev")
-
-
-def as_table(kline: list):
-    col_len = len(kline)
-    ret = {
-        k: vector([fn(row[k]) for row in kline], str(ty))
-        for k, fn, ty in
-        [
-            ("symbol", str, "str"),
-            ("period", str, "str"),
-            ("open_time", int, "int"),
-            ("open", float, "float"),
-            ("high", float, "float"),
-            ("low", float, "float"),
-            ("close", float, "float")
-        ]
-    }
-    return ret
-
-@coprocessor(args=["open_time", "close"], returns=[
-    "rv_7d",
-    "rv_15d",
-    "rv_30d",
-    "rv_60d",
-    "rv_90d",
-    "rv_180d"
-])
-def calc_rvs(open_time, close):
-    from greptime import vector, log, prev, sqrt, datetime, pow, sum, last
-    import greptime as g
-    def calc_rv(close, open_time, time, interval):
-        mask = (open_time < time) & (open_time > time - interval)
-        close = close[mask]
-        open_time = open_time[mask]
-        close = g.interval(open_time, close, datetime("10m"), lambda x:last(x))
-
-        avg_time_interval = (open_time[-1] - open_time[0])/(len(open_time)-1)
-        ref = log(close/prev(close))
-        var = sum(pow(ref, 2)/(len(ref)-1))
-        return sqrt(var/avg_time_interval)
-
-    # how to get env var,
-    # maybe through accessing scope and serde then send to remote?
-    timepoint = open_time[-1]
-    rv_7d = vector([calc_rv(close, open_time, timepoint, datetime("7d"))])
-    rv_15d = vector([calc_rv(close, open_time, timepoint, datetime("15d"))])
-    rv_30d = vector([calc_rv(close, open_time, timepoint, datetime("30d"))])
-    rv_60d = vector([calc_rv(close, open_time, timepoint, datetime("60d"))])
-    rv_90d = vector([calc_rv(close, open_time, timepoint, datetime("90d"))])
-    rv_180d = vector([calc_rv(close, open_time, timepoint, datetime("180d"))])
-    return rv_7d, rv_15d, rv_30d, rv_60d, rv_90d, rv_180d

component/script/python/example/fetch_kline.sh

@@ -1 +0,0 @@
-curl "https://api.bybit.com/v2/public/index-price-kline?symbol=BTCUSD&interval=1&limit=$1&from=1581231260" > kline.json

component/script/python/example/kline.json

@@ -1,108 +0,0 @@
-{
-    "ret_code": 0,
-    "ret_msg": "OK",
-    "ext_code": "",
-    "ext_info": "",
-    "result": [
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 300,
-            "open": "10107",
-            "high": "10109.34",
-            "low": "10106.71",
-            "close": "10106.79"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 900,
-            "open": "10106.79",
-            "high": "10109.27",
-            "low": "10105.92",
-            "close": "10106.09"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1200,
-            "open": "10106.09",
-            "high": "10108.75",
-            "low": "10104.66",
-            "close": "10108.73"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1800,
-            "open": "10108.73",
-            "high": "10109.52",
-            "low": "10106.07",
-            "close": "10106.38"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 2400,
-            "open": "10106.38",
-            "high": "10109.48",
-            "low": "10104.81",
-            "close": "10106.95"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 3000,
-            "open": "10106.95",
-            "high": "10109.48",
-            "low": "10106.6",
-            "close": "10107.55"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 3600,
-            "open": "10107.55",
-            "high": "10109.28",
-            "low": "10104.68",
-            "close": "10104.68"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 4200,
-            "open": "10104.68",
-            "high": "10109.18",
-            "low": "10104.14",
-            "close": "10108.8"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 4800,
-            "open": "10108.8",
-            "high": "10117.36",
-            "low": "10108.8",
-            "close": "10115.96"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 5400,
-            "open": "10115.96",
-            "high": "10119.19",
-            "low": "10115.96",
-            "close": "10117.08"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 6000,
-            "open": "10117.08",
-            "high": "10120.73",
-            "low": "10116.96",
-            "close": "10120.43"
-        }
-    ],
-    "time_now": "1661225351.158190"
-}

component/script/python/greptime/__init__.py

@@ -1,4 +0,0 @@
-from .greptime import coprocessor, copr
-from .greptime import vector, log, prev, next, first, last, sqrt, pow, datetime, sum, interval
-from .mock import mock_tester
-from .cfg import set_conn_addr, get_conn_addr

component/script/python/greptime/cfg.py

@@ -1,11 +0,0 @@
-GREPTIME_DB_CONN_ADDRESS = "localhost:3000"
-"""The Global Variable for address for conntect to database"""
-
-def set_conn_addr(addr: str):
-    """set database address to given `addr`"""
-    global GREPTIME_DB_CONN_ADDRESS
-    GREPTIME_DB_CONN_ADDRESS = addr
-
-def get_conn_addr()->str:
-    global GREPTIME_DB_CONN_ADDRESS
-    return GREPTIME_DB_CONN_ADDRESS

component/script/python/greptime/greptime.py

@@ -1,207 +0,0 @@
-"""
-Be note that this is a mock library, if not connected to database,
-it can only run on mock data and mock function which is supported by numpy
-"""
-import functools
-import numpy as np
-import json
-from urllib import request
-import inspect
-import requests
-
-from .cfg import set_conn_addr, get_conn_addr
-
-log = np.log
-sum = np.nansum
-sqrt = np.sqrt
-pow = np.power
-nan = np.nan
-
-
-class TimeStamp(str):
-    """
-    TODO: impl date time
-    """
-    pass
-
-
-class i32(int):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A signed 32-bit integer.
-    """
-
-    def __repr__(self) -> str:
-        return "i32"
-
-
-class i64(int):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A signed 64-bit integer.
-    """
-
-    def __repr__(self) -> str:
-        return "i64"
-
-
-class f32(float):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A 32-bit floating point number.
-    """
-
-    def __repr__(self) -> str:
-        return "f32"
-
-
-class f64(float):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A 64-bit floating point number.
-    """
-
-    def __repr__(self) -> str:
-        return "f64"
-
-
-class vector(np.ndarray):
-    """
-    A compact Vector with all elements of same Data type.
-    """
-    _datatype: str | None = None
-
-    def __new__(
-        cls,
-        lst,
-        dtype=None
-    ) -> ...:
-        self = np.asarray(lst).view(cls)
-        self._datatype = dtype
-        return self
-
-    def __str__(self) -> str:
-        return "vector({}, \"{}\")".format(super().__str__(), self.datatype())
-
-    def datatype(self):
-        return self._datatype
-
-    def filter(self, lst_bool):
-        return self[lst_bool]
-
-def last(lst):
-    return lst[-1]
-
-def first(lst):
-    return lst[0]
-
-def prev(lst):
-    ret = np.zeros(len(lst))
-    ret[1:] = lst[0:-1]
-    ret[0] = nan
-    return ret
-
-def next(lst):
-    ret = np.zeros(len(lst))
-    ret[:-1] = lst[1:]
-    ret[-1] = nan
-    return ret
-
-def interval(ts: vector, arr: vector, duration: int, func):
-    """
-    Note that this is a mock function with same functionailty to the actual Python Coprocessor
-    `arr` is a vector of integral or temporal type.
-    """
-    start = np.min(ts)
-    end = np.max(ts)
-    masks = [(ts >= i) & (ts <= (i+duration)) for i in range(start, end, duration)]
-    lst_res = [func(arr[mask]) for mask in masks]
-    return lst_res
-
-
-def factor(unit: str) -> int:
-    if unit == "d":
-        return 24 * 60 * 60
-    elif unit == "h":
-        return 60 * 60
-    elif unit == "m":
-        return 60
-    elif unit == "s":
-        return 1
-    else:
-        raise Exception("Only d,h,m,s, found{}".format(unit))
-
-
-def datetime(input_time: str) -> int:
-    """
-    support `d`(day) `h`(hour) `m`(minute) `s`(second)
-
-    support format:
-    `12s` `7d` `12d2h7m`
-    """
-
-    prev = 0
-    cur = 0
-    state = "Num"
-    parse_res = []
-    for idx, ch in enumerate(input_time):
-        if ch.isdigit():
-            cur = idx
-
-            if state != "Num":
-                parse_res.append((state, input_time[prev:cur], (prev, cur)))
-                prev = idx
-                state = "Num"
-        else:
-            cur = idx
-            if state != "Symbol":
-                parse_res.append((state, input_time[prev:cur], (prev, cur)))
-                prev = idx
-                state = "Symbol"
-    parse_res.append((state, input_time[prev:cur+1], (prev, cur+1)))
-
-    cur_idx = 0
-    res_time = 0
-    while cur_idx < len(parse_res):
-        pair = parse_res[cur_idx]
-        if pair[0] == "Num":
-            val = int(pair[1])
-            nxt = parse_res[cur_idx+1]
-            res_time += val * factor(nxt[1])
-            cur_idx += 2
-        else:
-            raise Exception("Two symbol in a row is impossible")
-
-    return res_time
-
-
-def coprocessor(args=None, returns=None, sql=None):
-    """
-    The actual coprocessor, which will connect to database and update
-    whatever function decorated with `@coprocessor(args=[...], returns=[...], sql=...)`
-    """
-    def decorator_copr(func):
-        @functools.wraps(func)
-        def wrapper_do_actual(*args, **kwargs):
-            if len(args)!=0 or len(kwargs)!=0:
-                raise Exception("Expect call with no arguements(for all args are given by coprocessor itself)")
-            source = inspect.getsource(func)
-            url = "http://{}/v1/scripts".format(get_conn_addr())
-            print("Posting to {}".format(url))
-            data = {
-                    "script": source,
-                    "engine": None,
-                }
-
-            res = requests.post(
-                url,
-                headers={"Content-Type": "application/json"},
-                json=data
-            )
-            return res
-        return wrapper_do_actual
-    return decorator_copr
-
-
-# make a alias for short
-copr = coprocessor

component/script/python/greptime/mock.py

@@ -1,82 +0,0 @@
-"""
-Note this is a mock library, if not connected to database, 
-it can only run on mock data and support by numpy
-"""
-from typing import Any
-import numpy as np
-from .greptime import i32,i64,f32,f64, vector, interval, prev, datetime, log, sum, sqrt, pow, nan, copr, coprocessor
-
-import inspect
-import functools
-import ast
-
-
-
-def mock_tester(
-    func,
-    env:dict, 
-    table=None
-):
-    """
-    Mock tester helper function,
-    What it does is replace `@coprocessor` with `@mock_cpor` and add a keyword `env=env`
-    like `@mock_copr(args=...,returns=...,env=env)`
-    """
-    code = inspect.getsource(func)
-    tree = ast.parse(code)
-    tree = HackyReplaceDecorator("env").visit(tree)
-    new_func = tree.body[0]
-    fn_name = new_func.name
-
-    code_obj = compile(tree, "<embedded>", "exec")
-    exec(code_obj)
-
-    ret = eval("{}()".format(fn_name))
-    return ret
-
-def mock_copr(args, returns, sql=None, env:None|dict=None):
-    """
-    This should not be used directly by user
-    """
-    def decorator_copr(func):
-        @functools.wraps(func)
-        def wrapper_do_actual(*fn_args, **fn_kwargs):
-
-            real_args = [env[name] for name in args]
-            ret = func(*real_args)
-            return ret
-        
-        return wrapper_do_actual
-    return decorator_copr
-
-class HackyReplaceDecorator(ast.NodeTransformer):
-    """
-    This class accept a `env` dict for environment to extract args from,
-    and put `env` dict in the param list of `mock_copr` decorator, i.e:
-
-    a `@copr(args=["a", "b"], returns=["c"])` with call like mock_helper(abc, env={"a":2, "b":3})
-    
-    will be transform into `@mock_copr(args=["a", "b"], returns=["c"], env={"a":2, "b":3})`
-    """
-    def __init__(self, env: str) -> None:
-        # just for add `env` keyword
-        self.env = env
-
-    def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
-        new_node = node
-        decorator_list = new_node.decorator_list
-        if len(decorator_list)!=1:
-            return node
-
-        deco = decorator_list[0]
-        if deco.func.id!="coprocessor" and deco.func.id !="copr":
-            raise Exception("Expect a @copr or @coprocessor, found {}.".format(deco.func.id))
-        deco.func = ast.Name(id="mock_copr", ctx=ast.Load())
-        new_kw = ast.keyword(arg="env", value=ast.Name(id=self.env, ctx=ast.Load()))
-        deco.keywords.append(new_kw)
-
-        # Tie up loose ends in the AST.
-        ast.copy_location(new_node, node)
-        ast.fix_missing_locations(new_node)
-        self.generic_visit(node)
-        return new_node

component/script/python/test.py

@@ -1,60 +0,0 @@
-from example.calc_rv import as_table, calc_rvs
-from greptime import coprocessor, set_conn_addr, get_conn_addr, mock_tester
-import sys
-import json
-import requests
-'''
-To run this script, you need to first start a http server of greptime, and 
-`
-python3 component/script/python/test.py 地址:端口
-`
-
-'''
-@coprocessor(sql='select number from numbers limit 10', args=['number'], returns=['n'])
-def test(n):
-    return n+2
-
-def init_table(close, open_time):
-    req_init = "/v1/sql?sql=create table k_line (close double, open_time bigint, TIME INDEX (open_time))"
-    print(get_db(req_init).text)
-    for c1, c2 in zip(close, open_time):
-        req = "/v1/sql?sql=INSERT INTO k_line(close, open_time) VALUES ({}, {})".format(c1, c2)
-        print(get_db(req).text)
-    print(get_db("/v1/sql?sql=select * from k_line").text)
-
-def get_db(req:str):
-    return requests.get("http://{}{}".format(get_conn_addr(), req))
-
-if __name__ == "__main__":
-    with open("component/script/python/example/kline.json", "r") as kline_file:
-        kline = json.load(kline_file)
-        table = as_table(kline["result"])
-        close = table["close"]
-        open_time = table["open_time"]
-        env = {"close":close, "open_time": open_time}
-
-        res = mock_tester(calc_rvs, env=env)
-        print("Mock result:", [i[0] for i in res])
-        exit()
-    if len(sys.argv)!=2:
-        raise Exception("Expect only one address as cmd's args")
-    set_conn_addr(sys.argv[1])
-    res = test()
-    print(res.headers)
-    print(res.text)
-    with open("component/script/python/example/kline.json", "r") as kline_file:
-        kline = json.load(kline_file)
-        # vec = vector([1,2,3], int)
-        # print(vec, vec.datatype())
-        table = as_table(kline["result"])
-        # print(table)
-        close = table["close"]
-        open_time = table["open_time"]
-        init_table(close, open_time)
-
-        real = calc_rvs()
-        print(real)
-        try:
-            print(real.text["error"])
-        except:
-            print(real.text)

config/datanode.example.toml

@@ -1,14 +1,50 @@
-http_addr = '0.0.0.0:3000'
-rpc_addr = '0.0.0.0:3001'
-wal_dir = '/tmp/greptimedb/wal'
+# Node running mode, see `standalone.example.toml`.
+mode = "distributed"
+# Whether to use in-memory catalog, see `standalone.example.toml`.
+enable_memory_catalog = false
+# The datanode identifier, should be unique.
+node_id = 42
+# gRPC server address, "127.0.0.1:3001" by default.
+rpc_addr = "127.0.0.1:3001"
+# Hostname of this node.
+rpc_hostname = "127.0.0.1"
+# The number of gRPC server worker threads, 8 by default.
+rpc_runtime_size = 8
 
-mysql_addr = '0.0.0.0:3306'
-mysql_runtime_size = 4
+# Metasrv client options.
+[meta_client_options]
+# Metasrv address list.
+metasrv_addrs = ["127.0.0.1:3002"]
+# Operation timeout in milliseconds, 3000 by default.
+timeout_millis = 3000
+# Connect server timeout in milliseconds, 5000 by default.
+connect_timeout_millis = 5000
+# `TCP_NODELAY` option for accepted connections, true by default.
+tcp_nodelay = true
 
-# applied when postgres feature enbaled
-postgres_addr = '0.0.0.0:5432'
-postgres_runtime_size = 4
+# WAL options, see `standalone.example.toml`.
+[wal]
+dir = "/tmp/greptimedb/wal"
+file_size = "1GB"
+purge_threshold = "50GB"
+purge_interval = "10m"
+read_batch_size = 128
+sync_write = false
 
+# Storage options, see `standalone.example.toml`.
 [storage]
-type = 'File'
-data_dir = '/tmp/greptimedb/data/'
+type = "File"
+data_dir = "/tmp/greptimedb/data/"
+
+# Compaction options, see `standalone.example.toml`.
+[compaction]
+max_inflight_tasks = 4
+max_files_in_level0 = 8
+max_purge_tasks = 32
+
+# Procedure storage options, see `standalone.example.toml`.
+# [procedure.store]
+# type = "File"
+# data_dir = "/tmp/greptimedb/procedure/"
+# max_retry_times = 3
+# retry_delay = "500ms"

config/frontend.example.toml

@@ -1,4 +1,58 @@
-http_addr = '0.0.0.0:4000'
-grpc_addr = '0.0.0.0:4001'
-mysql_addr = '0.0.0.0:4003'
-mysql_runtime_size = 4
+# Node running mode, see `standalone.example.toml`.
+mode = "distributed"
+
+# HTTP server options, see `standalone.example.toml`.
+[http_options]
+addr = "127.0.0.1:4000"
+timeout = "30s"
+
+# gRPC server options, see `standalone.example.toml`.
+[grpc_options]
+addr = "127.0.0.1:4001"
+runtime_size = 8
+
+# MySQL server options, see `standalone.example.toml`.
+[mysql_options]
+addr = "127.0.0.1:4002"
+runtime_size = 2
+
+# MySQL server TLS options, see `standalone.example.toml`.
+[mysql_options.tls]
+mode = "disable"
+cert_path = ""
+key_path = ""
+
+# PostgresSQL server options, see `standalone.example.toml`.
+[postgres_options]
+addr = "127.0.0.1:4003"
+runtime_size = 2
+
+# PostgresSQL server TLS options, see `standalone.example.toml`.
+[postgres_options.tls]
+mode = "disable"
+cert_path = ""
+key_path = ""
+
+# OpenTSDB protocol options, see `standalone.example.toml`.
+[opentsdb_options]
+addr = "127.0.0.1:4242"
+runtime_size = 2
+
+# InfluxDB protocol options, see `standalone.example.toml`.
+[influxdb_options]
+enable = true
+
+# Prometheus protocol options, see `standalone.example.toml`.
+[prometheus_options]
+enable = true
+
+# Prometheus protocol options, see `standalone.example.toml`.
+[prom_options]
+addr = "127.0.0.1:4004"
+
+# Metasrv client options, see `datanode.example.toml`.
+[meta_client_options]
+metasrv_addrs = ["127.0.0.1:3002"]
+timeout_millis = 3000
+connect_timeout_millis = 5000
+tcp_nodelay = true

config/metasrv.example.toml

@@ -0,0 +1,15 @@
+# The bind address of metasrv, "127.0.0.1:3002" by default.
+bind_addr = "127.0.0.1:3002"
+# The communication server address for frontend and datanode to connect to metasrv,  "127.0.0.1:3002" by default for localhost.
+server_addr = "127.0.0.1:3002"
+# Etcd server address, "127.0.0.1:2379" by default.
+store_addr = "127.0.0.1:2379"
+# Datanode lease in seconds, 15 seconds by default.
+datanode_lease_secs = 15
+# Datanode selector type.
+# - "LeaseBased" (default value).
+# - "LoadBased"
+# For details, please see "https://docs.greptime.com/developer-guide/meta/selector".
+selector = "LeaseBased"
+# Store data in memory, false by default.
+use_memory_store = false

config/standalone.example.toml

@@ -0,0 +1,120 @@
+# Node running mode, "standalone" or "distributed".
+mode = "standalone"
+# Whether to use in-memory catalog, `false` by default.
+enable_memory_catalog = false
+
+# HTTP server options.
+[http_options]
+# Server address, "127.0.0.1:4000" by default.
+addr = "127.0.0.1:4000"
+# HTTP request timeout, 30s by default.
+timeout = "30s"
+
+# gRPC server options.
+[grpc_options]
+# Server address, "127.0.0.1:4001" by default.
+addr = "127.0.0.1:4001"
+# The number of server worker threads, 8 by default.
+runtime_size = 8
+
+# MySQL server options.
+[mysql_options]
+# Server address, "127.0.0.1:4002" by default.
+addr = "127.0.0.1:4002"
+# The number of server worker threads, 2 by default.
+runtime_size = 2
+
+# MySQL server TLS options.
+[mysql_options.tls]
+# TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html
+# - "disable" (default value)
+# - "prefer"
+# - "require"
+# - "verify-ca"
+# - "verify-full"
+mode = "disable"
+# Certificate file path.
+cert_path = ""
+# Private key file path.
+key_path = ""
+
+# PostgresSQL server options.
+[postgres_options]
+# Server address, "127.0.0.1:4003" by default.
+addr = "127.0.0.1:4003"
+# The number of server worker threads, 2 by default.
+runtime_size = 2
+
+# PostgresSQL server TLS options, see `[mysql_options.tls]` section.
+[postgres_options.tls]
+# TLS mode.
+mode = "disable"
+# certificate file path.
+cert_path = ""
+# private key file path.
+key_path = ""
+
+# OpenTSDB protocol options.
+[opentsdb_options]
+# OpenTSDB telnet API server address, "127.0.0.1:4242" by default.
+addr = "127.0.0.1:4242"
+# The number of server worker threads, 2 by default.
+runtime_size = 2
+
+# InfluxDB protocol options.
+[influxdb_options]
+# Whether to enable InfluxDB protocol in HTTP API, true by default.
+enable = true
+
+# Prometheus protocol options.
+[prometheus_options]
+# Whether to enable Prometheus remote write and read in HTTP API, true by default.
+enable = true
+
+# Prom protocol options.
+[prom_options]
+# Prometheus API server address, "127.0.0.1:4004" by default.
+addr = "127.0.0.1:4004"
+
+# WAL options.
+[wal]
+# WAL data directory.
+dir = "/tmp/greptimedb/wal"
+# WAL file size in bytes.
+file_size = "1GB"
+# WAL purge threshold in bytes.
+purge_threshold = "50GB"
+# WAL purge interval in seconds.
+purge_interval = "10m"
+# WAL read batch size.
+read_batch_size = 128
+# Whether to sync log file after every write.
+sync_write = false
+
+# Storage options.
+[storage]
+# Storage type.
+type = "File"
+# Data directory, "/tmp/greptimedb/data" by default.
+data_dir = "/tmp/greptimedb/data/"
+
+# Compaction options.
+[compaction]
+# Max task number that can concurrently run.
+max_inflight_tasks = 4
+# Max files in level 0 to trigger compaction.
+max_files_in_level0 = 8
+# Max task number for SST purge task after compaction.
+max_purge_tasks = 32
+
+# Procedure storage options.
+# Uncomment to enable.
+# [procedure.store]
+# # Storage type.
+# type = "File"
+# # Procedure data path.
+# data_dir = "/tmp/greptimedb/procedure/"
+# # Procedure max retry time.
+# max_retry_times = 3
+# # Initial retry delay of procedures, increases exponentially
+# retry_delay = "500ms"

docker/Dockerfile

@@ -9,7 +9,10 @@ RUN apt-get update && apt-get install -y \
     protobuf-compiler \
     curl \
     build-essential \
-    pkg-config
+    pkg-config \
+    python3 \
+    python3-dev \
+    && pip install pyarrow
 
 # Install Rust.
 SHELL ["/bin/bash", "-c"]
@@ -24,9 +27,10 @@ RUN cargo build --release
 # TODO(zyy17): Maybe should use the more secure container image.
 FROM ubuntu:22.04 as base
 
-WORKDIR /greptimedb
-COPY --from=builder /greptimedb/target/release/greptime /greptimedb/bin/
-ENV PATH /greptimedb/bin/:$PATH
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates
 
-ENTRYPOINT [ "greptime" ]
-CMD [ "datanode", "start"]
+WORKDIR /greptime
+COPY --from=builder /greptimedb/target/release/greptime /greptime/bin/
+ENV PATH /greptime/bin/:$PATH
+
+ENTRYPOINT ["greptime"]

docker/aarch64/Dockerfile

@@ -0,0 +1,57 @@
+FROM ubuntu:22.04 as builder
+
+ENV LANG en_US.utf8
+WORKDIR /greptimedb
+
+# Install dependencies.
+RUN apt-get update && apt-get install -y \
+    libssl-dev \
+    protobuf-compiler \
+    curl \
+    build-essential \
+    pkg-config \
+    wget
+
+# Install Rust.
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /root/.cargo/bin/:$PATH
+
+# Install cross platform toolchain
+RUN apt-get -y update && \
+    apt-get -y install g++-aarch64-linux-gnu gcc-aarch64-linux-gnu && \
+    apt-get install binutils-aarch64-linux-gnu
+
+COPY ./docker/aarch64/compile-python.sh ./docker/aarch64/
+RUN chmod +x ./docker/aarch64/compile-python.sh && \
+    ./docker/aarch64/compile-python.sh
+
+COPY ./rust-toolchain.toml .
+# Install rustup target for cross compiling.
+RUN rustup target add aarch64-unknown-linux-gnu
+COPY . .
+# Update dependency, using separate `RUN` to separate cache
+RUN cargo fetch
+
+# This three env var is set in script, so I set it manually in dockerfile.
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
+ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib/
+ENV PY_INSTALL_PATH=/greptimedb/python_arm64_build
+
+# Set the environment variable for cross compiling and compile it
+# cross compiled python is `python3` in path, but pyo3 need `python` in path so alias it
+# Build the project in release mode.
+RUN export PYO3_CROSS_LIB_DIR=$PY_INSTALL_PATH/lib && \ 
+    alias python=python3 && \
+    cargo build --target aarch64-unknown-linux-gnu --release -F pyo3_backend
+
+# Exporting the binary to the clean image
+FROM ubuntu:22.04 as base
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates
+
+WORKDIR /greptime
+COPY --from=builder /greptimedb/target/aarch64-unknown-linux-gnu/release/greptime /greptime/bin/
+ENV PATH /greptime/bin/:$PATH
+
+ENTRYPOINT ["greptime"]

docker/aarch64/compile-python.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+set -e
+
+# this script will download Python source code, compile it, and install it to /usr/local/lib
+# then use this python to compile cross-compiled python for aarch64
+ARCH=$1
+PYTHON_VERSION=3.10.10
+PYTHON_SOURCE_DIR=Python-${PYTHON_VERSION}
+PYTHON_INSTALL_PATH_AMD64=${PWD}/python-${PYTHON_VERSION}/amd64
+PYTHON_INSTALL_PATH_AARCH64=${PWD}/python-${PYTHON_VERSION}/aarch64
+
+function download_python_source_code() {
+  wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz
+  tar -xvf Python-$PYTHON_VERSION.tgz
+}
+
+function compile_for_amd64_platform() {
+  mkdir -p "$PYTHON_INSTALL_PATH_AMD64"
+
+  echo "Compiling for amd64 platform..."
+
+  ./configure \
+    --prefix="$PYTHON_INSTALL_PATH_AMD64" \
+    --enable-shared \
+    ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \
+    ac_cv_have_long_long_format=yes \
+    --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no
+
+  make
+  make install
+}
+
+# explain Python compile options here a bit:s
+# --enable-shared: enable building a shared Python library (default is no) but we do need it for calling from rust
+# CC, CXX, AR, LD, RANLIB: set the compiler, archiver, linker, and ranlib programs to use
+# build: the machine you are building on, host: the machine you will run the compiled program on
+# --with-system-ffi: build _ctypes module using an installed ffi library, see Doc/library/ctypes.rst, not used in here TODO: could remove
+# ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes:
+# allow cross-compiled python to have -pthread set for CXX, see https://github.com/python/cpython/pull/22525
+# ac_cv_have_long_long_format=yes: target platform supports long long type
+# disable-ipv6: disable ipv6 support, we don't need it in here
+# ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no: disable pty support, we don't need it in here
+function compile_for_aarch64_platform() {
+  export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
+  export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
+  export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
+
+  mkdir -p "$PYTHON_INSTALL_PATH_AARCH64"
+
+  echo "Compiling for aarch64 platform..."
+  echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
+  echo "LIBRARY_PATH: $LIBRARY_PATH"
+  echo "PATH: $PATH"
+
+  ./configure --build=x86_64-linux-gnu --host=aarch64-linux-gnu \
+    --prefix="$PYTHON_INSTALL_PATH_AARCH64" --enable-optimizations \
+    CC=aarch64-linux-gnu-gcc \
+    CXX=aarch64-linux-gnu-g++ \
+    AR=aarch64-linux-gnu-ar \
+    LD=aarch64-linux-gnu-ld \
+    RANLIB=aarch64-linux-gnu-ranlib \
+    --enable-shared \
+    ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \
+    ac_cv_have_long_long_format=yes \
+    --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no
+
+  make
+  make altinstall
+}
+
+# Main script starts here.
+download_python_source_code
+
+# Enter the python source code directory.
+cd $PYTHON_SOURCE_DIR || exit 1
+
+# Build local python first, then build cross-compiled python.
+compile_for_amd64_platform
+
+# Clean the build directory.
+make clean && make distclean
+
+# Cross compile python for aarch64.
+if [ "$ARCH" = "aarch64-unknown-linux-gnu" ]; then
+  compile_for_aarch64_platform
+fi

docker/ci/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    ca-certificates \
+    python3.10 \
+    python3.10-dev \
+    python3-pip
+
+COPY requirements.txt /etc/greptime/requirements.txt
+
+RUN python3 -m pip install -r /etc/greptime/requirements.txt
+
+ARG TARGETARCH
+
+ADD $TARGETARCH/greptime /greptime/bin/
+
+ENV PATH /greptime/bin/:$PATH
+
+ENTRYPOINT ["greptime"]

docker/ci/requirements.txt

@@ -0,0 +1,5 @@
+numpy>=1.24.2
+pandas>=1.5.3
+pyarrow>=11.0.0
+requests>=2.28.2
+scipy>=1.10.1

docs/how-to/how-to-write-aggregate-function.md

@@ -55,7 +55,7 @@ The DataFusion basically execute aggregate like this:
 2. Call `update_batch` on each accumulator with partitioned data, to let you update your aggregate calculation.
 3. Call `state` to get each accumulator's internal state, the medial calculation result.
 4. Call `merge_batch` to merge all accumulator's internal state to one.
-5. Execute `evalute` on the chosen one to get the final calculation result.
+5. Execute `evaluate` on the chosen one to get the final calculation result.
 
 Once you know the meaning of each method, you can easily write your accumulator. You can refer to `Median` accumulator or `SUM` accumulator defined in  file `my_sum_udaf_example.rs` for more details.
 
@@ -63,7 +63,7 @@ Once you know the meaning of each method, you can easily write your accumulator.
 
 You can call `register_aggregate_function` method in query engine to register your aggregate function. To do that, you have to new an instance of struct `AggregateFunctionMeta`. The struct has three fields, first is the name of your aggregate function's name. The function name is case-sensitive due to DataFusion's restriction. We strongly recommend using lowercase for your name. If you have to use uppercase name, wrap your aggregate function with quotation marks. For example, if you define an aggregate function named "my_aggr", you can use "`SELECT MY_AGGR(x)`"; if you define "my_AGGR", you have to use "`SELECT "my_AGGR"(x)`".
 
-The second field is arg_counts ,the count of the arguments. Like accumulator `percentile`, caculating the p_number of the column. We need to input the value of column and the value of p to cacalate, and so the count of the arguments is two.
+The second field is arg_counts ,the count of the arguments. Like accumulator `percentile`, calculating the p_number of the column. We need to input the value of column and the value of p to cacalate, and so the count of the arguments is two.
 
 The third field is a function about how to create your accumulator creator that you defined in step 1 above. Create creator, that's a bit intertwined, but it is how we make DataFusion use a newly created aggregate function each time it executes a SQL, preventing the stored input types from affecting each other. The key detail can be starting looking at our `DfContextProviderAdapter` struct's `get_aggregate_meta` method.
 

docs/rfcs/2022-12-20-promql-in-rust/rfc.md

@@ -0,0 +1,175 @@
+---
+Feature Name: "promql-in-rust"
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/596
+Date: 2022-12-20
+Author: "Ruihang Xia <waynestxia@gmail.com>"
+---
+
+Rewrite PromQL in Rust
+----------------------
+
+# Summary
+A Rust native implementation of PromQL, for GreptimeDB.
+
+# Motivation
+Prometheus and its query language PromQL prevails in the cloud-native observability area, which is an important scenario for time series database like GreptimeDB. We already have support for its remote read and write protocols. Users can now integrate GreptimeDB as the storage backend to existing Prometheus deployment, but cannot run PromQL query directly on GreptimeDB like SQL.
+
+This RFC proposes to add support for PromQL. Because it was created in Go, we can't use the existing code easily. For interoperability, performance and extendability, porting its logic to Rust is a good choice.
+
+# Details
+
+## Overview
+One of the goals is to make use of our existing basic operators, execution model and runtime to reduce the work. So the entire proposal is built on top of Apache Arrow DataFusion. The rewrote PromQL logic is manifested as `Expr` or `Execution Plan` in DataFusion. And both the intermediate data structure and the result is in the format of `Arrow`'s `RecordBatch`.
+
+The following sections are organized in a top-down manner. Starts with evaluation procedure. Then introduces the building blocks of our new PromQL operation. Follows by an explanation of data model. And end with an example logic plan.
+
+*This RFC is heavily related to Prometheus and PromQL. It won't repeat some basic concepts of them.*
+
+## Evaluation
+
+The original implementation is like an interpreter of parsed PromQL AST. It has two characteristics: (1) Operations are evaluated in place after they are parsed to AST. And some key parameters are separated from the AST because they do not present in the query, but come from other places like another field in the HTTP payload. (2) calculation is performed per timestamp. You can see this pattern many times:
+```go
+for ts := ev.startTimestamp; ts <= ev.endTimestamp; ts += ev.interval {}
+```
+
+These bring out two differences in the proposed implementation. First, to make it more general and clear, the evaluation procedure is reorganized into serval phases (and is the same as DataFusion's). And second, data are evaluated by time series (corresponding to "columnar calculation", if think timestamp as row number).
+
+```
+                                      Logic
+   Query             AST              Plan
+ ─────────► Parser ───────► Logical ────────► Physical ────┐
+                            Planner           Planner      │
+                                                           │
+ ◄───────────────────────────── Executor  ◄────────────────┘
+        Evaluation Result                     Execution
+                                                Plan
+```
+
+- Parser 
+
+    Provided by [`promql-parser`](https://github.com/GreptimeTeam/promql-parser) crate. Same as the original implementation.
+
+- Logical Planner
+
+    Generates a logical plan with all the needed parameters. It should accept something like `EvalStmt` in Go's implementation, which contains query time range, evaluation interval and lookback range.
+
+    Another important thing done here is assembling the logic plan, with all the operations baked into logically. Like what's the filter and time range to read, how the data then flows through a selector into a binary operation, etc. Or what's the output schema of every single step. The generated logic plan is deterministic without variables, and can be `EXPLAIN`ed clearly.
+
+- Physical Planner
+
+    This step converts a logic plan into evaluatable execution plan. There are not many special things like the previous step. Except when a query is going to be executed distributedly. In this case, a logic plan will be divided into serval parts and sent to serval nodes. One physical planner only sees its own part.
+
+- Executor
+
+    As its name shows, this step calculates data to result. And all new calculation logic, the implementation of PromQL in rust, is placed here. And the rewrote functions are using `RecordBatch` and `Array` from `Arrow` as the intermediate data structure.
+
+    Each "batch" contains only data from single time series. This is from the underlying storage implementation. Though it's not a requirement of this RFC, having this property can simplify some functions.
+    
+    Another thing to mention is the rewrote functions don't aware of timestamp or value columns, they are defined only based on the input data types. For example, `increase()` function in PromQL calculates the unbiased delta of data, its implementation here only does this single thing. Let's compare the signature of two implementations:
+
+    - Go
+        ```go
+        func funcIncrease(vals []parser.Value, args parser.Expressions) Vector {}
+        ```
+    - Rust
+        ```rust
+        fn prom_increase(input: Array) -> Array {}
+        ```
+
+    Some unimportant parameters are omitted. The original Go version only writes the logic for `Point`'s value, either float or histogram. But the proposed rewritten one accepts a generic `Array` as input, which can be any type that suits, from `i8` to `u64` to `TimestampNanosecond`.
+
+## Plan and Expression
+
+They are structures to express logic from PromQL. The proposed implementation is built on top of DataFusion, thus our plan and expression are in form of `ExtensionPlan` and `ScalarUDF`. The only difference between them in this context is the return type: plan returns a record batch while expression returns a single column.
+
+This RFC proposes to add four new plans, they are fundamental building blocks that mainly handle data selection logic in PromQL, for the following calculation expressions. 
+
+- `SeriesNormalize`
+
+    Sort data inside one series on the timestamp column, and bias "offset" if has. This plan usually comes after `TableScan` (or `TableScan` and `Filter`) plan.
+
+- `VectorManipulator` and `MatrixManipulator`
+
+    Corresponding to `InstantSelector` and `RangeSelector`. We don't calculate timestamp by timestamp, thus use "vector" instead of "instant", this image shows the difference. And "matrix" is another name for "range vector", for not confused with our "vector". The following section will detail how they are implemented using Arrow.
+
+    ![instant_and_vector](instant-and-vector.png)
+
+    Due to "interval" parameter in PromQL, data after "selector" (or "manipulator" here) are usually shorter than input. And we have to modify the entire record batch to shorten both timestamp, value and tag columns. So they are formed as plan.
+
+- `PromAggregator`
+
+    The carrier of aggregator expressions. This should not be very different from the DataFusion built-in `Aggregate` plan, except PromQL can use "group without" to do reverse selection.
+
+PromQL has around 70 expressions and functions. But luckily we can reuse lots of them from DataFusion. Like unary expression, binary expression and aggregator. We only need to implement those PromQL-specific expressions, like `rate` or `percentile`. The following table lists some typical functions in PromQL, and their signature in the proposed implementation. Other function should be the same.
+
+| Name               	| In Param(s)                                          	| Out Param(s) 	| Explain            	|
+|--------------------	|------------------------------------------------------	|--------------	|--------------------	|
+| instant_delta      	| Matrix T                                             	| Array T      	| idelta in PromQL   	|
+| increase           	| Matrix T                                             	| Array T      	| increase in PromQL 	|
+| extrapolate_factor 	| - Matrix T<br>- Array Timestamp<br>- Array Timestamp 	| Array T      	| *                   	|
+
+*: *`extrapolate_factor` is one of the "dark sides" in PromQL. In short it's a translation of this [paragraph](https://github.com/prometheus/prometheus/blob/0372e259baf014bbade3134fd79bcdfd8cbdef2c/promql/functions.go#L134-L159)*
+
+To reuse those common calculation logic, we can break them into serval expressions, and assemble in the logic planning phase. Like `rate()` in PromQL can be represented as `increase / extrapolate_factor`.
+
+## Data Model
+
+This part explains how data is represented. Following the data model in GreptimeDB, all the data are stored as table, with tag columns, timestamp column and value column. Table to record batch is very straightforward. So an instant vector can be thought of as a row (though as said before, we don't use instant vectors) in the table. Given four basic types in PromQL: scalar, string, instant vector and range vector, only the last "range vector" need some tricks to adapt our columnar calculation.
+
+Range vector is some sort of matrix, it's consisted of small one-dimension vectors, with each being an input of range function. And, applying range function to a range vector can be thought of kind of convolution.
+
+![range-vector-with-matrix](range-vector-with-matrix.png)
+
+(Left is an illustration of range vector. Notice the Y-axis has no meaning, it's just put different pieces separately. The right side is an imagined "matrix" as range function. Multiplying the left side to it can get a one-dimension "matrix" with four elements. That's the evaluation result of a range vector.)
+
+To adapt this range vector to record batch, it should be represented by a column. This RFC proposes to use `DictionaryArray` from Arrow to represent range vector, or `Matrix`. This is "misusing" `DictionaryArray` to ship some additional information about an array. Because the range vector is sliding over one series, we only need to know the `offset` and `length` of each slides to reconstruct the matrix from an array:
+
+![matrix-from-array](matrix-from-array.png)
+
+The length is not fixed, it depends on the input's timestamp. An PoC implementation of `Matrix` and `increase()` can be found in [this repo](https://github.com/waynexia/corroding-prometheus).
+
+## Example
+
+The logic plan of PromQL query
+```promql
+# start: 2022-12-20T10:00:00
+# end: 2022-12-21T10:00:00
+# interval: 1m
+# lookback: 30s
+sum (rate(request_duration[5m])) by (idc)
+```
+looks like
+
+<!-- title: 'PromAggregator: \naggr = sum, column = idc'
+operator: prom
+inputs:
+- title: 'Matrix Manipulator: \ninterval = 1m, range = 5m, expr = div(increase(value), extrapolate_factor(timestamp))'
+  operator: prom
+  inputs:
+  - title: 'Series Normalize: \noffset = 0'
+    operator: prom
+    inputs:
+    - title: 'Filter: \ntimestamp > 2022-12-20T10:00:00 && timestamp < 2022-12-21T10:00:00'
+      operator: filter
+      inputs:
+      - title: 'Table Scan: \ntable = request_duration, timestamp > 2022-12-20T10:00:00 && timestamp < 2022-12-21T10:00:00'
+        operator: scan -->
+
+![example](example.png)
+
+# Drawbacks
+
+Human-being is always error-prone. It's harder to endeavor to rewrite from the ground and requires more attention to ensure correctness, than translate line-by-line. And, since the evaluator's architecture are different, it might be painful to catch up with PromQL's breaking update (if any) in the future.
+
+Misusing Arrow's DictionaryVector as Matrix is another point. This hack needs some `unsafe` function call to bypass Arrow's check. And though Arrow's API is stable, this is still an undocumented behavior.
+
+# Alternatives
+
+There are a few alternatives we've considered:
+- Wrap the existing PromQL's implementation via FFI, and import it to GreptimeDB.
+- Translate its evaluator engine line-by-line, rather than rewrite one.
+- Integrate the Prometheus server into GreptimeDB via RPC, making it a detached execution engine for PromQL.
+
+The first and second options are making a separate execution engine in GreptimeDB, they may alleviate the pain during rewriting, but will have negative impacts to afterward evolve like resource management. And introduce another deploy component in the last option will bring a complex deploy architecture.
+
+And all of them are more or less redundant in data transportation that affects performance and resources. The proposed built-in executing procedure is also easy to integrate and expose to the existing SQL interface GreptimeDB currently provides. Some concepts in PromQL like sliding windows (range vector in PromQL) are very convenient and ergonomic in analyzing series data. This makes it not only a PromQL evaluator, but also an enhancement to our query system.

docs/rfcs/2023-01-03-procedure-framework.md

@@ -0,0 +1,151 @@
+---
+Feature Name: "procedure-framework"
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/286
+Date: 2023-01-03
+Author: "Yingwen <realevenyag@gmail.com>"
+---
+
+Procedure Framework
+----------------------
+
+# Summary
+A framework for executing operations in a fault-tolerant manner.
+
+# Motivation
+Some operations in GreptimeDB require multiple steps to implement. For example, creating a table needs:
+1. Check whether the table exists
+2. Create the table in the table engine
+  1. Create a region for the table in the storage engine
+  2. Persist the metadata of the table to the table manifest
+3. Add the table to the catalog manager
+
+If the node dies or restarts in the middle of creating a table, it could leave the system in an inconsistent state. The procedure framework, inspired by [Apache HBase's ProcedureV2 framework](https://github.com/apache/hbase/blob/bfc9fc9605de638785435e404430a9408b99a8d0/src/main/asciidoc/_chapters/pv2.adoc) and [Apache Accumulo’s FATE framework](https://accumulo.apache.org/docs/2.x/administration/fate), aims to provide a unified way to implement multi-step operations that is tolerant to failure.
+
+# Details
+## Overview
+The procedure framework consists of the following primary components:
+- A `Procedure` represents an operation or a set of operations to be performed step-by-step
+- `ProcedureManager`, the runtime to run `Procedures`. It executes the submitted procedures, stores procedures' states to the `ProcedureStore` and restores procedures from `ProcedureStore` while the database restarts.
+- `ProcedureStore` is a storage layer for persisting the procedure state
+
+
+## Procedures
+The `ProcedureManager` keeps calling `Procedure::execute()` until the Procedure is done, so the operation of the Procedure should be [idempotent](https://developer.mozilla.org/en-US/docs/Glossary/Idempotent): it needs to be able to undo or replay a partial execution of itself.
+
+```rust
+trait Procedure {
+    fn execute(&mut self, ctx: &Context) -> Result<Status>;
+
+    fn dump(&self) -> Result<String>;
+
+    fn rollback(&self) -> Result<()>;
+
+    // other methods...
+}
+```
+
+The `Status` is an enum that has the following variants:
+```rust
+enum Status {
+    Executing {
+        persist: bool,
+    },
+    Suspended {
+        subprocedures: Vec<ProcedureWithId>,
+        persist: bool,
+    },
+    Done,
+}
+```
+
+A call to `execute()` can result in the following possibilities:
+- `Ok(Status::Done)`: we are done
+- `Ok(Status::Executing { .. })`: there are remaining steps to do
+- `Ok(Status::Suspend { sub_procedure, .. })`: execution is suspended and can be resumed later after the sub-procedure is done.
+- `Err(e)`: error occurs during execution and the procedure is unable to proceed anymore.
+
+Users need to assign a unique `ProcedureId` to the procedure and the procedure can get this id via the `Context`. The `ProcedureId` is typically a UUID.
+
+```rust
+struct Context {
+    id: ProcedureId,
+    // other fields ...
+}
+```
+
+The `ProcedureManager` calls `Procedure::dump()` to serialize the internal state of the procedure and writes to the `ProcedureStore`. The `Status` has a field `persist` to tell the `ProcedureManager` whether it needs persistence.
+
+## Sub-procedures
+A procedure may need to create some sub-procedures to process its subtasks. For example, creating a distributed table with multiple regions (partitions) needs to set up the regions in each node, thus the parent procedure should instantiate a sub-procedure for each region. The `ProcedureManager` makes sure that the parent procedure does not proceed till all sub-procedures are successfully finished.
+
+The procedure can submit sub-procedures to the `ProcedureManager` by returning `Status::Suspended`. It needs to assign a procedure id to each procedure manually so it can track the status of the sub-procedures.
+```rust
+struct ProcedureWithId {
+    id: ProcedureId,
+    procedure: BoxedProcedure,
+}
+```
+
+## ProcedureStore
+We might need to provide two different ProcedureStore implementations:
+- In standalone mode, it stores data on the local disk.
+- In distributed mode, it stores data on the meta server or the object store service.
+
+These implementations should share the same storage structure. They store each procedure's state in a unique path based on the procedure id:
+
+```
+Sample paths:
+
+/procedures/{PROCEDURE_ID}/000001.step
+/procedures/{PROCEDURE_ID}/000002.step
+/procedures/{PROCEDURE_ID}/000003.commit
+```
+
+`ProcedureStore` behaves like a WAL. Before performing each step, the `ProcedureManager` can write the procedure's current state to the ProcedureStore, which stores the state in the `.step` file. The `000001` in the path is a monotonic increasing sequence of the step. After the procedure is done, the `ProcedureManager` puts a `.commit` file to indicate the procedure is finished (committed).
+
+The `ProcedureManager` can remove the procedure's files once the procedure is done, but it needs to leave the `.commit` as the last file to remove in case of failure during removal.
+
+## ProcedureManager
+`ProcedureManager` executes procedures submitted to it.
+
+```rust
+trait ProcedureManager {
+    fn register_loader(&self, name: &str, loader: BoxedProcedureLoader) -> Result<()>;
+
+    async fn submit(&self, procedure: ProcedureWithId) -> Result<()>;
+}
+```
+
+It supports the following operations:
+- Register a `ProcedureLoader` by the type name of the `Procedure`.
+- Submit a `Procedure` to the manager and execute it.
+
+When `ProcedureManager` starts, it loads procedures from the `ProcedureStore` and restores the procedures by the `ProcedureLoader`. The manager stores the type name from `Procedure::type_name()` with the data from `Procedure::dump()` in the `.step` file and uses the type name to find a `ProcedureLoader` to recover the procedure from its data.
+
+```rust
+type BoxedProcedureLoader = Box<dyn Fn(&str) -> Result<BoxedProcedure> + Send>;
+```
+
+## Rollback
+The rollback step is supposed to clean up the resources created during the execute() step. When a procedure has failed, the `ProcedureManager` puts a `rollback` file and calls the `Procedure::rollback()` method.
+
+
+```text
+/procedures/{PROCEDURE_ID}/000001.step
+/procedures/{PROCEDURE_ID}/000002.rollback
+```
+
+Rollback is complicated to implement so some procedures might not support rollback or only provide a best-efforts approach.
+
+## Locking
+The `ProcedureManager` can provide a locking mechanism that gives a procedure read/write access to a database object such as a table so other procedures are unable to modify the same table while the current one is executing.
+
+# Drawbacks
+The `Procedure` framework introduces additional complexity and overhead to our database.
+- To execute a `Procedure`, we need to write to the `ProcedureStore` multiple times, which may slow down the server
+- We need to rewrite the logic of creating/dropping/altering a table using the procedure framework
+
+# Alternatives
+Another approach is to tolerate failure during execution and allow users to retry the operation until it succeeds. But we still need to:
+- Make each step idempotent
+- Record the status in some place to check whether we are done

docs/rfcs/2023-02-01-table-compaction.md

@@ -0,0 +1,92 @@
+---
+Feature Name: "table-compaction"
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/930
+Date: 2023-02-01
+Author: "Lei, HUANG <mrsatangel@gmail.com>"
+---
+
+# Table Compaction
+
+--- 
+
+## Background
+
+GreptimeDB uses an LSM-tree based storage engine that flushes memtables to SSTs for persistence. 
+But currently it only supports level 0. SST files in level 0 does not guarantee to contain only rows with disjoint time ranges. 
+That is to say, different SST files in level 0 may contain overlapped timestamps. 
+The consequence is, in order to retrieve rows in some time range, all files need to be scanned, which brings a lot of IO overhead.
+
+Also, just like other LSMT engines, delete/update to existing primary keys are converted to new rows with delete/update mark and appended to SSTs on flushing. 
+We need to merge the operations to same primary keys so that we don't have to go through all SST files to find the final state of these primary keys.  
+
+## Goal
+
+Implement a compaction framework to: 
+- maintain SSTs in timestamp order to accelerate queries with timestamp condition;
+- merge rows with same primary key;
+- purge expired SSTs;
+- accommodate other tasks like data rollup/indexing.
+
+
+## Overview
+
+Table compaction involves following components:
+- Compaction scheduler: run compaction tasks, limit the consumed resources;
+- Compaction strategy: find the SSTs to compact and determine the output files of compaction.
+- Compaction task: read the rows from input SSTs and write to the output files.
+
+## Implementation
+
+### Compaction scheduler
+
+`CompactionScheduler` is an executor that continuously polls and executes compaction request from a task queue. 
+
+```rust
+#[async_trait]
+pub trait CompactionScheduler {
+    /// Schedules a compaction task.
+    async fn schedule(&self, task: CompactionRequest) -> Result<()>;
+
+    /// Stops compaction scheduler.
+    async fn stop(&self) -> Result<()>;
+}
+```
+
+
+
+### Compaction triggering
+
+Currently, we can check whether to compact tables when memtable is flushed to SST.
+
+https://github.com/GreptimeTeam/greptimedb/blob/4015dd80752e1e6aaa3d7cacc3203cb67ed9be6d/src/storage/src/flush.rs#L245
+
+
+### Compaction strategy
+
+`CompactionStrategy` defines how to pick SSTs in all levels for compaction.   
+
+```rust
+pub trait CompactionStrategy {
+    fn pick(
+        &self,
+        ctx: CompactionContext,
+        levels: &LevelMetas,
+    ) -> Result<CompactionTask>;
+}
+```
+
+The most suitable compaction strategy for time-series scenario would be 
+a hybrid strategy that combines time window compaction with size-tired compaction, just like [Cassandra](https://cassandra.apache.org/doc/latest/cassandra/operating/compaction/twcs.html) and [ScyllaDB](https://docs.scylladb.com/stable/architecture/compaction/compaction-strategies.html#time-window-compaction-strategy-twcs) does.
+
+We can first group SSTs in level n into buckets according to some predefined time window. Within that window, 
+SSTs are compacted in a size-tired manner (find SSTs with similar size and compact them to level n+1). 
+SSTs from different time windows are neven compacted together.
+That strategy guarantees SSTs in each level are mainly sorted in timestamp order which boosts queries with 
+explicit timestamp condition, while size-tired compaction minimizes the impact to foreground writes. 
+
+### Alternatives
+
+Currently, GreptimeDB's storage engine [only support two levels](https://github.com/GreptimeTeam/greptimedb/blob/43aefc5d74dfa73b7819cae77b7eb546d8534a41/src/storage/src/sst.rs#L32).
+For level 0, we can start with a simple time-window based leveled compaction, which reads from all SSTs in level 0, 
+align them to time windows with a fixed duration, merge them with SSTs in level 1 within the same time window 
+to ensure there is only one sorted run in level 1.

docs/rfcs/2023-03-08-region-fault-tolerance.md

@@ -0,0 +1,196 @@
+---
+Feature Name: "Fault Tolerance for Region"
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/1126
+Date: 2023-03-08
+Author: "Luo Fucong <luofucong@greptime.com>"
+---
+
+Fault Tolerance for Region
+----------------------
+
+# Summary
+
+This RFC proposes a method to achieve fault tolerance for regions in GreptimeDB's distributed mode. Or, put it in another way, achieving region high availability("HA") for GreptimeDB cluster. 
+
+In this RFC, we mainly describe two aspects of region HA: how region availability is detected, and what recovery process is need to be taken. We also discuss some alternatives and future work.
+
+When this feature is done, our users could expect a GreptimeDB cluster that can always handle their requests to regions, despite some requests may failed during the region failover. The optimization to reduce the MTTR(Mean Time To Recovery) is not a concern of this RPC, and is left for future work.
+
+# Motivation
+
+Fault tolerance for regions is a critical feature for our clients to use the GreptimeDB cluster confidently. High availability for users to interact with their stored data is a "must have" for any TSDB products, that include our GreptimeDB cluster.
+
+# Details
+
+## Background
+
+Some backgrounds about region in distributed mode:
+
+- A table is logically split into multiple regions. Each region stores a part of non-overlapping table data.
+- Regions are distributed in Datanodes, the mappings are not static, are assigned and governed by Metasrv.
+- In distributed mode, client requests are scoped in regions. To be more specific, when a request that needs to scan multiple regions arrived in Frontend, Frontend splits the request into multiple sub-requests, each of which scans one region only, and submits them to Datanodes that hold corresponding regions. 
+
+In conclusion, as long as regions remain available, and regions could regain availability when failures do occur, the overall region HA could be achieved. With this in mind, let's see how region failures are detected first. 
+
+## Failure Detection
+
+We detect region failures in Metasrv, and do it both passively and actively. Passively means that Metasrv do not fire some "are you healthy" requests to regions. Instead, we carry region healthy information in the heartbeat requests that are submit to Metasrv by Datanodes. 
+
+Datanode already carries its regions stats in the heartbeat request (the non-relevant fields are omitted):
+
+```protobuf
+message HeartbeatRequest {
+  ...
+  // Region stats on this node
+  repeated RegionStat region_stats = 6;
+  ...
+}
+
+message RegionStat {
+  uint64 region_id = 1;
+  TableName table_name = 2;
+  ...
+}
+```
+
+For the sake of simplicity, we don't add another field `bool available = 3` to the `RegionStat` message; instead, if the region were unavailable in the view of the Datanode that contains it, the Datanode just not includes the `RegionStat` of it in the heartbeat request. Or, if the Datanode itself is not unavailable, the heartbeat request is not submitted, effectively the same with not carrying the `RegionStat`.
+
+> The heartbeat interval is now hardcoded to five seconds.
+
+Metasrv gathers the heartbeat requests, extracts the `RegionStat`s, and treat them as region heartbeat. In this way, Metasrv maintains all regions healthy information. If some region's heartbeats were not received in a period of time, Metasrv speculates the region might be unavailable. To make the decision whether a region is failed or not, Metasrv uses a failure detection algorithm called the "[Phi φ Accrual Failure Detection](https://medium.com/@arpitbhayani/phi-%CF%86-accrual-failure-detection-79c21ce53a7a)". Basically, the algorithm calculates a value called "phi" to represent the possibility of a region's unavailability, based on the historical heartbeats' arrived rate. Once the "phi" is above some pre-defined threshold, Metasrv knows the region is failed.
+
+> This algorithm has been widely adopted in some well known products, like Akka and Cassandra.
+
+When Metasrv decides some region is failed from heartbeats, it's not the final decision. Here comes the "actively" detection. Before Metasrv decides to do region failover, it actively invokes the healthy check interface of the Datanode that the failure region resides. Only this healthy check is failed does Metasrv actually start doing failover upon the region.
+
+To conclude, the failure detection pseudo-codes are like this:
+
+```rust
+// in Metasrv:
+fn failure_detection() {
+    loop {
+        // passive detection
+        let failed_regions = all_regions.iter().filter(|r| r.estimated_failure_possibility() > config.phi).collect();
+
+        // find the datanodes that contains the failed regions
+        let datanodes_and_regions = find_region_resides_datanodes(failed_regions);
+
+        // active detection  
+        for (datanode, regions) in datanodes_and_regions {
+            if !datanode.is_healthy(regions) {
+                do_failover(datanode, regions);
+            }
+        }
+
+        sleep(config.detect_interval);
+    }
+}
+```
+
+Some design considerations:
+
+- Why active detecting while we have passively detection? Because it could be happened that the network is singly connectable sometimes (especially in the complex Cloud environment), then the Datanode's heartbeats cannot reach Metasrv, while Metasrv could request Datanode. Active detecting avoid this false positive situation.
+- Why the detection works on region instead of Datanode? Because we might face the possibility that only part of the regions in the Datanode are not available, not ALL regions. Especially the situation that Datanodes are used by multiple tenants. If this is the case, it's better to do failover upon the designated regions instead of the whole regions that reside on the Datanode. All in all, we want a more subtle control over region failover. 
+
+So we detect some regions are not available. How to regain the availability back?
+
+## Region Failover
+
+Region Failover largely relies on remote WAL, aka "[Bunshin](https://github.com/GreptimeTeam/bunshin)". I'm not including any of the details of it in this RFC, let's just assume we already have it.
+
+In general, region failover is fairly simple. Once Metasrv decides to do failover upon some regions, it first chooses one or more Datanodes to hold the failed region. This can be done easily, as the Metasrv already has the whole picture of Datanodes: it knows which Datanode has the minimum regions, what Datanode historically had the lowest CPU usage and IO rate, and how the Datanodes are assigned to tenants, among other information that can all help the Metasrv choose the most suitable Datanodes. Let's call these chosen Datanodes as "candidates".
+
+> The strategy to choose the most suitable candidates required careful design, but it's another RFC.
+
+Then, Metasrv sets the states of these failed regions as "passive". We should add a field to `Region`:
+
+```protobuf
+message Region {
+  uint64 id = 1;
+  string name = 2;
+  Partition partition = 3;
+  
+  message State {
+    Active,
+    Passive,
+  }
+  State state = 4;
+  
+  map<string, string> attrs = 100;
+}
+```
+
+Here `Region` is used in message `RegionRoute`, which indicates how the write request is split among regions. When a region is set as "passive", Frontend knows the write to it should be rejected at the moment (the region read is not blocked, however).
+
+> Making a region "passive" here is effectively blocking the write to it. It's ok in the failover situation, the region is failed anyway. However, when dealing with active maintenance operations, region state requires more refined design. But that's another story. 
+
+Third, Metasrv fires the "close region" requests to the failed Datanodes, and fires the "open region" requests to those candidates. "Close region" requests might be failed due to the unavailability of Datanodes, but that's fine, it's just a best-effort attempt to reduce the chance of any in-flight writes got handled unintentionally after the region is set as "passive". The "open region" requests must have succeeded though. Datanodes open regions from remote WAL.
+
+> Currently the "close region" is undefined in Datanode. It could be a local cache clean up of region data or other resources tidy up.
+
+Finally, when a candidate successfully opens its region, it calls back to Metasrv, indicating it is ready to handle region. "call back" here is backed by its heartbeat to Metasrv. Metasrv updates the region's state to "active", so as to let Frontend lifts the restrictions of region writes (again, the read part of region is untouched).
+
+All the above steps should be managed by remote procedure framework. It's another implementation challenge in the region failover feature. (One is the remote WAL of course.)
+
+A picture is worth a 1000 words:
+
+```text
+                                    +-------------------------+                                        
+                                    | Metasrv detects region  |                                        
+                                    | failure                 |                                        
+                                    +-------------------------+                                        
+                                                 |                                                     
+                                                 v                                                     
+                                    +----------------------------+                                        
+                                    | Metasrv chooses candidates |                                        
+                                    | to hold failed regions     |                                        
+                                    +----------------------------+                                        
+                                                 |                                                     
+                                                 v                                                     
+                                    +-------------------------+       +-------------------------+      
+                                    | Metasrv "passive" the   |------>| Frontend rejects writes |      
+                                    | failed regions          |       | to "passive" regions    |      
+                                    +-------------------------+       +-------------------------+      
+                                                 |                                                     
+                                                 v                                                     
++--------------------------+        +---------------------------+                                        
+| Candidate Datanodes open |<-------| Metasrv fires "close" and |                                        
+| regions from remote WAL  |        | "open" region requests    |                                        
++--------------------------+        +---------------------------+                                        
+             |                                                                                         
+             |                                                                                         
+             |                      +-------------------------+       +-------------------------+      
+             +--------------------->| Metasrv "active" the    |------>| Frontend lifts write    |      
+                                    | failed regions          |       | restriction to regions  |      
+                                    +-------------------------+       +-------------------------+      
+                                                 |                                                     
+                                                 v                                                     
+                                    +-------------------------+                                        
+                                    | Region failover done,   |                                        
+                                    | HA regain               |                                        
+                                    +-------------------------+                                        
+```
+
+# Alternatives
+
+## The "Neon" Way
+
+Remote WAL raises a problem that could harm the write throughput of GreptimeDB cluster: each write request has to do at least two remote call, one is from Frontend to Datanode, and one is from Datanode to remote WAL. What if we do it the "[Neon](https://github.com/neondatabase/neon)" way, making remote WAL sits in between the Frontend and Datanode, couldn't that improve our write throughput? It could, though there're some consistency issues like "read-your-writes" to solve.
+
+However, the main concerns we don't adopt this method are two-fold:
+
+1. Remote WAL is planned to be quorum based, it can be efficiently written;
+2. More importantly, we are planning to make the remote WAL an option that users could choose not to enable it (at the cost of some reliability reduction).
+
+## No WAL, Replication instead
+
+This method replicates region across Datanodes directly, like the common way in shared-nothing database. Were the main region failed, a standby region in the replicate group is elected as new "main" and take the read/write requests. The main concern to this method is the incompatibility to our current architecture and code structure. It requires a major redesign, but gains no significant advantage over the remote WAL method. 
+
+However, the replication does have its own advantage that we can learn from to optimize this failover procedure.
+
+# Future Work
+
+Some optimizations we could take:
+
+- To reduce the MTTR, we could make Metasrv chooses the candidate to each region at normal time. The candidate does some preparation works to reduce the open region time, effectively accelerate the failover procedure.
+- We can adopt the replication method, to the degree that region replicas are used as the fast catch-up candidates. The data difference among replicas is minor, region failover does not need to load or exchange too much data, greatly reduced the region failover time.

rust-toolchain

@@ -1 +0,0 @@
-nightly-2022-07-14

rust-toolchain.toml

@@ -0,0 +1,2 @@
+[toolchain]
+channel = "nightly-2023-02-26"

rustfmt.toml

@@ -1,3 +1,2 @@
 group_imports = "StdExternalCrate"
-
-
+imports_granularity = "Module"

scripts/fetch-dashboard-assets.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# This script is used to download built dashboard assets from the "GreptimeTeam/dashboard" repository.
+
+set -e
+
+declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
+declare -r ROOT_DIR=$(dirname ${SCRIPT_DIR})
+declare -r STATIC_DIR="$ROOT_DIR/src/servers/dashboard"
+
+RELEASE_VERSION="$(cat $STATIC_DIR/VERSION)"
+
+# Download the SHA256 checksum attached to the release. To verify the integrity
+# of the download, this checksum will be used to check the download tar file
+# containing the built dashboard assets.
+curl -Ls https://github.com/GreptimeTeam/dashboard/releases/download/$RELEASE_VERSION/sha256.txt --output sha256.txt
+
+# Download the tar file containing the built dashboard assets.
+curl -L https://github.com/GreptimeTeam/dashboard/releases/download/$RELEASE_VERSION/build.tar.gz --output build.tar.gz
+
+# Verify the checksums match; exit if they don't.
+case "$(uname -s)" in
+    FreeBSD | Darwin)
+        echo "$(cat sha256.txt)" | shasum --algorithm 256 --check \
+            || { echo "Checksums did not match for downloaded dashboard assets!"; exit 1; } ;;
+    Linux)
+        echo "$(cat sha256.txt)" | sha256sum --check -- \
+            || { echo "Checksums did not match for downloaded dashboard assets!"; exit 1; } ;;
+    *)
+        echo "The '$(uname -s)' operating system is not supported as a build host for the dashboard" >&2
+        exit 1
+esac
+
+# Extract the assets and clean up.
+tar -xzf build.tar.gz -C "$STATIC_DIR"
+rm sha256.txt
+rm build.tar.gz
+
+echo "Successfully download dashboard assets to $STATIC_DIR"

scripts/install.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+
+set -ue
+
+OS_TYPE=
+ARCH_TYPE=
+VERSION=${1:-latest}
+GITHUB_ORG=GreptimeTeam
+GITHUB_REPO=greptimedb
+BIN=greptime
+
+get_os_type() {
+    os_type="$(uname -s)"
+
+    case "$os_type" in
+    Darwin)
+        OS_TYPE=darwin
+        ;;
+    Linux)
+        OS_TYPE=linux
+        ;;
+    *)
+        echo "Error: Unknown OS type: $os_type"
+        exit 1
+    esac
+}
+
+get_arch_type() {
+    arch_type="$(uname -m)"
+
+    case "$arch_type" in
+    arm64)
+        ARCH_TYPE=arm64
+        ;;
+    aarch64)
+        ARCH_TYPE=arm64
+        ;;
+    x86_64)
+        ARCH_TYPE=amd64
+        ;;
+    amd64)
+        ARCH_TYPE=amd64
+        ;;
+    *)
+        echo "Error: Unknown CPU type: $arch_type"
+        exit 1
+    esac
+}
+
+get_os_type
+get_arch_type
+
+if [ -n "${OS_TYPE}" ] && [ -n "${ARCH_TYPE}" ]; then
+    echo "Downloading ${BIN}, OS: ${OS_TYPE}, Arch: ${ARCH_TYPE}, Version: ${VERSION}"
+
+    if [ "${VERSION}" = "latest" ]; then
+        wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/latest/download/${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz"
+    else
+        wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz"
+    fi
+
+    tar xvf ${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz && rm ${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz && echo "Run './${BIN} --help' to get started"
+fi

src/api/Cargo.toml

@@ -1,14 +1,19 @@
 [package]
 name = "api"
-version = "0.1.0"
-edition = "2021"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+version.workspace = true
+edition.workspace = true
+license.workspace = true
 
 [dependencies]
+arrow-flight.workspace = true
+common-base = { path = "../common/base" }
+common-error = { path = "../common/error" }
+common-time = { path = "../common/time" }
 datatypes = { path = "../datatypes" }
-prost = "0.11"
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "d3861c34f7920238869d0d4e50dc1e6b189d2a6b" }
+prost.workspace = true
 snafu = { version = "0.7", features = ["backtraces"] }
-tonic = "0.8"
+tonic.workspace = true
 
 [build-dependencies]
 tonic-build = "0.8"

src/api/build.rs

@@ -1,13 +0,0 @@
-fn main() {
-    tonic_build::configure()
-        .compile(
-            &[
-                "greptime/v1/insert.proto",
-                "greptime/v1/select.proto",
-                "greptime/v1/physical_plan.proto",
-                "greptime/v1/greptime.proto",
-            ],
-            &["."],
-        )
-        .expect("compile proto");
-}

src/api/greptime/v1/admin.proto

@@ -1,55 +0,0 @@
-syntax = "proto3";
-
-package greptime.v1;
-
-import "greptime/v1/column.proto";
-import "greptime/v1/common.proto";
-
-message AdminRequest {
-  string name = 1;
-  repeated AdminExpr exprs = 2;
-}
-
-message AdminResponse {
-  repeated AdminResult results = 1;
-}
-
-message AdminExpr {
-  ExprHeader header = 1;
-  oneof expr {
-    CreateExpr create = 2;
-    AlterExpr alter = 3;
-  }
-}
-
-message AdminResult {
-  ResultHeader header = 1;
-  oneof result {
-    MutateResult mutate = 2;
-  }
-}
-
-message CreateExpr {
-  optional string catalog_name = 1;
-  optional string schema_name = 2;
-  string table_name = 3;
-  optional string desc = 4;
-  repeated ColumnDef column_defs = 5;
-  string time_index = 6;
-  repeated string primary_keys = 7;
-  bool create_if_not_exists = 8;
-  map<string, string> table_options = 9;
-}
-
-message AlterExpr {
-  optional string catalog_name = 1;
-  optional string schema_name = 2;
-  string table_name = 3;
-  oneof kind {
-    AddColumn add_column = 4;
-  }
-}
-
-message AddColumn {
-  ColumnDef column_def = 1;
-}

src/api/greptime/v1/column.proto

@@ -1,79 +0,0 @@
-syntax = "proto3";
-
-package greptime.v1;
-
-message Column {
-  string column_name = 1;
-
-  enum SemanticType {
-    TAG = 0;
-    FIELD = 1;
-    TIMESTAMP = 2;
-  }
-  SemanticType semantic_type = 2;
-
-  message Values {
-    repeated int32 i8_values = 1;
-    repeated int32 i16_values = 2;
-    repeated int32 i32_values = 3;
-    repeated int64 i64_values = 4;
-
-    repeated uint32 u8_values = 5;
-    repeated uint32 u16_values = 6;
-    repeated uint32 u32_values = 7;
-    repeated uint64 u64_values = 8;
-
-    repeated float f32_values = 9;
-    repeated double f64_values = 10;
-
-    repeated bool bool_values = 11;
-    repeated bytes binary_values = 12;
-    repeated string string_values = 13;
-
-    repeated int32 date_values = 14;
-    repeated int64 datetime_values = 15;
-    repeated int64 ts_millis_values = 16;
-  }
-  // The array of non-null values in this column.
-  //
-  // For example: suppose there is a column "foo" that contains some int32 values (1, 2, 3, 4, 5, null, 7, 8, 9, null);
-  //   column:
-  //     column_name: foo
-  //     semantic_type: Tag
-  //     values: 1, 2, 3, 4, 5, 7, 8, 9
-  //     null_masks: 00100000 00000010
-  Values values = 3;
-
-  // Mask maps the positions of null values.
-  // If a bit in null_mask is 1, it indicates that the column value at that position is null.
-  bytes null_mask = 4;
-
-  // Helpful in creating vector from column.
-  optional ColumnDataType datatype = 5;
-}
-
-message ColumnDef {
-  string name = 1;
-  ColumnDataType datatype = 2;
-  bool is_nullable = 3;
-  optional bytes default_constraint = 4;
-}
-
-enum ColumnDataType {
-  BOOLEAN = 0;
-  INT8 = 1;
-  INT16 = 2;
-  INT32 = 3;
-  INT64 = 4;
-  UINT8 = 5;
-  UINT16 = 6;
-  UINT32 = 7;
-  UINT64 = 8;
-  FLOAT32 = 9;
-  FLOAT64 = 10;
-  BINARY = 11;
-  STRING = 12;
-  DATE = 13;
-  DATETIME = 14;
-  TIMESTAMP = 15;
-}

src/api/greptime/v1/common.proto

@@ -1,18 +0,0 @@
-syntax = "proto3";
-
-package greptime.v1;
-
-message ExprHeader {
-  uint32 version = 1;
-}
-
-message ResultHeader {
-  uint32 version = 1;
-  uint32 code = 2;
-  string err_msg = 3;
-}
-
-message MutateResult {
-  uint32 success = 1;
-  uint32 failure = 2;
-}

src/api/greptime/v1/database.proto

@@ -1,75 +0,0 @@
-syntax = "proto3";
-
-package greptime.v1;
-
-import "greptime/v1/common.proto";
-
-message DatabaseRequest {
-  string name = 1;
-  repeated ObjectExpr exprs = 2;
-}
-
-message DatabaseResponse {
-  repeated ObjectResult results = 1;
-}
-
-message ObjectExpr {
-  ExprHeader header = 1;
-  oneof expr {
-    InsertExpr insert = 2;
-    SelectExpr select = 3;
-    UpdateExpr update = 4;
-    DeleteExpr delete = 5;
-  }
-}
-
-// TODO(fys): Only support sql now, and will support promql etc in the future
-message SelectExpr {
-  oneof expr {
-    string sql = 1;
-    PhysicalPlan physical_plan = 15;
-  }
-}
-
-message PhysicalPlan {
-  bytes original_ql = 1;
-  bytes plan = 2;
-}
-
-message InsertExpr {
-  string table_name = 1;
-
-  message Values {
-    repeated bytes values = 1;
-  }
-
-  oneof expr {
-    Values values = 2;
-
-    // TODO(LFC): Remove field "sql" in InsertExpr.
-    // When Frontend instance received an insertion SQL (`insert into ...`), it's anticipated to parse the SQL and
-    // assemble the values to insert to feed Datanode. In other words, inserting data through Datanode instance's GRPC
-    // interface shouldn't use SQL directly.
-    // Then why the "sql" field exists here? It's because the Frontend needs table schema to create the values to insert,
-    // which is currently not able to find anywhere. (Maybe the table schema is suppose to be fetched from Meta?)
-    // The "sql" field is meant to be removed in the future.
-    string sql = 3;
-  }
-}
-
-// TODO(jiachun)
-message UpdateExpr {}
-// TODO(jiachun)
-message DeleteExpr {}
-
-message ObjectResult {
-  ResultHeader header = 1;
-  oneof result {
-    SelectResult select = 2;
-    MutateResult mutate = 3;
-  }
-}
-
-message SelectResult {
-  bytes raw_data = 1;
-}

src/api/greptime/v1/greptime.proto

@@ -1,20 +0,0 @@
-syntax = "proto3";
-
-package greptime.v1;
-
-import "greptime/v1/admin.proto";
-import "greptime/v1/database.proto";
-
-service Greptime {
-  rpc Batch(BatchRequest) returns (BatchResponse) {}
-}
-
-message BatchRequest {
-  repeated AdminRequest admins = 1;
-  repeated DatabaseRequest databases = 2;
-}
-
-message BatchResponse {
-  repeated AdminResponse admins = 1;
-  repeated DatabaseResponse databases = 2;
-}

src/api/greptime/v1/insert.proto

@@ -1,10 +0,0 @@
-syntax = "proto3";
-
-package greptime.v1.codec;
-
-import "greptime/v1/column.proto";
-
-message InsertBatch {
-  repeated Column columns = 1;
-  uint32 row_count = 2;
-}

src/api/greptime/v1/physical_plan.proto

@@ -1,33 +0,0 @@
-syntax = "proto3";
-
-package greptime.v1.codec;
-
-message PhysicalPlanNode {
-  oneof PhysicalPlanType {
-    ProjectionExecNode projection = 1;
-    MockInputExecNode mock = 99;
-    // TODO(fys): impl other physical plan node
-  }
-}
-
-message ProjectionExecNode {
-  PhysicalPlanNode input = 1;
-  repeated PhysicalExprNode expr = 2;
-  repeated string expr_name = 3;
-}
-
-message PhysicalExprNode {
-  oneof ExprType {
-    PhysicalColumn column = 1;
-    // TODO(fys): impl other physical expr node
-  }
-}
-
-message PhysicalColumn {
-  string name = 1;
-  uint64 index = 2;
-}
-
-message MockInputExecNode {
-  string name = 1;
-}

src/api/greptime/v1/select.proto

@@ -1,10 +0,0 @@
-syntax = "proto3";
-
-package greptime.v1.codec;
-
-import "greptime/v1/column.proto";
-
-message SelectResult {
-  repeated Column columns = 1;
-  uint32 row_count = 2;
-}

src/api/src/error.rs

@@ -1,6 +1,24 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+
+use common_error::ext::ErrorExt;
+use common_error::prelude::StatusCode;
 use datatypes::prelude::ConcreteDataType;
 use snafu::prelude::*;
-use snafu::Backtrace;
+use snafu::{Backtrace, ErrorCompat};
 
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -15,4 +33,44 @@ pub enum Error {
         from: ConcreteDataType,
         backtrace: Backtrace,
     },
+
+    #[snafu(display(
+        "Failed to convert column default constraint, column: {}, source: {}",
+        column,
+        source
+    ))]
+    ConvertColumnDefaultConstraint {
+        column: String,
+        #[snafu(backtrace)]
+        source: datatypes::error::Error,
+    },
+
+    #[snafu(display(
+        "Invalid column default constraint, column: {}, source: {}",
+        column,
+        source
+    ))]
+    InvalidColumnDefaultConstraint {
+        column: String,
+        #[snafu(backtrace)]
+        source: datatypes::error::Error,
+    },
+}
+
+impl ErrorExt for Error {
+    fn status_code(&self) -> StatusCode {
+        match self {
+            Error::UnknownColumnDataType { .. } => StatusCode::InvalidArguments,
+            Error::IntoColumnDataType { .. } => StatusCode::Unexpected,
+            Error::ConvertColumnDefaultConstraint { source, .. }
+            | Error::InvalidColumnDefaultConstraint { source, .. } => source.status_code(),
+        }
+    }
+    fn backtrace_opt(&self) -> Option<&Backtrace> {
+        ErrorCompat::backtrace(self)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }

src/api/src/helper.rs

@@ -1,9 +1,28 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_base::BitVec;
+use common_time::timestamp::TimeUnit;
 use datatypes::prelude::ConcreteDataType;
+use datatypes::types::TimestampType;
+use datatypes::value::Value;
+use datatypes::vectors::VectorRef;
 use snafu::prelude::*;
 
 use crate::error::{self, Result};
 use crate::v1::column::Values;
-use crate::v1::ColumnDataType;
+use crate::v1::{Column, ColumnDataType};
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct ColumnDataTypeWrapper(ColumnDataType);
@@ -38,7 +57,16 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
             ColumnDataType::String => ConcreteDataType::string_datatype(),
             ColumnDataType::Date => ConcreteDataType::date_datatype(),
             ColumnDataType::Datetime => ConcreteDataType::datetime_datatype(),
-            ColumnDataType::Timestamp => ConcreteDataType::timestamp_millis_datatype(),
+            ColumnDataType::TimestampSecond => ConcreteDataType::timestamp_second_datatype(),
+            ColumnDataType::TimestampMillisecond => {
+                ConcreteDataType::timestamp_millisecond_datatype()
+            }
+            ColumnDataType::TimestampMicrosecond => {
+                ConcreteDataType::timestamp_microsecond_datatype()
+            }
+            ColumnDataType::TimestampNanosecond => {
+                ConcreteDataType::timestamp_nanosecond_datatype()
+            }
         }
     }
 }
@@ -63,8 +91,15 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
             ConcreteDataType::String(_) => ColumnDataType::String,
             ConcreteDataType::Date(_) => ColumnDataType::Date,
             ConcreteDataType::DateTime(_) => ColumnDataType::Datetime,
-            ConcreteDataType::Timestamp(_) => ColumnDataType::Timestamp,
-            ConcreteDataType::Null(_) | ConcreteDataType::List(_) => {
+            ConcreteDataType::Timestamp(unit) => match unit {
+                TimestampType::Second(_) => ColumnDataType::TimestampSecond,
+                TimestampType::Millisecond(_) => ColumnDataType::TimestampMillisecond,
+                TimestampType::Microsecond(_) => ColumnDataType::TimestampMicrosecond,
+                TimestampType::Nanosecond(_) => ColumnDataType::TimestampNanosecond,
+            },
+            ConcreteDataType::Null(_)
+            | ConcreteDataType::List(_)
+            | ConcreteDataType::Dictionary(_) => {
                 return error::IntoColumnDataTypeSnafu { from: datatype }.fail()
             }
         });
@@ -72,137 +107,190 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
     }
 }
 
-impl Values {
-    pub fn with_capacity(datatype: ColumnDataType, capacity: usize) -> Self {
-        match datatype {
-            ColumnDataType::Boolean => Values {
-                bool_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Int8 => Values {
-                i8_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Int16 => Values {
-                i16_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Int32 => Values {
-                i32_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Int64 => Values {
-                i64_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Uint8 => Values {
-                u8_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Uint16 => Values {
-                u16_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Uint32 => Values {
-                u32_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Uint64 => Values {
-                u64_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Float32 => Values {
-                f32_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Float64 => Values {
-                f64_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Binary => Values {
-                binary_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::String => Values {
-                string_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Date => Values {
-                date_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Datetime => Values {
-                datetime_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-            ColumnDataType::Timestamp => Values {
-                ts_millis_values: Vec::with_capacity(capacity),
-                ..Default::default()
-            },
-        }
+pub fn values_with_capacity(datatype: ColumnDataType, capacity: usize) -> Values {
+    match datatype {
+        ColumnDataType::Boolean => Values {
+            bool_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Int8 => Values {
+            i8_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Int16 => Values {
+            i16_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Int32 => Values {
+            i32_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Int64 => Values {
+            i64_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Uint8 => Values {
+            u8_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Uint16 => Values {
+            u16_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Uint32 => Values {
+            u32_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Uint64 => Values {
+            u64_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Float32 => Values {
+            f32_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Float64 => Values {
+            f64_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Binary => Values {
+            binary_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::String => Values {
+            string_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Date => Values {
+            date_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::Datetime => Values {
+            datetime_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::TimestampSecond => Values {
+            ts_second_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::TimestampMillisecond => Values {
+            ts_millisecond_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::TimestampMicrosecond => Values {
+            ts_microsecond_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
+        ColumnDataType::TimestampNanosecond => Values {
+            ts_nanosecond_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
     }
 }
 
+// The type of vals must be same.
+pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
+    let values = column.values.get_or_insert_with(Values::default);
+    let mut null_mask = BitVec::from_slice(&column.null_mask);
+    let len = vector.len();
+    null_mask.reserve_exact(origin_count + len);
+    null_mask.extend(BitVec::repeat(false, len));
+
+    (0..len).for_each(|idx| match vector.get(idx) {
+        Value::Null => null_mask.set(idx + origin_count, true),
+        Value::Boolean(val) => values.bool_values.push(val),
+        Value::UInt8(val) => values.u8_values.push(val.into()),
+        Value::UInt16(val) => values.u16_values.push(val.into()),
+        Value::UInt32(val) => values.u32_values.push(val),
+        Value::UInt64(val) => values.u64_values.push(val),
+        Value::Int8(val) => values.i8_values.push(val.into()),
+        Value::Int16(val) => values.i16_values.push(val.into()),
+        Value::Int32(val) => values.i32_values.push(val),
+        Value::Int64(val) => values.i64_values.push(val),
+        Value::Float32(val) => values.f32_values.push(*val),
+        Value::Float64(val) => values.f64_values.push(*val),
+        Value::String(val) => values.string_values.push(val.as_utf8().to_string()),
+        Value::Binary(val) => values.binary_values.push(val.to_vec()),
+        Value::Date(val) => values.date_values.push(val.val()),
+        Value::DateTime(val) => values.datetime_values.push(val.val()),
+        Value::Timestamp(val) => match val.unit() {
+            TimeUnit::Second => values.ts_second_values.push(val.value()),
+            TimeUnit::Millisecond => values.ts_millisecond_values.push(val.value()),
+            TimeUnit::Microsecond => values.ts_microsecond_values.push(val.value()),
+            TimeUnit::Nanosecond => values.ts_nanosecond_values.push(val.value()),
+        },
+        Value::List(_) => unreachable!(),
+    });
+    column.null_mask = null_mask.into_vec();
+}
+
 #[cfg(test)]
 mod tests {
+    use std::sync::Arc;
+
+    use datatypes::vectors::{
+        BooleanVector, TimestampMicrosecondVector, TimestampMillisecondVector,
+        TimestampNanosecondVector, TimestampSecondVector,
+    };
+
     use super::*;
 
     #[test]
     fn test_values_with_capacity() {
-        let values = Values::with_capacity(ColumnDataType::Int8, 2);
+        let values = values_with_capacity(ColumnDataType::Int8, 2);
         let values = values.i8_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Int32, 2);
+        let values = values_with_capacity(ColumnDataType::Int32, 2);
         let values = values.i32_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Int64, 2);
+        let values = values_with_capacity(ColumnDataType::Int64, 2);
         let values = values.i64_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Uint8, 2);
+        let values = values_with_capacity(ColumnDataType::Uint8, 2);
         let values = values.u8_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Uint32, 2);
+        let values = values_with_capacity(ColumnDataType::Uint32, 2);
         let values = values.u32_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Uint64, 2);
+        let values = values_with_capacity(ColumnDataType::Uint64, 2);
         let values = values.u64_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Float32, 2);
+        let values = values_with_capacity(ColumnDataType::Float32, 2);
         let values = values.f32_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Float64, 2);
+        let values = values_with_capacity(ColumnDataType::Float64, 2);
         let values = values.f64_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Binary, 2);
+        let values = values_with_capacity(ColumnDataType::Binary, 2);
         let values = values.binary_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Boolean, 2);
+        let values = values_with_capacity(ColumnDataType::Boolean, 2);
         let values = values.bool_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::String, 2);
+        let values = values_with_capacity(ColumnDataType::String, 2);
         let values = values.string_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Date, 2);
+        let values = values_with_capacity(ColumnDataType::Date, 2);
         let values = values.date_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Datetime, 2);
+        let values = values_with_capacity(ColumnDataType::Datetime, 2);
         let values = values.datetime_values;
         assert_eq!(2, values.capacity());
 
-        let values = Values::with_capacity(ColumnDataType::Timestamp, 2);
-        let values = values.ts_millis_values;
+        let values = values_with_capacity(ColumnDataType::TimestampMillisecond, 2);
+        let values = values.ts_millisecond_values;
         assert_eq!(2, values.capacity());
     }
 
@@ -269,8 +357,8 @@ mod tests {
             ColumnDataTypeWrapper(ColumnDataType::Datetime).into()
         );
         assert_eq!(
-            ConcreteDataType::timestamp_millis_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Timestamp).into()
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::TimestampMillisecond).into()
         );
     }
 
@@ -337,8 +425,8 @@ mod tests {
             ConcreteDataType::datetime_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Timestamp),
-            ConcreteDataType::timestamp_millis_datatype()
+            ColumnDataTypeWrapper(ColumnDataType::TimestampMillisecond),
+            ConcreteDataType::timestamp_millisecond_datatype()
                 .try_into()
                 .unwrap()
         );
@@ -355,7 +443,73 @@ mod tests {
         assert!(result.is_err());
         assert_eq!(
             result.unwrap_err().to_string(),
-            "Failed to create column datatype from List(ListType { inner: Boolean(BooleanType) })"
+            "Failed to create column datatype from List(ListType { item_type: Boolean(BooleanType) })"
         );
     }
+
+    #[test]
+    fn test_column_put_timestamp_values() {
+        let mut column = Column {
+            column_name: "test".to_string(),
+            semantic_type: 0,
+            values: Some(Values {
+                ..Default::default()
+            }),
+            null_mask: vec![],
+            datatype: 0,
+        };
+
+        let vector = Arc::new(TimestampNanosecondVector::from_vec(vec![1, 2, 3]));
+        push_vals(&mut column, 3, vector);
+        assert_eq!(
+            vec![1, 2, 3],
+            column.values.as_ref().unwrap().ts_nanosecond_values
+        );
+
+        let vector = Arc::new(TimestampMillisecondVector::from_vec(vec![4, 5, 6]));
+        push_vals(&mut column, 3, vector);
+        assert_eq!(
+            vec![4, 5, 6],
+            column.values.as_ref().unwrap().ts_millisecond_values
+        );
+
+        let vector = Arc::new(TimestampMicrosecondVector::from_vec(vec![7, 8, 9]));
+        push_vals(&mut column, 3, vector);
+        assert_eq!(
+            vec![7, 8, 9],
+            column.values.as_ref().unwrap().ts_microsecond_values
+        );
+
+        let vector = Arc::new(TimestampSecondVector::from_vec(vec![10, 11, 12]));
+        push_vals(&mut column, 3, vector);
+        assert_eq!(
+            vec![10, 11, 12],
+            column.values.as_ref().unwrap().ts_second_values
+        );
+    }
+
+    #[test]
+    fn test_column_put_vector() {
+        use crate::v1::column::SemanticType;
+        // Some(false), None, Some(true), Some(true)
+        let mut column = Column {
+            column_name: "test".to_string(),
+            semantic_type: SemanticType::Field as i32,
+            values: Some(Values {
+                bool_values: vec![false, true, true],
+                ..Default::default()
+            }),
+            null_mask: vec![2],
+            datatype: ColumnDataType::Boolean as i32,
+        };
+        let row_count = 4;
+
+        let vector = Arc::new(BooleanVector::from(vec![Some(true), None, Some(false)]));
+        push_vals(&mut column, row_count, vector);
+        // Some(false), None, Some(true), Some(true), Some(true), None, Some(false)
+        let bool_values = column.values.unwrap().bool_values;
+        assert_eq!(vec![false, true, true, true, false], bool_values);
+        let null_mask = column.null_mask;
+        assert_eq!(34, null_mask[0]);
+    }
 }

src/api/src/lib.rs

@@ -1,6 +1,26 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 pub mod error;
 pub mod helper;
-pub mod serde;
+
+pub mod prometheus {
+    pub mod remote {
+        pub use greptime_proto::prometheus::remote::*;
+    }
+}
+
 pub mod v1;
 
 pub use prost::DecodeError;

src/api/src/serde.rs

@@ -1,167 +0,0 @@
-pub use prost::DecodeError;
-use prost::Message;
-
-use crate::v1::codec::{InsertBatch, PhysicalPlanNode, SelectResult};
-
-macro_rules! impl_convert_with_bytes {
-    ($data_type: ty) => {
-        impl From<$data_type> for Vec<u8> {
-            fn from(entity: $data_type) -> Self {
-                entity.encode_to_vec()
-            }
-        }
-
-        impl TryFrom<&[u8]> for $data_type {
-            type Error = DecodeError;
-
-            fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
-                <$data_type>::decode(value.as_ref())
-            }
-        }
-    };
-}
-
-impl_convert_with_bytes!(InsertBatch);
-impl_convert_with_bytes!(SelectResult);
-impl_convert_with_bytes!(PhysicalPlanNode);
-
-#[cfg(test)]
-mod tests {
-    use std::ops::Deref;
-
-    use crate::v1::codec::*;
-    use crate::v1::column;
-    use crate::v1::Column;
-
-    const SEMANTIC_TAG: i32 = 0;
-
-    #[test]
-    fn test_convert_insert_batch() {
-        let insert_batch = mock_insert_batch();
-
-        let bytes: Vec<u8> = insert_batch.into();
-        let insert: InsertBatch = bytes.deref().try_into().unwrap();
-
-        assert_eq!(8, insert.row_count);
-        assert_eq!(1, insert.columns.len());
-
-        let column = &insert.columns[0];
-        assert_eq!("foo", column.column_name);
-        assert_eq!(SEMANTIC_TAG, column.semantic_type);
-        assert_eq!(vec![1], column.null_mask);
-        assert_eq!(
-            vec![2, 3, 4, 5, 6, 7, 8],
-            column.values.as_ref().unwrap().i32_values
-        );
-    }
-
-    #[should_panic]
-    #[test]
-    fn test_convert_insert_batch_wrong() {
-        let insert_batch = mock_insert_batch();
-
-        let mut bytes: Vec<u8> = insert_batch.into();
-
-        // modify some bytes
-        bytes[0] = 0b1;
-        bytes[1] = 0b1;
-
-        let insert: InsertBatch = bytes.deref().try_into().unwrap();
-
-        assert_eq!(8, insert.row_count);
-        assert_eq!(1, insert.columns.len());
-
-        let column = &insert.columns[0];
-        assert_eq!("foo", column.column_name);
-        assert_eq!(SEMANTIC_TAG, column.semantic_type);
-        assert_eq!(vec![1], column.null_mask);
-        assert_eq!(
-            vec![2, 3, 4, 5, 6, 7, 8],
-            column.values.as_ref().unwrap().i32_values
-        );
-    }
-
-    #[test]
-    fn test_convert_select_result() {
-        let select_result = mock_select_result();
-
-        let bytes: Vec<u8> = select_result.into();
-        let result: SelectResult = bytes.deref().try_into().unwrap();
-
-        assert_eq!(8, result.row_count);
-        assert_eq!(1, result.columns.len());
-
-        let column = &result.columns[0];
-        assert_eq!("foo", column.column_name);
-        assert_eq!(SEMANTIC_TAG, column.semantic_type);
-        assert_eq!(vec![1], column.null_mask);
-        assert_eq!(
-            vec![2, 3, 4, 5, 6, 7, 8],
-            column.values.as_ref().unwrap().i32_values
-        );
-    }
-
-    #[should_panic]
-    #[test]
-    fn test_convert_select_result_wrong() {
-        let select_result = mock_select_result();
-
-        let mut bytes: Vec<u8> = select_result.into();
-
-        // modify some bytes
-        bytes[0] = 0b1;
-        bytes[1] = 0b1;
-
-        let result: SelectResult = bytes.deref().try_into().unwrap();
-
-        assert_eq!(8, result.row_count);
-        assert_eq!(1, result.columns.len());
-
-        let column = &result.columns[0];
-        assert_eq!("foo", column.column_name);
-        assert_eq!(SEMANTIC_TAG, column.semantic_type);
-        assert_eq!(vec![1], column.null_mask);
-        assert_eq!(
-            vec![2, 3, 4, 5, 6, 7, 8],
-            column.values.as_ref().unwrap().i32_values
-        );
-    }
-
-    fn mock_insert_batch() -> InsertBatch {
-        let values = column::Values {
-            i32_values: vec![2, 3, 4, 5, 6, 7, 8],
-            ..Default::default()
-        };
-        let null_mask = vec![1];
-        let column = Column {
-            column_name: "foo".to_string(),
-            semantic_type: SEMANTIC_TAG,
-            values: Some(values),
-            null_mask,
-            ..Default::default()
-        };
-        InsertBatch {
-            columns: vec![column],
-            row_count: 8,
-        }
-    }
-
-    fn mock_select_result() -> SelectResult {
-        let values = column::Values {
-            i32_values: vec![2, 3, 4, 5, 6, 7, 8],
-            ..Default::default()
-        };
-        let null_mask = vec![1];
-        let column = Column {
-            column_name: "foo".to_string(),
-            semantic_type: SEMANTIC_TAG,
-            values: Some(values),
-            null_mask,
-            ..Default::default()
-        };
-        SelectResult {
-            columns: vec![column],
-            row_count: 8,
-        }
-    }
-}

src/api/src/v1.rs

@@ -1,6 +1,21 @@
-#![allow(clippy::derive_partial_eq_without_eq)]
-tonic::include_proto!("greptime.v1");
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
-pub mod codec {
-    tonic::include_proto!("greptime.v1.codec");
+pub mod column_def;
+
+pub mod meta {
+    pub use greptime_proto::v1::meta::*;
 }
+
+pub use greptime_proto::v1::*;

src/api/src/v1/column_def.rs

@@ -0,0 +1,42 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema};
+use snafu::ResultExt;
+
+use crate::error::{self, Result};
+use crate::helper::ColumnDataTypeWrapper;
+use crate::v1::ColumnDef;
+
+pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
+    let data_type = ColumnDataTypeWrapper::try_new(column_def.datatype)?;
+
+    let constraint = if column_def.default_constraint.is_empty() {
+        None
+    } else {
+        Some(
+            ColumnDefaultConstraint::try_from(column_def.default_constraint.as_slice()).context(
+                error::ConvertColumnDefaultConstraintSnafu {
+                    column: &column_def.name,
+                },
+            )?,
+        )
+    };
+
+    ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
+        .with_default_constraint(constraint)
+        .context(error::InvalidColumnDefaultConstraintSnafu {
+            column: &column_def.name,
+        })
+}

src/catalog/Cargo.toml

@@ -1,25 +1,45 @@
 [package]
 name = "catalog"
-version = "0.1.0"
-edition = "2021"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+version.workspace = true
+edition.workspace = true
+license.workspace = true
 
 [dependencies]
-async-stream = "0.3"
+api = { path = "../api" }
+arc-swap = "1.0"
+async-stream.workspace = true
 async-trait = "0.1"
+backoff = { version = "0.4", features = ["tokio"] }
+common-catalog = { path = "../common/catalog" }
 common-error = { path = "../common/error" }
+common-grpc = { path = "../common/grpc" }
 common-query = { path = "../common/query" }
 common-recordbatch = { path = "../common/recordbatch" }
+common-runtime = { path = "../common/runtime" }
 common-telemetry = { path = "../common/telemetry" }
 common-time = { path = "../common/time" }
-datafusion = { git = "https://github.com/apache/arrow-datafusion.git", branch = "arrow2", features = ["simd"] }
+dashmap = "5.4"
+datafusion.workspace = true
 datatypes = { path = "../datatypes" }
 futures = "0.3"
-futures-util = "0.3"
+futures-util.workspace = true
+lazy_static = "1.4"
+meta-client = { path = "../meta-client" }
+parking_lot = "0.12"
+regex = "1.6"
 serde = "1.0"
 serde_json = "1.0"
+session = { path = "../session" }
 snafu = { version = "0.7", features = ["backtraces"] }
+storage = { path = "../storage" }
 table = { path = "../table" }
+tokio.workspace = true
 
 [dev-dependencies]
-tokio = { version = "1.0", features = ["full"] }
+common-test-util = { path = "../common/test-util" }
+chrono.workspace = true
+log-store = { path = "../log-store" }
+mito = { path = "../mito", features = ["test"] }
+object-store = { path = "../object-store" }
+storage = { path = "../storage" }
+tokio.workspace = true

src/catalog/src/consts.rs

@@ -1,13 +0,0 @@
-pub const SYSTEM_CATALOG_NAME: &str = "system";
-pub const INFORMATION_SCHEMA_NAME: &str = "information_schema";
-pub const SYSTEM_CATALOG_TABLE_NAME: &str = "system_catalog";
-pub const DEFAULT_CATALOG_NAME: &str = "greptime";
-pub const DEFAULT_SCHEMA_NAME: &str = "public";
-
-/// Reserves [0,MIN_USER_TABLE_ID) for internal usage.
-/// User defined table id starts from this value.
-pub const MIN_USER_TABLE_ID: u32 = 1024;
-/// system_catalog table id
-pub const SYSTEM_CATALOG_TABLE_ID: u32 = 0;
-/// scripts table id
-pub const SCRIPTS_TABLE_ID: u32 = 1;

src/catalog/src/error.rs

@@ -1,11 +1,28 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
+use std::fmt::Debug;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::prelude::{Snafu, StatusCode};
 use datafusion::error::DataFusionError;
-use datatypes::arrow;
+use datatypes::prelude::ConcreteDataType;
 use snafu::{Backtrace, ErrorCompat};
 
+use crate::DeregisterTableRequest;
+
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
@@ -36,24 +53,28 @@ pub enum Error {
     SystemCatalog { msg: String, backtrace: Backtrace },
 
     #[snafu(display(
-        "System catalog table type mismatch, expected: binary, found: {:?} source: {}",
+        "System catalog table type mismatch, expected: binary, found: {:?}",
         data_type,
-        source
     ))]
     SystemCatalogTypeMismatch {
-        data_type: arrow::datatypes::DataType,
-        #[snafu(backtrace)]
-        source: datatypes::error::Error,
+        data_type: ConcreteDataType,
+        backtrace: Backtrace,
     },
 
     #[snafu(display("Invalid system catalog entry type: {:?}", entry_type))]
-    InvalidEntryType { entry_type: Option<u8> },
+    InvalidEntryType {
+        entry_type: Option<u8>,
+        backtrace: Backtrace,
+    },
 
     #[snafu(display("Invalid system catalog key: {:?}", key))]
-    InvalidKey { key: Option<String> },
+    InvalidKey {
+        key: Option<String>,
+        backtrace: Backtrace,
+    },
 
     #[snafu(display("Catalog value is not present"))]
-    EmptyValue,
+    EmptyValue { backtrace: Backtrace },
 
     #[snafu(display("Failed to deserialize value, source: {}", source))]
     ValueDeserialize {
@@ -62,18 +83,34 @@ pub enum Error {
     },
 
     #[snafu(display("Cannot find catalog by name: {}", catalog_name))]
-    CatalogNotFound { catalog_name: String },
+    CatalogNotFound {
+        catalog_name: String,
+        backtrace: Backtrace,
+    },
 
-    #[snafu(display("Cannot find schema, schema info: {}", schema_info))]
-    SchemaNotFound { schema_info: String },
+    #[snafu(display("Cannot find schema {} in catalog {}", schema, catalog))]
+    SchemaNotFound {
+        catalog: String,
+        schema: String,
+        backtrace: Backtrace,
+    },
 
-    #[snafu(display("Table {} already exists", table))]
+    #[snafu(display("Table `{}` already exists", table))]
     TableExists { table: String, backtrace: Backtrace },
 
-    #[snafu(display("Failed to register table"))]
-    RegisterTable {
-        #[snafu(backtrace)]
-        source: BoxedError,
+    #[snafu(display("Table `{}` not exist", table))]
+    TableNotExist { table: String, backtrace: Backtrace },
+
+    #[snafu(display("Schema {} already exists", schema))]
+    SchemaExists {
+        schema: String,
+        backtrace: Backtrace,
+    },
+
+    #[snafu(display("Operation {} not implemented yet", operation))]
+    Unimplemented {
+        operation: String,
+        backtrace: Backtrace,
     },
 
     #[snafu(display("Failed to open table, table info: {}, source: {}", table_info, source))]
@@ -84,7 +121,10 @@ pub enum Error {
     },
 
     #[snafu(display("Table not found while opening table, table info: {}", table_info))]
-    TableNotFound { table_info: String },
+    TableNotFound {
+        table_info: String,
+        backtrace: Backtrace,
+    },
 
     #[snafu(display("Failed to read system catalog table records"))]
     ReadSystemCatalog {
@@ -96,13 +136,89 @@ pub enum Error {
         "Failed to insert table creation record to system catalog, source: {}",
         source
     ))]
-    InsertTableRecord {
+    InsertCatalogRecord {
+        #[snafu(backtrace)]
+        source: table::error::Error,
+    },
+
+    #[snafu(display(
+        "Failed to deregister table, request: {:?}, source: {}",
+        request,
+        source
+    ))]
+    DeregisterTable {
+        request: DeregisterTableRequest,
         #[snafu(backtrace)]
         source: table::error::Error,
     },
 
     #[snafu(display("Illegal catalog manager state: {}", msg))]
     IllegalManagerState { backtrace: Backtrace, msg: String },
+
+    #[snafu(display("Failed to scan system catalog table, source: {}", source))]
+    SystemCatalogTableScan {
+        #[snafu(backtrace)]
+        source: table::error::Error,
+    },
+
+    #[snafu(display("Failure during SchemaProvider operation, source: {}", source))]
+    SchemaProviderOperation {
+        #[snafu(backtrace)]
+        source: BoxedError,
+    },
+
+    #[snafu(display("{source}"))]
+    Internal {
+        #[snafu(backtrace)]
+        source: BoxedError,
+    },
+
+    #[snafu(display("Failed to execute system catalog table scan, source: {}", source))]
+    SystemCatalogTableScanExec {
+        #[snafu(backtrace)]
+        source: common_query::error::Error,
+    },
+    #[snafu(display("Cannot parse catalog value, source: {}", source))]
+    InvalidCatalogValue {
+        #[snafu(backtrace)]
+        source: common_catalog::error::Error,
+    },
+
+    #[snafu(display("Failed to perform metasrv operation, source: {}", source))]
+    MetaSrv {
+        #[snafu(backtrace)]
+        source: meta_client::error::Error,
+    },
+
+    #[snafu(display("Invalid table info in catalog, source: {}", source))]
+    InvalidTableInfoInCatalog {
+        #[snafu(backtrace)]
+        source: datatypes::error::Error,
+    },
+
+    #[snafu(display("Failed to serialize or deserialize catalog entry: {}", source))]
+    CatalogEntrySerde {
+        #[snafu(backtrace)]
+        source: common_catalog::error::Error,
+    },
+
+    #[snafu(display("Illegal access to catalog: {} and schema: {}", catalog, schema))]
+    QueryAccessDenied { catalog: String, schema: String },
+
+    #[snafu(display(
+        "Failed to get region stats, catalog: {}, schema: {}, table: {}, source: {}",
+        catalog,
+        schema,
+        table,
+        source
+    ))]
+    RegionStats {
+        catalog: String,
+        schema: String,
+        table: String,
+        #[snafu(backtrace)]
+        source: table::error::Error,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -117,21 +233,39 @@ impl ErrorExt for Error {
             | Error::CatalogNotFound { .. }
             | Error::InvalidEntryType { .. } => StatusCode::Unexpected,
 
-            Error::SystemCatalog { .. } | Error::EmptyValue | Error::ValueDeserialize { .. } => {
-                StatusCode::StorageUnavailable
-            }
+            Error::SystemCatalog { .. }
+            | Error::EmptyValue { .. }
+            | Error::ValueDeserialize { .. } => StatusCode::StorageUnavailable,
+
+            Error::SystemCatalogTypeMismatch { .. } => StatusCode::Internal,
 
             Error::ReadSystemCatalog { source, .. } => source.status_code(),
-            Error::SystemCatalogTypeMismatch { source, .. } => source.status_code(),
+            Error::InvalidCatalogValue { source, .. } | Error::CatalogEntrySerde { source } => {
+                source.status_code()
+            }
 
-            Error::RegisterTable { .. } => StatusCode::Internal,
             Error::TableExists { .. } => StatusCode::TableAlreadyExists,
+            Error::TableNotExist { .. } => StatusCode::TableNotFound,
+            Error::SchemaExists { .. } => StatusCode::InvalidArguments,
 
             Error::OpenSystemCatalog { source, .. }
             | Error::CreateSystemCatalog { source, .. }
-            | Error::InsertTableRecord { source, .. }
+            | Error::InsertCatalogRecord { source, .. }
             | Error::OpenTable { source, .. }
-            | Error::CreateTable { source, .. } => source.status_code(),
+            | Error::CreateTable { source, .. }
+            | Error::DeregisterTable { source, .. }
+            | Error::RegionStats { source, .. } => source.status_code(),
+
+            Error::MetaSrv { source, .. } => source.status_code(),
+            Error::SystemCatalogTableScan { source } => source.status_code(),
+            Error::SystemCatalogTableScanExec { source } => source.status_code(),
+            Error::InvalidTableInfoInCatalog { source } => source.status_code(),
+            Error::SchemaProviderOperation { source } | Error::Internal { source } => {
+                source.status_code()
+            }
+
+            Error::Unimplemented { .. } => StatusCode::Unsupported,
+            Error::QueryAccessDenied { .. } => StatusCode::AccessDenied,
         }
     }
 
@@ -152,8 +286,6 @@ impl From<Error> for DataFusionError {
 
 #[cfg(test)]
 mod tests {
-    use common_error::mock::MockError;
-    use datatypes::arrow::datatypes::DataType;
     use snafu::GenerateImplicitData;
 
     use super::*;
@@ -171,23 +303,7 @@ mod tests {
 
         assert_eq!(
             StatusCode::Unexpected,
-            Error::InvalidKey { key: None }.status_code()
-        );
-
-        assert_eq!(
-            StatusCode::StorageUnavailable,
-            Error::OpenSystemCatalog {
-                source: table::error::Error::new(MockError::new(StatusCode::StorageUnavailable))
-            }
-            .status_code()
-        );
-
-        assert_eq!(
-            StatusCode::StorageUnavailable,
-            Error::CreateSystemCatalog {
-                source: table::error::Error::new(MockError::new(StatusCode::StorageUnavailable))
-            }
-            .status_code()
+            InvalidKeySnafu { key: None }.build().status_code()
         );
 
         assert_eq!(
@@ -202,17 +318,14 @@ mod tests {
         assert_eq!(
             StatusCode::Internal,
             Error::SystemCatalogTypeMismatch {
-                data_type: DataType::Boolean,
-                source: datatypes::error::Error::UnsupportedArrowType {
-                    arrow_type: DataType::Boolean,
-                    backtrace: Backtrace::generate()
-                }
+                data_type: ConcreteDataType::binary_datatype(),
+                backtrace: Backtrace::generate(),
             }
             .status_code()
         );
         assert_eq!(
             StatusCode::StorageUnavailable,
-            Error::EmptyValue.status_code()
+            EmptyValueSnafu {}.build().status_code()
         );
     }
 

src/catalog/src/helper.rs

@@ -0,0 +1,379 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::fmt::{Display, Formatter};
+
+use common_catalog::error::{
+    DeserializeCatalogEntryValueSnafu, Error, InvalidCatalogSnafu, SerializeCatalogEntryValueSnafu,
+};
+use lazy_static::lazy_static;
+use regex::Regex;
+use serde::{Deserialize, Serialize, Serializer};
+use snafu::{ensure, OptionExt, ResultExt};
+use table::metadata::{RawTableInfo, TableId, TableVersion};
+
+pub const CATALOG_KEY_PREFIX: &str = "__c";
+pub const SCHEMA_KEY_PREFIX: &str = "__s";
+pub const TABLE_GLOBAL_KEY_PREFIX: &str = "__tg";
+pub const TABLE_REGIONAL_KEY_PREFIX: &str = "__tr";
+
+const ALPHANUMERICS_NAME_PATTERN: &str = "[a-zA-Z_][a-zA-Z0-9_]*";
+
+lazy_static! {
+    static ref CATALOG_KEY_PATTERN: Regex = Regex::new(&format!(
+        "^{CATALOG_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})$"
+    ))
+    .unwrap();
+}
+
+lazy_static! {
+    static ref SCHEMA_KEY_PATTERN: Regex = Regex::new(&format!(
+        "^{SCHEMA_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})$"
+    ))
+    .unwrap();
+}
+
+lazy_static! {
+    static ref TABLE_GLOBAL_KEY_PATTERN: Regex = Regex::new(&format!(
+        "^{TABLE_GLOBAL_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})$"
+    ))
+    .unwrap();
+}
+
+lazy_static! {
+    static ref TABLE_REGIONAL_KEY_PATTERN: Regex = Regex::new(&format!(
+        "^{TABLE_REGIONAL_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})-([0-9]+)$"
+    ))
+    .unwrap();
+}
+
+pub fn build_catalog_prefix() -> String {
+    format!("{CATALOG_KEY_PREFIX}-")
+}
+
+pub fn build_schema_prefix(catalog_name: impl AsRef<str>) -> String {
+    format!("{SCHEMA_KEY_PREFIX}-{}-", catalog_name.as_ref())
+}
+
+pub fn build_table_global_prefix(
+    catalog_name: impl AsRef<str>,
+    schema_name: impl AsRef<str>,
+) -> String {
+    format!(
+        "{TABLE_GLOBAL_KEY_PREFIX}-{}-{}-",
+        catalog_name.as_ref(),
+        schema_name.as_ref()
+    )
+}
+
+pub fn build_table_regional_prefix(
+    catalog_name: impl AsRef<str>,
+    schema_name: impl AsRef<str>,
+) -> String {
+    format!(
+        "{}-{}-{}-",
+        TABLE_REGIONAL_KEY_PREFIX,
+        catalog_name.as_ref(),
+        schema_name.as_ref()
+    )
+}
+
+/// Table global info has only one key across all datanodes so it does not have `node_id` field.
+#[derive(Clone)]
+pub struct TableGlobalKey {
+    pub catalog_name: String,
+    pub schema_name: String,
+    pub table_name: String,
+}
+
+impl Display for TableGlobalKey {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(TABLE_GLOBAL_KEY_PREFIX)?;
+        f.write_str("-")?;
+        f.write_str(&self.catalog_name)?;
+        f.write_str("-")?;
+        f.write_str(&self.schema_name)?;
+        f.write_str("-")?;
+        f.write_str(&self.table_name)
+    }
+}
+
+impl TableGlobalKey {
+    pub fn parse<S: AsRef<str>>(s: S) -> Result<Self, Error> {
+        let key = s.as_ref();
+        let captures = TABLE_GLOBAL_KEY_PATTERN
+            .captures(key)
+            .context(InvalidCatalogSnafu { key })?;
+        ensure!(captures.len() == 4, InvalidCatalogSnafu { key });
+
+        Ok(Self {
+            catalog_name: captures[1].to_string(),
+            schema_name: captures[2].to_string(),
+            table_name: captures[3].to_string(),
+        })
+    }
+}
+
+/// Table global info contains necessary info for a datanode to create table regions, including
+/// table id, table meta(schema...), region id allocation across datanodes.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct TableGlobalValue {
+    /// Id of datanode that created the global table info kv. only for debugging.
+    pub node_id: u64,
+    /// Allocation of region ids across all datanodes.
+    pub regions_id_map: HashMap<u64, Vec<u32>>,
+    pub table_info: RawTableInfo,
+}
+
+impl TableGlobalValue {
+    pub fn table_id(&self) -> TableId {
+        self.table_info.ident.table_id
+    }
+}
+
+/// Table regional info that varies between datanode, so it contains a `node_id` field.
+pub struct TableRegionalKey {
+    pub catalog_name: String,
+    pub schema_name: String,
+    pub table_name: String,
+    pub node_id: u64,
+}
+
+impl Display for TableRegionalKey {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(TABLE_REGIONAL_KEY_PREFIX)?;
+        f.write_str("-")?;
+        f.write_str(&self.catalog_name)?;
+        f.write_str("-")?;
+        f.write_str(&self.schema_name)?;
+        f.write_str("-")?;
+        f.write_str(&self.table_name)?;
+        f.write_str("-")?;
+        f.serialize_u64(self.node_id)
+    }
+}
+
+impl TableRegionalKey {
+    pub fn parse<S: AsRef<str>>(s: S) -> Result<Self, Error> {
+        let key = s.as_ref();
+        let captures = TABLE_REGIONAL_KEY_PATTERN
+            .captures(key)
+            .context(InvalidCatalogSnafu { key })?;
+        ensure!(captures.len() == 5, InvalidCatalogSnafu { key });
+        let node_id = captures[4]
+            .to_string()
+            .parse()
+            .map_err(|_| InvalidCatalogSnafu { key }.build())?;
+        Ok(Self {
+            catalog_name: captures[1].to_string(),
+            schema_name: captures[2].to_string(),
+            table_name: captures[3].to_string(),
+            node_id,
+        })
+    }
+}
+
+/// Regional table info of specific datanode, including table version on that datanode and
+/// region ids allocated by metasrv.
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct TableRegionalValue {
+    pub version: TableVersion,
+    pub regions_ids: Vec<u32>,
+}
+
+pub struct CatalogKey {
+    pub catalog_name: String,
+}
+
+impl Display for CatalogKey {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(CATALOG_KEY_PREFIX)?;
+        f.write_str("-")?;
+        f.write_str(&self.catalog_name)
+    }
+}
+
+impl CatalogKey {
+    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
+        let key = s.as_ref();
+        let captures = CATALOG_KEY_PATTERN
+            .captures(key)
+            .context(InvalidCatalogSnafu { key })?;
+        ensure!(captures.len() == 2, InvalidCatalogSnafu { key });
+        Ok(Self {
+            catalog_name: captures[1].to_string(),
+        })
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct CatalogValue;
+
+pub struct SchemaKey {
+    pub catalog_name: String,
+    pub schema_name: String,
+}
+
+impl Display for SchemaKey {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(SCHEMA_KEY_PREFIX)?;
+        f.write_str("-")?;
+        f.write_str(&self.catalog_name)?;
+        f.write_str("-")?;
+        f.write_str(&self.schema_name)
+    }
+}
+
+impl SchemaKey {
+    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
+        let key = s.as_ref();
+        let captures = SCHEMA_KEY_PATTERN
+            .captures(key)
+            .context(InvalidCatalogSnafu { key })?;
+        ensure!(captures.len() == 3, InvalidCatalogSnafu { key });
+        Ok(Self {
+            catalog_name: captures[1].to_string(),
+            schema_name: captures[2].to_string(),
+        })
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SchemaValue;
+
+macro_rules! define_catalog_value {
+    ( $($val_ty: ty), *) => {
+            $(
+                impl $val_ty {
+                    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
+                        serde_json::from_str(s.as_ref())
+                            .context(DeserializeCatalogEntryValueSnafu { raw: s.as_ref() })
+                    }
+
+                    pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result<Self, Error> {
+                         Self::parse(&String::from_utf8_lossy(bytes.as_ref()))
+                    }
+
+                    pub fn as_bytes(&self) -> Result<Vec<u8>, Error> {
+                        Ok(serde_json::to_string(self)
+                            .context(SerializeCatalogEntryValueSnafu)?
+                            .into_bytes())
+                    }
+                }
+            )*
+        }
+}
+
+define_catalog_value!(
+    TableRegionalValue,
+    TableGlobalValue,
+    CatalogValue,
+    SchemaValue
+);
+
+#[cfg(test)]
+mod tests {
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::schema::{ColumnSchema, RawSchema, Schema};
+    use table::metadata::{RawTableMeta, TableIdent, TableType};
+
+    use super::*;
+
+    #[test]
+    fn test_parse_catalog_key() {
+        let key = "__c-C";
+        let catalog_key = CatalogKey::parse(key).unwrap();
+        assert_eq!("C", catalog_key.catalog_name);
+        assert_eq!(key, catalog_key.to_string());
+    }
+
+    #[test]
+    fn test_parse_schema_key() {
+        let key = "__s-C-S";
+        let schema_key = SchemaKey::parse(key).unwrap();
+        assert_eq!("C", schema_key.catalog_name);
+        assert_eq!("S", schema_key.schema_name);
+        assert_eq!(key, schema_key.to_string());
+    }
+
+    #[test]
+    fn test_parse_table_key() {
+        let key = "__tg-C-S-T";
+        let entry = TableGlobalKey::parse(key).unwrap();
+        assert_eq!("C", entry.catalog_name);
+        assert_eq!("S", entry.schema_name);
+        assert_eq!("T", entry.table_name);
+        assert_eq!(key, &entry.to_string());
+    }
+
+    #[test]
+    fn test_build_prefix() {
+        assert_eq!("__c-", build_catalog_prefix());
+        assert_eq!("__s-CATALOG-", build_schema_prefix("CATALOG"));
+        assert_eq!(
+            "__tg-CATALOG-SCHEMA-",
+            build_table_global_prefix("CATALOG", "SCHEMA")
+        );
+    }
+
+    #[test]
+    fn test_serialize_schema() {
+        let schema = Schema::new(vec![ColumnSchema::new(
+            "name",
+            ConcreteDataType::string_datatype(),
+            true,
+        )]);
+
+        let meta = RawTableMeta {
+            schema: RawSchema::from(&schema),
+            engine: "mito".to_string(),
+            created_on: chrono::DateTime::default(),
+            primary_key_indices: vec![0, 1],
+            next_column_id: 3,
+            engine_options: Default::default(),
+            value_indices: vec![2, 3],
+            options: Default::default(),
+            region_numbers: vec![1],
+        };
+
+        let table_info = RawTableInfo {
+            ident: TableIdent {
+                table_id: 42,
+                version: 1,
+            },
+            name: "table_1".to_string(),
+            desc: Some("blah".to_string()),
+            catalog_name: "catalog_1".to_string(),
+            schema_name: "schema_1".to_string(),
+            meta,
+            table_type: TableType::Base,
+        };
+
+        let value = TableGlobalValue {
+            node_id: 0,
+            regions_id_map: HashMap::from([(0, vec![1, 2, 3])]),
+            table_info,
+        };
+        let serialized = serde_json::to_string(&value).unwrap();
+        let deserialized = TableGlobalValue::parse(serialized).unwrap();
+        assert_eq!(value, deserialized);
+    }
+
+    #[test]
+    fn test_table_global_value_compatibility() {
+        let s = r#"{"node_id":1,"regions_id_map":{"1":[0]},"table_info":{"ident":{"table_id":1098,"version":1},"name":"container_cpu_limit","desc":"Created on insertion","catalog_name":"greptime","schema_name":"dd","meta":{"schema":{"column_schemas":[{"name":"container_id","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"container_name","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"docker_image","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"host","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"image_name","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"image_tag","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"interval","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"runtime","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"short_image","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"type","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"dd_value","data_type":{"Float64":{}},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"ts","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":true,"default_constraint":null,"metadata":{"greptime:time_index":"true"}},{"name":"git.repository_url","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}}],"timestamp_index":11,"version":1},"primary_key_indices":[0,1,2,3,4,5,6,7,8,9,12],"value_indices":[10,11],"engine":"mito","next_column_id":12,"region_numbers":[],"engine_options":{},"options":{},"created_on":"1970-01-01T00:00:00Z"},"table_type":"Base"}}"#;
+        TableGlobalValue::parse(s).unwrap();
+    }
+}

src/catalog/src/lib.rs

@@ -1,22 +1,41 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #![feature(assert_matches)]
 
 use std::any::Any;
+use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
+use api::v1::meta::{RegionStat, TableName};
+use common_telemetry::{info, warn};
+use snafu::ResultExt;
+use table::engine::{EngineContext, TableEngineRef};
 use table::metadata::TableId;
 use table::requests::CreateTableRequest;
 use table::TableRef;
 
-pub use crate::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID};
-pub use crate::manager::LocalCatalogManager;
+use crate::error::{CreateTableSnafu, Result};
 pub use crate::schema::{SchemaProvider, SchemaProviderRef};
 
-pub mod consts;
 pub mod error;
-mod manager;
-pub mod memory;
+pub mod helper;
+pub mod local;
+pub mod remote;
 pub mod schema;
-mod system;
+pub mod system;
+pub mod table_source;
 pub mod tables;
 
 /// Represent a list of named catalogs
@@ -31,13 +50,13 @@ pub trait CatalogList: Sync + Send {
         &self,
         name: String,
         catalog: CatalogProviderRef,
-    ) -> Option<CatalogProviderRef>;
+    ) -> Result<Option<CatalogProviderRef>>;
 
     /// Retrieves the list of available catalog names
-    fn catalog_names(&self) -> Vec<String>;
+    fn catalog_names(&self) -> Result<Vec<String>>;
 
     /// Retrieves a specific catalog by name, provided it exists.
-    fn catalog(&self, name: &str) -> Option<CatalogProviderRef>;
+    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>>;
 }
 
 /// Represents a catalog, comprising a number of named schemas.
@@ -47,14 +66,17 @@ pub trait CatalogProvider: Sync + Send {
     fn as_any(&self) -> &dyn Any;
 
     /// Retrieves the list of available schema names in this catalog.
-    fn schema_names(&self) -> Vec<String>;
+    fn schema_names(&self) -> Result<Vec<String>>;
 
     /// Registers schema to this catalog.
-    fn register_schema(&self, name: String, schema: SchemaProviderRef)
-        -> Option<SchemaProviderRef>;
+    fn register_schema(
+        &self,
+        name: String,
+        schema: SchemaProviderRef,
+    ) -> Result<Option<SchemaProviderRef>>;
 
     /// Retrieves a specific schema from the catalog by name, provided it exists.
-    fn schema(&self, name: &str) -> Option<SchemaProviderRef>;
+    fn schema(&self, name: &str) -> Result<Option<SchemaProviderRef>>;
 }
 
 pub type CatalogListRef = Arc<dyn CatalogList>;
@@ -63,32 +85,42 @@ pub type CatalogProviderRef = Arc<dyn CatalogProvider>;
 #[async_trait::async_trait]
 pub trait CatalogManager: CatalogList {
     /// Starts a catalog manager.
-    async fn start(&self) -> error::Result<()>;
+    async fn start(&self) -> Result<()>;
 
-    /// Returns next available table id.
-    fn next_table_id(&self) -> TableId;
+    /// Registers a table within given catalog/schema to catalog manager,
+    /// returns whether the table registered.
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool>;
 
-    /// Registers a table given given catalog/schema to catalog manager,
-    /// returns table registered.
-    async fn register_table(&self, request: RegisterTableRequest) -> error::Result<usize>;
+    /// Deregisters a table within given catalog/schema to catalog manager,
+    /// returns whether the table deregistered.
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool>;
+
+    /// Register a schema with catalog name and schema name. Retuens whether the
+    /// schema registered.
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool>;
+
+    /// Rename a table to [RenameTableRequest::new_table_name], returns whether the table is renamed.
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool>;
 
     /// Register a system table, should be called before starting the manager.
     async fn register_system_table(&self, request: RegisterSystemTableRequest)
         -> error::Result<()>;
 
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>>;
+
     /// Returns the table by catalog, schema and table name.
-    fn table(
+    async fn table(
         &self,
-        catalog: Option<&str>,
-        schema: Option<&str>,
+        catalog: &str,
+        schema: &str,
         table_name: &str,
-    ) -> error::Result<Option<TableRef>>;
+    ) -> Result<Option<TableRef>>;
 }
 
 pub type CatalogManagerRef = Arc<dyn CatalogManager>;
 
 /// Hook called after system table opening.
-pub type OpenSystemTableHook = Arc<dyn Fn(TableRef) -> error::Result<()> + Send + Sync>;
+pub type OpenSystemTableHook = Arc<dyn Fn(TableRef) -> Result<()> + Send + Sync>;
 
 /// Register system table request:
 /// - When system table is already created and registered, the hook will be called
@@ -99,15 +131,147 @@ pub struct RegisterSystemTableRequest {
     pub open_hook: Option<OpenSystemTableHook>,
 }
 
+#[derive(Clone)]
 pub struct RegisterTableRequest {
-    pub catalog: Option<String>,
-    pub schema: Option<String>,
+    pub catalog: String,
+    pub schema: String,
     pub table_name: String,
     pub table_id: TableId,
     pub table: TableRef,
 }
 
-/// Formats table fully-qualified name
-pub fn format_full_table_name(catalog: &str, schema: &str, table: &str) -> String {
-    format!("{}.{}.{}", catalog, schema, table)
+impl Debug for RegisterTableRequest {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RegisterTableRequest")
+            .field("catalog", &self.catalog)
+            .field("schema", &self.schema)
+            .field("table_name", &self.table_name)
+            .field("table_id", &self.table_id)
+            .field("table", &self.table.table_info())
+            .finish()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct RenameTableRequest {
+    pub catalog: String,
+    pub schema: String,
+    pub table_name: String,
+    pub new_table_name: String,
+    pub table_id: TableId,
+}
+
+#[derive(Debug, Clone)]
+pub struct DeregisterTableRequest {
+    pub catalog: String,
+    pub schema: String,
+    pub table_name: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct RegisterSchemaRequest {
+    pub catalog: String,
+    pub schema: String,
+}
+
+pub trait CatalogProviderFactory {
+    fn create(&self, catalog_name: String) -> CatalogProviderRef;
+}
+
+pub trait SchemaProviderFactory {
+    fn create(&self, catalog_name: String, schema_name: String) -> SchemaProviderRef;
+}
+
+pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
+    manager: &'a M,
+    engine: TableEngineRef,
+    sys_table_requests: &'a mut Vec<RegisterSystemTableRequest>,
+) -> Result<()> {
+    for req in sys_table_requests.drain(..) {
+        let catalog_name = &req.create_table_request.catalog_name;
+        let schema_name = &req.create_table_request.schema_name;
+        let table_name = &req.create_table_request.table_name;
+        let table_id = req.create_table_request.id;
+
+        let table = manager.table(catalog_name, schema_name, table_name).await?;
+        let table = if let Some(table) = table {
+            table
+        } else {
+            let table = engine
+                .create_table(&EngineContext::default(), req.create_table_request.clone())
+                .await
+                .with_context(|_| CreateTableSnafu {
+                    table_info: common_catalog::format_full_table_name(
+                        catalog_name,
+                        schema_name,
+                        table_name,
+                    ),
+                })?;
+            manager
+                .register_table(RegisterTableRequest {
+                    catalog: catalog_name.clone(),
+                    schema: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    table_id,
+                    table: table.clone(),
+                })
+                .await?;
+            info!("Created and registered system table: {table_name}");
+            table
+        };
+        if let Some(hook) = req.open_hook {
+            (hook)(table)?;
+        }
+    }
+    Ok(())
+}
+
+/// The stat of regions in the datanode node.
+/// The number of regions can be got from len of vec.
+///
+/// Ignores any errors occurred during iterating regions. The intention of this method is to
+/// collect region stats that will be carried in Datanode's heartbeat to Metasrv, so it's a
+/// "try our best" job.
+pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<RegionStat>) {
+    let mut region_number: u64 = 0;
+    let mut region_stats = Vec::new();
+
+    let Ok(catalog_names) = catalog_manager.catalog_names() else { return (region_number, region_stats) };
+    for catalog_name in catalog_names {
+        let Ok(Some(catalog)) = catalog_manager.catalog(&catalog_name) else { continue };
+
+        let Ok(schema_names) = catalog.schema_names() else { continue };
+        for schema_name in schema_names {
+            let Ok(Some(schema)) = catalog.schema(&schema_name) else { continue };
+
+            let Ok(table_names) = schema.table_names() else { continue };
+            for table_name in table_names {
+                let Ok(Some(table)) = schema.table(&table_name).await else { continue };
+
+                let region_numbers = &table.table_info().meta.region_numbers;
+                region_number += region_numbers.len() as u64;
+
+                match table.region_stats() {
+                    Ok(stats) => {
+                        let stats = stats.into_iter().map(|stat| RegionStat {
+                            region_id: stat.region_id,
+                            table_name: Some(TableName {
+                                catalog_name: catalog_name.clone(),
+                                schema_name: schema_name.clone(),
+                                table_name: table_name.clone(),
+                            }),
+                            approximate_bytes: stat.disk_usage_bytes as i64,
+                            ..Default::default()
+                        });
+
+                        region_stats.extend(stats);
+                    }
+                    Err(e) => {
+                        warn!("Failed to get region status, err: {:?}", e);
+                    }
+                };
+            }
+        }
+    }
+    (region_number, region_stats)
 }

src/catalog/src/local.rs

@@ -0,0 +1,21 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod manager;
+pub mod memory;
+
+pub use manager::LocalCatalogManager;
+pub use memory::{
+    new_memory_catalog_list, MemoryCatalogManager, MemoryCatalogProvider, MemorySchemaProvider,
+};

src/catalog/src/local/manager.rs

@@ -0,0 +1,574 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
+    SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_NAME,
+};
+use common_catalog::format_full_table_name;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use common_telemetry::{error, info};
+use datatypes::prelude::ScalarVector;
+use datatypes::vectors::{BinaryVector, UInt8Vector};
+use futures_util::lock::Mutex;
+use snafu::{ensure, OptionExt, ResultExt};
+use table::engine::{EngineContext, TableEngineRef};
+use table::metadata::TableId;
+use table::requests::OpenTableRequest;
+use table::table::numbers::NumbersTable;
+use table::table::TableIdProvider;
+use table::TableRef;
+
+use crate::error::{
+    self, CatalogNotFoundSnafu, IllegalManagerStateSnafu, OpenTableSnafu, ReadSystemCatalogSnafu,
+    Result, SchemaExistsSnafu, SchemaNotFoundSnafu, SystemCatalogSnafu,
+    SystemCatalogTypeMismatchSnafu, TableExistsSnafu, TableNotFoundSnafu,
+};
+use crate::local::memory::{MemoryCatalogManager, MemoryCatalogProvider, MemorySchemaProvider};
+use crate::system::{
+    decode_system_catalog, Entry, SystemCatalogTable, TableEntry, ENTRY_TYPE_INDEX, KEY_INDEX,
+    VALUE_INDEX,
+};
+use crate::tables::SystemCatalog;
+use crate::{
+    handle_system_table_request, CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef,
+    DeregisterTableRequest, RegisterSchemaRequest, RegisterSystemTableRequest,
+    RegisterTableRequest, RenameTableRequest, SchemaProvider, SchemaProviderRef,
+};
+
+/// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
+pub struct LocalCatalogManager {
+    system: Arc<SystemCatalog>,
+    catalogs: Arc<MemoryCatalogManager>,
+    engine: TableEngineRef,
+    next_table_id: AtomicU32,
+    init_lock: Mutex<bool>,
+    register_lock: Mutex<()>,
+    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
+}
+
+impl LocalCatalogManager {
+    /// Create a new [CatalogManager] with given user catalogs and table engine
+    pub async fn try_new(engine: TableEngineRef) -> Result<Self> {
+        let table = SystemCatalogTable::new(engine.clone()).await?;
+        let memory_catalog_list = crate::local::memory::new_memory_catalog_list()?;
+        let system_catalog = Arc::new(SystemCatalog::new(
+            table,
+            memory_catalog_list.clone(),
+            engine.clone(),
+        ));
+        Ok(Self {
+            system: system_catalog,
+            catalogs: memory_catalog_list,
+            engine,
+            next_table_id: AtomicU32::new(MIN_USER_TABLE_ID),
+            init_lock: Mutex::new(false),
+            register_lock: Mutex::new(()),
+            system_table_requests: Mutex::new(Vec::default()),
+        })
+    }
+
+    /// Scan all entries from system catalog table
+    pub async fn init(&self) -> Result<()> {
+        self.init_system_catalog()?;
+        let system_records = self.system.information_schema.system.records().await?;
+        let entries = self.collect_system_catalog_entries(system_records).await?;
+        let max_table_id = self.handle_system_catalog_entries(entries).await?;
+
+        info!(
+            "All system catalog entries processed, max table id: {}",
+            max_table_id
+        );
+        self.next_table_id
+            .store((max_table_id + 1).max(MIN_USER_TABLE_ID), Ordering::Relaxed);
+        *self.init_lock.lock().await = true;
+
+        // Processing system table hooks
+        let mut sys_table_requests = self.system_table_requests.lock().await;
+        handle_system_table_request(self, self.engine.clone(), &mut sys_table_requests).await?;
+        Ok(())
+    }
+
+    fn init_system_catalog(&self) -> Result<()> {
+        let system_schema = Arc::new(MemorySchemaProvider::new());
+        system_schema.register_table(
+            SYSTEM_CATALOG_TABLE_NAME.to_string(),
+            self.system.information_schema.system.clone(),
+        )?;
+        let system_catalog = Arc::new(MemoryCatalogProvider::new());
+        system_catalog.register_schema(INFORMATION_SCHEMA_NAME.to_string(), system_schema)?;
+        self.catalogs
+            .register_catalog(SYSTEM_CATALOG_NAME.to_string(), system_catalog)?;
+
+        let default_catalog = Arc::new(MemoryCatalogProvider::new());
+        let default_schema = Arc::new(MemorySchemaProvider::new());
+
+        // Add numbers table for test
+        let table = Arc::new(NumbersTable::default());
+        default_schema.register_table("numbers".to_string(), table)?;
+
+        default_catalog.register_schema(DEFAULT_SCHEMA_NAME.to_string(), default_schema)?;
+        self.catalogs
+            .register_catalog(DEFAULT_CATALOG_NAME.to_string(), default_catalog)?;
+        Ok(())
+    }
+
+    /// Collect stream of system catalog entries to `Vec<Entry>`
+    async fn collect_system_catalog_entries(
+        &self,
+        stream: SendableRecordBatchStream,
+    ) -> Result<Vec<Entry>> {
+        let record_batch = common_recordbatch::util::collect(stream)
+            .await
+            .context(ReadSystemCatalogSnafu)?;
+        let rbs = record_batch
+            .into_iter()
+            .map(Self::record_batch_to_entry)
+            .collect::<Result<Vec<_>>>()?;
+        Ok(rbs.into_iter().flat_map(Vec::into_iter).collect::<_>())
+    }
+
+    /// Convert `RecordBatch` to a vector of `Entry`.
+    fn record_batch_to_entry(rb: RecordBatch) -> Result<Vec<Entry>> {
+        ensure!(
+            rb.num_columns() >= 6,
+            SystemCatalogSnafu {
+                msg: format!("Length mismatch: {}", rb.num_columns())
+            }
+        );
+
+        let entry_type = rb
+            .column(ENTRY_TYPE_INDEX)
+            .as_any()
+            .downcast_ref::<UInt8Vector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(ENTRY_TYPE_INDEX).data_type(),
+            })?;
+
+        let key = rb
+            .column(KEY_INDEX)
+            .as_any()
+            .downcast_ref::<BinaryVector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(KEY_INDEX).data_type(),
+            })?;
+
+        let value = rb
+            .column(VALUE_INDEX)
+            .as_any()
+            .downcast_ref::<BinaryVector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(VALUE_INDEX).data_type(),
+            })?;
+
+        let mut res = Vec::with_capacity(rb.num_rows());
+        for ((t, k), v) in entry_type
+            .iter_data()
+            .zip(key.iter_data())
+            .zip(value.iter_data())
+        {
+            let entry = decode_system_catalog(t, k, v)?;
+            res.push(entry);
+        }
+        Ok(res)
+    }
+
+    /// Processes records from system catalog table and returns the max table id persisted
+    /// in system catalog table.
+    async fn handle_system_catalog_entries(&self, entries: Vec<Entry>) -> Result<TableId> {
+        let entries = Self::sort_entries(entries);
+        let mut max_table_id = 0;
+        for entry in entries {
+            match entry {
+                Entry::Catalog(c) => {
+                    self.catalogs.register_catalog_if_absent(
+                        c.catalog_name.clone(),
+                        Arc::new(MemoryCatalogProvider::new()),
+                    );
+                    info!("Register catalog: {}", c.catalog_name);
+                }
+                Entry::Schema(s) => {
+                    let catalog =
+                        self.catalogs
+                            .catalog(&s.catalog_name)?
+                            .context(CatalogNotFoundSnafu {
+                                catalog_name: &s.catalog_name,
+                            })?;
+                    catalog.register_schema(
+                        s.schema_name.clone(),
+                        Arc::new(MemorySchemaProvider::new()),
+                    )?;
+                    info!("Registered schema: {:?}", s);
+                }
+                Entry::Table(t) => {
+                    self.open_and_register_table(&t).await?;
+                    info!("Registered table: {:?}", t);
+                    max_table_id = max_table_id.max(t.table_id);
+                }
+            }
+        }
+        Ok(max_table_id)
+    }
+
+    /// Sort catalog entries to ensure catalog entries comes first, then schema entries,
+    /// and table entries is the last.
+    fn sort_entries(mut entries: Vec<Entry>) -> Vec<Entry> {
+        entries.sort();
+        entries
+    }
+
+    async fn open_and_register_table(&self, t: &TableEntry) -> Result<()> {
+        let catalog = self
+            .catalogs
+            .catalog(&t.catalog_name)?
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &t.catalog_name,
+            })?;
+        let schema = catalog
+            .schema(&t.schema_name)?
+            .context(SchemaNotFoundSnafu {
+                catalog: &t.catalog_name,
+                schema: &t.schema_name,
+            })?;
+
+        let context = EngineContext {};
+        let request = OpenTableRequest {
+            catalog_name: t.catalog_name.clone(),
+            schema_name: t.schema_name.clone(),
+            table_name: t.table_name.clone(),
+            table_id: t.table_id,
+        };
+
+        let option = self
+            .engine
+            .open_table(&context, request)
+            .await
+            .with_context(|_| OpenTableSnafu {
+                table_info: format!(
+                    "{}.{}.{}, id: {}",
+                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
+                ),
+            })?
+            .with_context(|| TableNotFoundSnafu {
+                table_info: format!(
+                    "{}.{}.{}, id: {}",
+                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
+                ),
+            })?;
+
+        schema.register_table(t.table_name.clone(), option)?;
+        Ok(())
+    }
+}
+
+impl CatalogList for LocalCatalogManager {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn register_catalog(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Result<Option<CatalogProviderRef>> {
+        self.catalogs.register_catalog(name, catalog)
+    }
+
+    fn catalog_names(&self) -> Result<Vec<String>> {
+        let mut res = self.catalogs.catalog_names()?;
+        res.push(SYSTEM_CATALOG_NAME.to_string());
+        Ok(res)
+    }
+
+    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>> {
+        if name.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
+            Ok(Some(self.system.clone()))
+        } else {
+            self.catalogs.catalog(name)
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl TableIdProvider for LocalCatalogManager {
+    async fn next_table_id(&self) -> table::Result<TableId> {
+        Ok(self.next_table_id.fetch_add(1, Ordering::Relaxed))
+    }
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for LocalCatalogManager {
+    /// Start [LocalCatalogManager] to load all information from system catalog table.
+    /// Make sure table engine is initialized before starting [MemoryCatalogManager].
+    async fn start(&self) -> Result<()> {
+        self.init().await
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        let started = self.init_lock.lock().await;
+
+        ensure!(
+            *started,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager not started",
+            }
+        );
+
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+
+        let catalog = self
+            .catalogs
+            .catalog(catalog_name)?
+            .context(CatalogNotFoundSnafu { catalog_name })?;
+        let schema = catalog
+            .schema(schema_name)?
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            })?;
+
+        {
+            let _lock = self.register_lock.lock().await;
+            if let Some(existing) = schema.table(&request.table_name).await? {
+                if existing.table_info().ident.table_id != request.table_id {
+                    error!(
+                        "Unexpected table register request: {:?}, existing: {:?}",
+                        request,
+                        existing.table_info()
+                    );
+                    return TableExistsSnafu {
+                        table: format_full_table_name(
+                            catalog_name,
+                            schema_name,
+                            &request.table_name,
+                        ),
+                    }
+                    .fail();
+                }
+                // Try to register table with same table id, just ignore.
+                Ok(false)
+            } else {
+                // table does not exist
+                self.system
+                    .register_table(
+                        catalog_name.clone(),
+                        schema_name.clone(),
+                        request.table_name.clone(),
+                        request.table_id,
+                    )
+                    .await?;
+                schema.register_table(request.table_name, request.table)?;
+                Ok(true)
+            }
+        }
+    }
+
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
+        let started = self.init_lock.lock().await;
+
+        ensure!(
+            *started,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager not started",
+            }
+        );
+
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+
+        let catalog = self
+            .catalogs
+            .catalog(catalog_name)?
+            .context(CatalogNotFoundSnafu { catalog_name })?;
+
+        let schema = catalog
+            .schema(schema_name)?
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            })?;
+
+        // rename table in system catalog
+        self.system
+            .register_table(
+                catalog_name.clone(),
+                schema_name.clone(),
+                request.new_table_name.clone(),
+                request.table_id,
+            )
+            .await?;
+        Ok(schema
+            .rename_table(&request.table_name, request.new_table_name)
+            .is_ok())
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool> {
+        {
+            let started = *self.init_lock.lock().await;
+            ensure!(started, IllegalManagerStateSnafu { msg: "not started" });
+        }
+
+        {
+            let _ = self.register_lock.lock().await;
+
+            let DeregisterTableRequest {
+                catalog,
+                schema,
+                table_name,
+            } = &request;
+            let table_id = self
+                .catalogs
+                .table(catalog, schema, table_name)
+                .await?
+                .with_context(|| error::TableNotExistSnafu {
+                    table: format_full_table_name(catalog, schema, table_name),
+                })?
+                .table_info()
+                .ident
+                .table_id;
+
+            if !self.system.deregister_table(&request, table_id).await? {
+                return Ok(false);
+            }
+
+            self.catalogs.deregister_table(request).await
+        }
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        let started = self.init_lock.lock().await;
+        ensure!(
+            *started,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager not started",
+            }
+        );
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+
+        let catalog = self
+            .catalogs
+            .catalog(catalog_name)?
+            .context(CatalogNotFoundSnafu { catalog_name })?;
+
+        {
+            let _lock = self.register_lock.lock().await;
+            ensure!(
+                catalog.schema(schema_name)?.is_none(),
+                SchemaExistsSnafu {
+                    schema: schema_name,
+                }
+            );
+            self.system
+                .register_schema(request.catalog, schema_name.clone())
+                .await?;
+            catalog.register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))?;
+            Ok(true)
+        }
+    }
+
+    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
+        ensure!(
+            !*self.init_lock.lock().await,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager already started",
+            }
+        );
+
+        let mut sys_table_requests = self.system_table_requests.lock().await;
+        sys_table_requests.push(request);
+
+        Ok(())
+    }
+
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+        self.catalogs
+            .catalog(catalog)?
+            .context(CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .schema(schema)
+    }
+
+    async fn table(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        let catalog = self
+            .catalogs
+            .catalog(catalog_name)?
+            .context(CatalogNotFoundSnafu { catalog_name })?;
+        let schema = catalog
+            .schema(schema_name)?
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            })?;
+        schema.table(table_name).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+
+    use super::*;
+    use crate::system::{CatalogEntry, SchemaEntry};
+
+    #[test]
+    fn test_sort_entry() {
+        let vec = vec![
+            Entry::Table(TableEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+                table_name: "T1".to_string(),
+                table_id: 1,
+            }),
+            Entry::Catalog(CatalogEntry {
+                catalog_name: "C2".to_string(),
+            }),
+            Entry::Schema(SchemaEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+            }),
+            Entry::Schema(SchemaEntry {
+                catalog_name: "C2".to_string(),
+                schema_name: "S2".to_string(),
+            }),
+            Entry::Catalog(CatalogEntry {
+                catalog_name: "".to_string(),
+            }),
+            Entry::Table(TableEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+                table_name: "T2".to_string(),
+                table_id: 2,
+            }),
+        ];
+        let res = LocalCatalogManager::sort_entries(vec);
+        assert_matches!(res[0], Entry::Catalog(..));
+        assert_matches!(res[1], Entry::Catalog(..));
+        assert_matches!(res[2], Entry::Schema(..));
+        assert_matches!(res[3], Entry::Schema(..));
+        assert_matches!(res[4], Entry::Table(..));
+        assert_matches!(res[5], Entry::Table(..));
+    }
+}

src/catalog/src/local/memory.rs

@@ -0,0 +1,531 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::collections::hash_map::Entry;
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::{Arc, RwLock};
+
+use async_trait::async_trait;
+use common_catalog::consts::MIN_USER_TABLE_ID;
+use common_telemetry::error;
+use snafu::{ensure, OptionExt};
+use table::metadata::TableId;
+use table::table::TableIdProvider;
+use table::TableRef;
+
+use crate::error::{
+    self, CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu, TableNotFoundSnafu,
+};
+use crate::schema::SchemaProvider;
+use crate::{
+    CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef, DeregisterTableRequest,
+    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
+    SchemaProviderRef,
+};
+
+/// Simple in-memory list of catalogs
+pub struct MemoryCatalogManager {
+    /// Collection of catalogs containing schemas and ultimately Tables
+    pub catalogs: RwLock<HashMap<String, CatalogProviderRef>>,
+    pub table_id: AtomicU32,
+}
+
+impl Default for MemoryCatalogManager {
+    fn default() -> Self {
+        let manager = Self {
+            table_id: AtomicU32::new(MIN_USER_TABLE_ID),
+            catalogs: Default::default(),
+        };
+        let default_catalog = Arc::new(MemoryCatalogProvider::new());
+        manager
+            .register_catalog("greptime".to_string(), default_catalog.clone())
+            .unwrap();
+        default_catalog
+            .register_schema("public".to_string(), Arc::new(MemorySchemaProvider::new()))
+            .unwrap();
+        manager
+    }
+}
+
+#[async_trait::async_trait]
+impl TableIdProvider for MemoryCatalogManager {
+    async fn next_table_id(&self) -> table::error::Result<TableId> {
+        Ok(self.table_id.fetch_add(1, Ordering::Relaxed))
+    }
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for MemoryCatalogManager {
+    async fn start(&self) -> Result<()> {
+        self.table_id.store(MIN_USER_TABLE_ID, Ordering::Relaxed);
+        Ok(())
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        let catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get(&request.catalog)
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .clone();
+        let schema = catalog
+            .schema(&request.schema)?
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
+        schema
+            .register_table(request.table_name, request.table)
+            .map(|v| v.is_none())
+    }
+
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
+        let catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get(&request.catalog)
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .clone();
+        let schema = catalog
+            .schema(&request.schema)?
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
+        Ok(schema
+            .rename_table(&request.table_name, request.new_table_name)
+            .is_ok())
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool> {
+        let catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get(&request.catalog)
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .clone();
+        let schema = catalog
+            .schema(&request.schema)?
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
+        schema
+            .deregister_table(&request.table_name)
+            .map(|v| v.is_some())
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        let catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get(&request.catalog)
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?;
+        catalog.register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))?;
+        Ok(true)
+    }
+
+    async fn register_system_table(&self, _request: RegisterSystemTableRequest) -> Result<()> {
+        // TODO(ruihang): support register system table request
+        Ok(())
+    }
+
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+        let catalogs = self.catalogs.read().unwrap();
+        if let Some(c) = catalogs.get(catalog) {
+            c.schema(schema)
+        } else {
+            Ok(None)
+        }
+    }
+
+    async fn table(
+        &self,
+        catalog: &str,
+        schema: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        let catalog = {
+            let c = self.catalogs.read().unwrap();
+            let Some(c) = c.get(catalog) else { return Ok(None) };
+            c.clone()
+        };
+        match catalog.schema(schema)? {
+            None => Ok(None),
+            Some(s) => s.table(table_name).await,
+        }
+    }
+}
+
+impl MemoryCatalogManager {
+    /// Registers a catalog and return `None` if no catalog with the same name was already
+    /// registered, or `Some` with the previously registered catalog.
+    pub fn register_catalog_if_absent(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Option<CatalogProviderRef> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let entry = catalogs.entry(name);
+        match entry {
+            Entry::Occupied(v) => Some(v.get().clone()),
+            Entry::Vacant(v) => {
+                v.insert(catalog);
+                None
+            }
+        }
+    }
+}
+
+impl CatalogList for MemoryCatalogManager {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn register_catalog(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Result<Option<CatalogProviderRef>> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        Ok(catalogs.insert(name, catalog))
+    }
+
+    fn catalog_names(&self) -> Result<Vec<String>> {
+        let catalogs = self.catalogs.read().unwrap();
+        Ok(catalogs.keys().map(|s| s.to_string()).collect())
+    }
+
+    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>> {
+        let catalogs = self.catalogs.read().unwrap();
+        Ok(catalogs.get(name).cloned())
+    }
+}
+
+impl Default for MemoryCatalogProvider {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Simple in-memory implementation of a catalog.
+pub struct MemoryCatalogProvider {
+    schemas: RwLock<HashMap<String, Arc<dyn SchemaProvider>>>,
+}
+
+impl MemoryCatalogProvider {
+    /// Instantiates a new MemoryCatalogProvider with an empty collection of schemas.
+    pub fn new() -> Self {
+        Self {
+            schemas: RwLock::new(HashMap::new()),
+        }
+    }
+}
+
+impl CatalogProvider for MemoryCatalogProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema_names(&self) -> Result<Vec<String>> {
+        let schemas = self.schemas.read().unwrap();
+        Ok(schemas.keys().cloned().collect())
+    }
+
+    fn register_schema(
+        &self,
+        name: String,
+        schema: SchemaProviderRef,
+    ) -> Result<Option<SchemaProviderRef>> {
+        let mut schemas = self.schemas.write().unwrap();
+        ensure!(
+            !schemas.contains_key(&name),
+            error::SchemaExistsSnafu { schema: &name }
+        );
+        Ok(schemas.insert(name, schema))
+    }
+
+    fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>> {
+        let schemas = self.schemas.read().unwrap();
+        Ok(schemas.get(name).cloned())
+    }
+}
+
+/// Simple in-memory implementation of a schema.
+pub struct MemorySchemaProvider {
+    tables: RwLock<HashMap<String, TableRef>>,
+}
+
+impl MemorySchemaProvider {
+    /// Instantiates a new MemorySchemaProvider with an empty collection of tables.
+    pub fn new() -> Self {
+        Self {
+            tables: RwLock::new(HashMap::new()),
+        }
+    }
+}
+
+impl Default for MemorySchemaProvider {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[async_trait]
+impl SchemaProvider for MemorySchemaProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn table_names(&self) -> Result<Vec<String>> {
+        let tables = self.tables.read().unwrap();
+        Ok(tables.keys().cloned().collect())
+    }
+
+    async fn table(&self, name: &str) -> Result<Option<TableRef>> {
+        let tables = self.tables.read().unwrap();
+        Ok(tables.get(name).cloned())
+    }
+
+    fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
+        let mut tables = self.tables.write().unwrap();
+        if let Some(existing) = tables.get(name.as_str()) {
+            // if table with the same name but different table id exists, then it's a fatal bug
+            if existing.table_info().ident.table_id != table.table_info().ident.table_id {
+                error!(
+                    "Unexpected table register: {:?}, existing: {:?}",
+                    table.table_info(),
+                    existing.table_info()
+                );
+                return TableExistsSnafu { table: name }.fail()?;
+            }
+            Ok(Some(existing.clone()))
+        } else {
+            Ok(tables.insert(name, table))
+        }
+    }
+
+    fn rename_table(&self, name: &str, new_name: String) -> Result<TableRef> {
+        let mut tables = self.tables.write().unwrap();
+        if tables.get(name).is_some() {
+            let table = tables.remove(name).unwrap();
+            tables.insert(new_name, table.clone());
+            Ok(table)
+        } else {
+            TableNotFoundSnafu {
+                table_info: name.to_string(),
+            }
+            .fail()?
+        }
+    }
+
+    fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
+        let mut tables = self.tables.write().unwrap();
+        Ok(tables.remove(name))
+    }
+
+    fn table_exist(&self, name: &str) -> Result<bool> {
+        let tables = self.tables.read().unwrap();
+        Ok(tables.contains_key(name))
+    }
+}
+
+/// Create a memory catalog list contains a numbers table for test
+pub fn new_memory_catalog_list() -> Result<Arc<MemoryCatalogManager>> {
+    Ok(Arc::new(MemoryCatalogManager::default()))
+}
+
+#[cfg(test)]
+mod tests {
+    use common_catalog::consts::*;
+    use common_error::ext::ErrorExt;
+    use common_error::prelude::StatusCode;
+    use table::table::numbers::NumbersTable;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_new_memory_catalog_list() {
+        let catalog_list = new_memory_catalog_list().unwrap();
+        let default_catalog = catalog_list.catalog(DEFAULT_CATALOG_NAME).unwrap().unwrap();
+
+        let default_schema = default_catalog
+            .schema(DEFAULT_SCHEMA_NAME)
+            .unwrap()
+            .unwrap();
+
+        default_schema
+            .register_table("numbers".to_string(), Arc::new(NumbersTable::default()))
+            .unwrap();
+
+        let table = default_schema.table("numbers").await.unwrap();
+        assert!(table.is_some());
+        assert!(default_schema.table("not_exists").await.unwrap().is_none());
+    }
+
+    #[tokio::test]
+    async fn test_mem_provider() {
+        let provider = MemorySchemaProvider::new();
+        let table_name = "numbers";
+        assert!(!provider.table_exist(table_name).unwrap());
+        assert!(provider.deregister_table(table_name).unwrap().is_none());
+        let test_table = NumbersTable::default();
+        // register table successfully
+        assert!(provider
+            .register_table(table_name.to_string(), Arc::new(test_table))
+            .unwrap()
+            .is_none());
+        assert!(provider.table_exist(table_name).unwrap());
+        let other_table = NumbersTable::new(12);
+        let result = provider.register_table(table_name.to_string(), Arc::new(other_table));
+        let err = result.err().unwrap();
+        assert!(err.backtrace_opt().is_some());
+        assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
+    }
+
+    #[tokio::test]
+    async fn test_mem_provider_rename_table() {
+        let provider = MemorySchemaProvider::new();
+        let table_name = "num";
+        assert!(!provider.table_exist(table_name).unwrap());
+        let test_table: TableRef = Arc::new(NumbersTable::default());
+        // register test table
+        assert!(provider
+            .register_table(table_name.to_string(), test_table.clone())
+            .unwrap()
+            .is_none());
+        assert!(provider.table_exist(table_name).unwrap());
+
+        // rename test table
+        let new_table_name = "numbers";
+        provider
+            .rename_table(table_name, new_table_name.to_string())
+            .unwrap();
+
+        // test old table name not exist
+        assert!(!provider.table_exist(table_name).unwrap());
+        assert!(provider.deregister_table(table_name).unwrap().is_none());
+
+        // test new table name exists
+        assert!(provider.table_exist(new_table_name).unwrap());
+        let registered_table = provider.table(new_table_name).await.unwrap().unwrap();
+        assert_eq!(
+            registered_table.table_info().ident.table_id,
+            test_table.table_info().ident.table_id
+        );
+
+        let other_table = Arc::new(NumbersTable::new(2));
+        let result = provider.register_table(new_table_name.to_string(), other_table);
+        let err = result.err().unwrap();
+        assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
+    }
+
+    #[tokio::test]
+    async fn test_catalog_rename_table() {
+        let catalog = MemoryCatalogManager::default();
+        let schema = catalog
+            .schema(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
+            .unwrap()
+            .unwrap();
+
+        // register table
+        let table_name = "num";
+        let table_id = 2333;
+        let table: TableRef = Arc::new(NumbersTable::new(table_id));
+        let register_table_req = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id,
+            table,
+        };
+        assert!(catalog.register_table(register_table_req).await.unwrap());
+        assert!(schema.table_exist(table_name).unwrap());
+
+        // rename table
+        let new_table_name = "numbers";
+        let rename_table_req = RenameTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            new_table_name: new_table_name.to_string(),
+            table_id,
+        };
+        assert!(catalog.rename_table(rename_table_req).await.unwrap());
+        assert!(!schema.table_exist(table_name).unwrap());
+        assert!(schema.table_exist(new_table_name).unwrap());
+
+        let registered_table = catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(registered_table.table_info().ident.table_id, table_id);
+    }
+
+    #[test]
+    pub fn test_register_if_absent() {
+        let list = MemoryCatalogManager::default();
+        assert!(list
+            .register_catalog_if_absent(
+                "test_catalog".to_string(),
+                Arc::new(MemoryCatalogProvider::new())
+            )
+            .is_none());
+        list.register_catalog_if_absent(
+            "test_catalog".to_string(),
+            Arc::new(MemoryCatalogProvider::new()),
+        )
+        .unwrap();
+        list.as_any()
+            .downcast_ref::<MemoryCatalogManager>()
+            .unwrap();
+    }
+
+    #[tokio::test]
+    pub async fn test_catalog_deregister_table() {
+        let catalog = MemoryCatalogManager::default();
+        let schema = catalog
+            .schema(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
+            .unwrap()
+            .unwrap();
+
+        let register_table_req = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "numbers".to_string(),
+            table_id: 2333,
+            table: Arc::new(NumbersTable::default()),
+        };
+        catalog.register_table(register_table_req).await.unwrap();
+        assert!(schema.table_exist("numbers").unwrap());
+
+        let deregister_table_req = DeregisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "numbers".to_string(),
+        };
+        catalog
+            .deregister_table(deregister_table_req)
+            .await
+            .unwrap();
+        assert!(!schema.table_exist("numbers").unwrap());
+    }
+}

src/catalog/src/manager.rs

@@ -1,404 +0,0 @@
-use std::any::Any;
-use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::Arc;
-
-use common_recordbatch::RecordBatch;
-use common_telemetry::{debug, info};
-use datatypes::prelude::ScalarVector;
-use datatypes::vectors::{BinaryVector, UInt8Vector};
-use futures_util::lock::Mutex;
-use futures_util::StreamExt;
-use snafu::{ensure, OptionExt, ResultExt};
-use table::engine::{EngineContext, TableEngineRef};
-use table::metadata::TableId;
-use table::requests::OpenTableRequest;
-use table::table::numbers::NumbersTable;
-use table::TableRef;
-
-use super::error::Result;
-use crate::consts::{
-    INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_NAME,
-};
-use crate::error::{
-    CatalogNotFoundSnafu, CreateTableSnafu, IllegalManagerStateSnafu, OpenTableSnafu,
-    ReadSystemCatalogSnafu, SchemaNotFoundSnafu, SystemCatalogSnafu,
-    SystemCatalogTypeMismatchSnafu, TableExistsSnafu, TableNotFoundSnafu,
-};
-use crate::memory::{MemoryCatalogList, MemoryCatalogProvider, MemorySchemaProvider};
-use crate::system::{
-    decode_system_catalog, Entry, SystemCatalogTable, TableEntry, ENTRY_TYPE_INDEX, KEY_INDEX,
-    VALUE_INDEX,
-};
-use crate::tables::SystemCatalog;
-use crate::{
-    format_full_table_name, CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef,
-    RegisterSystemTableRequest, RegisterTableRequest, SchemaProvider, DEFAULT_CATALOG_NAME,
-    DEFAULT_SCHEMA_NAME,
-};
-
-/// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
-pub struct LocalCatalogManager {
-    system: Arc<SystemCatalog>,
-    catalogs: Arc<MemoryCatalogList>,
-    engine: TableEngineRef,
-    next_table_id: AtomicU32,
-    init_lock: Mutex<bool>,
-    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
-}
-
-impl LocalCatalogManager {
-    /// Create a new [CatalogManager] with given user catalogs and table engine
-    pub async fn try_new(engine: TableEngineRef) -> Result<Self> {
-        let table = SystemCatalogTable::new(engine.clone()).await?;
-        let memory_catalog_list = crate::memory::new_memory_catalog_list()?;
-        let system_catalog = Arc::new(SystemCatalog::new(
-            table,
-            memory_catalog_list.clone(),
-            engine.clone(),
-        ));
-        Ok(Self {
-            system: system_catalog,
-            catalogs: memory_catalog_list,
-            engine,
-            next_table_id: AtomicU32::new(MIN_USER_TABLE_ID),
-            init_lock: Mutex::new(false),
-            system_table_requests: Mutex::new(Vec::default()),
-        })
-    }
-
-    /// Scan all entries from system catalog table
-    pub async fn init(&self) -> Result<()> {
-        self.init_system_catalog()?;
-        let mut system_records = self.system.information_schema.system.records().await?;
-        let mut max_table_id = 0;
-        while let Some(records) = system_records
-            .next()
-            .await
-            .transpose()
-            .context(ReadSystemCatalogSnafu)?
-        {
-            let table_id = self.handle_system_catalog_entries(records).await?;
-            max_table_id = max_table_id.max(table_id);
-        }
-        info!(
-            "All system catalog entries processed, max table id: {}",
-            max_table_id
-        );
-        self.next_table_id
-            .store((max_table_id + 1).max(MIN_USER_TABLE_ID), Ordering::Relaxed);
-        *self.init_lock.lock().await = true;
-
-        // Processing system table hooks
-        let mut sys_table_requests = self.system_table_requests.lock().await;
-        for req in sys_table_requests.drain(..) {
-            let catalog_name = &req.create_table_request.catalog_name;
-            let schema_name = &req.create_table_request.schema_name;
-            let table_name = &req.create_table_request.table_name;
-            let table_id = req.create_table_request.id;
-
-            let table = if let Some(table) =
-                self.table(catalog_name.as_deref(), schema_name.as_deref(), table_name)?
-            {
-                table
-            } else {
-                let table = self
-                    .engine
-                    .create_table(&EngineContext::default(), req.create_table_request.clone())
-                    .await
-                    .with_context(|_| CreateTableSnafu {
-                        table_info: format!(
-                            "{}.{}.{}, id: {}",
-                            catalog_name.as_deref().unwrap_or(DEFAULT_CATALOG_NAME),
-                            schema_name.as_deref().unwrap_or(DEFAULT_SCHEMA_NAME),
-                            table_name,
-                            table_id,
-                        ),
-                    })?;
-                self.register_table(RegisterTableRequest {
-                    catalog: catalog_name.clone(),
-                    schema: schema_name.clone(),
-                    table_name: table_name.clone(),
-                    table_id,
-                    table: table.clone(),
-                })
-                .await?;
-
-                info!("Created and registered system table: {}", table_name);
-
-                table
-            };
-
-            if let Some(hook) = req.open_hook {
-                (hook)(table)?;
-            }
-        }
-
-        Ok(())
-    }
-
-    fn init_system_catalog(&self) -> Result<()> {
-        let system_schema = Arc::new(MemorySchemaProvider::new());
-        system_schema.register_table(
-            SYSTEM_CATALOG_TABLE_NAME.to_string(),
-            self.system.information_schema.system.clone(),
-        )?;
-        let system_catalog = Arc::new(MemoryCatalogProvider::new());
-        system_catalog.register_schema(INFORMATION_SCHEMA_NAME.to_string(), system_schema);
-        self.catalogs
-            .register_catalog(SYSTEM_CATALOG_NAME.to_string(), system_catalog);
-
-        let default_catalog = Arc::new(MemoryCatalogProvider::new());
-        let default_schema = Arc::new(MemorySchemaProvider::new());
-
-        // Add numbers table for test
-        // TODO(hl): remove this registration
-        let table = Arc::new(NumbersTable::default());
-        default_schema.register_table("numbers".to_string(), table)?;
-
-        default_catalog.register_schema(DEFAULT_SCHEMA_NAME.to_string(), default_schema);
-        self.catalogs
-            .register_catalog(DEFAULT_CATALOG_NAME.to_string(), default_catalog);
-        Ok(())
-    }
-
-    /// Processes records from system catalog table and returns the max table id persisted
-    /// in system catalog table.
-    async fn handle_system_catalog_entries(&self, records: RecordBatch) -> Result<TableId> {
-        ensure!(
-            records.df_recordbatch.columns().len() >= 6,
-            SystemCatalogSnafu {
-                msg: format!(
-                    "Length mismatch: {}",
-                    records.df_recordbatch.columns().len()
-                )
-            }
-        );
-
-        let entry_type = UInt8Vector::try_from_arrow_array(&records.df_recordbatch.columns()[0])
-            .with_context(|_| SystemCatalogTypeMismatchSnafu {
-                data_type: records.df_recordbatch.columns()[ENTRY_TYPE_INDEX]
-                    .data_type()
-                    .clone(),
-            })?;
-
-        let key = BinaryVector::try_from_arrow_array(&records.df_recordbatch.columns()[1])
-            .with_context(|_| SystemCatalogTypeMismatchSnafu {
-                data_type: records.df_recordbatch.columns()[KEY_INDEX]
-                    .data_type()
-                    .clone(),
-            })?;
-
-        let value = BinaryVector::try_from_arrow_array(&records.df_recordbatch.columns()[3])
-            .with_context(|_| SystemCatalogTypeMismatchSnafu {
-                data_type: records.df_recordbatch.columns()[VALUE_INDEX]
-                    .data_type()
-                    .clone(),
-            })?;
-
-        let mut max_table_id = 0;
-        for ((t, k), v) in entry_type
-            .iter_data()
-            .zip(key.iter_data())
-            .zip(value.iter_data())
-        {
-            let entry = decode_system_catalog(t, k, v)?;
-            match entry {
-                Entry::Catalog(c) => {
-                    self.catalogs.register_catalog_if_absent(
-                        c.catalog_name.clone(),
-                        Arc::new(MemoryCatalogProvider::new()),
-                    );
-                    info!("Register catalog: {}", c.catalog_name);
-                }
-                Entry::Schema(s) => {
-                    let catalog =
-                        self.catalogs
-                            .catalog(&s.catalog_name)
-                            .context(CatalogNotFoundSnafu {
-                                catalog_name: &s.catalog_name,
-                            })?;
-                    catalog.register_schema(
-                        s.schema_name.clone(),
-                        Arc::new(MemorySchemaProvider::new()),
-                    );
-                    info!("Registered schema: {:?}", s);
-                }
-                Entry::Table(t) => {
-                    debug!("t: {:?}", t);
-                    self.open_and_register_table(&t).await?;
-                    info!("Registered table: {:?}", t);
-                    max_table_id = max_table_id.max(t.table_id);
-                }
-            }
-        }
-        Ok(max_table_id)
-    }
-
-    async fn open_and_register_table(&self, t: &TableEntry) -> Result<()> {
-        let catalog = self
-            .catalogs
-            .catalog(&t.catalog_name)
-            .context(CatalogNotFoundSnafu {
-                catalog_name: &t.catalog_name,
-            })?;
-        let schema = catalog
-            .schema(&t.schema_name)
-            .context(SchemaNotFoundSnafu {
-                schema_info: format!("{}.{}", &t.catalog_name, &t.schema_name),
-            })?;
-
-        let context = EngineContext {};
-        let request = OpenTableRequest {
-            catalog_name: t.catalog_name.clone(),
-            schema_name: t.schema_name.clone(),
-            table_name: t.table_name.clone(),
-            table_id: t.table_id,
-        };
-
-        let option = self
-            .engine
-            .open_table(&context, request)
-            .await
-            .with_context(|_| OpenTableSnafu {
-                table_info: format!(
-                    "{}.{}.{}, id: {}",
-                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
-                ),
-            })?
-            .with_context(|| TableNotFoundSnafu {
-                table_info: format!(
-                    "{}.{}.{}, id: {}",
-                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
-                ),
-            })?;
-
-        schema.register_table(t.table_name.clone(), option)?;
-        Ok(())
-    }
-}
-
-impl CatalogList for LocalCatalogManager {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn register_catalog(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Option<Arc<dyn CatalogProvider>> {
-        self.catalogs.register_catalog(name, catalog)
-    }
-
-    fn catalog_names(&self) -> Vec<String> {
-        let mut res = self.catalogs.catalog_names();
-        res.push(SYSTEM_CATALOG_NAME.to_string());
-        res
-    }
-
-    fn catalog(&self, name: &str) -> Option<Arc<dyn CatalogProvider>> {
-        if name.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
-            Some(self.system.clone())
-        } else {
-            self.catalogs.catalog(name)
-        }
-    }
-}
-
-#[async_trait::async_trait]
-impl CatalogManager for LocalCatalogManager {
-    /// Start [MemoryCatalogManager] to load all information from system catalog table.
-    /// Make sure table engine is initialized before starting [MemoryCatalogManager].
-    async fn start(&self) -> Result<()> {
-        self.init().await
-    }
-
-    #[inline]
-    fn next_table_id(&self) -> TableId {
-        self.next_table_id.fetch_add(1, Ordering::Relaxed)
-    }
-
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<usize> {
-        let started = self.init_lock.lock().await;
-
-        ensure!(
-            *started,
-            IllegalManagerStateSnafu {
-                msg: "Catalog manager not started",
-            }
-        );
-
-        let catalog_name = request
-            .catalog
-            .unwrap_or_else(|| DEFAULT_CATALOG_NAME.to_string());
-        let schema_name = request
-            .schema
-            .unwrap_or_else(|| DEFAULT_SCHEMA_NAME.to_string());
-
-        let catalog = self
-            .catalogs
-            .catalog(&catalog_name)
-            .context(CatalogNotFoundSnafu {
-                catalog_name: &catalog_name,
-            })?;
-        let schema = catalog
-            .schema(&schema_name)
-            .with_context(|| SchemaNotFoundSnafu {
-                schema_info: format!("{}.{}", catalog_name, schema_name),
-            })?;
-
-        if schema.table_exist(&request.table_name) {
-            return TableExistsSnafu {
-                table: format_full_table_name(&catalog_name, &schema_name, &request.table_name),
-            }
-            .fail();
-        }
-
-        self.system
-            .register_table(
-                catalog_name,
-                schema_name,
-                request.table_name.clone(),
-                request.table_id,
-            )
-            .await?;
-
-        schema.register_table(request.table_name, request.table)?;
-        Ok(1)
-    }
-
-    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
-        ensure!(
-            !*self.init_lock.lock().await,
-            IllegalManagerStateSnafu {
-                msg: "Catalog manager already started",
-            }
-        );
-
-        let mut sys_table_requests = self.system_table_requests.lock().await;
-        sys_table_requests.push(request);
-
-        Ok(())
-    }
-
-    fn table(
-        &self,
-        catalog: Option<&str>,
-        schema: Option<&str>,
-        table_name: &str,
-    ) -> Result<Option<TableRef>> {
-        let catalog_name = catalog.unwrap_or(DEFAULT_CATALOG_NAME);
-        let schema_name = schema.unwrap_or(DEFAULT_SCHEMA_NAME);
-
-        let catalog = self
-            .catalogs
-            .catalog(catalog_name)
-            .context(CatalogNotFoundSnafu { catalog_name })?;
-        let schema = catalog
-            .schema(schema_name)
-            .with_context(|| SchemaNotFoundSnafu {
-                schema_info: format!("{}.{}", catalog_name, schema_name),
-            })?;
-        Ok(schema.table(table_name))
-    }
-}

src/catalog/src/memory.rs

@@ -1,236 +0,0 @@
-use std::any::Any;
-use std::collections::hash_map::Entry;
-use std::collections::HashMap;
-use std::sync::Arc;
-use std::sync::RwLock;
-
-use table::TableRef;
-
-use crate::error::{Result, TableExistsSnafu};
-use crate::schema::SchemaProvider;
-use crate::{CatalogList, CatalogProvider, CatalogProviderRef, SchemaProviderRef};
-
-/// Simple in-memory list of catalogs
-#[derive(Default)]
-pub struct MemoryCatalogList {
-    /// Collection of catalogs containing schemas and ultimately Tables
-    pub catalogs: RwLock<HashMap<String, CatalogProviderRef>>,
-}
-
-impl MemoryCatalogList {
-    /// Registers a catalog and return `None` if no catalog with the same name was already
-    /// registered, or `Some` with the previously registered catalog.
-    pub fn register_catalog_if_absent(
-        &self,
-        name: String,
-        catalog: Arc<dyn CatalogProvider>,
-    ) -> Option<CatalogProviderRef> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let entry = catalogs.entry(name);
-        match entry {
-            Entry::Occupied(v) => Some(v.get().clone()),
-            Entry::Vacant(v) => {
-                v.insert(catalog);
-                None
-            }
-        }
-    }
-}
-
-impl CatalogList for MemoryCatalogList {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn register_catalog(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Option<CatalogProviderRef> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        catalogs.insert(name, catalog)
-    }
-
-    fn catalog_names(&self) -> Vec<String> {
-        let catalogs = self.catalogs.read().unwrap();
-        catalogs.keys().map(|s| s.to_string()).collect()
-    }
-
-    fn catalog(&self, name: &str) -> Option<CatalogProviderRef> {
-        let catalogs = self.catalogs.read().unwrap();
-        catalogs.get(name).cloned()
-    }
-}
-
-impl Default for MemoryCatalogProvider {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-/// Simple in-memory implementation of a catalog.
-pub struct MemoryCatalogProvider {
-    schemas: RwLock<HashMap<String, Arc<dyn SchemaProvider>>>,
-}
-
-impl MemoryCatalogProvider {
-    /// Instantiates a new MemoryCatalogProvider with an empty collection of schemas.
-    pub fn new() -> Self {
-        Self {
-            schemas: RwLock::new(HashMap::new()),
-        }
-    }
-}
-
-impl CatalogProvider for MemoryCatalogProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema_names(&self) -> Vec<String> {
-        let schemas = self.schemas.read().unwrap();
-        schemas.keys().cloned().collect()
-    }
-
-    fn register_schema(
-        &self,
-        name: String,
-        schema: SchemaProviderRef,
-    ) -> Option<SchemaProviderRef> {
-        let mut schemas = self.schemas.write().unwrap();
-        schemas.insert(name, schema)
-    }
-
-    fn schema(&self, name: &str) -> Option<Arc<dyn SchemaProvider>> {
-        let schemas = self.schemas.read().unwrap();
-        schemas.get(name).cloned()
-    }
-}
-
-/// Simple in-memory implementation of a schema.
-pub struct MemorySchemaProvider {
-    tables: RwLock<HashMap<String, TableRef>>,
-}
-
-impl MemorySchemaProvider {
-    /// Instantiates a new MemorySchemaProvider with an empty collection of tables.
-    pub fn new() -> Self {
-        Self {
-            tables: RwLock::new(HashMap::new()),
-        }
-    }
-}
-
-impl Default for MemorySchemaProvider {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl SchemaProvider for MemorySchemaProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn table_names(&self) -> Vec<String> {
-        let tables = self.tables.read().unwrap();
-        tables.keys().cloned().collect()
-    }
-
-    fn table(&self, name: &str) -> Option<TableRef> {
-        let tables = self.tables.read().unwrap();
-        tables.get(name).cloned()
-    }
-
-    fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
-        if self.table_exist(name.as_str()) {
-            return TableExistsSnafu { table: name }.fail()?;
-        }
-        let mut tables = self.tables.write().unwrap();
-        Ok(tables.insert(name, table))
-    }
-
-    fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
-        let mut tables = self.tables.write().unwrap();
-        Ok(tables.remove(name))
-    }
-
-    fn table_exist(&self, name: &str) -> bool {
-        let tables = self.tables.read().unwrap();
-        tables.contains_key(name)
-    }
-}
-
-/// Create a memory catalog list contains a numbers table for test
-pub fn new_memory_catalog_list() -> Result<Arc<MemoryCatalogList>> {
-    Ok(Arc::new(MemoryCatalogList::default()))
-}
-
-#[cfg(test)]
-mod tests {
-    use common_error::ext::ErrorExt;
-    use common_error::prelude::StatusCode;
-    use table::table::numbers::NumbersTable;
-
-    use super::*;
-    use crate::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-
-    #[test]
-    fn test_new_memory_catalog_list() {
-        let catalog_list = new_memory_catalog_list().unwrap();
-
-        assert!(catalog_list.catalog(DEFAULT_CATALOG_NAME).is_none());
-        let default_catalog = Arc::new(MemoryCatalogProvider::default());
-        catalog_list.register_catalog(DEFAULT_CATALOG_NAME.to_string(), default_catalog.clone());
-
-        assert!(default_catalog.schema(DEFAULT_SCHEMA_NAME).is_none());
-        let default_schema = Arc::new(MemorySchemaProvider::default());
-        default_catalog.register_schema(DEFAULT_SCHEMA_NAME.to_string(), default_schema.clone());
-
-        default_schema
-            .register_table("numbers".to_string(), Arc::new(NumbersTable::default()))
-            .unwrap();
-
-        let table = default_schema.table("numbers");
-        assert!(table.is_some());
-
-        assert!(default_schema.table("not_exists").is_none());
-    }
-
-    #[tokio::test]
-    async fn test_mem_provider() {
-        let provider = MemorySchemaProvider::new();
-        let table_name = "numbers";
-        assert!(!provider.table_exist(table_name));
-        assert!(provider.deregister_table(table_name).unwrap().is_none());
-        let test_table = NumbersTable::default();
-        // register table successfully
-        assert!(provider
-            .register_table(table_name.to_string(), Arc::new(test_table))
-            .unwrap()
-            .is_none());
-        assert!(provider.table_exist(table_name));
-        let other_table = NumbersTable::default();
-        let result = provider.register_table(table_name.to_string(), Arc::new(other_table));
-        let err = result.err().unwrap();
-        assert!(err.backtrace_opt().is_some());
-        assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
-    }
-
-    #[test]
-    pub fn test_register_if_absent() {
-        let list = MemoryCatalogList::default();
-        assert!(list
-            .register_catalog_if_absent(
-                "test_catalog".to_string(),
-                Arc::new(MemoryCatalogProvider::new())
-            )
-            .is_none());
-        list.register_catalog_if_absent(
-            "test_catalog".to_string(),
-            Arc::new(MemoryCatalogProvider::new()),
-        )
-        .unwrap();
-        list.as_any().downcast_ref::<MemoryCatalogList>().unwrap();
-    }
-}

src/catalog/src/remote.rs

@@ -0,0 +1,131 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Debug;
+use std::pin::Pin;
+use std::sync::Arc;
+
+pub use client::MetaKvBackend;
+use futures::Stream;
+use futures_util::StreamExt;
+pub use manager::{RemoteCatalogManager, RemoteCatalogProvider, RemoteSchemaProvider};
+
+use crate::error::Error;
+
+mod client;
+mod manager;
+
+#[derive(Debug, Clone)]
+pub struct Kv(pub Vec<u8>, pub Vec<u8>);
+
+pub type ValueIter<'a, E> = Pin<Box<dyn Stream<Item = Result<Kv, E>> + Send + 'a>>;
+
+#[async_trait::async_trait]
+pub trait KvBackend: Send + Sync {
+    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
+    where
+        'a: 'b;
+
+    async fn set(&self, key: &[u8], val: &[u8]) -> Result<(), Error>;
+
+    /// Compare and set value of key. `expect` is the expected value, if backend's current value associated
+    /// with key is the same as `expect`, the value will be updated to `val`.
+    ///
+    /// - If the compare-and-set operation successfully updated value, this method will return an `Ok(Ok())`
+    /// - If associated value is not the same as `expect`, no value will be updated and an `Ok(Err(Vec<u8>))`
+    /// will be returned, the `Err(Vec<u8>)` indicates the current associated value of key.
+    /// - If any error happens during operation, an `Err(Error)` will be returned.
+    async fn compare_and_set(
+        &self,
+        key: &[u8],
+        expect: &[u8],
+        val: &[u8],
+    ) -> Result<Result<(), Option<Vec<u8>>>, Error>;
+
+    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Error>;
+
+    async fn delete(&self, key: &[u8]) -> Result<(), Error> {
+        self.delete_range(key, &[]).await
+    }
+
+    /// Default get is implemented based on `range` method.
+    async fn get(&self, key: &[u8]) -> Result<Option<Kv>, Error> {
+        let mut iter = self.range(key);
+        while let Some(r) = iter.next().await {
+            let kv = r?;
+            if kv.0 == key {
+                return Ok(Some(kv));
+            }
+        }
+        return Ok(None);
+    }
+}
+
+pub type KvBackendRef = Arc<dyn KvBackend>;
+
+#[cfg(test)]
+mod tests {
+    use async_stream::stream;
+
+    use super::*;
+
+    struct MockKvBackend {}
+
+    #[async_trait::async_trait]
+    impl KvBackend for MockKvBackend {
+        fn range<'a, 'b>(&'a self, _key: &[u8]) -> ValueIter<'b, Error>
+        where
+            'a: 'b,
+        {
+            Box::pin(stream!({
+                for i in 0..3 {
+                    yield Ok(Kv(
+                        i.to_string().as_bytes().to_vec(),
+                        i.to_string().as_bytes().to_vec(),
+                    ))
+                }
+            }))
+        }
+
+        async fn set(&self, _key: &[u8], _val: &[u8]) -> Result<(), Error> {
+            unimplemented!()
+        }
+
+        async fn compare_and_set(
+            &self,
+            _key: &[u8],
+            _expect: &[u8],
+            _val: &[u8],
+        ) -> Result<Result<(), Option<Vec<u8>>>, Error> {
+            unimplemented!()
+        }
+
+        async fn delete_range(&self, _key: &[u8], _end: &[u8]) -> Result<(), Error> {
+            unimplemented!()
+        }
+    }
+
+    #[tokio::test]
+    async fn test_get() {
+        let backend = MockKvBackend {};
+        let result = backend.get(0.to_string().as_bytes()).await;
+        assert_eq!(0.to_string().as_bytes(), result.unwrap().unwrap().0);
+        let result = backend.get(1.to_string().as_bytes()).await;
+        assert_eq!(1.to_string().as_bytes(), result.unwrap().unwrap().0);
+        let result = backend.get(2.to_string().as_bytes()).await;
+        assert_eq!(2.to_string().as_bytes(), result.unwrap().unwrap().0);
+        let result = backend.get(3.to_string().as_bytes()).await;
+        assert!(result.unwrap().is_none());
+    }
+}

src/catalog/src/remote/client.rs

@@ -0,0 +1,108 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use async_stream::stream;
+use common_telemetry::info;
+use meta_client::client::MetaClient;
+use meta_client::rpc::{CompareAndPutRequest, DeleteRangeRequest, PutRequest, RangeRequest};
+use snafu::ResultExt;
+
+use crate::error::{Error, MetaSrvSnafu};
+use crate::remote::{Kv, KvBackend, ValueIter};
+#[derive(Debug)]
+pub struct MetaKvBackend {
+    pub client: Arc<MetaClient>,
+}
+
+/// Implement `KvBackend` trait for `MetaKvBackend` instead of opendal's `Accessor` since
+/// `MetaClient`'s range method can return both keys and values, which can reduce IO overhead
+/// comparing to `Accessor`'s list and get method.
+#[async_trait::async_trait]
+impl KvBackend for MetaKvBackend {
+    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
+    where
+        'a: 'b,
+    {
+        let key = key.to_vec();
+        Box::pin(stream!({
+            let mut resp = self
+                .client
+                .range(RangeRequest::new().with_prefix(key))
+                .await
+                .context(MetaSrvSnafu)?;
+            let kvs = resp.take_kvs();
+            for mut kv in kvs.into_iter() {
+                yield Ok(Kv(kv.take_key(), kv.take_value()))
+            }
+        }))
+    }
+
+    async fn get(&self, key: &[u8]) -> Result<Option<Kv>, Error> {
+        let mut response = self
+            .client
+            .range(RangeRequest::new().with_key(key))
+            .await
+            .context(MetaSrvSnafu)?;
+        Ok(response
+            .take_kvs()
+            .get_mut(0)
+            .map(|kv| Kv(kv.take_key(), kv.take_value())))
+    }
+
+    async fn set(&self, key: &[u8], val: &[u8]) -> Result<(), Error> {
+        let req = PutRequest::new()
+            .with_key(key.to_vec())
+            .with_value(val.to_vec());
+        let _ = self.client.put(req).await.context(MetaSrvSnafu)?;
+        Ok(())
+    }
+
+    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Error> {
+        let req = DeleteRangeRequest::new().with_range(key.to_vec(), end.to_vec());
+        let resp = self.client.delete_range(req).await.context(MetaSrvSnafu)?;
+        info!(
+            "Delete range, key: {}, end: {}, deleted: {}",
+            String::from_utf8_lossy(key),
+            String::from_utf8_lossy(end),
+            resp.deleted()
+        );
+
+        Ok(())
+    }
+
+    async fn compare_and_set(
+        &self,
+        key: &[u8],
+        expect: &[u8],
+        val: &[u8],
+    ) -> Result<Result<(), Option<Vec<u8>>>, Error> {
+        let request = CompareAndPutRequest::new()
+            .with_key(key.to_vec())
+            .with_expect(expect.to_vec())
+            .with_value(val.to_vec());
+        let mut response = self
+            .client
+            .compare_and_put(request)
+            .await
+            .context(MetaSrvSnafu)?;
+        if response.is_success() {
+            Ok(Ok(()))
+        } else {
+            Ok(Err(response.take_prev_kv().map(|v| v.value().to_vec())))
+        }
+    }
+}

src/catalog/src/remote/manager.rs

@@ -0,0 +1,850 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::collections::{HashMap, HashSet};
+use std::pin::Pin;
+use std::sync::Arc;
+
+use arc_swap::ArcSwap;
+use async_stream::stream;
+use async_trait::async_trait;
+use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID};
+use common_telemetry::{debug, error, info};
+use dashmap::DashMap;
+use futures::Stream;
+use futures_util::StreamExt;
+use parking_lot::RwLock;
+use snafu::{OptionExt, ResultExt};
+use table::engine::{EngineContext, TableEngineRef};
+use table::metadata::TableId;
+use table::requests::{CreateTableRequest, OpenTableRequest};
+use table::table::numbers::NumbersTable;
+use table::TableRef;
+use tokio::sync::Mutex;
+
+use crate::error::{
+    CatalogNotFoundSnafu, CreateTableSnafu, InvalidCatalogValueSnafu, OpenTableSnafu, Result,
+    SchemaNotFoundSnafu, TableExistsSnafu, UnimplementedSnafu,
+};
+use crate::helper::{
+    build_catalog_prefix, build_schema_prefix, build_table_global_prefix, CatalogKey, CatalogValue,
+    SchemaKey, SchemaValue, TableGlobalKey, TableGlobalValue, TableRegionalKey, TableRegionalValue,
+    CATALOG_KEY_PREFIX,
+};
+use crate::remote::{Kv, KvBackendRef};
+use crate::{
+    handle_system_table_request, CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef,
+    DeregisterTableRequest, RegisterSchemaRequest, RegisterSystemTableRequest,
+    RegisterTableRequest, RenameTableRequest, SchemaProvider, SchemaProviderRef,
+};
+
+/// Catalog manager based on metasrv.
+pub struct RemoteCatalogManager {
+    node_id: u64,
+    backend: KvBackendRef,
+    catalogs: Arc<RwLock<DashMap<String, CatalogProviderRef>>>,
+    engine: TableEngineRef,
+    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
+}
+
+impl RemoteCatalogManager {
+    pub fn new(engine: TableEngineRef, node_id: u64, backend: KvBackendRef) -> Self {
+        Self {
+            engine,
+            node_id,
+            backend,
+            catalogs: Default::default(),
+            system_table_requests: Default::default(),
+        }
+    }
+
+    fn build_catalog_key(&self, catalog_name: impl AsRef<str>) -> CatalogKey {
+        CatalogKey {
+            catalog_name: catalog_name.as_ref().to_string(),
+        }
+    }
+
+    fn new_catalog_provider(&self, catalog_name: &str) -> CatalogProviderRef {
+        Arc::new(RemoteCatalogProvider {
+            node_id: self.node_id,
+            catalog_name: catalog_name.to_string(),
+            backend: self.backend.clone(),
+            schemas: Default::default(),
+            mutex: Default::default(),
+        }) as _
+    }
+
+    fn new_schema_provider(&self, catalog_name: &str, schema_name: &str) -> SchemaProviderRef {
+        Arc::new(RemoteSchemaProvider {
+            catalog_name: catalog_name.to_string(),
+            schema_name: schema_name.to_string(),
+            tables: Default::default(),
+            node_id: self.node_id,
+            backend: self.backend.clone(),
+            mutex: Default::default(),
+        }) as _
+    }
+
+    async fn iter_remote_catalogs(
+        &self,
+    ) -> Pin<Box<dyn Stream<Item = Result<CatalogKey>> + Send + '_>> {
+        let catalog_range_prefix = build_catalog_prefix();
+        info!("catalog_range_prefix: {}", catalog_range_prefix);
+        let mut catalogs = self.backend.range(catalog_range_prefix.as_bytes());
+        Box::pin(stream!({
+            while let Some(r) = catalogs.next().await {
+                let Kv(k, _) = r?;
+                if !k.starts_with(catalog_range_prefix.as_bytes()) {
+                    debug!("Ignoring non-catalog key: {}", String::from_utf8_lossy(&k));
+                    continue;
+                }
+
+                let catalog_key = String::from_utf8_lossy(&k);
+                if let Ok(key) = CatalogKey::parse(&catalog_key) {
+                    yield Ok(key)
+                } else {
+                    error!("Invalid catalog key: {:?}", catalog_key);
+                }
+            }
+        }))
+    }
+
+    async fn iter_remote_schemas(
+        &self,
+        catalog_name: &str,
+    ) -> Pin<Box<dyn Stream<Item = Result<SchemaKey>> + Send + '_>> {
+        let schema_prefix = build_schema_prefix(catalog_name);
+        let mut schemas = self.backend.range(schema_prefix.as_bytes());
+
+        Box::pin(stream!({
+            while let Some(r) = schemas.next().await {
+                let Kv(k, _) = r?;
+                if !k.starts_with(schema_prefix.as_bytes()) {
+                    debug!("Ignoring non-schema key: {}", String::from_utf8_lossy(&k));
+                    continue;
+                }
+
+                let schema_key = SchemaKey::parse(&String::from_utf8_lossy(&k))
+                    .context(InvalidCatalogValueSnafu)?;
+                yield Ok(schema_key)
+            }
+        }))
+    }
+
+    /// Iterate over all table entries on metasrv
+    async fn iter_remote_tables(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+    ) -> Pin<Box<dyn Stream<Item = Result<(TableGlobalKey, TableGlobalValue)>> + Send + '_>> {
+        let table_prefix = build_table_global_prefix(catalog_name, schema_name);
+        let mut tables = self.backend.range(table_prefix.as_bytes());
+        Box::pin(stream!({
+            while let Some(r) = tables.next().await {
+                let Kv(k, v) = r?;
+                if !k.starts_with(table_prefix.as_bytes()) {
+                    debug!("Ignoring non-table prefix: {}", String::from_utf8_lossy(&k));
+                    continue;
+                }
+                let table_key = TableGlobalKey::parse(&String::from_utf8_lossy(&k))
+                    .context(InvalidCatalogValueSnafu)?;
+                let table_value =
+                    TableGlobalValue::from_bytes(&v).context(InvalidCatalogValueSnafu)?;
+
+                info!(
+                    "Found catalog table entry, key: {}, value: {:?}",
+                    table_key, table_value
+                );
+                // metasrv has allocated region ids to current datanode
+                if table_value
+                    .regions_id_map
+                    .get(&self.node_id)
+                    .map(|v| !v.is_empty())
+                    .unwrap_or(false)
+                {
+                    yield Ok((table_key, table_value))
+                }
+            }
+        }))
+    }
+
+    /// Fetch catalogs/schemas/tables from remote catalog manager along with max table id allocated.
+    async fn initiate_catalogs(&self) -> Result<(HashMap<String, CatalogProviderRef>, TableId)> {
+        let mut res = HashMap::new();
+        let max_table_id = MIN_USER_TABLE_ID - 1;
+
+        // initiate default catalog and schema
+        let default_catalog = self.initiate_default_catalog().await?;
+        res.insert(DEFAULT_CATALOG_NAME.to_string(), default_catalog);
+        info!("Default catalog and schema registered");
+
+        let mut catalogs = self.iter_remote_catalogs().await;
+        while let Some(r) = catalogs.next().await {
+            let CatalogKey { catalog_name, .. } = r?;
+            info!("Fetch catalog from metasrv: {}", catalog_name);
+            let catalog = res
+                .entry(catalog_name.clone())
+                .or_insert_with(|| self.new_catalog_provider(&catalog_name))
+                .clone();
+
+            self.initiate_schemas(catalog_name, catalog, max_table_id)
+                .await?;
+        }
+
+        Ok((res, max_table_id))
+    }
+
+    async fn initiate_schemas(
+        &self,
+        catalog_name: String,
+        catalog: CatalogProviderRef,
+        max_table_id: TableId,
+    ) -> Result<()> {
+        let mut schemas = self.iter_remote_schemas(&catalog_name).await;
+        while let Some(r) = schemas.next().await {
+            let SchemaKey {
+                catalog_name,
+                schema_name,
+                ..
+            } = r?;
+            info!("Found schema: {}.{}", catalog_name, schema_name);
+            let schema = match catalog.schema(&schema_name)? {
+                None => {
+                    let schema = self.new_schema_provider(&catalog_name, &schema_name);
+                    catalog.register_schema(schema_name.clone(), schema.clone())?;
+                    info!("Registered schema: {}", &schema_name);
+                    schema
+                }
+                Some(schema) => schema,
+            };
+
+            info!(
+                "Fetch schema from metasrv: {}.{}",
+                &catalog_name, &schema_name
+            );
+            self.initiate_tables(&catalog_name, &schema_name, schema, max_table_id)
+                .await?;
+        }
+        Ok(())
+    }
+
+    /// Initiates all tables inside a catalog by fetching data from metasrv.
+    async fn initiate_tables<'a>(
+        &'a self,
+        catalog_name: &'a str,
+        schema_name: &'a str,
+        schema: SchemaProviderRef,
+        mut max_table_id: TableId,
+    ) -> Result<()> {
+        info!("initializing tables in {}.{}", catalog_name, schema_name);
+        let mut table_num = 0;
+        let mut tables = self.iter_remote_tables(catalog_name, schema_name).await;
+        while let Some(r) = tables.next().await {
+            let (table_key, table_value) = r?;
+            let table_ref = self.open_or_create_table(&table_key, &table_value).await?;
+            schema.register_table(table_key.table_name.to_string(), table_ref)?;
+            info!("Registered table {}", &table_key.table_name);
+            max_table_id = max_table_id.max(table_value.table_id());
+            table_num += 1;
+        }
+        info!(
+            "initialized tables in {}.{}, total: {}",
+            catalog_name, schema_name, table_num
+        );
+        Ok(())
+    }
+
+    async fn initiate_default_catalog(&self) -> Result<CatalogProviderRef> {
+        let default_catalog = self.new_catalog_provider(DEFAULT_CATALOG_NAME);
+        let default_schema = self.new_schema_provider(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
+        default_catalog.register_schema(DEFAULT_SCHEMA_NAME.to_string(), default_schema.clone())?;
+        let schema_key = SchemaKey {
+            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        }
+        .to_string();
+        self.backend
+            .set(
+                schema_key.as_bytes(),
+                &SchemaValue {}
+                    .as_bytes()
+                    .context(InvalidCatalogValueSnafu)?,
+            )
+            .await?;
+        info!("Registered default schema");
+
+        let catalog_key = CatalogKey {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        }
+        .to_string();
+        self.backend
+            .set(
+                catalog_key.as_bytes(),
+                &CatalogValue {}
+                    .as_bytes()
+                    .context(InvalidCatalogValueSnafu)?,
+            )
+            .await?;
+        info!("Registered default catalog");
+        Ok(default_catalog)
+    }
+
+    async fn open_or_create_table(
+        &self,
+        table_key: &TableGlobalKey,
+        table_value: &TableGlobalValue,
+    ) -> Result<TableRef> {
+        let context = EngineContext {};
+        let TableGlobalKey {
+            catalog_name,
+            schema_name,
+            table_name,
+            ..
+        } = table_key;
+
+        let table_id = table_value.table_id();
+
+        let TableGlobalValue {
+            table_info,
+            regions_id_map,
+            ..
+        } = table_value;
+
+        // unwrap safety: checked in yielding this table when `iter_remote_tables`
+        let region_numbers = regions_id_map.get(&self.node_id).unwrap();
+
+        let request = OpenTableRequest {
+            catalog_name: catalog_name.clone(),
+            schema_name: schema_name.clone(),
+            table_name: table_name.clone(),
+            table_id,
+        };
+        match self
+            .engine
+            .open_table(&context, request)
+            .await
+            .with_context(|_| OpenTableSnafu {
+                table_info: format!("{catalog_name}.{schema_name}.{table_name}, id:{table_id}"),
+            })? {
+            Some(table) => {
+                info!(
+                    "Table opened: {}.{}.{}",
+                    catalog_name, schema_name, table_name
+                );
+                Ok(table)
+            }
+            None => {
+                info!(
+                    "Try create table: {}.{}.{}",
+                    catalog_name, schema_name, table_name
+                );
+
+                let meta = &table_info.meta;
+                let req = CreateTableRequest {
+                    id: table_id,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    desc: None,
+                    schema: meta.schema.clone(),
+                    region_numbers: region_numbers.clone(),
+                    primary_key_indices: meta.primary_key_indices.clone(),
+                    create_if_not_exists: true,
+                    table_options: meta.options.clone(),
+                };
+
+                self.engine
+                    .create_table(&context, req)
+                    .await
+                    .context(CreateTableSnafu {
+                        table_info: format!(
+                            "{}.{}.{}, id:{}",
+                            &catalog_name, &schema_name, &table_name, table_id
+                        ),
+                    })
+            }
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for RemoteCatalogManager {
+    async fn start(&self) -> Result<()> {
+        let (catalogs, max_table_id) = self.initiate_catalogs().await?;
+        info!(
+            "Initialized catalogs: {:?}",
+            catalogs.keys().cloned().collect::<Vec<_>>()
+        );
+
+        {
+            let self_catalogs = self.catalogs.read();
+            catalogs.into_iter().for_each(|(k, v)| {
+                self_catalogs.insert(k, v);
+            });
+        }
+
+        info!("Max table id allocated: {}", max_table_id);
+
+        let mut system_table_requests = self.system_table_requests.lock().await;
+        handle_system_table_request(self, self.engine.clone(), &mut system_table_requests).await?;
+        info!("All system table opened");
+
+        self.catalog(DEFAULT_CATALOG_NAME)
+            .unwrap()
+            .unwrap()
+            .schema(DEFAULT_SCHEMA_NAME)
+            .unwrap()
+            .unwrap()
+            .register_table("numbers".to_string(), Arc::new(NumbersTable::default()))
+            .unwrap();
+        Ok(())
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        let catalog_name = request.catalog;
+        let schema_name = request.schema;
+        let catalog_provider = self.catalog(&catalog_name)?.context(CatalogNotFoundSnafu {
+            catalog_name: &catalog_name,
+        })?;
+        let schema_provider =
+            catalog_provider
+                .schema(&schema_name)?
+                .with_context(|| SchemaNotFoundSnafu {
+                    catalog: &catalog_name,
+                    schema: &schema_name,
+                })?;
+        if schema_provider.table_exist(&request.table_name)? {
+            return TableExistsSnafu {
+                table: format!("{}.{}.{}", &catalog_name, &schema_name, &request.table_name),
+            }
+            .fail();
+        }
+        schema_provider.register_table(request.table_name, request.table)?;
+        Ok(true)
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool> {
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+        let schema = self
+            .schema(catalog_name, schema_name)?
+            .context(SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            })?;
+
+        let result = schema.deregister_table(&request.table_name)?;
+        Ok(result.is_none())
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        let catalog_name = request.catalog;
+        let schema_name = request.schema;
+        let catalog_provider = self.catalog(&catalog_name)?.context(CatalogNotFoundSnafu {
+            catalog_name: &catalog_name,
+        })?;
+        let schema_provider = self.new_schema_provider(&catalog_name, &schema_name);
+        catalog_provider.register_schema(schema_name, schema_provider)?;
+        Ok(true)
+    }
+
+    async fn rename_table(&self, _request: RenameTableRequest) -> Result<bool> {
+        UnimplementedSnafu {
+            operation: "rename table",
+        }
+        .fail()
+    }
+
+    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
+        let mut requests = self.system_table_requests.lock().await;
+        requests.push(request);
+        Ok(())
+    }
+
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+        self.catalog(catalog)?
+            .context(CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .schema(schema)
+    }
+
+    async fn table(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        let catalog = self
+            .catalog(catalog_name)?
+            .with_context(|| CatalogNotFoundSnafu { catalog_name })?;
+        let schema = catalog
+            .schema(schema_name)?
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            })?;
+        schema.table(table_name).await
+    }
+}
+
+impl CatalogList for RemoteCatalogManager {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn register_catalog(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Result<Option<CatalogProviderRef>> {
+        let key = self.build_catalog_key(&name).to_string();
+        let backend = self.backend.clone();
+        let catalogs = self.catalogs.clone();
+
+        std::thread::spawn(|| {
+            common_runtime::block_on_write(async move {
+                backend
+                    .set(
+                        key.as_bytes(),
+                        &CatalogValue {}
+                            .as_bytes()
+                            .context(InvalidCatalogValueSnafu)?,
+                    )
+                    .await?;
+
+                let catalogs = catalogs.read();
+                let prev = catalogs.insert(name, catalog.clone());
+
+                Ok(prev)
+            })
+        })
+        .join()
+        .unwrap()
+    }
+
+    /// List all catalogs from metasrv
+    fn catalog_names(&self) -> Result<Vec<String>> {
+        let catalogs = self.catalogs.read();
+        Ok(catalogs.iter().map(|k| k.key().to_string()).collect())
+    }
+
+    /// Read catalog info of given name from metasrv.
+    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>> {
+        {
+            let catalogs = self.catalogs.read();
+            let catalog = catalogs.get(name);
+
+            if let Some(catalog) = catalog {
+                return Ok(Some(catalog.clone()));
+            }
+        }
+
+        let catalogs = self.catalogs.write();
+
+        let catalog = catalogs.get(name);
+        if let Some(catalog) = catalog {
+            return Ok(Some(catalog.clone()));
+        }
+
+        // It's for lack of incremental catalog syncing between datanode and meta. Here we fetch catalog
+        // from meta on demand. This can be removed when incremental catalog syncing is done in datanode.
+
+        let backend = self.backend.clone();
+
+        let catalogs_from_meta: HashSet<String> = std::thread::spawn(|| {
+            common_runtime::block_on_read(async move {
+                let mut stream = backend.range(CATALOG_KEY_PREFIX.as_bytes());
+                let mut catalogs = HashSet::new();
+
+                while let Some(catalog) = stream.next().await {
+                    if let Ok(catalog) = catalog {
+                        let catalog_key = String::from_utf8_lossy(&catalog.0);
+
+                        if let Ok(key) = CatalogKey::parse(&catalog_key) {
+                            catalogs.insert(key.catalog_name);
+                        }
+                    }
+                }
+
+                catalogs
+            })
+        })
+        .join()
+        .unwrap();
+
+        catalogs.retain(|catalog_name, _| catalogs_from_meta.get(catalog_name).is_some());
+
+        for catalog in catalogs_from_meta {
+            catalogs
+                .entry(catalog.clone())
+                .or_insert(self.new_catalog_provider(&catalog));
+        }
+
+        let catalog = catalogs.get(name);
+
+        Ok(catalog.as_deref().cloned())
+    }
+}
+
+pub struct RemoteCatalogProvider {
+    node_id: u64,
+    catalog_name: String,
+    backend: KvBackendRef,
+    schemas: Arc<ArcSwap<HashMap<String, SchemaProviderRef>>>,
+    mutex: Arc<Mutex<()>>,
+}
+
+impl RemoteCatalogProvider {
+    pub fn new(catalog_name: String, backend: KvBackendRef, node_id: u64) -> Self {
+        Self {
+            node_id,
+            catalog_name,
+            backend,
+            schemas: Default::default(),
+            mutex: Default::default(),
+        }
+    }
+
+    pub fn refresh_schemas(&self) -> Result<()> {
+        let schemas = self.schemas.clone();
+        let schema_prefix = build_schema_prefix(&self.catalog_name);
+        let catalog_name = self.catalog_name.clone();
+        let mutex = self.mutex.clone();
+        let backend = self.backend.clone();
+        let node_id = self.node_id;
+
+        std::thread::spawn(move || {
+            common_runtime::block_on_write(async move {
+                let _guard = mutex.lock().await;
+                let prev_schemas = schemas.load();
+                let mut new_schemas = HashMap::with_capacity(prev_schemas.len() + 1);
+                new_schemas.clone_from(&prev_schemas);
+
+                let mut remote_schemas = backend.range(schema_prefix.as_bytes());
+                while let Some(r) = remote_schemas.next().await {
+                    let Kv(k, _) = r?;
+                    let schema_key = SchemaKey::parse(&String::from_utf8_lossy(&k))
+                        .context(InvalidCatalogValueSnafu)?;
+                    if !new_schemas.contains_key(&schema_key.schema_name) {
+                        new_schemas.insert(
+                            schema_key.schema_name.clone(),
+                            Arc::new(RemoteSchemaProvider::new(
+                                catalog_name.clone(),
+                                schema_key.schema_name,
+                                node_id,
+                                backend.clone(),
+                            )),
+                        );
+                    }
+                }
+                schemas.store(Arc::new(new_schemas));
+                Ok(())
+            })
+        })
+        .join()
+        .unwrap()?;
+
+        Ok(())
+    }
+
+    fn build_schema_key(&self, schema_name: impl AsRef<str>) -> SchemaKey {
+        SchemaKey {
+            catalog_name: self.catalog_name.clone(),
+            schema_name: schema_name.as_ref().to_string(),
+        }
+    }
+}
+
+impl CatalogProvider for RemoteCatalogProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema_names(&self) -> Result<Vec<String>> {
+        self.refresh_schemas()?;
+        Ok(self.schemas.load().keys().cloned().collect::<Vec<_>>())
+    }
+
+    fn register_schema(
+        &self,
+        name: String,
+        schema: SchemaProviderRef,
+    ) -> Result<Option<SchemaProviderRef>> {
+        let key = self.build_schema_key(&name).to_string();
+        let backend = self.backend.clone();
+        let mutex = self.mutex.clone();
+        let schemas = self.schemas.clone();
+
+        std::thread::spawn(|| {
+            common_runtime::block_on_write(async move {
+                let _guard = mutex.lock().await;
+                backend
+                    .set(
+                        key.as_bytes(),
+                        &SchemaValue {}
+                            .as_bytes()
+                            .context(InvalidCatalogValueSnafu)?,
+                    )
+                    .await?;
+
+                let prev_schemas = schemas.load();
+                let mut new_schemas = HashMap::with_capacity(prev_schemas.len() + 1);
+                new_schemas.clone_from(&prev_schemas);
+                let prev_schema = new_schemas.insert(name, schema);
+                schemas.store(Arc::new(new_schemas));
+                Ok(prev_schema)
+            })
+        })
+        .join()
+        .unwrap()
+    }
+
+    fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>> {
+        // TODO(hl): We should refresh whole catalog before calling datafusion's query engine.
+        self.refresh_schemas()?;
+        Ok(self.schemas.load().get(name).cloned())
+    }
+}
+
+pub struct RemoteSchemaProvider {
+    catalog_name: String,
+    schema_name: String,
+    node_id: u64,
+    backend: KvBackendRef,
+    tables: Arc<ArcSwap<HashMap<String, TableRef>>>,
+    mutex: Arc<Mutex<()>>,
+}
+
+impl RemoteSchemaProvider {
+    pub fn new(
+        catalog_name: String,
+        schema_name: String,
+        node_id: u64,
+        backend: KvBackendRef,
+    ) -> Self {
+        Self {
+            catalog_name,
+            schema_name,
+            node_id,
+            backend,
+            tables: Default::default(),
+            mutex: Default::default(),
+        }
+    }
+
+    fn build_regional_table_key(&self, table_name: impl AsRef<str>) -> TableRegionalKey {
+        TableRegionalKey {
+            catalog_name: self.catalog_name.clone(),
+            schema_name: self.schema_name.clone(),
+            table_name: table_name.as_ref().to_string(),
+            node_id: self.node_id,
+        }
+    }
+}
+
+#[async_trait]
+impl SchemaProvider for RemoteSchemaProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn table_names(&self) -> Result<Vec<String>> {
+        Ok(self.tables.load().keys().cloned().collect::<Vec<_>>())
+    }
+
+    async fn table(&self, name: &str) -> Result<Option<TableRef>> {
+        Ok(self.tables.load().get(name).cloned())
+    }
+
+    fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
+        let table_info = table.table_info();
+        let table_version = table_info.ident.version;
+        let table_value = TableRegionalValue {
+            version: table_version,
+            regions_ids: table.table_info().meta.region_numbers.clone(),
+        };
+        let backend = self.backend.clone();
+        let mutex = self.mutex.clone();
+        let tables = self.tables.clone();
+        let table_key = self.build_regional_table_key(&name).to_string();
+
+        let prev = std::thread::spawn(move || {
+            common_runtime::block_on_read(async move {
+                let _guard = mutex.lock().await;
+                backend
+                    .set(
+                        table_key.as_bytes(),
+                        &table_value.as_bytes().context(InvalidCatalogValueSnafu)?,
+                    )
+                    .await?;
+                debug!(
+                    "Successfully set catalog table entry, key: {}, table value: {:?}",
+                    table_key, table_value
+                );
+
+                let prev_tables = tables.load();
+                let mut new_tables = HashMap::with_capacity(prev_tables.len() + 1);
+                new_tables.clone_from(&prev_tables);
+                let prev = new_tables.insert(name, table);
+                tables.store(Arc::new(new_tables));
+                Ok(prev)
+            })
+        })
+        .join()
+        .unwrap();
+        prev
+    }
+
+    fn rename_table(&self, _name: &str, _new_name: String) -> Result<TableRef> {
+        UnimplementedSnafu {
+            operation: "rename table",
+        }
+        .fail()
+    }
+
+    fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
+        let table_name = name.to_string();
+        let table_key = self.build_regional_table_key(&table_name).to_string();
+        let backend = self.backend.clone();
+        let mutex = self.mutex.clone();
+        let tables = self.tables.clone();
+        let prev = std::thread::spawn(move || {
+            common_runtime::block_on_read(async move {
+                let _guard = mutex.lock().await;
+                backend.delete(table_key.as_bytes()).await?;
+                debug!(
+                    "Successfully deleted catalog table entry, key: {}",
+                    table_key
+                );
+
+                let prev_tables = tables.load();
+                let mut new_tables = HashMap::with_capacity(prev_tables.len() + 1);
+                new_tables.clone_from(&prev_tables);
+                let prev = new_tables.remove(&table_name);
+                tables.store(Arc::new(new_tables));
+                Ok(prev)
+            })
+        })
+        .join()
+        .unwrap();
+        prev
+    }
+
+    /// Checks if table exists in schema provider based on locally opened table map.
+    fn table_exist(&self, name: &str) -> Result<bool> {
+        Ok(self.tables.load().contains_key(name))
+    }
+}

src/catalog/src/schema.rs

@@ -1,26 +1,46 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::sync::Arc;
 
+use async_trait::async_trait;
 use table::TableRef;
 
 use crate::error::Result;
 
 /// Represents a schema, comprising a number of named tables.
+#[async_trait]
 pub trait SchemaProvider: Sync + Send {
     /// Returns the schema provider as [`Any`](std::any::Any)
     /// so that it can be downcast to a specific implementation.
     fn as_any(&self) -> &dyn Any;
 
     /// Retrieves the list of available table names in this schema.
-    fn table_names(&self) -> Vec<String>;
+    fn table_names(&self) -> Result<Vec<String>>;
 
     /// Retrieves a specific table from the schema by name, provided it exists.
-    fn table(&self, name: &str) -> Option<TableRef>;
+    async fn table(&self, name: &str) -> Result<Option<TableRef>>;
 
     /// If supported by the implementation, adds a new table to this schema.
     /// If a table of the same name existed before, it returns "Table already exists" error.
     fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>>;
 
+    /// If supported by the implementation, renames an existing table from this schema and returns it.
+    /// If no table of that name exists, returns "Table not found" error.
+    fn rename_table(&self, name: &str, new_name: String) -> Result<TableRef>;
+
     /// If supported by the implementation, removes an existing table from this schema and returns it.
     /// If no table of that name exists, returns Ok(None).
     fn deregister_table(&self, name: &str) -> Result<Option<TableRef>>;
@@ -28,7 +48,7 @@ pub trait SchemaProvider: Sync + Send {
     /// If supported by the implementation, checks the table exist in the schema provider or not.
     /// If no matched table in the schema provider, return false.
     /// Otherwise, return true.
-    fn table_exist(&self, name: &str) -> bool;
+    fn table_exist(&self, name: &str) -> Result<bool>;
 }
 
 pub type SchemaProviderRef = Arc<dyn SchemaProvider>;

src/catalog/src/system.rs

@@ -1,40 +1,53 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_NAME,
+    SYSTEM_CATALOG_TABLE_ID, SYSTEM_CATALOG_TABLE_NAME,
+};
 use common_query::logical_plan::Expr;
+use common_query::physical_plan::{PhysicalPlanRef, SessionContext};
 use common_recordbatch::SendableRecordBatchStream;
 use common_telemetry::debug;
-use common_time::timestamp::Timestamp;
 use common_time::util;
-use datatypes::prelude::{ConcreteDataType, ScalarVector};
-use datatypes::schema::{ColumnSchema, Schema, SchemaBuilder, SchemaRef};
-use datatypes::vectors::{BinaryVector, TimestampVector, UInt8Vector};
+use datatypes::prelude::{ConcreteDataType, ScalarVector, VectorRef};
+use datatypes::schema::{ColumnSchema, RawSchema, SchemaRef};
+use datatypes::vectors::{BinaryVector, TimestampMillisecondVector, UInt8Vector};
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
 use table::engine::{EngineContext, TableEngineRef};
-use table::metadata::TableId;
-use table::requests::{CreateTableRequest, InsertRequest, OpenTableRequest};
+use table::metadata::{TableId, TableInfoRef};
+use table::requests::{
+    CreateTableRequest, DeleteRequest, InsertRequest, OpenTableRequest, TableOptions,
+};
 use table::{Table, TableRef};
 
-use crate::consts::{
-    INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID,
-    SYSTEM_CATALOG_TABLE_NAME,
-};
 use crate::error::{
-    CreateSystemCatalogSnafu, EmptyValueSnafu, Error, InvalidEntryTypeSnafu, InvalidKeySnafu,
+    self, CreateSystemCatalogSnafu, EmptyValueSnafu, Error, InvalidEntryTypeSnafu, InvalidKeySnafu,
     OpenSystemCatalogSnafu, Result, ValueDeserializeSnafu,
 };
+use crate::DeregisterTableRequest;
 
 pub const ENTRY_TYPE_INDEX: usize = 0;
 pub const KEY_INDEX: usize = 1;
-pub const TIMESTAMP_INDEX: usize = 2;
 pub const VALUE_INDEX: usize = 3;
 
-pub struct SystemCatalogTable {
-    schema: SchemaRef,
-    pub table: TableRef,
-}
+pub struct SystemCatalogTable(TableRef);
 
 #[async_trait::async_trait]
 impl Table for SystemCatalogTable {
@@ -43,21 +56,29 @@ impl Table for SystemCatalogTable {
     }
 
     fn schema(&self) -> SchemaRef {
-        self.schema.clone()
+        self.0.schema()
     }
 
     async fn scan(
         &self,
-        _projection: &Option<Vec<usize>>,
-        _filters: &[Expr],
-        _limit: Option<usize>,
-    ) -> table::Result<SendableRecordBatchStream> {
-        panic!("System catalog table does not support scan!")
+        projection: Option<&Vec<usize>>,
+        filters: &[Expr],
+        limit: Option<usize>,
+    ) -> table::Result<PhysicalPlanRef> {
+        self.0.scan(projection, filters, limit).await
     }
 
     /// Insert values into table.
     async fn insert(&self, request: InsertRequest) -> table::error::Result<usize> {
-        self.table.insert(request).await
+        self.0.insert(request).await
+    }
+
+    fn table_info(&self) -> TableInfoRef {
+        self.0.table_info()
+    }
+
+    async fn delete(&self, request: DeleteRequest) -> table::Result<usize> {
+        self.0.delete(request).await
     }
 }
 
@@ -69,7 +90,7 @@ impl SystemCatalogTable {
             table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
             table_id: SYSTEM_CATALOG_TABLE_ID,
         };
-        let schema = Arc::new(build_system_catalog_schema());
+        let schema = build_system_catalog_schema();
         let ctx = EngineContext::default();
 
         if let Some(table) = engine
@@ -77,33 +98,41 @@ impl SystemCatalogTable {
             .await
             .context(OpenSystemCatalogSnafu)?
         {
-            Ok(Self { table, schema })
+            Ok(Self(table))
         } else {
             // system catalog table is not yet created, try to create
             let request = CreateTableRequest {
                 id: SYSTEM_CATALOG_TABLE_ID,
-                catalog_name: Some(SYSTEM_CATALOG_NAME.to_string()),
-                schema_name: Some(INFORMATION_SCHEMA_NAME.to_string()),
+                catalog_name: SYSTEM_CATALOG_NAME.to_string(),
+                schema_name: INFORMATION_SCHEMA_NAME.to_string(),
                 table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
                 desc: Some("System catalog table".to_string()),
-                schema: schema.clone(),
-                primary_key_indices: vec![ENTRY_TYPE_INDEX, KEY_INDEX, TIMESTAMP_INDEX],
+                schema,
+                region_numbers: vec![0],
+                primary_key_indices: vec![ENTRY_TYPE_INDEX, KEY_INDEX],
                 create_if_not_exists: true,
-                table_options: HashMap::new(),
+                table_options: TableOptions::default(),
             };
 
             let table = engine
                 .create_table(&ctx, request)
                 .await
                 .context(CreateSystemCatalogSnafu)?;
-            Ok(Self { table, schema })
+            Ok(Self(table))
         }
     }
 
     /// Create a stream of all entries inside system catalog table
     pub async fn records(&self) -> Result<SendableRecordBatchStream> {
         let full_projection = None;
-        let stream = self.table.scan(&full_projection, &[], None).await.unwrap();
+        let ctx = SessionContext::new();
+        let scan = self
+            .scan(full_projection, &[], None)
+            .await
+            .context(error::SystemCatalogTableScanSnafu)?;
+        let stream = scan
+            .execute(0, ctx.task_ctx())
+            .context(error::SystemCatalogTableScanExecSnafu)?;
         Ok(stream)
     }
 }
@@ -116,7 +145,7 @@ impl SystemCatalogTable {
 /// - value: JSON-encoded value of entry's metadata.
 /// - gmt_created: create time of this metadata.
 /// - gmt_modified: last updated time of this metadata.
-fn build_system_catalog_schema() -> Schema {
+fn build_system_catalog_schema() -> RawSchema {
     let cols = vec![
         ColumnSchema::new(
             "entry_type".to_string(),
@@ -130,9 +159,10 @@ fn build_system_catalog_schema() -> Schema {
         ),
         ColumnSchema::new(
             "timestamp".to_string(),
-            ConcreteDataType::timestamp_millis_datatype(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
             false,
-        ),
+        )
+        .with_time_index(true),
         ColumnSchema::new(
             "value".to_string(),
             ConcreteDataType::binary_datatype(),
@@ -140,68 +170,108 @@ fn build_system_catalog_schema() -> Schema {
         ),
         ColumnSchema::new(
             "gmt_created".to_string(),
-            ConcreteDataType::timestamp_millis_datatype(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
             false,
         ),
         ColumnSchema::new(
             "gmt_modified".to_string(),
-            ConcreteDataType::timestamp_millis_datatype(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
             false,
         ),
     ];
 
-    // The schema of this table must be valid.
-    SchemaBuilder::try_from(cols)
-        .unwrap()
-        .timestamp_index(2)
-        .build()
-        .unwrap()
+    RawSchema::new(cols)
 }
 
-pub fn build_table_insert_request(full_table_name: String, table_id: TableId) -> InsertRequest {
-    let mut columns_values = HashMap::with_capacity(6);
-    columns_values.insert(
+/// Formats key string for table entry in system catalog
+#[inline]
+pub fn format_table_entry_key(catalog: &str, schema: &str, table_id: TableId) -> String {
+    format!("{catalog}.{schema}.{table_id}")
+}
+
+pub fn build_table_insert_request(
+    catalog: String,
+    schema: String,
+    table_name: String,
+    table_id: TableId,
+) -> InsertRequest {
+    let entry_key = format_table_entry_key(&catalog, &schema, table_id);
+    build_insert_request(
+        EntryType::Table,
+        entry_key.as_bytes(),
+        serde_json::to_string(&TableEntryValue { table_name })
+            .unwrap()
+            .as_bytes(),
+    )
+}
+
+pub(crate) fn build_table_deletion_request(
+    request: &DeregisterTableRequest,
+    table_id: TableId,
+) -> DeleteRequest {
+    let table_key = format_table_entry_key(&request.catalog, &request.schema, table_id);
+    DeleteRequest {
+        key_column_values: build_primary_key_columns(EntryType::Table, table_key.as_bytes()),
+    }
+}
+
+fn build_primary_key_columns(entry_type: EntryType, key: &[u8]) -> HashMap<String, VectorRef> {
+    let mut m = HashMap::with_capacity(3);
+    m.insert(
         "entry_type".to_string(),
-        Arc::new(UInt8Vector::from_slice(&[EntryType::Table as u8])) as _,
+        Arc::new(UInt8Vector::from_slice([entry_type as u8])) as _,
     );
-
-    columns_values.insert(
+    m.insert(
         "key".to_string(),
-        Arc::new(BinaryVector::from_slice(&[full_table_name.as_bytes()])) as _,
+        Arc::new(BinaryVector::from_slice(&[key])) as _,
     );
-
     // Timestamp in key part is intentionally left to 0
-    columns_values.insert(
+    m.insert(
         "timestamp".to_string(),
-        Arc::new(TimestampVector::from_slice(&[Timestamp::from_millis(0)])) as _,
+        Arc::new(TimestampMillisecondVector::from_slice([0])) as _,
     );
+    m
+}
+
+pub fn build_schema_insert_request(catalog_name: String, schema_name: String) -> InsertRequest {
+    let full_schema_name = format!("{catalog_name}.{schema_name}");
+    build_insert_request(
+        EntryType::Schema,
+        full_schema_name.as_bytes(),
+        serde_json::to_string(&SchemaEntryValue {})
+            .unwrap()
+            .as_bytes(),
+    )
+}
+
+pub fn build_insert_request(entry_type: EntryType, key: &[u8], value: &[u8]) -> InsertRequest {
+    let primary_key_columns = build_primary_key_columns(entry_type, key);
+
+    let mut columns_values = HashMap::with_capacity(6);
+    columns_values.extend(primary_key_columns.into_iter());
 
     columns_values.insert(
         "value".to_string(),
-        Arc::new(BinaryVector::from_slice(&[serde_json::to_string(
-            &TableEntryValue { table_id },
-        )
-        .unwrap()
-        .as_bytes()])) as _,
+        Arc::new(BinaryVector::from_slice(&[value])) as _,
     );
 
+    let now = util::current_time_millis();
     columns_values.insert(
         "gmt_created".to_string(),
-        Arc::new(TimestampVector::from_slice(&[Timestamp::from_millis(
-            util::current_time_millis(),
-        )])) as _,
+        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
     );
 
     columns_values.insert(
         "gmt_modified".to_string(),
-        Arc::new(TimestampVector::from_slice(&[Timestamp::from_millis(
-            util::current_time_millis(),
-        )])) as _,
+        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
     );
 
     InsertRequest {
+        catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        schema_name: DEFAULT_SCHEMA_NAME.to_string(),
         table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
         columns_values,
+        region_number: 0, // system catalog table has only one region
     }
 }
 
@@ -241,8 +311,8 @@ pub fn decode_system_catalog(
         }
 
         EntryType::Table => {
-            // As for table entry, the key is a string with format: `<catalog_name>.<schema_name>.<table_name>`
-            // and the value is a JSON string with format: `{"table_id": <table_id>}`
+            // As for table entry, the key is a string with format: `<catalog_name>.<schema_name>.<table_id>`
+            // and the value is a JSON string with format: `{"table_name": <table_name>}`
             let table_parts = key.split('.').collect::<Vec<_>>();
             ensure!(
                 table_parts.len() >= 3,
@@ -254,11 +324,12 @@ pub fn decode_system_catalog(
             debug!("Table meta value: {}", String::from_utf8_lossy(value));
             let table_meta: TableEntryValue =
                 serde_json::from_slice(value).context(ValueDeserializeSnafu)?;
+            let table_id = table_parts[2].parse::<TableId>().unwrap();
             Ok(Entry::Table(TableEntry {
                 catalog_name: table_parts[0].to_string(),
                 schema_name: table_parts[1].to_string(),
-                table_name: table_parts[2].to_string(),
-                table_id: table_meta.table_id,
+                table_name: table_meta.table_name,
+                table_id,
             }))
         }
     }
@@ -287,25 +358,28 @@ impl TryFrom<u8> for EntryType {
     }
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
 pub enum Entry {
     Catalog(CatalogEntry),
     Schema(SchemaEntry),
     Table(TableEntry),
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
 pub struct CatalogEntry {
     pub catalog_name: String,
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
 pub struct SchemaEntry {
     pub catalog_name: String,
     pub schema_name: String,
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SchemaEntryValue;
+
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
 pub struct TableEntry {
     pub catalog_name: String,
     pub schema_name: String,
@@ -315,15 +389,28 @@ pub struct TableEntry {
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct TableEntryValue {
-    pub table_id: TableId,
+    pub table_name: String,
 }
 
 #[cfg(test)]
 mod tests {
+    use common_recordbatch::RecordBatches;
+    use common_test_util::temp_dir::{create_temp_dir, TempDir};
+    use datatypes::value::Value;
+    use log_store::NoopLogStore;
+    use mito::config::EngineConfig;
+    use mito::engine::MitoEngine;
+    use object_store::ObjectStore;
+    use storage::compaction::noop::NoopCompactionScheduler;
+    use storage::config::EngineConfig as StorageEngineConfig;
+    use storage::EngineImpl;
+    use table::metadata::TableType;
+    use table::metadata::TableType::Base;
+
     use super::*;
 
     #[test]
-    pub fn test_decode_catalog_enrty() {
+    pub fn test_decode_catalog_entry() {
         let entry = decode_system_catalog(
             Some(EntryType::Catalog as u8),
             Some("some_catalog".as_bytes()),
@@ -333,7 +420,7 @@ mod tests {
         if let Entry::Catalog(e) = entry {
             assert_eq!("some_catalog", e.catalog_name);
         } else {
-            panic!("Unexpected type: {:?}", entry);
+            panic!("Unexpected type: {entry:?}");
         }
     }
 
@@ -350,7 +437,7 @@ mod tests {
             assert_eq!("some_catalog", e.catalog_name);
             assert_eq!("some_schema", e.schema_name);
         } else {
-            panic!("Unexpected type: {:?}", entry);
+            panic!("Unexpected type: {entry:?}");
         }
     }
 
@@ -358,8 +445,8 @@ mod tests {
     pub fn test_decode_table() {
         let entry = decode_system_catalog(
             Some(EntryType::Table as u8),
-            Some("some_catalog.some_schema.some_table".as_bytes()),
-            Some("{\"table_id\":42}".as_bytes()),
+            Some("some_catalog.some_schema.42".as_bytes()),
+            Some("{\"table_name\":\"some_table\"}".as_bytes()),
         )
         .unwrap();
 
@@ -369,7 +456,7 @@ mod tests {
             assert_eq!("some_table", e.table_name);
             assert_eq!(42, e.table_id);
         } else {
-            panic!("Unexpected type: {:?}", entry);
+            panic!("Unexpected type: {entry:?}");
         }
     }
 
@@ -378,7 +465,7 @@ mod tests {
     pub fn test_decode_mismatch() {
         decode_system_catalog(
             Some(EntryType::Table as u8),
-            Some("some_catalog.some_schema.some_table".as_bytes()),
+            Some("some_catalog.some_schema.42".as_bytes()),
             None,
         )
         .unwrap();
@@ -391,4 +478,92 @@ mod tests {
         assert_eq!(EntryType::Table, EntryType::try_from(3).unwrap());
         assert!(EntryType::try_from(4).is_err());
     }
+
+    pub async fn prepare_table_engine() -> (TempDir, TableEngineRef) {
+        let dir = create_temp_dir("system-table-test");
+        let store_dir = dir.path().to_string_lossy();
+        let mut builder = object_store::services::Fs::default();
+        builder.root(&store_dir);
+        let object_store = ObjectStore::new(builder).unwrap().finish();
+        let noop_compaction_scheduler = Arc::new(NoopCompactionScheduler::default());
+        let table_engine = Arc::new(MitoEngine::new(
+            EngineConfig::default(),
+            EngineImpl::new(
+                StorageEngineConfig::default(),
+                Arc::new(NoopLogStore::default()),
+                object_store.clone(),
+                noop_compaction_scheduler,
+            ),
+            object_store,
+        ));
+        (dir, table_engine)
+    }
+
+    #[tokio::test]
+    async fn test_system_table_type() {
+        let (_dir, table_engine) = prepare_table_engine().await;
+        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
+        assert_eq!(Base, system_table.table_type());
+    }
+
+    #[tokio::test]
+    async fn test_system_table_info() {
+        let (_dir, table_engine) = prepare_table_engine().await;
+        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
+        let info = system_table.table_info();
+        assert_eq!(TableType::Base, info.table_type);
+        assert_eq!(SYSTEM_CATALOG_TABLE_NAME, info.name);
+        assert_eq!(SYSTEM_CATALOG_TABLE_ID, info.ident.table_id);
+        assert_eq!(SYSTEM_CATALOG_NAME, info.catalog_name);
+        assert_eq!(INFORMATION_SCHEMA_NAME, info.schema_name);
+    }
+
+    #[tokio::test]
+    async fn test_system_catalog_table_records() {
+        let (_, table_engine) = prepare_table_engine().await;
+        let catalog_table = SystemCatalogTable::new(table_engine).await.unwrap();
+
+        let table_insertion = build_table_insert_request(
+            DEFAULT_CATALOG_NAME.to_string(),
+            DEFAULT_SCHEMA_NAME.to_string(),
+            "my_table".to_string(),
+            1,
+        );
+        let result = catalog_table.insert(table_insertion).await.unwrap();
+        assert_eq!(result, 1);
+
+        let records = catalog_table.records().await.unwrap();
+        let mut batches = RecordBatches::try_collect(records).await.unwrap().take();
+        assert_eq!(batches.len(), 1);
+        let batch = batches.remove(0);
+        assert_eq!(batch.num_rows(), 1);
+
+        let row = batch.rows().next().unwrap();
+        let Value::UInt8(entry_type) = row[0] else { unreachable!() };
+        let Value::Binary(key) = row[1].clone() else { unreachable!() };
+        let Value::Binary(value) = row[3].clone() else { unreachable!() };
+        let entry = decode_system_catalog(Some(entry_type), Some(&*key), Some(&*value)).unwrap();
+        let expected = Entry::Table(TableEntry {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "my_table".to_string(),
+            table_id: 1,
+        });
+        assert_eq!(entry, expected);
+
+        let table_deletion = build_table_deletion_request(
+            &DeregisterTableRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_SCHEMA_NAME.to_string(),
+                table_name: "my_table".to_string(),
+            },
+            1,
+        );
+        let result = catalog_table.delete(table_deletion).await.unwrap();
+        assert_eq!(result, 1);
+
+        let records = catalog_table.records().await.unwrap();
+        let batches = RecordBatches::try_collect(records).await.unwrap().take();
+        assert_eq!(batches.len(), 0);
+    }
 }

src/catalog/src/table_source.rs

@@ -0,0 +1,178 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use common_catalog::format_full_table_name;
+use datafusion::common::{OwnedTableReference, ResolvedTableReference, TableReference};
+use datafusion::datasource::provider_as_source;
+use datafusion::logical_expr::TableSource;
+use session::context::QueryContext;
+use snafu::{ensure, OptionExt};
+use table::table::adapter::DfTableProviderAdapter;
+
+use crate::error::{
+    CatalogNotFoundSnafu, QueryAccessDeniedSnafu, Result, SchemaNotFoundSnafu, TableNotExistSnafu,
+};
+use crate::CatalogListRef;
+
+pub struct DfTableSourceProvider {
+    catalog_list: CatalogListRef,
+    resolved_tables: HashMap<String, Arc<dyn TableSource>>,
+    disallow_cross_schema_query: bool,
+    default_catalog: String,
+    default_schema: String,
+}
+
+impl DfTableSourceProvider {
+    pub fn new(
+        catalog_list: CatalogListRef,
+        disallow_cross_schema_query: bool,
+        query_ctx: &QueryContext,
+    ) -> Self {
+        Self {
+            catalog_list,
+            disallow_cross_schema_query,
+            resolved_tables: HashMap::new(),
+            default_catalog: query_ctx.current_catalog(),
+            default_schema: query_ctx.current_schema(),
+        }
+    }
+
+    pub fn resolve_table_ref<'a>(
+        &'a self,
+        table_ref: TableReference<'a>,
+    ) -> Result<ResolvedTableReference<'a>> {
+        if self.disallow_cross_schema_query {
+            match &table_ref {
+                TableReference::Bare { .. } => (),
+                TableReference::Partial { schema, .. } => {
+                    ensure!(
+                        schema.as_ref() == self.default_schema,
+                        QueryAccessDeniedSnafu {
+                            catalog: &self.default_catalog,
+                            schema: schema.as_ref(),
+                        }
+                    );
+                }
+                TableReference::Full {
+                    catalog, schema, ..
+                } => {
+                    ensure!(
+                        catalog.as_ref() == self.default_catalog
+                            && schema.as_ref() == self.default_schema,
+                        QueryAccessDeniedSnafu {
+                            catalog: catalog.as_ref(),
+                            schema: schema.as_ref()
+                        }
+                    );
+                }
+            };
+        }
+
+        Ok(table_ref.resolve(&self.default_catalog, &self.default_schema))
+    }
+
+    pub async fn resolve_table(
+        &mut self,
+        table_ref: OwnedTableReference,
+    ) -> Result<Arc<dyn TableSource>> {
+        let table_ref = table_ref.as_table_reference();
+        let table_ref = self.resolve_table_ref(table_ref)?;
+
+        let resolved_name = table_ref.to_string();
+        if let Some(table) = self.resolved_tables.get(&resolved_name) {
+            return Ok(table.clone());
+        }
+
+        let catalog_name = table_ref.catalog.as_ref();
+        let schema_name = table_ref.schema.as_ref();
+        let table_name = table_ref.table.as_ref();
+
+        let catalog = self
+            .catalog_list
+            .catalog(catalog_name)?
+            .context(CatalogNotFoundSnafu { catalog_name })?;
+        let schema = catalog.schema(schema_name)?.context(SchemaNotFoundSnafu {
+            catalog: catalog_name,
+            schema: schema_name,
+        })?;
+        let table = schema
+            .table(table_name)
+            .await?
+            .with_context(|| TableNotExistSnafu {
+                table: format_full_table_name(catalog_name, schema_name, table_name),
+            })?;
+
+        let table = DfTableProviderAdapter::new(table);
+        let table = provider_as_source(Arc::new(table));
+        self.resolved_tables.insert(resolved_name, table.clone());
+        Ok(table)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::borrow::Cow;
+
+    use session::context::QueryContext;
+
+    use super::*;
+    use crate::local::MemoryCatalogManager;
+
+    #[test]
+    fn test_validate_table_ref() {
+        let query_ctx = &QueryContext::with("greptime", "public");
+
+        let table_provider =
+            DfTableSourceProvider::new(Arc::new(MemoryCatalogManager::default()), true, query_ctx);
+
+        let table_ref = TableReference::Bare {
+            table: Cow::Borrowed("table_name"),
+        };
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_ok());
+
+        let table_ref = TableReference::Partial {
+            schema: Cow::Borrowed("public"),
+            table: Cow::Borrowed("table_name"),
+        };
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_ok());
+
+        let table_ref = TableReference::Partial {
+            schema: Cow::Borrowed("wrong_schema"),
+            table: Cow::Borrowed("table_name"),
+        };
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_err());
+
+        let table_ref = TableReference::Full {
+            catalog: Cow::Borrowed("greptime"),
+            schema: Cow::Borrowed("public"),
+            table: Cow::Borrowed("table_name"),
+        };
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_ok());
+
+        let table_ref = TableReference::Full {
+            catalog: Cow::Borrowed("wrong_catalog"),
+            schema: Cow::Borrowed("public"),
+            table: Cow::Borrowed("table_name"),
+        };
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_err());
+    }
+}

src/catalog/src/tables.rs

@@ -1,3 +1,17 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // The `tables` table in system catalog keeps a record of all tables created by user.
 
 use std::any::Any;
@@ -6,24 +20,32 @@ use std::sync::Arc;
 use std::task::{Context, Poll};
 
 use async_stream::stream;
+use async_trait::async_trait;
+use common_catalog::consts::{INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_TABLE_NAME};
+use common_error::ext::BoxedError;
 use common_query::logical_plan::Expr;
+use common_query::physical_plan::PhysicalPlanRef;
 use common_recordbatch::error::Result as RecordBatchResult;
-use common_recordbatch::{RecordBatch, RecordBatchStream, SendableRecordBatchStream};
-use datatypes::prelude::{ConcreteDataType, VectorBuilder};
+use common_recordbatch::{RecordBatch, RecordBatchStream};
+use datatypes::prelude::{ConcreteDataType, DataType};
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
-use datatypes::value::Value;
+use datatypes::value::ValueRef;
 use datatypes::vectors::VectorRef;
 use futures::Stream;
 use snafu::ResultExt;
 use table::engine::TableEngineRef;
-use table::metadata::TableId;
+use table::error::TablesRecordBatchSnafu;
+use table::metadata::{TableId, TableInfoRef};
+use table::table::scan::SimpleTableScan;
 use table::{Table, TableRef};
 
-use crate::consts::{INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_TABLE_NAME};
-use crate::error::InsertTableRecordSnafu;
-use crate::system::{build_table_insert_request, SystemCatalogTable};
+use crate::error::{self, Error, InsertCatalogRecordSnafu, Result as CatalogResult};
+use crate::system::{
+    build_schema_insert_request, build_table_deletion_request, build_table_insert_request,
+    SystemCatalogTable,
+};
 use crate::{
-    format_full_table_name, CatalogListRef, CatalogProvider, SchemaProvider, SchemaProviderRef,
+    CatalogListRef, CatalogProvider, DeregisterTableRequest, SchemaProvider, SchemaProviderRef,
 };
 
 /// Tables holds all tables created by user.
@@ -53,23 +75,53 @@ impl Table for Tables {
         self.schema.clone()
     }
 
+    fn table_info(&self) -> TableInfoRef {
+        unreachable!("Tables does not support table_info method")
+    }
+
     async fn scan(
         &self,
-        _projection: &Option<Vec<usize>>,
+        _projection: Option<&Vec<usize>>,
         _filters: &[Expr],
         _limit: Option<usize>,
-    ) -> table::error::Result<SendableRecordBatchStream> {
+    ) -> table::error::Result<PhysicalPlanRef> {
         let catalogs = self.catalogs.clone();
         let schema_ref = self.schema.clone();
         let engine_name = self.engine_name.clone();
 
         let stream = stream!({
-            for catalog_name in catalogs.catalog_names() {
-                let catalog = catalogs.catalog(&catalog_name).unwrap();
-                for schema_name in catalog.schema_names() {
-                    let mut tables_in_schema = Vec::with_capacity(catalog.schema_names().len());
-                    let schema = catalog.schema(&schema_name).unwrap();
-                    for table_name in schema.table_names() {
+            for catalog_name in catalogs
+                .catalog_names()
+                .map_err(BoxedError::new)
+                .context(TablesRecordBatchSnafu)?
+            {
+                let catalog = catalogs
+                    .catalog(&catalog_name)
+                    .map_err(BoxedError::new)
+                    .context(TablesRecordBatchSnafu)?
+                    .unwrap();
+                for schema_name in catalog
+                    .schema_names()
+                    .map_err(BoxedError::new)
+                    .context(TablesRecordBatchSnafu)?
+                {
+                    let mut tables_in_schema = Vec::with_capacity(
+                        catalog
+                            .schema_names()
+                            .map_err(BoxedError::new)
+                            .context(TablesRecordBatchSnafu)?
+                            .len(),
+                    );
+                    let schema = catalog
+                        .schema(&schema_name)
+                        .map_err(BoxedError::new)
+                        .context(TablesRecordBatchSnafu)?
+                        .unwrap();
+                    for table_name in schema
+                        .table_names()
+                        .map_err(BoxedError::new)
+                        .context(TablesRecordBatchSnafu)?
+                    {
                         tables_in_schema.push(table_name);
                     }
 
@@ -85,10 +137,11 @@ impl Table for Tables {
             }
         });
 
-        Ok(Box::pin(TablesRecordBatchStream {
+        let stream = Box::pin(TablesRecordBatchStream {
             schema: self.schema.clone(),
             stream: Box::pin(stream),
-        }))
+        });
+        Ok(Arc::new(SimpleTableScan::new(stream)))
     }
 }
 
@@ -100,26 +153,27 @@ fn tables_to_record_batch(
     engine: &str,
 ) -> Vec<VectorRef> {
     let mut catalog_vec =
-        VectorBuilder::with_capacity(ConcreteDataType::string_datatype(), table_names.len());
+        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
     let mut schema_vec =
-        VectorBuilder::with_capacity(ConcreteDataType::string_datatype(), table_names.len());
+        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
     let mut table_name_vec =
-        VectorBuilder::with_capacity(ConcreteDataType::string_datatype(), table_names.len());
+        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
     let mut engine_vec =
-        VectorBuilder::with_capacity(ConcreteDataType::string_datatype(), table_names.len());
+        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
 
     for table_name in table_names {
-        catalog_vec.push(&Value::String(catalog_name.into()));
-        schema_vec.push(&Value::String(schema_name.into()));
-        table_name_vec.push(&Value::String(table_name.into()));
-        engine_vec.push(&Value::String(engine.into()));
+        // Safety: All these vectors are string type.
+        catalog_vec.push_value_ref(ValueRef::String(catalog_name));
+        schema_vec.push_value_ref(ValueRef::String(schema_name));
+        table_name_vec.push_value_ref(ValueRef::String(&table_name));
+        engine_vec.push_value_ref(ValueRef::String(engine));
     }
 
     vec![
-        catalog_vec.finish(),
-        schema_vec.finish(),
-        table_name_vec.finish(),
-        engine_vec.finish(),
+        catalog_vec.to_vector(),
+        schema_vec.to_vector(),
+        table_name_vec.to_vector(),
+        engine_vec.to_vector(),
     ]
 }
 
@@ -147,22 +201,26 @@ pub struct InformationSchema {
     pub system: Arc<SystemCatalogTable>,
 }
 
+#[async_trait]
 impl SchemaProvider for InformationSchema {
     fn as_any(&self) -> &dyn Any {
         self
     }
 
-    fn table_names(&self) -> Vec<String> {
-        vec!["tables".to_string(), SYSTEM_CATALOG_TABLE_NAME.to_string()]
+    fn table_names(&self) -> Result<Vec<String>, Error> {
+        Ok(vec![
+            "tables".to_string(),
+            SYSTEM_CATALOG_TABLE_NAME.to_string(),
+        ])
     }
 
-    fn table(&self, name: &str) -> Option<TableRef> {
+    async fn table(&self, name: &str) -> Result<Option<TableRef>, Error> {
         if name.eq_ignore_ascii_case("tables") {
-            Some(self.tables.clone())
+            Ok(Some(self.tables.clone()))
         } else if name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME) {
-            Some(self.system.clone())
+            Ok(Some(self.system.clone()))
         } else {
-            None
+            Ok(None)
         }
     }
 
@@ -174,12 +232,17 @@ impl SchemaProvider for InformationSchema {
         panic!("System catalog & schema does not support register table")
     }
 
+    fn rename_table(&self, _name: &str, _new_name: String) -> crate::error::Result<TableRef> {
+        unimplemented!("System catalog & schema does not support rename table")
+    }
+
     fn deregister_table(&self, _name: &str) -> crate::error::Result<Option<TableRef>> {
         panic!("System catalog & schema does not support deregister table")
     }
 
-    fn table_exist(&self, name: &str) -> bool {
-        name.eq_ignore_ascii_case("tables") || name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME)
+    fn table_exist(&self, name: &str) -> Result<bool, Error> {
+        Ok(name.eq_ignore_ascii_case("tables")
+            || name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME))
     }
 }
 
@@ -209,13 +272,40 @@ impl SystemCatalog {
         table_name: String,
         table_id: TableId,
     ) -> crate::error::Result<usize> {
-        let full_table_name = format_full_table_name(&catalog, &schema, &table_name);
-        let request = build_table_insert_request(full_table_name, table_id);
+        let request = build_table_insert_request(catalog, schema, table_name, table_id);
         self.information_schema
             .system
             .insert(request)
             .await
-            .context(InsertTableRecordSnafu)
+            .context(InsertCatalogRecordSnafu)
+    }
+
+    pub(crate) async fn deregister_table(
+        &self,
+        request: &DeregisterTableRequest,
+        table_id: TableId,
+    ) -> CatalogResult<bool> {
+        self.information_schema
+            .system
+            .delete(build_table_deletion_request(request, table_id))
+            .await
+            .map(|x| x == 1)
+            .with_context(|_| error::DeregisterTableSnafu {
+                request: request.clone(),
+            })
+    }
+
+    pub async fn register_schema(
+        &self,
+        catalog: String,
+        schema: String,
+    ) -> crate::error::Result<usize> {
+        let request = build_schema_insert_request(catalog, schema);
+        self.information_schema
+            .system
+            .insert(request)
+            .await
+            .context(InsertCatalogRecordSnafu)
     }
 }
 
@@ -224,23 +314,23 @@ impl CatalogProvider for SystemCatalog {
         self
     }
 
-    fn schema_names(&self) -> Vec<String> {
-        vec![INFORMATION_SCHEMA_NAME.to_string()]
+    fn schema_names(&self) -> Result<Vec<String>, Error> {
+        Ok(vec![INFORMATION_SCHEMA_NAME.to_string()])
     }
 
     fn register_schema(
         &self,
         _name: String,
         _schema: SchemaProviderRef,
-    ) -> Option<SchemaProviderRef> {
+    ) -> Result<Option<SchemaProviderRef>, Error> {
         panic!("System catalog does not support registering schema!")
     }
 
-    fn schema(&self, name: &str) -> Option<Arc<dyn SchemaProvider>> {
+    fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>, Error> {
         if name.eq_ignore_ascii_case(INFORMATION_SCHEMA_NAME) {
-            Some(self.information_schema.clone())
+            Ok(Some(self.information_schema.clone()))
         } else {
-            None
+            Ok(None)
         }
     }
 }
@@ -273,74 +363,72 @@ fn build_schema_for_tables() -> Schema {
 
 #[cfg(test)]
 mod tests {
-    use datatypes::arrow::array::Utf8Array;
-    use datatypes::arrow::datatypes::DataType;
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use common_query::physical_plan::SessionContext;
     use futures_util::StreamExt;
     use table::table::numbers::NumbersTable;
 
     use super::*;
-    use crate::memory::{new_memory_catalog_list, MemoryCatalogProvider, MemorySchemaProvider};
+    use crate::local::memory::new_memory_catalog_list;
     use crate::CatalogList;
 
     #[tokio::test]
     async fn test_tables() {
         let catalog_list = new_memory_catalog_list().unwrap();
-        let catalog_provider = Arc::new(MemoryCatalogProvider::default());
-        let schema = Arc::new(MemorySchemaProvider::new());
+        let schema = catalog_list
+            .catalog(DEFAULT_CATALOG_NAME)
+            .unwrap()
+            .unwrap()
+            .schema(DEFAULT_SCHEMA_NAME)
+            .unwrap()
+            .unwrap();
         schema
             .register_table("test_table".to_string(), Arc::new(NumbersTable::default()))
             .unwrap();
-        catalog_provider.register_schema("test_schema".to_string(), schema);
-        catalog_list.register_catalog("test_catalog".to_string(), catalog_provider);
-        let tables = Tables::new(catalog_list, "test_engine".to_string());
 
-        let mut tables_stream = tables.scan(&None, &[], None).await.unwrap();
+        let tables = Tables::new(catalog_list, "test_engine".to_string());
+        let tables_stream = tables.scan(None, &[], None).await.unwrap();
+        let session_ctx = SessionContext::new();
+        let mut tables_stream = tables_stream.execute(0, session_ctx.task_ctx()).unwrap();
+
         if let Some(t) = tables_stream.next().await {
-            let batch = t.unwrap().df_recordbatch;
+            let batch = t.unwrap();
             assert_eq!(1, batch.num_rows());
             assert_eq!(4, batch.num_columns());
-            assert_eq!(&DataType::Utf8, batch.column(0).data_type());
-            assert_eq!(&DataType::Utf8, batch.column(1).data_type());
-            assert_eq!(&DataType::Utf8, batch.column(2).data_type());
-            assert_eq!(&DataType::Utf8, batch.column(3).data_type());
             assert_eq!(
-                "test_catalog",
-                batch
-                    .column(0)
-                    .as_any()
-                    .downcast_ref::<Utf8Array<i32>>()
-                    .unwrap()
-                    .value(0)
+                ConcreteDataType::string_datatype(),
+                batch.column(0).data_type()
+            );
+            assert_eq!(
+                ConcreteDataType::string_datatype(),
+                batch.column(1).data_type()
+            );
+            assert_eq!(
+                ConcreteDataType::string_datatype(),
+                batch.column(2).data_type()
+            );
+            assert_eq!(
+                ConcreteDataType::string_datatype(),
+                batch.column(3).data_type()
+            );
+            assert_eq!(
+                "greptime",
+                batch.column(0).get_ref(0).as_string().unwrap().unwrap()
             );
 
             assert_eq!(
-                "test_schema",
-                batch
-                    .column(1)
-                    .as_any()
-                    .downcast_ref::<Utf8Array<i32>>()
-                    .unwrap()
-                    .value(0)
+                "public",
+                batch.column(1).get_ref(0).as_string().unwrap().unwrap()
             );
 
             assert_eq!(
                 "test_table",
-                batch
-                    .column(2)
-                    .as_any()
-                    .downcast_ref::<Utf8Array<i32>>()
-                    .unwrap()
-                    .value(0)
+                batch.column(2).get_ref(0).as_string().unwrap().unwrap()
             );
 
             assert_eq!(
                 "test_engine",
-                batch
-                    .column(3)
-                    .as_any()
-                    .downcast_ref::<Utf8Array<i32>>()
-                    .unwrap()
-                    .value(0)
+                batch.column(3).get_ref(0).as_string().unwrap().unwrap()
             );
         } else {
             panic!("Record batch should not be empty!")

src/catalog/tests/local_catalog_tests.rs

@@ -0,0 +1,171 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use catalog::local::LocalCatalogManager;
+    use catalog::{CatalogManager, RegisterTableRequest, RenameTableRequest};
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use common_telemetry::{error, info};
+    use mito::config::EngineConfig;
+    use table::table::numbers::NumbersTable;
+    use table::TableRef;
+    use tokio::sync::Mutex;
+
+    async fn create_local_catalog_manager() -> Result<LocalCatalogManager, catalog::error::Error> {
+        let (_dir, object_store) =
+            mito::table::test_util::new_test_object_store("setup_mock_engine_and_table").await;
+        let mock_engine = Arc::new(mito::table::test_util::MockMitoEngine::new(
+            EngineConfig::default(),
+            mito::table::test_util::MockEngine::default(),
+            object_store,
+        ));
+        let catalog_manager = LocalCatalogManager::try_new(mock_engine).await.unwrap();
+        catalog_manager.start().await?;
+        Ok(catalog_manager)
+    }
+
+    #[tokio::test]
+    async fn test_rename_table() {
+        common_telemetry::init_default_ut_logging();
+        let catalog_manager = create_local_catalog_manager().await.unwrap();
+        // register table
+        let table_name = "test_table";
+        let table_id = 42;
+        let table = Arc::new(NumbersTable::new(table_id));
+        let request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id,
+            table: table.clone(),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        // rename table
+        let new_table_name = "table_t";
+        let rename_table_req = RenameTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            new_table_name: new_table_name.to_string(),
+            table_id,
+        };
+        assert!(catalog_manager
+            .rename_table(rename_table_req)
+            .await
+            .unwrap());
+
+        let registered_table = catalog_manager
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(registered_table.table_info().ident.table_id, table_id);
+    }
+
+    #[tokio::test]
+    async fn test_duplicate_register() {
+        let catalog_manager = create_local_catalog_manager().await.unwrap();
+        let request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "test_table".to_string(),
+            table_id: 42,
+            table: Arc::new(NumbersTable::new(42)),
+        };
+        assert!(catalog_manager
+            .register_table(request.clone())
+            .await
+            .unwrap());
+
+        // register table with same table id will succeed with 0 as return val.
+        assert!(!catalog_manager.register_table(request).await.unwrap());
+
+        let err = catalog_manager
+            .register_table(RegisterTableRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_SCHEMA_NAME.to_string(),
+                table_name: "test_table".to_string(),
+                table_id: 43,
+                table: Arc::new(NumbersTable::new(43)),
+            })
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("Table `greptime.public.test_table` already exists"),
+            "Actual error message: {err}",
+        );
+    }
+
+    #[test]
+    fn test_concurrent_register() {
+        common_telemetry::init_default_ut_logging();
+        let rt = Arc::new(tokio::runtime::Builder::new_multi_thread().build().unwrap());
+        let catalog_manager =
+            Arc::new(rt.block_on(async { create_local_catalog_manager().await.unwrap() }));
+
+        let succeed: Arc<Mutex<Option<TableRef>>> = Arc::new(Mutex::new(None));
+
+        let mut handles = Vec::with_capacity(8);
+        for i in 0..8 {
+            let catalog = catalog_manager.clone();
+            let succeed = succeed.clone();
+            let handle = rt.spawn(async move {
+                let table_id = 42 + i;
+                let table = Arc::new(NumbersTable::new(table_id));
+                let req = RegisterTableRequest {
+                    catalog: DEFAULT_CATALOG_NAME.to_string(),
+                    schema: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "test_table".to_string(),
+                    table_id,
+                    table: table.clone(),
+                };
+                match catalog.register_table(req).await {
+                    Ok(res) => {
+                        if res {
+                            let mut succeed = succeed.lock().await;
+                            info!("Successfully registered table: {}", table_id);
+                            *succeed = Some(table);
+                        }
+                    }
+                    Err(_) => {
+                        error!("Failed to register table {}", table_id);
+                    }
+                }
+            });
+            handles.push(handle);
+        }
+
+        rt.block_on(async move {
+            for handle in handles {
+                handle.await.unwrap();
+            }
+            let guard = succeed.lock().await;
+            let table = guard.as_ref().unwrap();
+            let table_registered = catalog_manager
+                .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "test_table")
+                .await
+                .unwrap()
+                .unwrap();
+            assert_eq!(
+                table_registered.table_info().ident.table_id,
+                table.table_info().ident.table_id
+            );
+        });
+    }
+}