refactor: Rename table-engine to mito (#539)

* refactor: Rename table-engine to mito

* style: Format codes

* docs: Update mito engine comment

* docs: Explain what is mito in README

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,86 @@
+---
+name: Bug report
+description: Is something not working? Help us fix it!
+labels: [ "bug" ]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Take some time to fill out this bug report. Thank you!
+
+  - type: dropdown
+    id: type
+    attributes:
+      label: What type of bug is this?
+      multiple: true
+      options:
+        - Configuration
+        - Crash
+        - Data corruption
+        - Incorrect result
+        - Locking issue
+        - Performance issue
+        - Unexpected error
+        - Other
+    validations:
+      required: true
+
+  - type: dropdown
+    id: subsystem
+    attributes:
+      label: What subsystems are affected?
+      description: You can pick multiple subsystems.
+      multiple: true
+      options:
+        - Standalone mode
+        - Frontend
+        - Datanode
+        - Meta
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: |
+        Tell us what happened and also what you would have expected to
+        happen instead.
+      placeholder: "Describe the bug"
+    validations:
+      required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: What operating system did you use?
+      description: |
+        Please provide OS, version, and architecture. For example:
+        Windows 10 x64, Ubuntu 21.04 x64, Mac OS X 10.5 ARM, Rasperry
+        Pi i386, etc.
+      placeholder: "Ubuntu 21.04 x64"
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output and stack trace
+      description: |
+        Please copy and paste any relevant log output or a stack
+        trace. This will be automatically formatted into code, so no
+        need for backticks.
+      render: bash
+
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: How can we reproduce the bug?
+      description: |
+        Please walk us through and provide steps and details on how
+        to reproduce the issue. If possible, provide scripts that we
+        can run to trigger the bug.
+      render: bash
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Greptime Community Slack
+    url: https://greptime.com/slack
+    about: Get free help from the Greptime community
+  - name: Greptime Community Discussion
+    url: https://github.com/greptimeTeam/greptimedb/discussions
+    about: Get free help from the Greptime community

.github/ISSUE_TEMPLATE/enhancement.yml

@@ -0,0 +1,39 @@
+---
+name: Enhancement
+description: Suggest an enhancement to existing functionality
+labels: [ "enhancement" ]
+body:
+  - type: dropdown
+    id: type
+    attributes:
+      label: What type of enhancement is this?
+      multiple: true
+      options:
+        - API improvement
+        - Configuration
+        - Performance
+        - Refactor
+        - Tech debt reduction
+        - User experience
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: what
+    attributes:
+      label: What does the enhancement do?
+      description: |
+        Give a high-level overview of how you
+        suggest improving an existing feature or functionality.
+    validations:
+     required: true
+
+  - type: textarea
+    id: implementation
+    attributes:
+      label: Implementation challenges
+      description: |
+        Share any ideas of how to implement the enhancement.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,42 @@
+---
+name: Feature request
+description: Suggest a new feature for GreptimeDB
+labels: [ "feature request" ]
+body:
+  - type: markdown
+    id: info
+    attributes:
+      value: |
+        Only use this template to suggest a new feature that doesn't already exist in GreptimeDB.
+        For enhancements to existing features, use the "Enhancement" issue template. For bugs,
+        use the bug report template.
+
+  - type: textarea
+    id: what
+    attributes:
+      label: What problem does the new feature solve?
+      description: |
+        Describe the problem and why it is important to solve. Did you consider alternative
+        solutions, perhaps outside the database? Why is it better to add the feature to
+        GreptimeDB?
+    validations:
+      required: true
+
+  - type: textarea
+    id: how
+    attributes:
+      label: What does the feature do?
+      description: |
+        Give a high-level overview of what the feature does and how it would work.
+    validations:
+     required: true
+
+  - type: textarea
+    id: implementation
+    attributes:
+      label: Implementation challenges
+      description: |
+        If you have ideas of how to implement the feature, and any particularly
+        challenging issues to overcome, then provide them here.
+    validations:
+      required: false

.github/pull_request_template.md

@@ -0,0 +1,19 @@
+I hereby agree to the terms of the [GreptimeDB CLA](https://gist.github.com/xtang/6378857777706e568c1949c7578592cc)
+
+## What's changed and what's your intention?
+
+_PLEASE DO NOT LEAVE THIS EMPTY !!!_
+
+Please explain IN DETAIL what the changes are in this PR and why they are needed:
+
+- Summarize your change (**mandatory**)
+- How does this PR work? Need a brief introduction for the changed logic (optional)
+- Describe clearly one logical change and avoid lazy messages (optional)
+- Describe any limitations of the current code (optional)
+
+## Checklist
+
+- []  I have written the necessary rustdoc comments.
+- []  I have added the necessary unit tests and integration tests.
+
+## Refer to a related PR or issue link (optional)

.github/workflows/coverage.yml

@@ -1,10 +1,24 @@
 on:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
   push:
     branches:
       - "main"
       - "develop"
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
   workflow_dispatch:
 
 name: Code coverage
@@ -15,7 +29,7 @@ env:
 jobs:
   coverage:
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-8-cores
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -27,10 +41,6 @@ jobs:
           components: llvm-tools-preview
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-      - name: Cleanup disk
-        uses: curoky/cleanup-disk-action@v2.0
-        with:
-          retain: 'rust'
       - name: Install latest nextest release
         uses: taiki-e/install-action@nextest
       - name: Install cargo-llvm-cov

.github/workflows/develop.yml

@@ -1,6 +1,12 @@
 on:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
   push:
     branches:
       - develop
@@ -8,14 +14,13 @@ on:
     paths-ignore:
       - 'docs/**'
       - 'config/**'
-      - '.github/**'
       - '**.md'
-      - '**.yml'
       - '.dockerignore'
       - 'docker/**'
+      - '.gitignore'
   workflow_dispatch:
 
-name: Continuous integration for developing
+name: CI
 
 env:
   RUST_TOOLCHAIN: nightly-2022-07-14
@@ -37,44 +42,45 @@ jobs:
       - name: Run cargo check
         run: cargo check --workspace --all-targets
 
-  test:
-    name: Test Suite
-    if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v3
-      - name: Cache LLVM and Clang
-        id: cache-llvm
-        uses: actions/cache@v3
-        with:
-          path: ./llvm
-          key: llvm
-      - uses: arduino/setup-protoc@v1
-      - uses: KyleMayes/install-llvm-action@v1
-        with:
-          version: "14.0"
-          cached: ${{ steps.cache-llvm.outputs.cache-hit }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-      - name: Cleanup disk
-        uses: curoky/cleanup-disk-action@v2.0
-        with:
-          retain: 'rust,llvm'
-      - name: Install latest nextest release
-        uses: taiki-e/install-action@nextest
-      - name: Run tests
-        run: cargo nextest run
-        env:
-          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
-          RUST_BACKTRACE: 1
-          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-          UNITTEST_LOG_DIR: "__unittest_logs"
+  # Use coverage to run test.
+  # test:
+  #   name: Test Suite
+  #   if: github.event.pull_request.draft == false
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 60
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - name: Cache LLVM and Clang
+  #       id: cache-llvm
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: ./llvm
+  #         key: llvm
+  #     - uses: arduino/setup-protoc@v1
+  #     - uses: KyleMayes/install-llvm-action@v1
+  #       with:
+  #         version: "14.0"
+  #         cached: ${{ steps.cache-llvm.outputs.cache-hit }}
+  #     - uses: dtolnay/rust-toolchain@master
+  #       with:
+  #         toolchain: ${{ env.RUST_TOOLCHAIN }}
+  #     - name: Rust Cache
+  #       uses: Swatinem/rust-cache@v2
+  #     - name: Cleanup disk
+  #       uses: curoky/cleanup-disk-action@v2.0
+  #       with:
+  #         retain: 'rust,llvm'
+  #     - name: Install latest nextest release
+  #       uses: taiki-e/install-action@nextest
+  #     - name: Run tests
+  #       run: cargo nextest run
+  #       env:
+  #         CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
+  #         RUST_BACKTRACE: 1
+  #         GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+  #         GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+  #         GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+  #         UNITTEST_LOG_DIR: "__unittest_logs"
 
   fmt:
     name: Rustfmt

.github/workflows/license.yaml

@@ -0,0 +1,16 @@
+name: License checker
+
+on:
+  push:
+    branches:
+    - develop
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+jobs:
+  license-header-check:
+    runs-on: ubuntu-latest
+    name: license-header-check
+    steps:
+    - uses: actions/checkout@v2
+    - name: Check License Header
+      uses: apache/skywalking-eyes/header@main

.github/workflows/release.yml

@@ -17,10 +17,10 @@ jobs:
         # The file format is greptime-<os>-<arch>
         include:
           - arch: x86_64-unknown-linux-gnu
-            os: ubuntu-latest
+            os: ubuntu-latest-16-cores
             file: greptime-linux-amd64
           - arch: aarch64-unknown-linux-gnu
-            os: ubuntu-latest
+            os: ubuntu-latest-16-cores
             file: greptime-linux-arm64
           - arch: aarch64-apple-darwin
             os: macos-latest

.licenserc.yaml

@@ -0,0 +1,14 @@
+header:
+  license:
+    spdx-id: Apache-2.0
+    copyright-owner: Greptime Team
+
+  paths:
+    - "**/*.rs"
+    - "**/*.py"
+
+  comment: on-failure
+
+dependency:
+  files:
+    - Cargo.toml

CONTRIBUTING.md

@@ -1,12 +1,55 @@
-# Contributing to GreptimeDB
+# Welcome!
 
-Much appreciate for your interest in contributing to GreptimeDB! This document list some guidelines for contributing to our code base.
+Thanks a lot for considering contributing to GreptimeDB. We believe people like you would make GreptimeDB a great product. We intend to build a community where individuals can have open talks, show respect for one another, and speak with true ❤️. Meanwhile, we are to keep transparency and make your effort count here.
 
-To learn about the design of GreptimeDB, please refer to the [design docs](https://github.com/GrepTimeTeam/docs).
+Read the guidelines, and they can help you get started. Communicate with respect to developers maintaining and developing the project. In return, they should reciprocate that respect by addressing your issue, reviewing changes, as well as helping finalize and merge your pull requests.
 
-## Pull Requests
+Follow our [README](https://github.com/GreptimeTeam/greptimedb#readme) to get the whole picture of the project. To learn about the design of GreptimeDB, please refer to the [design docs](https://github.com/GrepTimeTeam/docs).
+
+## Your First Contribution
+
+It can feel intimidating to contribute to a complex project, but it can also be exciting and fun. These general notes will help everyone participate in this communal activity.
+
+- Follow the [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/develop/CODE_OF_CONDUCT.md)
+- Small changes make huge differences. We will happily accept a PR making a single character change if it helps move forward. Don't wait to have everything working.
+- Check the closed issues before opening your issue.
+- Try to follow the existing style of the code.
+- More importantly, when in doubt, ask away.
+
+Pull requests are great, but we accept all kinds of other help if you like. Such as
+
+- Write tutorials or blog posts. Blog, speak about, or create tutorials about one of GreptimeDB's many features. Mention [@greptime](https://twitter.com/greptime) on Twitter and email info@greptime.com so we can give pointers and tips and help you spread the word by promoting your content on Greptime communication channels.
+- Improve the documentation. [Submit documentation](http://github.com/greptimeTeam/docs/) updates, enhancements, designs, or bug fixes, and fixing any spelling or grammar errors will be very much appreciated.
+- Present at meetups and conferences about your GreptimeDB projects. Your unique challenges and successes in building things with GreptimeDB can provide great speaking material. We'd love to review your talk abstract, so get in touch with us if you'd like some help!
+- Submit bug reports. To report a bug or a security issue, you can [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new).
+- Speak up feature requests. Send feedback is a great way for us to understand your different use cases of GreptimeDB better. If you want to share your experience with GreptimeDB, or if you want to discuss any ideas, you can start a discussion on [GitHub discussions](https://github.com/GreptimeTeam/greptimedb/discussions), chat with the Greptime team on [Slack](https://greptime.com/slack), or you can tweet [@greptime](https://twitter.com/greptime) on Twitter.
+
+## Code of Conduct
+
+Also, there are things that we are not looking for because they don't match the goals of the product or benefit the community. Please read [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/develop/CODE_OF_CONDUCT.md); we hope everyone can keep good manners and become an honored member.
+
+## License
+
+GreptimeDB uses the [Apache 2.0 license](https://github.com/GreptimeTeam/greptimedb/blob/master/LICENSE) to strike a balance between open contributions and allowing you to use the software however you want.
+
+## Getting Started
+
+### Submitting Issues
+
+- Check if an issue already exists. Before filing an issue report, see whether it's already covered. Use the search bar and check out existing issues.
+- File an issue:
+  - To report a bug, a security issue, or anything that you think is a problem and that isn't under the radar, go ahead and [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new).
+  - In the given templates, look for the one that suits you.
+  - If you bump into anything, reach out to our [Slack](https://greptime.com/slack) for a wider audience and ask for help.
+- What happens after:
+  - Once we spot a new issue, we identify and categorize it as soon as possible.
+  - Usually, it gets assigned to other developers. Follow up and see what folks are talking about and how they take care of it.
+  - Please be patient and offer as much information as you can to help reach a solution or a consensus. You are not alone and embrace team power.
 
 ### Before PR
+
+- To ensure that community is free and confident in its ability to use your contributions, please sign the Contributor License Agreement (CLA) which will be incorporated in the pull request process.
+- Make sure all your codes are formatted and follow the [coding style](https://pingcap.github.io/style-guide/rust/).
 - Make sure all unit tests are passed.
 - Make sure all clippy warnings are fixed (you can check it locally by running `cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr`).
 
@@ -38,19 +81,31 @@ now `pre-commit` will run automatically on `git commit`.
 
 ### Title
 
-The titles of pull requests should be prefixed with one of the change types listed in [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0) 
-like `feat`/`fix`/`docs`, with a concise summary of code change follows. The following scope field is optional, you can fill it with the name of sub-crate if the pull request only changes one, or just leave it blank.
+The titles of pull requests should be prefixed with category names listed in [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0)
+like `feat`/`fix`/`docs`, with a concise summary of code change following. DO NOT use last commit message as pull request title.
 
 ### Description
 
-- If your pull request is small, like a typo fix, feel free to go brief.
+- Feel free to go brief if your pull request is small, like a typo fix.
 - But if it contains large code change, make sure to state the motivation/design details of this PR so that reviewers can understand what you're trying to do.
 - If the PR contains any breaking change or API change, make sure that is clearly listed in your description.
 
-## Getting help
+### Commit Messages
+
+All commit messages SHOULD adhere to the [Conventional Commits specification](https://conventionalcommits.org/).
+
+## Getting Help
 
 There are many ways to get help when you're stuck. It is recommended to ask for help by opening an issue, with a detailed description
-of what you were trying to do and what went wrong. You can also reach for help in our Slack channel.
+of what you were trying to do and what went wrong. You can also reach for help in our [Slack channel](https://greptime.com/slack).
 
-## Bug report
-To report a bug or a security issue, you can [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new).
+## Community
+
+The core team will be thrilled if you participate in any way you like. When you are stuck, try ask for help by filing an issue, with a detailed description of what you were trying to do and what went wrong. If you have any questions or if you would like to get involved in our community, please check out:
+- [GreptimeDB Community Slack](https://greptime.com/slack)
+- [GreptimeDB Github Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)
+
+Also, see some extra GreptimeDB content:
+- [GreptimeDB Docs](https://greptime.com/docs)
+- [Learn GreptimeDB](https://greptime.com/products/db)
+- [Greptime Inc. Website](https://greptime.com)

Cargo.lock

@@ -119,6 +119,7 @@ name = "api"
 version = "0.1.0"
 dependencies = [
  "common-base",
+ "common-error",
  "common-time",
  "datatypes",
  "prost 0.11.0",
@@ -276,6 +277,36 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "async-io"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8121296a9f05be7f34aa4196b1747243b3b62e048bb7906f644f3fbfc490cf7"
+dependencies = [
+ "async-lock",
+ "autocfg",
+ "concurrent-queue",
+ "futures-lite",
+ "libc",
+ "log",
+ "parking",
+ "polling",
+ "slab",
+ "socket2",
+ "waker-fn",
+ "winapi",
+]
+
+[[package]]
+name = "async-lock"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685"
+dependencies = [
+ "event-listener",
+ "futures-lite",
+]
+
 [[package]]
 name = "async-stream"
 version = "0.3.3"
@@ -657,6 +688,12 @@ version = "3.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
 
+[[package]]
+name = "bytecount"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c"
+
 [[package]]
 name = "bytemuck"
 version = "1.12.1"
@@ -698,6 +735,37 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c"
 
+[[package]]
+name = "camino"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88ad0e1e3e88dd237a156ab9f571021b8a158caa0ae44b1968a241efb5144c1e"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo-platform"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo_metadata"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
+dependencies = [
+ "camino",
+ "cargo-platform",
+ "semver",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "caseless"
 version = "0.2.1"
@@ -739,6 +807,7 @@ dependencies = [
  "lazy_static",
  "log-store",
  "meta-client",
+ "mito",
  "object-store",
  "opendal",
  "regex",
@@ -747,7 +816,6 @@ dependencies = [
  "snafu",
  "storage",
  "table",
- "table-engine",
  "tempdir",
  "tokio",
 ]
@@ -958,6 +1026,7 @@ dependencies = [
  "common-base",
  "common-error",
  "common-grpc",
+ "common-insert",
  "common-query",
  "common-recordbatch",
  "common-time",
@@ -1110,10 +1179,13 @@ dependencies = [
  "async-trait",
  "common-base",
  "common-error",
+ "common-query",
+ "common-recordbatch",
  "common-runtime",
  "criterion 0.4.0",
  "dashmap",
  "datafusion",
+ "datatypes",
  "rand 0.8.5",
  "snafu",
  "tokio",
@@ -1660,12 +1732,14 @@ dependencies = [
  "datafusion",
  "datafusion-common",
  "datatypes",
+ "frontend",
  "futures",
  "hyper",
  "log-store",
  "meta-client",
  "meta-srv",
  "metrics",
+ "mito",
  "object-store",
  "query",
  "script",
@@ -1678,7 +1752,6 @@ dependencies = [
  "store-api",
  "substrait 0.1.0",
  "table",
- "table-engine",
  "tempdir",
  "tokio",
  "tokio-stream",
@@ -1878,6 +1951,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "error-chain"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc"
+dependencies = [
+ "version_check",
+]
+
 [[package]]
 name = "error-code"
 version = "2.3.1"
@@ -2015,11 +2097,13 @@ dependencies = [
  "async-stream",
  "async-trait",
  "catalog",
+ "chrono",
  "client",
  "common-base",
  "common-catalog",
  "common-error",
  "common-grpc",
+ "common-insert",
  "common-query",
  "common-recordbatch",
  "common-runtime",
@@ -2031,8 +2115,11 @@ dependencies = [
  "datanode",
  "datatypes",
  "futures",
+ "futures-util",
  "itertools",
  "meta-client",
+ "meta-srv",
+ "moka",
  "openmetrics-parser",
  "prost 0.11.0",
  "query",
@@ -2174,6 +2261,21 @@ version = "0.3.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68"
 
+[[package]]
+name = "futures-lite"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48"
+dependencies = [
+ "fastrand",
+ "futures-core",
+ "futures-io",
+ "memchr",
+ "parking",
+ "pin-project-lite",
+ "waker-fn",
+]
+
 [[package]]
 name = "futures-macro"
 version = "0.3.24"
@@ -2945,6 +3047,7 @@ dependencies = [
  "futures",
  "meta-srv",
  "rand 0.8.5",
+ "serde",
  "snafu",
  "tokio",
  "tokio-stream",
@@ -3084,6 +3187,62 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "mito"
+version = "0.1.0"
+dependencies = [
+ "arc-swap",
+ "async-stream",
+ "async-trait",
+ "chrono",
+ "common-catalog",
+ "common-error",
+ "common-query",
+ "common-recordbatch",
+ "common-telemetry",
+ "common-time",
+ "datafusion",
+ "datafusion-common",
+ "datatypes",
+ "futures",
+ "log-store",
+ "object-store",
+ "serde",
+ "serde_json",
+ "snafu",
+ "storage",
+ "store-api",
+ "table",
+ "tempdir",
+ "tokio",
+]
+
+[[package]]
+name = "moka"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b49a05f67020456541f4f29cbaa812016a266a86ec76f96d3873d459c68fe5e"
+dependencies = [
+ "async-io",
+ "async-lock",
+ "crossbeam-channel",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "futures-util",
+ "num_cpus",
+ "once_cell",
+ "parking_lot",
+ "quanta",
+ "rustc_version",
+ "scheduled-thread-pool",
+ "skeptic",
+ "smallvec",
+ "tagptr",
+ "thiserror",
+ "triomphe",
+ "uuid",
+]
+
 [[package]]
 name = "multimap"
 version = "0.8.3"
@@ -3558,6 +3717,12 @@ version = "6.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff"
 
+[[package]]
+name = "parking"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72"
+
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -3738,9 +3903,9 @@ dependencies = [
 
 [[package]]
 name = "pgwire"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e30e99a0b8acf60a6815aa8178e9ffb08178ef3ca1366673bb0d6c7ababe4c2"
+checksum = "5dacbf864d6cb6a0e676c9a1162ab7b315b5c8e6c87fa9b6e0ba9ba0a569adb1"
 dependencies = [
  "async-trait",
  "bytes",
@@ -3753,6 +3918,7 @@ dependencies = [
  "thiserror",
  "time 0.3.14",
  "tokio",
+ "tokio-rustls",
  "tokio-util",
 ]
 
@@ -3919,6 +4085,20 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "polling"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "libc",
+ "log",
+ "wepoll-ffi",
+ "winapi",
+]
+
 [[package]]
 name = "portable-atomic"
 version = "0.3.15"
@@ -4126,6 +4306,17 @@ dependencies = [
  "prost 0.11.0",
 ]
 
+[[package]]
+name = "pulldown-cmark"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d9cc634bc78768157b5cbfe988ffcd1dcba95cd2b2f03a88316c08c6d00ed63"
+dependencies = [
+ "bitflags",
+ "memchr",
+ "unicase",
+]
+
 [[package]]
 name = "quanta"
 version = "0.10.1"
@@ -4729,6 +4920,10 @@ dependencies = [
 name = "rustpython-pylib"
 version = "0.1.0"
 source = "git+https://github.com/RustPython/RustPython?rev=02a1d1d#02a1d1d7db57afbb78049599c2585cc7cd59e6d3"
+dependencies = [
+ "rustpython-bytecode",
+ "rustpython-derive",
+]
 
 [[package]]
 name = "rustpython-vm"
@@ -4865,6 +5060,15 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "scheduled-thread-pool"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf"
+dependencies = [
+ "parking_lot",
+]
+
 [[package]]
 name = "schemars"
 version = "0.8.11"
@@ -4918,6 +5122,7 @@ dependencies = [
  "futures",
  "futures-util",
  "log-store",
+ "mito",
  "paste",
  "query",
  "ron",
@@ -4932,7 +5137,6 @@ dependencies = [
  "sql",
  "storage",
  "table",
- "table-engine",
  "tempdir",
  "tokio",
  "tokio-test",
@@ -4976,6 +5180,9 @@ name = "semver"
 version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "serde"
@@ -5189,6 +5396,21 @@ version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
 
+[[package]]
+name = "skeptic"
+version = "0.13.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8"
+dependencies = [
+ "bytecount",
+ "cargo_metadata",
+ "error-chain",
+ "glob",
+ "pulldown-cmark",
+ "tempfile",
+ "walkdir",
+]
+
 [[package]]
 name = "sketches-ddsketch"
 version = "0.2.0"
@@ -5270,16 +5492,17 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 name = "sql"
 version = "0.1.0"
 dependencies = [
+ "api",
  "catalog",
  "common-catalog",
  "common-error",
  "common-time",
  "datatypes",
  "itertools",
+ "mito",
  "once_cell",
  "snafu",
  "sqlparser",
- "table-engine",
 ]
 
 [[package]]
@@ -5586,33 +5809,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "table-engine"
-version = "0.1.0"
-dependencies = [
- "arc-swap",
- "async-stream",
- "async-trait",
- "chrono",
- "common-error",
- "common-query",
- "common-recordbatch",
- "common-telemetry",
- "common-time",
- "datafusion",
- "datafusion-common",
- "datatypes",
- "futures",
- "log-store",
- "object-store",
- "serde",
- "serde_json",
- "snafu",
- "storage",
- "store-api",
- "table",
- "tempdir",
- "tokio",
-]
+name = "tagptr"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
 
 [[package]]
 name = "tap"
@@ -6187,6 +6387,12 @@ dependencies = [
  "tracing-log",
 ]
 
+[[package]]
+name = "triomphe"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1ee9bd9239c339d714d657fac840c6d2a4f9c45f4f9ec7b0975113458be78db"
+
 [[package]]
 name = "try-lock"
 version = "0.2.3"
@@ -6443,6 +6649,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8e76fae08f03f96e166d2dfda232190638c10e0383841252416f9cfe2ae60e6"
 
+[[package]]
+name = "waker-fn"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
+
 [[package]]
 name = "walkdir"
 version = "2.3.2"
@@ -6571,6 +6783,15 @@ dependencies = [
  "webpki",
 ]
 
+[[package]]
+name = "wepoll-ffi"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "which"
 version = "4.3.0"

Cargo.toml

@@ -32,7 +32,7 @@ members = [
     "src/storage",
     "src/store-api",
     "src/table",
-    "src/table-engine",
+    "src/mito",
 ]
 
 [profile.release]

README.md

@@ -1,101 +1,94 @@
-# GreptimeDB
+<p align="center">
+    <img src="/docs/logo-text-padding.png" alt="GreptimeDB Logo" width="400px"></img>
+</p>
 
-[![codecov](https://codecov.io/gh/GrepTimeTeam/greptimedb/branch/develop/graph/badge.svg?token=FITFDI3J3C)](https://codecov.io/gh/GrepTimeTeam/greptimedb)
+<h3 align="center">
+    The next-generation hybrid timeseries/analytics processing database in the cloud
+</h3>
 
-GreptimeDB: the next-generation hybrid timeseries/analytics processing database in the cloud.
+<p align="center">
+    <a href="https://codecov.io/gh/GrepTimeTeam/greptimedb"><img src="https://codecov.io/gh/GrepTimeTeam/greptimedb/branch/develop/graph/badge.svg?token=FITFDI3J3C"></img></a>
+    &nbsp;
+    <a href="https://github.com/GreptimeTeam/greptimedb/actions/workflows/develop.yml"><img src="https://github.com/GreptimeTeam/greptimedb/actions/workflows/develop.yml/badge.svg" alt="CI"></img></a>
+    &nbsp;
+    <a href="https://github.com/greptimeTeam/greptimedb/blob/develop/LICENSE"><img src="https://img.shields.io/github/license/greptimeTeam/greptimedb"></a>
+</p>
 
-## Getting Started
+<p align="center">
+    <a href="https://twitter.com/greptime"><img src="https://img.shields.io/badge/twitter-follow_us-1d9bf0.svg"></a>
+    &nbsp;
+    <a href="https://www.linkedin.com/company/greptime/"><img src="https://img.shields.io/badge/linkedin-connect_with_us-0a66c2.svg"></a>
+</p>
 
-### Prerequisites
+## What is GreptimeDB
 
-To compile GreptimeDB from source, you'll need the following:
-- Rust
-- Protobuf
+GreptimeDB is an open-source time-series database with a special focus on
+scalability, analytical capabilities and efficiency. It's designed to work on
+infrastructure of the cloud era, and users benefit from its elasticity and commodity
+storage.
 
-#### Rust
+Our core developers have been building time-series data platform
+for years. Based on their best-practices, GreptimeDB is born to give you:
 
-The easiest way to install Rust is to use [`rustup`](https://rustup.rs/), which will check our `rust-toolchain` file and install correct Rust version for you.
+- A standalone binary that scales to highly-available distributed cluster, providing a transparent experience for cluster users
+- Optimized columnar layout for handling time-series data; compacted, compressed, stored on various storage backends
+- Flexible index options, tackling high cardinality issues down
+- Distributed, parallel query execution, leveraging elastic computing resource
+- Native SQL, and Python scripting for advanced analytical scenarios
+- Widely adopted database protocols and APIs
+- Extensible table engine architecture for extensive workloads
 
-#### Protobuf
+## Quick Start
 
-`protoc` is required for compiling `.proto` files. `protobuf` is available from
-major package manager on macos and linux distributions. You can find an
-installation instructions [here](https://grpc.io/docs/protoc-installation/).
+### Build
 
-### Build the Docker Image
+#### Build from Source
+
+To compile GreptimeDB from source, you'll need:
+
+- C/C++ Toolchain: provides basic tools for compiling and linking. This is
+  available as `build-essential` on ubuntu and similar name on other platforms.
+- Rust: the easiest way to install Rust is to use
+  [`rustup`](https://rustup.rs/), which will check our `rust-toolchain` file and
+  install correct Rust version for you.
+- Protobuf: `protoc` is required for compiling `.proto` files. `protobuf` is
+  available from major package manager on macos and linux distributions. You can
+  find an installation instructions
+  [here](https://grpc.io/docs/protoc-installation/).
+
+#### Build with Docker
+
+A docker image with necessary dependencies is provided:
 
 ```
 docker build --network host -f docker/Dockerfile -t greptimedb .
 ```
 
-## Usage
+### Run
 
-### Start Datanode
+Start GreptimeDB from source code, in standalone mode:
 
 ```
-// Start datanode with default options.
-cargo run -- datanode start
-
-OR
-
-// Start datanode with `http-addr` option.
-cargo run -- datanode start --http-addr=0.0.0.0:9999
-
-OR
-
-// Start datanode with `log-dir` and `log-level` options.
-cargo run -- --log-dir=logs --log-level=debug datanode start
+cargo run -- standalone start
 ```
 
-Start datanode with config file:
+Or if you built from docker:
 
 ```
-cargo run -- --log-dir=logs --log-level=debug datanode start -c ./config/datanode.example.toml
+docker run -p 4002:4002 -v "$(pwd):/tmp/greptimedb" greptime/greptimedb standalone start
 ```
 
-Start datanode by runing docker container:
+For more startup options, greptimedb's **distributed mode** and information
+about Kubernetes deployment, check our [docs](https://docs.greptime.com/).
 
-```
-docker run -p 3000:3000 \
--p 3001:3001 \
--p 3306:3306 \
-greptimedb
-```
+### Connect
 
-### Start Frontend
-
-Frontend should connect to Datanode, so **Datanode must have been started** at first!
-
-```
-// Connects to local Datanode at its default GRPC port: 3001
-
-// Start Frontend with default options.
-cargo run -- frontend start
-
-OR
-
-// Start Frontend with `mysql-addr` option.
-cargo run -- frontend start --mysql-addr=0.0.0.0:9999
-
-OR
-
-// Start datanode with `log-dir` and `log-level` options.
-cargo run -- --log-dir=logs --log-level=debug frontend start
-```
-
-Start datanode with config file:
-
-```
-cargo run -- --log-dir=logs --log-level=debug frontend start -c ./config/frontend.example.toml
-```
-
-### SQL Operations
-
-1. Connecting DB by [mysql client](https://dev.mysql.com/downloads/mysql/):
+1. Connect to GreptimeDB via standard [MySQL
+   client](https://dev.mysql.com/downloads/mysql/):
 
    ```
-   # The datanode listen on port 3306 by default.
-   mysql -h 127.0.0.1 -P 3306
+   # The standalone instance listen on port 4002 by default.
+   mysql -h 127.0.0.1 -P 4002
    ```
 
 2. Create table:
@@ -110,29 +103,89 @@ cargo run -- --log-dir=logs --log-level=debug frontend start -c ./config/fronten
      PRIMARY KEY(host)) ENGINE=mito WITH(regions=1);
    ```
 
-3. Insert data:
+3. Insert some data:
 
    ```SQL
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host1', 66.6, 1024, 1660897955);
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host2', 77.7, 2048, 1660897956);
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host3', 88.8, 4096, 1660897957);
+   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host1', 66.6, 1024, 1660897955000);
+   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host2', 77.7, 2048, 1660897956000);
+   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host3', 88.8, 4096, 1660897957000);
    ```
 
-4. Query data:
+4. Query the data:
 
    ```SQL
    mysql> SELECT * FROM monitor;
-   +-------+------------+------+--------+
-   | host  | ts         | cpu  | memory |
-   +-------+------------+------+--------+
-   | host1 | 1660897955 | 66.6 |   1024 |
-   | host2 | 1660897956 | 77.7 |   2048 |
-   | host3 | 1660897957 | 88.8 |   4096 |
-   +-------+------------+------+--------+
+   +-------+---------------------+------+--------+
+   | host  | ts                  | cpu  | memory |
+   +-------+---------------------+------+--------+
+   | host1 | 2022-08-19 08:32:35 | 66.6 |   1024 |
+   | host2 | 2022-08-19 08:32:36 | 77.7 |   2048 |
+   | host3 | 2022-08-19 08:32:37 | 88.8 |   4096 |
+   +-------+---------------------+------+--------+
    3 rows in set (0.01 sec)
    ```
-   You can delete your data by removing `/tmp/greptimedb`.
+
+You can always cleanup test database by removing `/tmp/greptimedb`.
+
+## Resources
+
+### Installation
+
+- [Pre-built Binaries](https://github.com/GreptimeTeam/greptimedb/releases):
+  downloadable pre-built binaries for Linux and MacOS
+- [Docker Images](https://hub.docker.com/r/greptime/greptimedb): pre-built
+  Docker images
+- [`gtctl`](https://github.com/GreptimeTeam/gtctl): the command-line tool for
+  Kubernetes deployment
+
+### Documentation
+
+- GreptimeDB [User Guide](https://docs.greptime.com/user-guide/concepts.html)
+- GreptimeDB [Developer
+  Guide](https://docs.greptime.com/developer-guide/overview.html)
+
+### SDK
+
+- [GreptimeDB Java
+  Client](https://github.com/GreptimeTeam/greptimedb-client-java)
+
+## Project Status
+
+This project is in its early stage and under heavy development. We move fast and
+break things. Benchmark on development branch may not represent its potential
+performance. We release pre-built binaries constantly for functional
+evaluation. Do not use it in production at the moment.
+
+## Community
+
+Our core team is thrilled too see you participate in any ways you like. When you are stuck, try to
+ask for help by filling an issue with a detailed description of what you were trying to do
+and what went wrong. If you have any questions or if you would like to get involved in our
+community, please check out:
+
+- GreptimeDB Community on [Slack](https://greptime.com/slack)
+- GreptimeDB GitHub [Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)
+- Greptime official [Website](https://greptime.com)
+
+In addition, you may:
+
+- View our official [Blog](https://greptime.com/blog)
+- Connect us with [Linkedin](https://www.linkedin.com/company/greptime/)
+- Follow us on [Twitter](https://twitter.com/greptime)
+
+## License
+
+GreptimeDB uses the [Apache 2.0 license][1] to strike a balance between
+open contributions and allowing you to use the software however you want.
+
+[1]: <https://github.com/greptimeTeam/greptimedb/blob/develop/LICENSE>
 
 ## Contributing
 
 Please refer to [contribution guidelines](CONTRIBUTING.md) for more information.
+
+## Acknowledgement
+- GreptimeDB uses [Apache Arrow](https://arrow.apache.org/) as the memory model and [Apache Parquet](https://parquet.apache.org/) as the persistent file format.
+- GreptimeDB's query engine is powered by [Apache Arrow DataFusion](https://github.com/apache/arrow-datafusion).
+- [OpenDAL](https://github.com/datafuselabs/opendal) from [Datafuse Labs](https://github.com/datafuselabs) gives GreptimeDB a very general and elegant data access abstraction layer.
+- GreptimeDB’s meta service is based on [etcd](https://etcd.io/).

benchmarks/Cargo.toml

@@ -2,6 +2,7 @@
 name = "benchmarks"
 version = "0.1.0"
 edition = "2021"
+license = "Apache-2.0"
 
 [dependencies]
 arrow = "10"

benchmarks/src/bin/nyc-taxi.rs

@@ -1,35 +1,41 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //! Use the taxi trip records from New York City dataset to bench. You can download the dataset from
 //! [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page).
 
 #![feature(once_cell)]
 #![allow(clippy::print_stdout)]
 
-use std::{
-    collections::HashMap,
-    path::{Path, PathBuf},
-    sync::Arc,
-    time::Instant,
-};
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::Instant;
 
-use arrow::{
-    array::{ArrayRef, PrimitiveArray, StringArray, TimestampNanosecondArray},
-    datatypes::{DataType, Float64Type, Int64Type},
-    record_batch::RecordBatch,
-};
+use arrow::array::{ArrayRef, PrimitiveArray, StringArray, TimestampNanosecondArray};
+use arrow::datatypes::{DataType, Float64Type, Int64Type};
+use arrow::record_batch::RecordBatch;
 use clap::Parser;
-use client::{
-    admin::Admin,
-    api::v1::{
-        codec::InsertBatch, column::Values, insert_expr, Column, ColumnDataType, ColumnDef,
-        CreateExpr, InsertExpr,
-    },
-    Client, Database, Select,
-};
+use client::admin::Admin;
+use client::api::v1::codec::InsertBatch;
+use client::api::v1::column::Values;
+use client::api::v1::{insert_expr, Column, ColumnDataType, ColumnDef, CreateExpr, InsertExpr};
+use client::{Client, Database, Select};
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
-use parquet::{
-    arrow::{ArrowReader, ParquetFileArrowReader},
-    file::{reader::FileReader, serialized_reader::SerializedFileReader},
-};
+use parquet::arrow::{ArrowReader, ParquetFileArrowReader};
+use parquet::file::reader::FileReader;
+use parquet::file::serialized_reader::SerializedFileReader;
 use tokio::task::JoinSet;
 
 const DATABASE_NAME: &str = "greptime";
@@ -97,11 +103,13 @@ async fn write_data(
         let row_count = record_batch.num_rows();
         let insert_batch = convert_record_batch(record_batch).into();
         let insert_expr = InsertExpr {
+            schema_name: "public".to_string(),
             table_name: TABLE_NAME.to_string(),
             expr: Some(insert_expr::Expr::Values(insert_expr::Values {
                 values: vec![insert_batch],
             })),
             options: HashMap::default(),
+            region_number: 0,
         };
         let now = Instant::now();
         db.insert(insert_expr).await.unwrap();
@@ -342,6 +350,8 @@ fn create_table_expr() -> CreateExpr {
         primary_keys: vec!["VendorID".to_string()],
         create_if_not_exists: false,
         table_options: Default::default(),
+        region_ids: vec![0],
+        table_id: Some(0),
     }
 }
 

codecov.yml

@@ -1,10 +1,9 @@
 # codecov config
 coverage:
   status:
-    patch: off # disable patch status
     project:
       default:
-        enable: yes
         threshold: 1%
+    patch: off
 ignore:
   - "**/error*.rs" # ignore all error.rs files

component/script/python/example/calc_rv.py

@@ -1,71 +0,0 @@
-import sys
-# for annoying releative import beyond top-level package
-sys.path.insert(0, "../")
-from greptime import mock_tester, coprocessor, greptime as gt_builtin
-from greptime.greptime import interval, vector, log, prev, sqrt, datetime
-import greptime.greptime as greptime
-import json
-import numpy as np
-
-
-def data_sample(k_lines, symbol, density=5 * 30 * 86400):
-    """
-    Only return close data for simplicty for now
-    """
-    k_lines = k_lines["result"] if k_lines["ret_msg"] == "OK" else None
-    if k_lines is None:
-        raise Exception("Expect a `OK`ed message")
-    close = [float(i["close"]) for i in k_lines]
-
-    return interval(close, density, "prev")
-
-
-def as_table(kline: list):
-    col_len = len(kline)
-    ret = {
-        k: vector([fn(row[k]) for row in kline], str(ty))
-        for k, fn, ty in
-        [
-            ("symbol", str, "str"),
-            ("period", str, "str"),
-            ("open_time", int, "int"),
-            ("open", float, "float"),
-            ("high", float, "float"),
-            ("low", float, "float"),
-            ("close", float, "float")
-        ]
-    }
-    return ret
-
-@coprocessor(args=["open_time", "close"], returns=[
-    "rv_7d",
-    "rv_15d",
-    "rv_30d",
-    "rv_60d",
-    "rv_90d",
-    "rv_180d"
-])
-def calc_rvs(open_time, close):
-    from greptime import vector, log, prev, sqrt, datetime, pow, sum, last
-    import greptime as g
-    def calc_rv(close, open_time, time, interval):
-        mask = (open_time < time) & (open_time > time - interval)
-        close = close[mask]
-        open_time = open_time[mask]
-        close = g.interval(open_time, close, datetime("10m"), lambda x:last(x))
-
-        avg_time_interval = (open_time[-1] - open_time[0])/(len(open_time)-1)
-        ref = log(close/prev(close))
-        var = sum(pow(ref, 2)/(len(ref)-1))
-        return sqrt(var/avg_time_interval)
-
-    # how to get env var,
-    # maybe through accessing scope and serde then send to remote?
-    timepoint = open_time[-1]
-    rv_7d = vector([calc_rv(close, open_time, timepoint, datetime("7d"))])
-    rv_15d = vector([calc_rv(close, open_time, timepoint, datetime("15d"))])
-    rv_30d = vector([calc_rv(close, open_time, timepoint, datetime("30d"))])
-    rv_60d = vector([calc_rv(close, open_time, timepoint, datetime("60d"))])
-    rv_90d = vector([calc_rv(close, open_time, timepoint, datetime("90d"))])
-    rv_180d = vector([calc_rv(close, open_time, timepoint, datetime("180d"))])
-    return rv_7d, rv_15d, rv_30d, rv_60d, rv_90d, rv_180d

component/script/python/example/fetch_kline.sh

@@ -1 +0,0 @@
-curl "https://api.bybit.com/v2/public/index-price-kline?symbol=BTCUSD&interval=1&limit=$1&from=1581231260" > kline.json

component/script/python/example/kline.json

@@ -1,108 +0,0 @@
-{
-    "ret_code": 0,
-    "ret_msg": "OK",
-    "ext_code": "",
-    "ext_info": "",
-    "result": [
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 300,
-            "open": "10107",
-            "high": "10109.34",
-            "low": "10106.71",
-            "close": "10106.79"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 900,
-            "open": "10106.79",
-            "high": "10109.27",
-            "low": "10105.92",
-            "close": "10106.09"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1200,
-            "open": "10106.09",
-            "high": "10108.75",
-            "low": "10104.66",
-            "close": "10108.73"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 1800,
-            "open": "10108.73",
-            "high": "10109.52",
-            "low": "10106.07",
-            "close": "10106.38"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 2400,
-            "open": "10106.38",
-            "high": "10109.48",
-            "low": "10104.81",
-            "close": "10106.95"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 3000,
-            "open": "10106.95",
-            "high": "10109.48",
-            "low": "10106.6",
-            "close": "10107.55"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 3600,
-            "open": "10107.55",
-            "high": "10109.28",
-            "low": "10104.68",
-            "close": "10104.68"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 4200,
-            "open": "10104.68",
-            "high": "10109.18",
-            "low": "10104.14",
-            "close": "10108.8"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 4800,
-            "open": "10108.8",
-            "high": "10117.36",
-            "low": "10108.8",
-            "close": "10115.96"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 5400,
-            "open": "10115.96",
-            "high": "10119.19",
-            "low": "10115.96",
-            "close": "10117.08"
-        },
-        {
-            "symbol": "BTCUSD",
-            "period": "1",
-            "open_time": 6000,
-            "open": "10117.08",
-            "high": "10120.73",
-            "low": "10116.96",
-            "close": "10120.43"
-        }
-    ],
-    "time_now": "1661225351.158190"
-}

component/script/python/greptime/__init__.py

@@ -1,4 +0,0 @@
-from .greptime import coprocessor, copr
-from .greptime import vector, log, prev, next, first, last, sqrt, pow, datetime, sum, interval
-from .mock import mock_tester
-from .cfg import set_conn_addr, get_conn_addr

component/script/python/greptime/cfg.py

@@ -1,11 +0,0 @@
-GREPTIME_DB_CONN_ADDRESS = "localhost:3000"
-"""The Global Variable for address for conntect to database"""
-
-def set_conn_addr(addr: str):
-    """set database address to given `addr`"""
-    global GREPTIME_DB_CONN_ADDRESS
-    GREPTIME_DB_CONN_ADDRESS = addr
-
-def get_conn_addr()->str:
-    global GREPTIME_DB_CONN_ADDRESS
-    return GREPTIME_DB_CONN_ADDRESS

component/script/python/greptime/greptime.py

@@ -1,207 +0,0 @@
-"""
-Be note that this is a mock library, if not connected to database,
-it can only run on mock data and mock function which is supported by numpy
-"""
-import functools
-import numpy as np
-import json
-from urllib import request
-import inspect
-import requests
-
-from .cfg import set_conn_addr, get_conn_addr
-
-log = np.log
-sum = np.nansum
-sqrt = np.sqrt
-pow = np.power
-nan = np.nan
-
-
-class TimeStamp(str):
-    """
-    TODO: impl date time
-    """
-    pass
-
-
-class i32(int):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A signed 32-bit integer.
-    """
-
-    def __repr__(self) -> str:
-        return "i32"
-
-
-class i64(int):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A signed 64-bit integer.
-    """
-
-    def __repr__(self) -> str:
-        return "i64"
-
-
-class f32(float):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A 32-bit floating point number.
-    """
-
-    def __repr__(self) -> str:
-        return "f32"
-
-
-class f64(float):
-    """
-    For Python Coprocessor Type Annotation ONLY
-    A 64-bit floating point number.
-    """
-
-    def __repr__(self) -> str:
-        return "f64"
-
-
-class vector(np.ndarray):
-    """
-    A compact Vector with all elements of same Data type.
-    """
-    _datatype: str | None = None
-
-    def __new__(
-        cls,
-        lst,
-        dtype=None
-    ) -> ...:
-        self = np.asarray(lst).view(cls)
-        self._datatype = dtype
-        return self
-
-    def __str__(self) -> str:
-        return "vector({}, \"{}\")".format(super().__str__(), self.datatype())
-
-    def datatype(self):
-        return self._datatype
-
-    def filter(self, lst_bool):
-        return self[lst_bool]
-
-def last(lst):
-    return lst[-1]
-
-def first(lst):
-    return lst[0]
-
-def prev(lst):
-    ret = np.zeros(len(lst))
-    ret[1:] = lst[0:-1]
-    ret[0] = nan
-    return ret
-
-def next(lst):
-    ret = np.zeros(len(lst))
-    ret[:-1] = lst[1:]
-    ret[-1] = nan
-    return ret
-
-def interval(ts: vector, arr: vector, duration: int, func):
-    """
-    Note that this is a mock function with same functionailty to the actual Python Coprocessor
-    `arr` is a vector of integral or temporal type.
-    """
-    start = np.min(ts)
-    end = np.max(ts)
-    masks = [(ts >= i) & (ts <= (i+duration)) for i in range(start, end, duration)]
-    lst_res = [func(arr[mask]) for mask in masks]
-    return lst_res
-
-
-def factor(unit: str) -> int:
-    if unit == "d":
-        return 24 * 60 * 60
-    elif unit == "h":
-        return 60 * 60
-    elif unit == "m":
-        return 60
-    elif unit == "s":
-        return 1
-    else:
-        raise Exception("Only d,h,m,s, found{}".format(unit))
-
-
-def datetime(input_time: str) -> int:
-    """
-    support `d`(day) `h`(hour) `m`(minute) `s`(second)
-
-    support format:
-    `12s` `7d` `12d2h7m`
-    """
-
-    prev = 0
-    cur = 0
-    state = "Num"
-    parse_res = []
-    for idx, ch in enumerate(input_time):
-        if ch.isdigit():
-            cur = idx
-
-            if state != "Num":
-                parse_res.append((state, input_time[prev:cur], (prev, cur)))
-                prev = idx
-                state = "Num"
-        else:
-            cur = idx
-            if state != "Symbol":
-                parse_res.append((state, input_time[prev:cur], (prev, cur)))
-                prev = idx
-                state = "Symbol"
-    parse_res.append((state, input_time[prev:cur+1], (prev, cur+1)))
-
-    cur_idx = 0
-    res_time = 0
-    while cur_idx < len(parse_res):
-        pair = parse_res[cur_idx]
-        if pair[0] == "Num":
-            val = int(pair[1])
-            nxt = parse_res[cur_idx+1]
-            res_time += val * factor(nxt[1])
-            cur_idx += 2
-        else:
-            raise Exception("Two symbol in a row is impossible")
-
-    return res_time
-
-
-def coprocessor(args=None, returns=None, sql=None):
-    """
-    The actual coprocessor, which will connect to database and update
-    whatever function decorated with `@coprocessor(args=[...], returns=[...], sql=...)`
-    """
-    def decorator_copr(func):
-        @functools.wraps(func)
-        def wrapper_do_actual(*args, **kwargs):
-            if len(args)!=0 or len(kwargs)!=0:
-                raise Exception("Expect call with no arguements(for all args are given by coprocessor itself)")
-            source = inspect.getsource(func)
-            url = "http://{}/v1/scripts".format(get_conn_addr())
-            print("Posting to {}".format(url))
-            data = {
-                    "script": source,
-                    "engine": None,
-                }
-
-            res = requests.post(
-                url,
-                headers={"Content-Type": "application/json"},
-                json=data
-            )
-            return res
-        return wrapper_do_actual
-    return decorator_copr
-
-
-# make a alias for short
-copr = coprocessor

component/script/python/greptime/mock.py

@@ -1,82 +0,0 @@
-"""
-Note this is a mock library, if not connected to database, 
-it can only run on mock data and support by numpy
-"""
-from typing import Any
-import numpy as np
-from .greptime import i32,i64,f32,f64, vector, interval, prev, datetime, log, sum, sqrt, pow, nan, copr, coprocessor
-
-import inspect
-import functools
-import ast
-
-
-
-def mock_tester(
-    func,
-    env:dict, 
-    table=None
-):
-    """
-    Mock tester helper function,
-    What it does is replace `@coprocessor` with `@mock_cpor` and add a keyword `env=env`
-    like `@mock_copr(args=...,returns=...,env=env)`
-    """
-    code = inspect.getsource(func)
-    tree = ast.parse(code)
-    tree = HackyReplaceDecorator("env").visit(tree)
-    new_func = tree.body[0]
-    fn_name = new_func.name
-
-    code_obj = compile(tree, "<embedded>", "exec")
-    exec(code_obj)
-
-    ret = eval("{}()".format(fn_name))
-    return ret
-
-def mock_copr(args, returns, sql=None, env:None|dict=None):
-    """
-    This should not be used directly by user
-    """
-    def decorator_copr(func):
-        @functools.wraps(func)
-        def wrapper_do_actual(*fn_args, **fn_kwargs):
-
-            real_args = [env[name] for name in args]
-            ret = func(*real_args)
-            return ret
-        
-        return wrapper_do_actual
-    return decorator_copr
-
-class HackyReplaceDecorator(ast.NodeTransformer):
-    """
-    This class accept a `env` dict for environment to extract args from,
-    and put `env` dict in the param list of `mock_copr` decorator, i.e:
-
-    a `@copr(args=["a", "b"], returns=["c"])` with call like mock_helper(abc, env={"a":2, "b":3})
-    
-    will be transform into `@mock_copr(args=["a", "b"], returns=["c"], env={"a":2, "b":3})`
-    """
-    def __init__(self, env: str) -> None:
-        # just for add `env` keyword
-        self.env = env
-
-    def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
-        new_node = node
-        decorator_list = new_node.decorator_list
-        if len(decorator_list)!=1:
-            return node
-
-        deco = decorator_list[0]
-        if deco.func.id!="coprocessor" and deco.func.id !="copr":
-            raise Exception("Expect a @copr or @coprocessor, found {}.".format(deco.func.id))
-        deco.func = ast.Name(id="mock_copr", ctx=ast.Load())
-        new_kw = ast.keyword(arg="env", value=ast.Name(id=self.env, ctx=ast.Load()))
-        deco.keywords.append(new_kw)
-
-        # Tie up loose ends in the AST.
-        ast.copy_location(new_node, node)
-        ast.fix_missing_locations(new_node)
-        self.generic_visit(node)
-        return new_node

component/script/python/test.py

@@ -1,60 +0,0 @@
-from example.calc_rv import as_table, calc_rvs
-from greptime import coprocessor, set_conn_addr, get_conn_addr, mock_tester
-import sys
-import json
-import requests
-'''
-To run this script, you need to first start a http server of greptime, and 
-`
-python3 component/script/python/test.py 地址:端口
-`
-
-'''
-@coprocessor(sql='select number from numbers limit 10', args=['number'], returns=['n'])
-def test(n):
-    return n+2
-
-def init_table(close, open_time):
-    req_init = "/v1/sql?sql=create table k_line (close double, open_time bigint, TIME INDEX (open_time))"
-    print(get_db(req_init).text)
-    for c1, c2 in zip(close, open_time):
-        req = "/v1/sql?sql=INSERT INTO k_line(close, open_time) VALUES ({}, {})".format(c1, c2)
-        print(get_db(req).text)
-    print(get_db("/v1/sql?sql=select * from k_line").text)
-
-def get_db(req:str):
-    return requests.get("http://{}{}".format(get_conn_addr(), req))
-
-if __name__ == "__main__":
-    with open("component/script/python/example/kline.json", "r") as kline_file:
-        kline = json.load(kline_file)
-        table = as_table(kline["result"])
-        close = table["close"]
-        open_time = table["open_time"]
-        env = {"close":close, "open_time": open_time}
-
-        res = mock_tester(calc_rvs, env=env)
-        print("Mock result:", [i[0] for i in res])
-        exit()
-    if len(sys.argv)!=2:
-        raise Exception("Expect only one address as cmd's args")
-    set_conn_addr(sys.argv[1])
-    res = test()
-    print(res.headers)
-    print(res.text)
-    with open("component/script/python/example/kline.json", "r") as kline_file:
-        kline = json.load(kline_file)
-        # vec = vector([1,2,3], int)
-        # print(vec, vec.datatype())
-        table = as_table(kline["result"])
-        # print(table)
-        close = table["close"]
-        open_time = table["open_time"]
-        init_table(close, open_time)
-
-        real = calc_rvs()
-        print(real)
-        try:
-            print(real.text["error"])
-        except:
-            print(real.text)

config/datanode.example.toml

@@ -1,22 +1,17 @@
 node_id = 42
-http_addr = '0.0.0.0:3000'
-rpc_addr = '0.0.0.0:3001'
+mode = 'distributed'
+rpc_addr = '127.0.0.1:3001'
 wal_dir = '/tmp/greptimedb/wal'
 rpc_runtime_size = 8
-mode = "standalone"
-mysql_addr = '0.0.0.0:3306'
+mysql_addr = '127.0.0.1:3306'
 mysql_runtime_size = 4
 
-# applied when postgres feature enbaled
-postgres_addr = '0.0.0.0:5432'
-postgres_runtime_size = 4
-
 [storage]
 type = 'File'
 data_dir = '/tmp/greptimedb/data/'
 
 [meta_client_opts]
-metasrv_addr = "1.1.1.1:3002"
+metasrv_addr = '1.1.1.1:3002'
 timeout_millis = 3000
 connect_timeout_millis = 5000
-tcp_nodelay = true
+tcp_nodelay = false

config/frontend.example.toml

@@ -1,4 +1,9 @@
-http_addr = '0.0.0.0:4000'
-grpc_addr = '0.0.0.0:4001'
-mysql_addr = '0.0.0.0:4003'
-mysql_runtime_size = 4
+mode = 'distributed'
+datanode_rpc_addr = '127.0.0.1:3001'
+http_addr = '127.0.0.1:4000'
+
+[meta_client_opts]
+metasrv_addr = '1.1.1.1:3002'
+timeout_millis = 3000
+connect_timeout_millis = 5000
+tcp_nodelay = false

config/metasrv.example.toml

@@ -1,4 +1,4 @@
 bind_addr = '127.0.0.1:3002'
-server_addr = '0.0.0.0:3002'
-store_addr = '127.0.0.1:2380'
-datanode_lease_secs = 30
+server_addr = '127.0.0.1:3002'
+store_addr = '127.0.0.1:2379'
+datanode_lease_secs = 15

config/standalone.example.toml

@@ -0,0 +1,34 @@
+node_id = 0
+mode = 'standalone'
+http_addr = '127.0.0.1:4000'
+datanode_mysql_addr = '127.0.0.1:3306'
+datanode_mysql_runtime_size = 4
+wal_dir = '/tmp/greptimedb/wal/'
+
+[storage]
+type = 'File'
+data_dir = '/tmp/greptimedb/data/'
+
+[grpc_options]
+addr = '127.0.0.1:4001'
+runtime_size = 8
+
+[mysql_options]
+addr = '127.0.0.1:4002'
+runtime_size = 2
+
+[influxdb_options]
+enable = true
+
+[opentsdb_options]
+addr = '127.0.0.1:4242'
+enable = true
+runtime_size = 2
+
+[prometheus_options]
+enable = true
+
+[postgres_options]
+addr = '127.0.0.1:4003'
+runtime_size = 2
+check_pwd = false

rustfmt.toml

@@ -1,3 +1,2 @@
 group_imports = "StdExternalCrate"
-
-
+imports_granularity = "Module"

scripts/install.sh

@@ -7,7 +7,7 @@ ARCH_TYPE=
 VERSION=${1:-latest}
 GITHUB_ORG=GreptimeTeam
 GITHUB_REPO=greptimedb
-BIN=greptimedb
+BIN=greptime
 
 get_os_type() {
     os_type="$(uname -s)"

src/api/Cargo.toml

@@ -2,11 +2,13 @@
 name = "api"
 version = "0.1.0"
 edition = "2021"
+license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 common-base = { path = "../common/base" }
 common-time = { path = "../common/time" }
+common-error = { path = "../common/error" }
 datatypes = { path = "../datatypes" }
 prost = "0.11"
 snafu = { version = "0.7", features = ["backtraces"] }

src/api/build.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::path::PathBuf;
 
 fn main() {

src/api/greptime/v1/admin.proto

@@ -19,6 +19,7 @@ message AdminExpr {
   oneof expr {
     CreateExpr create = 2;
     AlterExpr alter = 3;
+    CreateDatabaseExpr create_database = 4;
   }
 }
 
@@ -29,6 +30,7 @@ message AdminResult {
   }
 }
 
+// TODO(hl): rename to CreateTableExpr
 message CreateExpr {
   optional string catalog_name = 1;
   optional string schema_name = 2;
@@ -39,6 +41,8 @@ message CreateExpr {
   repeated string primary_keys = 7;
   bool create_if_not_exists = 8;
   map<string, string> table_options = 9;
+  optional uint32 table_id = 10;
+  repeated uint32 region_ids = 11;
 }
 
 message AlterExpr {
@@ -46,10 +50,20 @@ message AlterExpr {
   optional string schema_name = 2;
   string table_name = 3;
   oneof kind {
-    AddColumn add_column = 4;
+    AddColumns add_columns = 4;
   }
 }
 
+message AddColumns {
+  repeated AddColumn add_columns = 1;
+}
+
 message AddColumn {
   ColumnDef column_def = 1;
+  bool is_key = 2;
+}
+
+message CreateDatabaseExpr {
+  //TODO(hl): maybe rename to schema_name?
+  string database_name = 1;
 }

src/api/greptime/v1/database.proto

@@ -38,14 +38,15 @@ message PhysicalPlan {
 }
 
 message InsertExpr {
-  string table_name = 1;
+  string schema_name = 1;
+  string table_name = 2;
 
   message Values {
     repeated bytes values = 1;
   }
 
   oneof expr {
-    Values values = 2;
+    Values values = 3;
 
     // TODO(LFC): Remove field "sql" in InsertExpr.
     // When Frontend instance received an insertion SQL (`insert into ...`), it's anticipated to parse the SQL and
@@ -54,10 +55,12 @@ message InsertExpr {
     // Then why the "sql" field exists here? It's because the Frontend needs table schema to create the values to insert,
     // which is currently not able to find anywhere. (Maybe the table schema is suppose to be fetched from Meta?)
     // The "sql" field is meant to be removed in the future.
-    string sql = 3;
+    string sql = 4;
   }
-  
-  map<string, bytes> options = 4;
+
+  /// The region number of current insert request.
+  uint32 region_number = 5;
+  map<string, bytes> options = 6;
 }
 
 // TODO(jiachun)

src/api/greptime/v1/meta/route.proto

@@ -69,6 +69,7 @@ message Table {
 }
 
 message Region {
+  // TODO(LFC): Maybe use message RegionNumber?
   uint64 id = 1;
   string name = 2;
   Partition partition = 3;

src/api/src/error.rs

@@ -1,6 +1,24 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+
+use common_error::ext::ErrorExt;
+use common_error::prelude::StatusCode;
 use datatypes::prelude::ConcreteDataType;
 use snafu::prelude::*;
-use snafu::Backtrace;
+use snafu::{Backtrace, ErrorCompat};
 
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -16,3 +34,19 @@ pub enum Error {
         backtrace: Backtrace,
     },
 }
+
+impl ErrorExt for Error {
+    fn status_code(&self) -> StatusCode {
+        match self {
+            Error::UnknownColumnDataType { .. } => StatusCode::InvalidArguments,
+            Error::IntoColumnDataType { .. } => StatusCode::Unexpected,
+        }
+    }
+    fn backtrace_opt(&self) -> Option<&Backtrace> {
+        ErrorCompat::backtrace(self)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/api/src/helper.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use common_base::BitVec;
 use common_time::timestamp::TimeUnit;
 use datatypes::prelude::ConcreteDataType;
@@ -7,8 +21,7 @@ use snafu::prelude::*;
 
 use crate::error::{self, Result};
 use crate::v1::column::Values;
-use crate::v1::Column;
-use crate::v1::ColumnDataType;
+use crate::v1::{Column, ColumnDataType};
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct ColumnDataTypeWrapper(ColumnDataType);

src/api/src/lib.rs

@@ -1,6 +1,21 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 pub mod error;
 pub mod helper;
 pub mod prometheus;
+pub mod result;
 pub mod serde;
 pub mod v1;
 

src/api/src/prometheus.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #![allow(clippy::derive_partial_eq_without_eq)]
 
 pub mod remote {

src/api/src/result.rs

@@ -1,23 +1,39 @@
-use api::v1::{
-    admin_result, codec::SelectResult, object_result, AdminResult, MutateResult, ObjectResult,
-    ResultHeader, SelectResult as SelectResultRaw,
-};
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use common_error::prelude::ErrorExt;
 
+use crate::v1::codec::SelectResult;
+use crate::v1::{
+    admin_result, object_result, AdminResult, MutateResult, ObjectResult, ResultHeader,
+    SelectResult as SelectResultRaw,
+};
+
 pub const PROTOCOL_VERSION: u32 = 1;
 
 pub type Success = u32;
 pub type Failure = u32;
 
 #[derive(Default)]
-pub(crate) struct ObjectResultBuilder {
+pub struct ObjectResultBuilder {
     version: u32,
     code: u32,
     err_msg: Option<String>,
     result: Option<Body>,
 }
 
-pub(crate) enum Body {
+pub enum Body {
     Mutate((Success, Failure)),
     Select(SelectResult),
 }
@@ -80,7 +96,7 @@ impl ObjectResultBuilder {
     }
 }
 
-pub(crate) fn build_err_result(err: &impl ErrorExt) -> ObjectResult {
+pub fn build_err_result(err: &impl ErrorExt) -> ObjectResult {
     ObjectResultBuilder::new()
         .status_code(err.status_code() as u32)
         .err_msg(err.to_string())
@@ -88,7 +104,7 @@ pub(crate) fn build_err_result(err: &impl ErrorExt) -> ObjectResult {
 }
 
 #[derive(Debug)]
-pub(crate) struct AdminResultBuilder {
+pub struct AdminResultBuilder {
     version: u32,
     code: u32,
     err_msg: Option<String>,
@@ -144,11 +160,11 @@ impl Default for AdminResultBuilder {
 
 #[cfg(test)]
 mod tests {
-    use api::v1::{object_result, MutateResult};
     use common_error::status_code::StatusCode;
 
     use super::*;
-    use crate::error::UnsupportedExprSnafu;
+    use crate::error::UnknownColumnDataTypeSnafu;
+    use crate::v1::{object_result, MutateResult};
 
     #[test]
     fn test_object_result_builder() {
@@ -175,14 +191,13 @@ mod tests {
 
     #[test]
     fn test_build_err_result() {
-        let err = UnsupportedExprSnafu { name: "select" }.build();
+        let err = UnknownColumnDataTypeSnafu { datatype: 1 }.build();
         let err_result = build_err_result(&err);
         let header = err_result.header.unwrap();
         let result = err_result.result;
 
         assert_eq!(PROTOCOL_VERSION, header.version);
-        assert_eq!(StatusCode::Internal as u32, header.code);
-        assert_eq!("Unsupported expr type: select", header.err_msg);
+        assert_eq!(StatusCode::InvalidArguments as u32, header.code);
         assert!(result.is_none());
     }
 }

src/api/src/serde.rs

@@ -1,10 +1,21 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 pub use prost::DecodeError;
 use prost::Message;
 
-use crate::v1::codec::InsertBatch;
-use crate::v1::codec::PhysicalPlanNode;
-use crate::v1::codec::RegionNumber;
-use crate::v1::codec::SelectResult;
+use crate::v1::codec::{InsertBatch, PhysicalPlanNode, RegionNumber, SelectResult};
 use crate::v1::meta::TableRouteValue;
 
 macro_rules! impl_convert_with_bytes {
@@ -36,8 +47,7 @@ mod tests {
     use std::ops::Deref;
 
     use crate::v1::codec::*;
-    use crate::v1::column;
-    use crate::v1::Column;
+    use crate::v1::{column, Column};
 
     const SEMANTIC_TAG: i32 = 0;
 

src/api/src/v1.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #![allow(clippy::derive_partial_eq_without_eq)]
 tonic::include_proto!("greptime.v1");
 

src/api/src/v1/meta.rs

@@ -1,8 +1,21 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 tonic::include_proto!("greptime.v1.meta");
 
 use std::collections::HashMap;
-use std::hash::Hash;
-use std::hash::Hasher;
+use std::hash::{Hash, Hasher};
 
 pub const PROTOCOL_VERSION: u64 = 1;
 
@@ -71,11 +84,22 @@ impl ResponseHeader {
             error: Some(error),
         }
     }
+
+    #[inline]
+    pub fn is_not_leader(&self) -> bool {
+        if let Some(error) = &self.error {
+            if error.code == ErrorCode::NotLeader as i32 {
+                return true;
+            }
+        }
+        false
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum ErrorCode {
     NoActiveDatanodes = 1,
+    NotLeader = 2,
 }
 
 impl Error {
@@ -86,6 +110,24 @@ impl Error {
             err_msg: "No active datanodes".to_string(),
         }
     }
+
+    #[inline]
+    pub fn is_not_leader() -> Self {
+        Self {
+            code: ErrorCode::NotLeader as i32,
+            err_msg: "Current server is not leader".to_string(),
+        }
+    }
+}
+
+impl HeartbeatResponse {
+    #[inline]
+    pub fn is_not_leader(&self) -> bool {
+        if let Some(header) = &self.header {
+            return header.is_not_leader();
+        }
+        false
+    }
 }
 
 macro_rules! gen_set_header {

src/catalog/Cargo.toml

@@ -2,6 +2,7 @@
 name = "catalog"
 version = "0.1.0"
 edition = "2021"
+license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -41,6 +42,6 @@ log-store = { path = "../log-store" }
 object-store = { path = "../object-store" }
 opendal = "0.17"
 storage = { path = "../storage" }
-table-engine = { path = "../table-engine" }
+mito = { path = "../mito" }
 tempdir = "0.3"
 tokio = { version = "1.0", features = ["full"] }

src/catalog/src/error.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
@@ -83,6 +97,12 @@ pub enum Error {
     #[snafu(display("Table {} already exists", table))]
     TableExists { table: String, backtrace: Backtrace },
 
+    #[snafu(display("Schema {} already exists", schema))]
+    SchemaExists {
+        schema: String,
+        backtrace: Backtrace,
+    },
+
     #[snafu(display("Failed to register table"))]
     RegisterTable {
         #[snafu(backtrace)]
@@ -112,7 +132,7 @@ pub enum Error {
         "Failed to insert table creation record to system catalog, source: {}",
         source
     ))]
-    InsertTableRecord {
+    InsertCatalogRecord {
         #[snafu(backtrace)]
         source: table::error::Error,
     },
@@ -165,19 +185,6 @@ pub enum Error {
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Failed to bump table id"))]
-    BumpTableId { msg: String, backtrace: Backtrace },
-
-    #[snafu(display("Failed to parse table id from metasrv, data: {:?}", data))]
-    ParseTableId { data: String, backtrace: Backtrace },
-
-    #[snafu(display("Failed to deserialize partition rule from string: {:?}", data))]
-    DeserializePartitionRule {
-        data: String,
-        source: serde_json::error::Error,
-        backtrace: Backtrace,
-    },
-
     #[snafu(display("Invalid table schema in catalog, source: {:?}", source))]
     InvalidSchemaInCatalog {
         #[snafu(backtrace)]
@@ -215,20 +222,17 @@ impl ErrorExt for Error {
 
             Error::RegisterTable { .. } => StatusCode::Internal,
             Error::TableExists { .. } => StatusCode::TableAlreadyExists,
+            Error::SchemaExists { .. } => StatusCode::InvalidArguments,
 
             Error::OpenSystemCatalog { source, .. }
             | Error::CreateSystemCatalog { source, .. }
-            | Error::InsertTableRecord { source, .. }
+            | Error::InsertCatalogRecord { source, .. }
             | Error::OpenTable { source, .. }
             | Error::CreateTable { source, .. } => source.status_code(),
             Error::MetaSrv { source, .. } => source.status_code(),
             Error::SystemCatalogTableScan { source } => source.status_code(),
             Error::SystemCatalogTableScanExec { source } => source.status_code(),
             Error::InvalidTableSchema { source, .. } => source.status_code(),
-            Error::BumpTableId { .. } | Error::ParseTableId { .. } => {
-                StatusCode::StorageUnavailable
-            }
-            Error::DeserializePartitionRule { .. } => StatusCode::Unexpected,
             Error::InvalidSchemaInCatalog { .. } => StatusCode::Unexpected,
             Error::Internal { source, .. } => source.status_code(),
         }

src/catalog/src/lib.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #![feature(assert_matches)]
 
 use std::any::Any;
@@ -69,17 +83,19 @@ pub trait CatalogManager: CatalogList {
     /// Starts a catalog manager.
     async fn start(&self) -> Result<()>;
 
-    /// Returns next available table id.
-    async fn next_table_id(&self) -> Result<TableId>;
-
     /// Registers a table given given catalog/schema to catalog manager,
     /// returns table registered.
     async fn register_table(&self, request: RegisterTableRequest) -> Result<usize>;
 
+    /// Register a schema with catalog name and schema name.
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<usize>;
+
     /// Register a system table, should be called before starting the manager.
     async fn register_system_table(&self, request: RegisterSystemTableRequest)
         -> error::Result<()>;
 
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>>;
+
     /// Returns the table by catalog, schema and table name.
     fn table(&self, catalog: &str, schema: &str, table_name: &str) -> Result<Option<TableRef>>;
 }
@@ -107,6 +123,12 @@ pub struct RegisterTableRequest {
     pub table: TableRef,
 }
 
+#[derive(Debug, Clone)]
+pub struct RegisterSchemaRequest {
+    pub catalog: String,
+    pub schema: String,
+}
+
 /// Formats table fully-qualified name
 pub fn format_full_table_name(catalog: &str, schema: &str, table: &str) -> String {
     format!("{}.{}.{}", catalog, schema, table)

src/catalog/src/local.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 pub mod manager;
 pub mod memory;
 

src/catalog/src/local/manager.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
@@ -16,13 +30,14 @@ use table::engine::{EngineContext, TableEngineRef};
 use table::metadata::TableId;
 use table::requests::OpenTableRequest;
 use table::table::numbers::NumbersTable;
+use table::table::TableIdProvider;
 use table::TableRef;
 
 use crate::error::{
-    CatalogNotFoundSnafu, IllegalManagerStateSnafu, OpenTableSnafu, SchemaNotFoundSnafu,
-    SystemCatalogSnafu, SystemCatalogTypeMismatchSnafu, TableExistsSnafu, TableNotFoundSnafu,
+    CatalogNotFoundSnafu, IllegalManagerStateSnafu, OpenTableSnafu, ReadSystemCatalogSnafu, Result,
+    SchemaExistsSnafu, SchemaNotFoundSnafu, SystemCatalogSnafu, SystemCatalogTypeMismatchSnafu,
+    TableExistsSnafu, TableNotFoundSnafu,
 };
-use crate::error::{ReadSystemCatalogSnafu, Result};
 use crate::local::memory::{MemoryCatalogManager, MemoryCatalogProvider, MemorySchemaProvider};
 use crate::system::{
     decode_system_catalog, Entry, SystemCatalogTable, TableEntry, ENTRY_TYPE_INDEX, KEY_INDEX,
@@ -31,8 +46,8 @@ use crate::system::{
 use crate::tables::SystemCatalog;
 use crate::{
     format_full_table_name, handle_system_table_request, CatalogList, CatalogManager,
-    CatalogProvider, CatalogProviderRef, RegisterSystemTableRequest, RegisterTableRequest,
-    SchemaProvider,
+    CatalogProvider, CatalogProviderRef, RegisterSchemaRequest, RegisterSystemTableRequest,
+    RegisterTableRequest, SchemaProvider, SchemaProviderRef,
 };
 
 /// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
@@ -278,6 +293,13 @@ impl CatalogList for LocalCatalogManager {
     }
 }
 
+#[async_trait::async_trait]
+impl TableIdProvider for LocalCatalogManager {
+    async fn next_table_id(&self) -> table::Result<TableId> {
+        Ok(self.next_table_id.fetch_add(1, Ordering::Relaxed))
+    }
+}
+
 #[async_trait::async_trait]
 impl CatalogManager for LocalCatalogManager {
     /// Start [LocalCatalogManager] to load all information from system catalog table.
@@ -286,11 +308,6 @@ impl CatalogManager for LocalCatalogManager {
         self.init().await
     }
 
-    #[inline]
-    async fn next_table_id(&self) -> Result<TableId> {
-        Ok(self.next_table_id.fetch_add(1, Ordering::Relaxed))
-    }
-
     async fn register_table(&self, request: RegisterTableRequest) -> Result<usize> {
         let started = self.init_lock.lock().await;
 
@@ -334,6 +351,34 @@ impl CatalogManager for LocalCatalogManager {
         Ok(1)
     }
 
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<usize> {
+        let started = self.init_lock.lock().await;
+        ensure!(
+            *started,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager not started",
+            }
+        );
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+
+        let catalog = self
+            .catalogs
+            .catalog(catalog_name)?
+            .context(CatalogNotFoundSnafu { catalog_name })?;
+        if catalog.schema(schema_name)?.is_some() {
+            return SchemaExistsSnafu {
+                schema: schema_name,
+            }
+            .fail();
+        }
+        self.system
+            .register_schema(request.catalog, schema_name.clone())
+            .await?;
+        catalog.register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))?;
+        Ok(1)
+    }
+
     async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
         ensure!(
             !*self.init_lock.lock().await,
@@ -348,6 +393,15 @@ impl CatalogManager for LocalCatalogManager {
         Ok(())
     }
 
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+        self.catalogs
+            .catalog(catalog)?
+            .context(CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .schema(schema)
+    }
+
     fn table(
         &self,
         catalog_name: &str,

src/catalog/src/local/memory.rs

@@ -1,20 +1,34 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::Arc;
-use std::sync::RwLock;
+use std::sync::{Arc, RwLock};
 
 use common_catalog::consts::MIN_USER_TABLE_ID;
 use snafu::OptionExt;
 use table::metadata::TableId;
+use table::table::TableIdProvider;
 use table::TableRef;
 
 use crate::error::{CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu};
 use crate::schema::SchemaProvider;
 use crate::{
-    CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef, RegisterSystemTableRequest,
-    RegisterTableRequest, SchemaProviderRef,
+    CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef, RegisterSchemaRequest,
+    RegisterSystemTableRequest, RegisterTableRequest, SchemaProviderRef,
 };
 
 /// Simple in-memory list of catalogs
@@ -41,6 +55,13 @@ impl Default for MemoryCatalogManager {
     }
 }
 
+#[async_trait::async_trait]
+impl TableIdProvider for MemoryCatalogManager {
+    async fn next_table_id(&self) -> table::error::Result<TableId> {
+        Ok(self.table_id.fetch_add(1, Ordering::Relaxed))
+    }
+}
+
 #[async_trait::async_trait]
 impl CatalogManager for MemoryCatalogManager {
     async fn start(&self) -> Result<()> {
@@ -48,10 +69,6 @@ impl CatalogManager for MemoryCatalogManager {
         Ok(())
     }
 
-    async fn next_table_id(&self) -> Result<TableId> {
-        Ok(self.table_id.fetch_add(1, Ordering::Relaxed))
-    }
-
     async fn register_table(&self, request: RegisterTableRequest) -> Result<usize> {
         let catalogs = self.catalogs.write().unwrap();
         let catalog = catalogs
@@ -70,10 +87,30 @@ impl CatalogManager for MemoryCatalogManager {
             .map(|v| if v.is_some() { 0 } else { 1 })
     }
 
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<usize> {
+        let catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get(&request.catalog)
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?;
+        catalog.register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))?;
+        Ok(1)
+    }
+
     async fn register_system_table(&self, _request: RegisterSystemTableRequest) -> Result<()> {
         unimplemented!()
     }
 
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+        let catalogs = self.catalogs.read().unwrap();
+        if let Some(c) = catalogs.get(catalog) {
+            c.schema(schema)
+        } else {
+            Ok(None)
+        }
+    }
+
     fn table(&self, catalog: &str, schema: &str, table_name: &str) -> Result<Option<TableRef>> {
         let c = self.catalogs.read().unwrap();
         let catalog = if let Some(c) = c.get(catalog) {

src/catalog/src/remote.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::fmt::Debug;
 use std::pin::Pin;
 use std::sync::Arc;

src/catalog/src/remote/client.rs

@@ -1,4 +1,19 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::fmt::Debug;
+use std::sync::Arc;
 
 use async_stream::stream;
 use common_telemetry::info;
@@ -10,7 +25,7 @@ use crate::error::{Error, MetaSrvSnafu};
 use crate::remote::{Kv, KvBackend, ValueIter};
 #[derive(Debug)]
 pub struct MetaKvBackend {
-    pub client: MetaClient,
+    pub client: Arc<MetaClient>,
 }
 
 /// Implement `KvBackend` trait for `MetaKvBackend` instead of opendal's `Accessor` since

src/catalog/src/remote/manager.rs

@@ -1,19 +1,30 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::collections::HashMap;
 use std::pin::Pin;
 use std::sync::Arc;
-use std::time::Duration;
 
 use arc_swap::ArcSwap;
 use async_stream::stream;
-use backoff::exponential::ExponentialBackoffBuilder;
-use backoff::ExponentialBackoff;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID};
 use common_catalog::{
     build_catalog_prefix, build_schema_prefix, build_table_global_prefix, CatalogKey, CatalogValue,
     SchemaKey, SchemaValue, TableGlobalKey, TableGlobalValue, TableRegionalKey, TableRegionalValue,
 };
-use common_telemetry::{debug, error, info};
+use common_telemetry::{debug, info};
 use futures::Stream;
 use futures_util::StreamExt;
 use snafu::{OptionExt, ResultExt};
@@ -25,14 +36,14 @@ use table::TableRef;
 use tokio::sync::Mutex;
 
 use crate::error::{
-    BumpTableIdSnafu, CatalogNotFoundSnafu, CreateTableSnafu, InvalidCatalogValueSnafu,
-    OpenTableSnafu, ParseTableIdSnafu, SchemaNotFoundSnafu, TableExistsSnafu,
+    CatalogNotFoundSnafu, CreateTableSnafu, InvalidCatalogValueSnafu, InvalidTableSchemaSnafu,
+    OpenTableSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu,
 };
-use crate::error::{InvalidTableSchemaSnafu, Result};
 use crate::remote::{Kv, KvBackendRef};
 use crate::{
     handle_system_table_request, CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef,
-    RegisterSystemTableRequest, RegisterTableRequest, SchemaProvider, SchemaProviderRef,
+    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, SchemaProvider,
+    SchemaProviderRef,
 };
 
 /// Catalog manager based on metasrv.
@@ -65,6 +76,7 @@ impl RemoteCatalogManager {
 
     fn new_catalog_provider(&self, catalog_name: &str) -> CatalogProviderRef {
         Arc::new(RemoteCatalogProvider {
+            node_id: self.node_id,
             catalog_name: catalog_name.to_string(),
             backend: self.backend.clone(),
             schemas: Default::default(),
@@ -126,8 +138,6 @@ impl RemoteCatalogManager {
     }
 
     /// Iterate over all table entries on metasrv
-    /// TODO(hl): table entries with different version is not currently considered.
-    /// Ideally deprecated table entry must be deleted when deregistering from catalog.
     async fn iter_remote_tables(
         &self,
         catalog_name: &str,
@@ -147,7 +157,7 @@ impl RemoteCatalogManager {
                 let table_value = TableGlobalValue::parse(&String::from_utf8_lossy(&v))
                     .context(InvalidCatalogValueSnafu)?;
 
-                debug!(
+                info!(
                     "Found catalog table entry, key: {}, value: {:?}",
                     table_key, table_value
                 );
@@ -232,6 +242,8 @@ impl RemoteCatalogManager {
         schema: SchemaProviderRef,
         mut max_table_id: TableId,
     ) -> Result<()> {
+        info!("initializing tables in {}.{}", catalog_name, schema_name);
+        let mut table_num = 0;
         let mut tables = self.iter_remote_tables(catalog_name, schema_name).await;
         while let Some(r) = tables.next().await {
             let (table_key, table_value) = r?;
@@ -242,7 +254,12 @@ impl RemoteCatalogManager {
                 info!("Max table id: {} -> {}", max_table_id, table_value.id);
                 max_table_id = table_value.id;
             }
+            table_num += 1;
         }
+        info!(
+            "initialized tables in {}.{}, total: {}",
+            catalog_name, schema_name, table_num
+        );
         Ok(())
     }
 
@@ -314,8 +331,19 @@ impl RemoteCatalogManager {
             .with_context(|_| OpenTableSnafu {
                 table_info: format!("{}.{}.{}, id:{}", catalog_name, schema_name, table_name, id,),
             })? {
-            Some(table) => Ok(table),
+            Some(table) => {
+                info!(
+                    "Table opened: {}.{}.{}",
+                    catalog_name, schema_name, table_name
+                );
+                Ok(table)
+            }
             None => {
+                info!(
+                    "Try create table: {}.{}.{}",
+                    catalog_name, schema_name, table_name
+                );
+
                 let schema = meta
                     .schema
                     .clone()
@@ -377,63 +405,6 @@ impl CatalogManager for RemoteCatalogManager {
         Ok(())
     }
 
-    /// Bump table id in a CAS manner with backoff.
-    async fn next_table_id(&self) -> Result<TableId> {
-        let key = common_catalog::consts::TABLE_ID_KEY_PREFIX.as_bytes();
-        let op = || async {
-            // TODO(hl): optimize this get
-            let (prev, prev_bytes) = match self.backend.get(key).await? {
-                None => (MIN_USER_TABLE_ID, vec![]),
-                Some(kv) => (parse_table_id(&kv.1)?, kv.1),
-            };
-
-            match self
-                .backend
-                .compare_and_set(key, &prev_bytes, &(prev + 1).to_le_bytes())
-                .await
-            {
-                Ok(cas_res) => match cas_res {
-                    Ok(_) => Ok(prev),
-                    Err(e) => {
-                        info!("Table id {:?} already occupied", e);
-                        Err(backoff::Error::transient(
-                            BumpTableIdSnafu {
-                                msg: "Table id occupied",
-                            }
-                            .build(),
-                        ))
-                    }
-                },
-                Err(e) => {
-                    error!(e;"Failed to CAS table id");
-                    Err(backoff::Error::permanent(
-                        BumpTableIdSnafu {
-                            msg: format!("Failed to perform CAS operation: {:?}", e),
-                        }
-                        .build(),
-                    ))
-                }
-            }
-        };
-
-        let retry_policy: ExponentialBackoff = ExponentialBackoffBuilder::new()
-            .with_initial_interval(Duration::from_millis(4))
-            .with_multiplier(2.0)
-            .with_max_interval(Duration::from_millis(1000))
-            .with_max_elapsed_time(Some(Duration::from_millis(3000)))
-            .build();
-
-        backoff::future::retry(retry_policy, op).await.map_err(|e| {
-            BumpTableIdSnafu {
-                msg: format!(
-                    "Bump table id exceeds max fail times, last error msg: {:?}",
-                    e
-                ),
-            }
-            .build()
-        })
-    }
-
     async fn register_table(&self, request: RegisterTableRequest) -> Result<usize> {
         let catalog_name = request.catalog;
         let schema_name = request.schema;
@@ -456,12 +427,31 @@ impl CatalogManager for RemoteCatalogManager {
         Ok(1)
     }
 
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<usize> {
+        let catalog_name = request.catalog;
+        let schema_name = request.schema;
+        let catalog_provider = self.catalog(&catalog_name)?.context(CatalogNotFoundSnafu {
+            catalog_name: &catalog_name,
+        })?;
+        let schema_provider = self.new_schema_provider(&catalog_name, &schema_name);
+        catalog_provider.register_schema(schema_name, schema_provider)?;
+        Ok(1)
+    }
+
     async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
         let mut requests = self.system_table_requests.lock().await;
         requests.push(request);
         Ok(())
     }
 
+    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+        self.catalog(catalog)?
+            .context(CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .schema(schema)
+    }
+
     fn table(
         &self,
         catalog_name: &str,
@@ -530,6 +520,7 @@ impl CatalogList for RemoteCatalogManager {
 }
 
 pub struct RemoteCatalogProvider {
+    node_id: u64,
     catalog_name: String,
     backend: KvBackendRef,
     schemas: Arc<ArcSwap<HashMap<String, SchemaProviderRef>>>,
@@ -537,8 +528,9 @@ pub struct RemoteCatalogProvider {
 }
 
 impl RemoteCatalogProvider {
-    pub fn new(catalog_name: String, backend: KvBackendRef) -> Self {
+    pub fn new(catalog_name: String, backend: KvBackendRef, node_id: u64) -> Self {
         Self {
+            node_id,
             catalog_name,
             backend,
             schemas: Default::default(),
@@ -546,6 +538,48 @@ impl RemoteCatalogProvider {
         }
     }
 
+    pub fn refresh_schemas(&self) -> Result<()> {
+        let schemas = self.schemas.clone();
+        let schema_prefix = build_schema_prefix(&self.catalog_name);
+        let catalog_name = self.catalog_name.clone();
+        let mutex = self.mutex.clone();
+        let backend = self.backend.clone();
+        let node_id = self.node_id;
+
+        std::thread::spawn(move || {
+            common_runtime::block_on_write(async move {
+                let _guard = mutex.lock().await;
+                let prev_schemas = schemas.load();
+                let mut new_schemas = HashMap::with_capacity(prev_schemas.len() + 1);
+                new_schemas.clone_from(&prev_schemas);
+
+                let mut remote_schemas = backend.range(schema_prefix.as_bytes());
+                while let Some(r) = remote_schemas.next().await {
+                    let Kv(k, _) = r?;
+                    let schema_key = SchemaKey::parse(&String::from_utf8_lossy(&k))
+                        .context(InvalidCatalogValueSnafu)?;
+                    if !new_schemas.contains_key(&schema_key.schema_name) {
+                        new_schemas.insert(
+                            schema_key.schema_name.clone(),
+                            Arc::new(RemoteSchemaProvider::new(
+                                catalog_name.clone(),
+                                schema_key.schema_name,
+                                node_id,
+                                backend.clone(),
+                            )),
+                        );
+                    }
+                }
+                schemas.store(Arc::new(new_schemas));
+                Ok(())
+            })
+        })
+        .join()
+        .unwrap()?;
+
+        Ok(())
+    }
+
     fn build_schema_key(&self, schema_name: impl AsRef<str>) -> SchemaKey {
         SchemaKey {
             catalog_name: self.catalog_name.clone(),
@@ -560,6 +594,7 @@ impl CatalogProvider for RemoteCatalogProvider {
     }
 
     fn schema_names(&self) -> Result<Vec<String>> {
+        self.refresh_schemas()?;
         Ok(self.schemas.load().keys().cloned().collect::<Vec<_>>())
     }
 
@@ -598,20 +633,12 @@ impl CatalogProvider for RemoteCatalogProvider {
     }
 
     fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>> {
+        // TODO(hl): We should refresh whole catalog before calling datafusion's query engine.
+        self.refresh_schemas()?;
         Ok(self.schemas.load().get(name).cloned())
     }
 }
 
-/// Parse u8 slice to `TableId`
-fn parse_table_id(val: &[u8]) -> Result<TableId> {
-    Ok(TableId::from_le_bytes(val.try_into().map_err(|_| {
-        ParseTableIdSnafu {
-            data: format!("{:?}", val),
-        }
-        .build()
-    })?))
-}
-
 pub struct RemoteSchemaProvider {
     catalog_name: String,
     schema_name: String,
@@ -733,17 +760,3 @@ impl SchemaProvider for RemoteSchemaProvider {
         Ok(self.tables.load().contains_key(name))
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_table_id() {
-        assert_eq!(12, parse_table_id(&12_i32.to_le_bytes()).unwrap());
-        let mut data = vec![];
-        data.extend_from_slice(&12_i32.to_le_bytes());
-        data.push(0);
-        assert!(parse_table_id(&data).is_err());
-    }
-}

src/catalog/src/schema.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::sync::Arc;
 

src/catalog/src/system.rs

@@ -1,14 +1,27 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::collections::HashMap;
 use std::sync::Arc;
 
 use common_catalog::consts::{
-    INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID,
-    SYSTEM_CATALOG_TABLE_NAME,
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_NAME,
+    SYSTEM_CATALOG_TABLE_ID, SYSTEM_CATALOG_TABLE_NAME,
 };
 use common_query::logical_plan::Expr;
-use common_query::physical_plan::PhysicalPlanRef;
-use common_query::physical_plan::RuntimeEnv;
+use common_query::physical_plan::{PhysicalPlanRef, RuntimeEnv};
 use common_recordbatch::SendableRecordBatchStream;
 use common_telemetry::debug;
 use common_time::timestamp::Timestamp;
@@ -151,7 +164,8 @@ fn build_system_catalog_schema() -> Schema {
             "timestamp".to_string(),
             ConcreteDataType::timestamp_millis_datatype(),
             false,
-        ),
+        )
+        .with_time_index(true),
         ColumnSchema::new(
             "value".to_string(),
             ConcreteDataType::binary_datatype(),
@@ -170,23 +184,40 @@ fn build_system_catalog_schema() -> Schema {
     ];
 
     // The schema of this table must be valid.
-    SchemaBuilder::try_from(cols)
-        .unwrap()
-        .timestamp_index(Some(2))
-        .build()
-        .unwrap()
+    SchemaBuilder::try_from(cols).unwrap().build().unwrap()
 }
 
 pub fn build_table_insert_request(full_table_name: String, table_id: TableId) -> InsertRequest {
+    build_insert_request(
+        EntryType::Table,
+        full_table_name.as_bytes(),
+        serde_json::to_string(&TableEntryValue { table_id })
+            .unwrap()
+            .as_bytes(),
+    )
+}
+
+pub fn build_schema_insert_request(catalog_name: String, schema_name: String) -> InsertRequest {
+    let full_schema_name = format!("{}.{}", catalog_name, schema_name);
+    build_insert_request(
+        EntryType::Schema,
+        full_schema_name.as_bytes(),
+        serde_json::to_string(&SchemaEntryValue {})
+            .unwrap()
+            .as_bytes(),
+    )
+}
+
+pub fn build_insert_request(entry_type: EntryType, key: &[u8], value: &[u8]) -> InsertRequest {
     let mut columns_values = HashMap::with_capacity(6);
     columns_values.insert(
         "entry_type".to_string(),
-        Arc::new(UInt8Vector::from_slice(&[EntryType::Table as u8])) as _,
+        Arc::new(UInt8Vector::from_slice(&[entry_type as u8])) as _,
     );
 
     columns_values.insert(
         "key".to_string(),
-        Arc::new(BinaryVector::from_slice(&[full_table_name.as_bytes()])) as _,
+        Arc::new(BinaryVector::from_slice(&[key])) as _,
     );
 
     // Timestamp in key part is intentionally left to 0
@@ -197,11 +228,7 @@ pub fn build_table_insert_request(full_table_name: String, table_id: TableId) ->
 
     columns_values.insert(
         "value".to_string(),
-        Arc::new(BinaryVector::from_slice(&[serde_json::to_string(
-            &TableEntryValue { table_id },
-        )
-        .unwrap()
-        .as_bytes()])) as _,
+        Arc::new(BinaryVector::from_slice(&[value])) as _,
     );
 
     columns_values.insert(
@@ -219,6 +246,8 @@ pub fn build_table_insert_request(full_table_name: String, table_id: TableId) ->
     );
 
     InsertRequest {
+        catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        schema_name: DEFAULT_SCHEMA_NAME.to_string(),
         table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
         columns_values,
     }
@@ -324,6 +353,9 @@ pub struct SchemaEntry {
     pub schema_name: String,
 }
 
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SchemaEntryValue;
+
 #[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
 pub struct TableEntry {
     pub catalog_name: String,
@@ -340,13 +372,13 @@ pub struct TableEntryValue {
 #[cfg(test)]
 mod tests {
     use log_store::fs::noop::NoopLogStore;
+    use mito::config::EngineConfig;
+    use mito::engine::MitoEngine;
     use object_store::ObjectStore;
     use storage::config::EngineConfig as StorageEngineConfig;
     use storage::EngineImpl;
     use table::metadata::TableType;
     use table::metadata::TableType::Base;
-    use table_engine::config::EngineConfig;
-    use table_engine::engine::MitoEngine;
     use tempdir::TempDir;
 
     use super::*;

src/catalog/src/tables.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // The `tables` table in system catalog keeps a record of all tables created by user.
 
 use std::any::Any;
@@ -24,8 +38,8 @@ use table::metadata::{TableId, TableInfoRef};
 use table::table::scan::SimpleTableScan;
 use table::{Table, TableRef};
 
-use crate::error::{Error, InsertTableRecordSnafu};
-use crate::system::{build_table_insert_request, SystemCatalogTable};
+use crate::error::{Error, InsertCatalogRecordSnafu};
+use crate::system::{build_schema_insert_request, build_table_insert_request, SystemCatalogTable};
 use crate::{
     format_full_table_name, CatalogListRef, CatalogProvider, SchemaProvider, SchemaProviderRef,
 };
@@ -254,7 +268,20 @@ impl SystemCatalog {
             .system
             .insert(request)
             .await
-            .context(InsertTableRecordSnafu)
+            .context(InsertCatalogRecordSnafu)
+    }
+
+    pub async fn register_schema(
+        &self,
+        catalog: String,
+        schema: String,
+    ) -> crate::error::Result<usize> {
+        let request = build_schema_insert_request(catalog, schema);
+        self.information_schema
+            .system
+            .insert(request)
+            .await
+            .context(InsertCatalogRecordSnafu)
     }
 }
 

src/catalog/tests/mock.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::collections::btree_map::Entry;
 use std::collections::{BTreeMap, HashMap};
 use std::fmt::{Display, Formatter};
@@ -13,7 +27,7 @@ use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, Schema};
 use datatypes::vectors::StringVector;
 use serde::Serializer;
-use table::engine::{EngineContext, TableEngine};
+use table::engine::{EngineContext, TableEngine, TableReference};
 use table::metadata::TableId;
 use table::requests::{AlterTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest};
 use table::test_util::MemTable;
@@ -151,6 +165,7 @@ impl TableEngine for MockTableEngine {
             table_id,
             catalog_name,
             schema_name,
+            vec![0],
         )) as Arc<_>;
 
         let mut tables = self.tables.write().await;
@@ -174,12 +189,28 @@ impl TableEngine for MockTableEngine {
         unimplemented!()
     }
 
-    fn get_table(&self, _ctx: &EngineContext, name: &str) -> table::Result<Option<TableRef>> {
-        futures::executor::block_on(async { Ok(self.tables.read().await.get(name).cloned()) })
+    fn get_table<'a>(
+        &self,
+        _ctx: &EngineContext,
+        table_ref: &'a TableReference,
+    ) -> table::Result<Option<TableRef>> {
+        futures::executor::block_on(async {
+            Ok(self
+                .tables
+                .read()
+                .await
+                .get(&table_ref.to_string())
+                .cloned())
+        })
     }
 
-    fn table_exists(&self, _ctx: &EngineContext, name: &str) -> bool {
-        futures::executor::block_on(async { self.tables.read().await.contains_key(name) })
+    fn table_exists<'a>(&self, _ctx: &EngineContext, table_ref: &'a TableReference) -> bool {
+        futures::executor::block_on(async {
+            self.tables
+                .read()
+                .await
+                .contains_key(&table_ref.to_string())
+        })
     }
 
     async fn drop_table(

src/catalog/tests/remote_catalog_tests.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #![feature(assert_matches)]
 
 mod mock;
@@ -11,8 +25,8 @@ mod tests {
     use catalog::remote::{
         KvBackend, KvBackendRef, RemoteCatalogManager, RemoteCatalogProvider, RemoteSchemaProvider,
     };
-    use catalog::{CatalogManager, CatalogManagerRef, RegisterTableRequest};
-    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID};
+    use catalog::{CatalogList, CatalogManager, RegisterTableRequest};
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
     use common_catalog::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
     use datatypes::schema::Schema;
     use futures_util::StreamExt;
@@ -61,7 +75,9 @@ mod tests {
         );
     }
 
-    async fn prepare_components(node_id: u64) -> (KvBackendRef, TableEngineRef, CatalogManagerRef) {
+    async fn prepare_components(
+        node_id: u64,
+    ) -> (KvBackendRef, TableEngineRef, Arc<RemoteCatalogManager>) {
         let backend = Arc::new(MockKvBackend::default()) as KvBackendRef;
         let table_engine = Arc::new(MockTableEngine::default());
         let catalog_manager =
@@ -207,6 +223,7 @@ mod tests {
         let catalog = Arc::new(RemoteCatalogProvider::new(
             catalog_name.clone(),
             backend.clone(),
+            node_id,
         ));
 
         // register catalog to catalog manager
@@ -277,19 +294,4 @@ mod tests {
             new_catalog.schema_names().unwrap().into_iter().collect()
         )
     }
-
-    #[tokio::test]
-    async fn test_next_table_id() {
-        let node_id = 42;
-        let (_, _, catalog_manager) = prepare_components(node_id).await;
-        assert_eq!(
-            MIN_USER_TABLE_ID,
-            catalog_manager.next_table_id().await.unwrap()
-        );
-
-        assert_eq!(
-            MIN_USER_TABLE_ID + 1,
-            catalog_manager.next_table_id().await.unwrap()
-        );
-    }
 }

src/client/Cargo.toml

@@ -2,6 +2,7 @@
 name = "client"
 version = "0.1.0"
 edition = "2021"
+license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -12,6 +13,7 @@ common-error = { path = "../common/error" }
 common-grpc = { path = "../common/grpc" }
 common-query = { path = "../common/query" }
 common-recordbatch = { path = "../common/recordbatch" }
+common-insert = { path = "../common/insert" }
 common-time = { path = "../common/time" }
 datafusion = { git = "https://github.com/apache/arrow-datafusion.git", branch = "arrow2", features = [
     "simd",

src/client/examples/insert.rs

@@ -1,8 +1,22 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::collections::HashMap;
 
-use api::v1::{codec::InsertBatch, *};
+use api::v1::codec::InsertBatch;
+use api::v1::*;
 use client::{Client, Database};
-
 fn main() {
     tracing::subscriber::set_global_default(tracing_subscriber::FmtSubscriber::builder().finish())
         .unwrap();
@@ -16,11 +30,13 @@ async fn run() {
     let db = Database::new("greptime", client);
 
     let expr = InsertExpr {
+        schema_name: "public".to_string(),
         table_name: "demo".to_string(),
         expr: Some(insert_expr::Expr::Values(insert_expr::Values {
             values: insert_batches(),
         })),
         options: HashMap::default(),
+        region_number: 0,
     };
     db.insert(expr).await.unwrap();
 }

src/client/examples/logical.rs

@@ -1,12 +1,25 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use api::v1::{ColumnDataType, ColumnDef, CreateExpr};
-use client::{admin::Admin, Client, Database};
+use client::admin::Admin;
+use client::{Client, Database};
 use prost_09::Message;
-use substrait_proto::protobuf::{
-    plan_rel::RelType as PlanRelType,
-    read_rel::{NamedTable, ReadType},
-    rel::RelType,
-    PlanRel, ReadRel, Rel,
-};
+use substrait_proto::protobuf::plan_rel::RelType as PlanRelType;
+use substrait_proto::protobuf::read_rel::{NamedTable, ReadType};
+use substrait_proto::protobuf::rel::RelType;
+use substrait_proto::protobuf::{PlanRel, ReadRel, Rel};
 use tracing::{event, Level};
 
 fn main() {
@@ -49,6 +62,8 @@ async fn run() {
         primary_keys: vec!["key".to_string()],
         create_if_not_exists: false,
         table_options: Default::default(),
+        table_id: Some(1024),
+        region_ids: vec![0],
     };
 
     let admin = Admin::new("create table", client.clone());

src/client/examples/physical.rs

@@ -1,10 +1,24 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::sync::Arc;
 
 use client::{Client, Database};
 use common_grpc::MockExecution;
-use datafusion::physical_plan::{
-    expressions::Column, projection::ProjectionExec, ExecutionPlan, PhysicalExpr,
-};
+use datafusion::physical_plan::expressions::Column;
+use datafusion::physical_plan::projection::ProjectionExec;
+use datafusion::physical_plan::{ExecutionPlan, PhysicalExpr};
 use tracing::{event, Level};
 
 fn main() {

src/client/examples/select.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use client::{Client, Database, Select};
 use tracing::{event, Level};
 

src/client/src/admin.rs

@@ -1,12 +1,24 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use api::v1::*;
 use common_error::prelude::StatusCode;
 use common_query::Output;
 use snafu::prelude::*;
 
 use crate::database::PROTOCOL_VERSION;
-use crate::error;
-use crate::Client;
-use crate::Result;
+use crate::{error, Client, Result};
 
 #[derive(Clone, Debug)]
 pub struct Admin {
@@ -70,6 +82,17 @@ impl Admin {
         );
         Ok(results)
     }
+
+    pub async fn create_database(&self, expr: CreateDatabaseExpr) -> Result<AdminResult> {
+        let header = ExprHeader {
+            version: PROTOCOL_VERSION,
+        };
+        let expr = AdminExpr {
+            header: Some(header),
+            expr: Some(admin_expr::Expr::CreateDatabase(expr)),
+        };
+        Ok(self.do_requests(vec![expr]).await?.remove(0))
+    }
 }
 
 pub fn admin_result_to_output(admin_result: AdminResult) -> Result<Output> {

src/client/src/client.rs

@@ -1,17 +1,28 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::sync::Arc;
 
 use api::v1::greptime_client::GreptimeClient;
 use api::v1::*;
 use common_grpc::channel_manager::ChannelManager;
 use parking_lot::RwLock;
-use snafu::OptionExt;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 use tonic::transport::Channel;
 
-use crate::error;
-use crate::load_balance::LoadBalance;
-use crate::load_balance::Loadbalancer;
-use crate::Result;
+use crate::load_balance::{LoadBalance, Loadbalancer};
+use crate::{error, Result};
 
 #[derive(Clone, Debug, Default)]
 pub struct Client {
@@ -128,8 +139,11 @@ impl Client {
             .context(error::IllegalGrpcClientStateSnafu {
                 err_msg: "No available peer found",
             })?;
-        let mut client = self.make_client(peer)?;
-        let result = client.batch(req).await.context(error::TonicStatusSnafu)?;
+        let mut client = self.make_client(&peer)?;
+        let result = client
+            .batch(req)
+            .await
+            .context(error::TonicStatusSnafu { addr: peer })?;
         Ok(result.into_inner())
     }
 

src/client/src/database.rs

@@ -1,31 +1,40 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::sync::Arc;
 
-use api::helper::ColumnDataTypeWrapper;
 use api::v1::codec::SelectResult as GrpcSelectResult;
+use api::v1::column::SemanticType;
 use api::v1::{
-    column::Values, object_expr, object_result, select_expr, Column, ColumnDataType,
-    DatabaseRequest, ExprHeader, InsertExpr, MutateResult as GrpcMutateResult, ObjectExpr,
-    ObjectResult as GrpcObjectResult, PhysicalPlan, SelectExpr,
+    object_expr, object_result, select_expr, DatabaseRequest, ExprHeader, InsertExpr,
+    MutateResult as GrpcMutateResult, ObjectExpr, ObjectResult as GrpcObjectResult, PhysicalPlan,
+    SelectExpr,
 };
-use common_base::BitVec;
 use common_error::status_code::StatusCode;
-use common_grpc::AsExcutionPlan;
-use common_grpc::DefaultAsPlanImpl;
+use common_grpc::{AsExcutionPlan, DefaultAsPlanImpl};
+use common_insert::column_to_vector;
 use common_query::Output;
 use common_recordbatch::{RecordBatch, RecordBatches};
-use common_time::date::Date;
-use common_time::datetime::DateTime;
-use common_time::timestamp::Timestamp;
 use datafusion::physical_plan::ExecutionPlan;
 use datatypes::prelude::*;
 use datatypes::schema::{ColumnSchema, Schema};
 use snafu::{ensure, OptionExt, ResultExt};
 
-use crate::error;
-use crate::{
-    error::{ConvertSchemaSnafu, DatanodeSnafu, DecodeSelectSnafu, EncodePhysicalSnafu},
-    Client, Result,
+use crate::error::{
+    ColumnToVectorSnafu, ConvertSchemaSnafu, DatanodeSnafu, DecodeSelectSnafu, EncodePhysicalSnafu,
 };
+use crate::{error, Client, Result};
 
 pub const PROTOCOL_VERSION: u32 = 1;
 
@@ -124,8 +133,6 @@ impl Database {
         obj_result.try_into()
     }
 
-    // TODO(jiachun) update/delete
-
     pub async fn object(&self, expr: ObjectExpr) -> Result<GrpcObjectResult> {
         let res = self.objects(vec![expr]).await?.pop().unwrap();
         Ok(res)
@@ -201,7 +208,9 @@ impl TryFrom<ObjectResult> for Output {
                 let vectors = select
                     .columns
                     .iter()
-                    .map(|column| column_to_vector(column, select.row_count))
+                    .map(|column| {
+                        column_to_vector(column, select.row_count).context(ColumnToVectorSnafu)
+                    })
                     .collect::<Result<Vec<VectorRef>>>()?;
 
                 let column_schemas = select
@@ -211,7 +220,12 @@ impl TryFrom<ObjectResult> for Output {
                     .map(|(column, vector)| {
                         let datatype = vector.data_type();
                         // nullable or not, does not affect the output
-                        ColumnSchema::new(&column.column_name, datatype, true)
+                        let mut column_schema =
+                            ColumnSchema::new(&column.column_name, datatype, true);
+                        if column.semantic_type == SemanticType::Timestamp as i32 {
+                            column_schema = column_schema.with_time_index(true);
+                        }
+                        column_schema
                     })
                     .collect::<Vec<ColumnSchema>>();
 
@@ -239,100 +253,11 @@ impl TryFrom<ObjectResult> for Output {
     }
 }
 
-fn column_to_vector(column: &Column, rows: u32) -> Result<VectorRef> {
-    let wrapper =
-        ColumnDataTypeWrapper::try_new(column.datatype).context(error::ColumnDataTypeSnafu)?;
-    let column_datatype = wrapper.datatype();
-
-    let rows = rows as usize;
-    let mut vector = VectorBuilder::with_capacity(wrapper.into(), rows);
-
-    if let Some(values) = &column.values {
-        let values = collect_column_values(column_datatype, values);
-        let mut values_iter = values.into_iter();
-
-        let null_mask = BitVec::from_slice(&column.null_mask);
-        let mut nulls_iter = null_mask.iter().by_vals().fuse();
-
-        for i in 0..rows {
-            if let Some(true) = nulls_iter.next() {
-                vector.push_null();
-            } else {
-                let value_ref = values_iter.next().context(error::InvalidColumnProtoSnafu {
-                    err_msg: format!(
-                        "value not found at position {} of column {}",
-                        i, &column.column_name
-                    ),
-                })?;
-                vector
-                    .try_push_ref(value_ref)
-                    .context(error::CreateVectorSnafu)?;
-            }
-        }
-    } else {
-        (0..rows).for_each(|_| vector.push_null());
-    }
-    Ok(vector.finish())
-}
-
-fn collect_column_values(column_datatype: ColumnDataType, values: &Values) -> Vec<ValueRef> {
-    macro_rules! collect_values {
-        ($value: expr, $mapper: expr) => {
-            $value.iter().map($mapper).collect::<Vec<ValueRef>>()
-        };
-    }
-
-    match column_datatype {
-        ColumnDataType::Boolean => collect_values!(values.bool_values, |v| ValueRef::from(*v)),
-        ColumnDataType::Int8 => collect_values!(values.i8_values, |v| ValueRef::from(*v as i8)),
-        ColumnDataType::Int16 => {
-            collect_values!(values.i16_values, |v| ValueRef::from(*v as i16))
-        }
-        ColumnDataType::Int32 => {
-            collect_values!(values.i32_values, |v| ValueRef::from(*v))
-        }
-        ColumnDataType::Int64 => {
-            collect_values!(values.i64_values, |v| ValueRef::from(*v as i64))
-        }
-        ColumnDataType::Uint8 => {
-            collect_values!(values.u8_values, |v| ValueRef::from(*v as u8))
-        }
-        ColumnDataType::Uint16 => {
-            collect_values!(values.u16_values, |v| ValueRef::from(*v as u16))
-        }
-        ColumnDataType::Uint32 => {
-            collect_values!(values.u32_values, |v| ValueRef::from(*v))
-        }
-        ColumnDataType::Uint64 => {
-            collect_values!(values.u64_values, |v| ValueRef::from(*v as u64))
-        }
-        ColumnDataType::Float32 => collect_values!(values.f32_values, |v| ValueRef::from(*v)),
-        ColumnDataType::Float64 => collect_values!(values.f64_values, |v| ValueRef::from(*v)),
-        ColumnDataType::Binary => {
-            collect_values!(values.binary_values, |v| ValueRef::from(v.as_slice()))
-        }
-        ColumnDataType::String => {
-            collect_values!(values.string_values, |v| ValueRef::from(v.as_str()))
-        }
-        ColumnDataType::Date => {
-            collect_values!(values.date_values, |v| ValueRef::Date(Date::new(*v)))
-        }
-        ColumnDataType::Datetime => {
-            collect_values!(values.datetime_values, |v| ValueRef::DateTime(
-                DateTime::new(*v)
-            ))
-        }
-        ColumnDataType::Timestamp => {
-            collect_values!(values.ts_millis_values, |v| ValueRef::Timestamp(
-                Timestamp::from_millis(*v)
-            ))
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
-    use datanode::server::grpc::select::{null_mask, values};
+    use api::helper::ColumnDataTypeWrapper;
+    use api::v1::Column;
+    use common_grpc::select::{null_mask, values};
     use datatypes::vectors::{
         BinaryVector, BooleanVector, DateTimeVector, DateVector, Float32Vector, Float64Vector,
         Int16Vector, Int32Vector, Int64Vector, Int8Vector, StringVector, UInt16Vector,

src/client/src/error.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::sync::Arc;
 
@@ -25,8 +39,9 @@ pub enum Error {
     #[snafu(display("Missing result header"))]
     MissingHeader,
 
-    #[snafu(display("Tonic internal error, source: {}", source))]
+    #[snafu(display("Tonic internal error, addr: {}, source: {}", addr, source))]
     TonicStatus {
+        addr: String,
         source: tonic::Status,
         backtrace: Backtrace,
     },
@@ -47,24 +62,12 @@ pub enum Error {
     #[snafu(display("Mutate result has failure {}", failure))]
     MutateFailure { failure: u32, backtrace: Backtrace },
 
-    #[snafu(display("Invalid column proto: {}", err_msg))]
-    InvalidColumnProto {
-        err_msg: String,
-        backtrace: Backtrace,
-    },
-
     #[snafu(display("Column datatype error, source: {}", source))]
     ColumnDataType {
         #[snafu(backtrace)]
         source: api::error::Error,
     },
 
-    #[snafu(display("Failed to create vector, source: {}", source))]
-    CreateVector {
-        #[snafu(backtrace)]
-        source: datatypes::error::Error,
-    },
-
     #[snafu(display("Failed to create RecordBatches, source: {}", source))]
     CreateRecordBatches {
         #[snafu(backtrace)]
@@ -96,6 +99,12 @@ pub enum Error {
         #[snafu(backtrace)]
         source: common_grpc::error::Error,
     },
+
+    #[snafu(display("Failed to convert column to vector, source: {}", source))]
+    ColumnToVector {
+        #[snafu(backtrace)]
+        source: common_insert::error::Error,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -111,15 +120,13 @@ impl ErrorExt for Error {
             | Error::Datanode { .. }
             | Error::EncodePhysical { .. }
             | Error::MutateFailure { .. }
-            | Error::InvalidColumnProto { .. }
             | Error::ColumnDataType { .. }
             | Error::MissingField { .. } => StatusCode::Internal,
-            Error::ConvertSchema { source } | Error::CreateVector { source } => {
-                source.status_code()
-            }
+            Error::ConvertSchema { source } => source.status_code(),
             Error::CreateRecordBatches { source } => source.status_code(),
             Error::CreateChannel { source, .. } => source.status_code(),
             Error::IllegalGrpcClientState { .. } => StatusCode::Unexpected,
+            Error::ColumnToVector { source, .. } => source.status_code(),
         }
     }
 

src/client/src/lib.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 pub mod admin;
 mod client;
 mod database;
@@ -6,8 +20,6 @@ pub mod load_balance;
 
 pub use api;
 
-pub use self::{
-    client::Client,
-    database::{Database, ObjectResult, Select},
-    error::{Error, Result},
-};
+pub use self::client::Client;
+pub use self::database::{Database, ObjectResult, Select};
+pub use self::error::{Error, Result};

src/client/src/load_balance.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use enum_dispatch::enum_dispatch;
 use rand::seq::SliceRandom;
 

src/cmd/Cargo.toml

@@ -3,6 +3,7 @@ name = "cmd"
 version = "0.1.0"
 edition = "2021"
 default-run = "greptime"
+license = "Apache-2.0"
 
 [[bin]]
 name = "greptime"
@@ -18,6 +19,7 @@ datanode = { path = "../datanode" }
 frontend = { path = "../frontend" }
 futures = "0.3"
 meta-srv = { path = "../meta-srv" }
+serde = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }
 tokio = { version = "1.18", features = ["full"] }
 toml = "0.5"

src/cmd/src/bin/greptime.rs

@@ -1,12 +1,23 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::fmt;
 
 use clap::Parser;
-use cmd::datanode;
 use cmd::error::Result;
-use cmd::frontend;
-use cmd::metasrv;
-use common_telemetry::logging::error;
-use common_telemetry::logging::info;
+use cmd::{datanode, frontend, metasrv, standalone};
+use common_telemetry::logging::{error, info};
 
 #[derive(Parser)]
 #[clap(name = "greptimedb")]
@@ -33,6 +44,8 @@ enum SubCommand {
     Frontend(frontend::Command),
     #[clap(name = "metasrv")]
     Metasrv(metasrv::Command),
+    #[clap(name = "standalone")]
+    Standalone(standalone::Command),
 }
 
 impl SubCommand {
@@ -41,6 +54,7 @@ impl SubCommand {
             SubCommand::Datanode(cmd) => cmd.run().await,
             SubCommand::Frontend(cmd) => cmd.run().await,
             SubCommand::Metasrv(cmd) => cmd.run().await,
+            SubCommand::Standalone(cmd) => cmd.run().await,
         }
     }
 }
@@ -51,6 +65,7 @@ impl fmt::Display for SubCommand {
             SubCommand::Datanode(..) => write!(f, "greptime-datanode"),
             SubCommand::Frontend(..) => write!(f, "greptime-frontend"),
             SubCommand::Metasrv(..) => write!(f, "greptime-metasrv"),
+            SubCommand::Standalone(..) => write!(f, "greptime-standalone"),
         }
     }
 }

src/cmd/src/datanode.rs

@@ -1,6 +1,21 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use clap::Parser;
 use common_telemetry::logging;
-use datanode::datanode::{Datanode, DatanodeOptions, Mode};
+use datanode::datanode::{Datanode, DatanodeOptions};
+use frontend::frontend::Mode;
 use snafu::ResultExt;
 
 use crate::error::{Error, MissingConfigSnafu, Result, StartDatanodeSnafu};
@@ -36,14 +51,10 @@ struct StartCommand {
     #[clap(long)]
     node_id: Option<u64>,
     #[clap(long)]
-    http_addr: Option<String>,
-    #[clap(long)]
     rpc_addr: Option<String>,
     #[clap(long)]
     mysql_addr: Option<String>,
     #[clap(long)]
-    postgres_addr: Option<String>,
-    #[clap(long)]
     metasrv_addr: Option<String>,
     #[clap(short, long)]
     config_file: Option<String>,
@@ -75,43 +86,27 @@ impl TryFrom<StartCommand> for DatanodeOptions {
             DatanodeOptions::default()
         };
 
-        if let Some(addr) = cmd.http_addr {
-            opts.http_addr = addr;
-        }
         if let Some(addr) = cmd.rpc_addr {
             opts.rpc_addr = addr;
         }
         if let Some(addr) = cmd.mysql_addr {
             opts.mysql_addr = addr;
         }
-        if let Some(addr) = cmd.postgres_addr {
-            opts.postgres_addr = addr;
+
+        if let Some(node_id) = cmd.node_id {
+            opts.node_id = Some(node_id);
         }
 
-        match (cmd.metasrv_addr, cmd.node_id) {
-            (Some(meta_addr), Some(node_id)) => {
-                // Running mode is only set to Distributed when
-                // both metasrv addr and node id are set in
-                // commandline options
-                opts.meta_client_opts.metasrv_addr = meta_addr;
-                opts.node_id = node_id;
-                opts.mode = Mode::Distributed;
-            }
-            (None, None) => {
-                opts.mode = Mode::Standalone;
-            }
-            (None, Some(_)) => {
-                return MissingConfigSnafu {
-                    msg: "Missing metasrv address option",
-                }
-                .fail();
-            }
-            (Some(_), None) => {
-                return MissingConfigSnafu {
-                    msg: "Missing node id option",
-                }
-                .fail();
+        if let Some(meta_addr) = cmd.metasrv_addr {
+            opts.meta_client_opts.metasrv_addr = meta_addr;
+            opts.mode = Mode::Distributed;
+        }
+
+        if let (Mode::Distributed, None) = (&opts.mode, &opts.node_id) {
+            return MissingConfigSnafu {
+                msg: "Missing node id option",
             }
+            .fail();
         }
         Ok(opts)
     }
@@ -119,7 +114,10 @@ impl TryFrom<StartCommand> for DatanodeOptions {
 
 #[cfg(test)]
 mod tests {
+    use std::assert_matches::assert_matches;
+
     use datanode::datanode::ObjectStoreConfig;
+    use frontend::frontend::Mode;
 
     use super::*;
 
@@ -127,10 +125,8 @@ mod tests {
     fn test_read_from_config_file() {
         let cmd = StartCommand {
             node_id: None,
-            http_addr: None,
             rpc_addr: None,
             mysql_addr: None,
-            postgres_addr: None,
             metasrv_addr: None,
             config_file: Some(format!(
                 "{}/../../config/datanode.example.toml",
@@ -138,10 +134,9 @@ mod tests {
             )),
         };
         let options: DatanodeOptions = cmd.try_into().unwrap();
-        assert_eq!("0.0.0.0:3000".to_string(), options.http_addr);
-        assert_eq!("0.0.0.0:3001".to_string(), options.rpc_addr);
+        assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
         assert_eq!("/tmp/greptimedb/wal".to_string(), options.wal_dir);
-        assert_eq!("0.0.0.0:3306".to_string(), options.mysql_addr);
+        assert_eq!("127.0.0.1:3306".to_string(), options.mysql_addr);
         assert_eq!(4, options.mysql_runtime_size);
         assert_eq!(
             "1.1.1.1:3002".to_string(),
@@ -149,10 +144,7 @@ mod tests {
         );
         assert_eq!(5000, options.meta_client_opts.connect_timeout_millis);
         assert_eq!(3000, options.meta_client_opts.timeout_millis);
-        assert!(options.meta_client_opts.tcp_nodelay);
-
-        assert_eq!("0.0.0.0:5432".to_string(), options.postgres_addr);
-        assert_eq!(4, options.postgres_runtime_size);
+        assert!(!options.meta_client_opts.tcp_nodelay);
 
         match options.storage {
             ObjectStoreConfig::File { data_dir } => {
@@ -167,10 +159,8 @@ mod tests {
             Mode::Standalone,
             DatanodeOptions::try_from(StartCommand {
                 node_id: None,
-                http_addr: None,
                 rpc_addr: None,
                 mysql_addr: None,
-                postgres_addr: None,
                 metasrv_addr: None,
                 config_file: None
             })
@@ -178,40 +168,51 @@ mod tests {
             .mode
         );
 
-        assert_eq!(
-            Mode::Distributed,
-            DatanodeOptions::try_from(StartCommand {
-                node_id: Some(42),
-                http_addr: None,
-                rpc_addr: None,
-                mysql_addr: None,
-                postgres_addr: None,
-                metasrv_addr: Some("127.0.0.1:3002".to_string()),
-                config_file: None
-            })
-            .unwrap()
-            .mode
-        );
+        let mode = DatanodeOptions::try_from(StartCommand {
+            node_id: Some(42),
+            rpc_addr: None,
+            mysql_addr: None,
+            metasrv_addr: Some("127.0.0.1:3002".to_string()),
+            config_file: None,
+        })
+        .unwrap()
+        .mode;
+        assert_matches!(mode, Mode::Distributed);
 
         assert!(DatanodeOptions::try_from(StartCommand {
             node_id: None,
-            http_addr: None,
             rpc_addr: None,
             mysql_addr: None,
-            postgres_addr: None,
             metasrv_addr: Some("127.0.0.1:3002".to_string()),
             config_file: None,
         })
         .is_err());
-        assert!(DatanodeOptions::try_from(StartCommand {
+
+        // Providing node_id but leave metasrv_addr absent is ok since metasrv_addr has default value
+        DatanodeOptions::try_from(StartCommand {
             node_id: Some(42),
-            http_addr: None,
             rpc_addr: None,
             mysql_addr: None,
-            postgres_addr: None,
             metasrv_addr: None,
             config_file: None,
         })
-        .is_err());
+        .unwrap();
+    }
+
+    #[test]
+    fn test_merge_config() {
+        let dn_opts = DatanodeOptions::try_from(StartCommand {
+            node_id: None,
+            rpc_addr: None,
+            mysql_addr: None,
+            metasrv_addr: None,
+            config_file: Some(format!(
+                "{}/../../config/datanode.example.toml",
+                std::env::current_dir().unwrap().as_path().to_str().unwrap()
+            )),
+        })
+        .unwrap();
+        assert_eq!(Some(42), dn_opts.node_id);
+        assert_eq!("1.1.1.1:3002", dn_opts.meta_client_opts.metasrv_addr);
     }
 }

src/cmd/src/error.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 
 use common_error::prelude::*;
@@ -11,6 +25,12 @@ pub enum Error {
         source: datanode::error::Error,
     },
 
+    #[snafu(display("Failed to build frontend, source: {}", source))]
+    BuildFrontend {
+        #[snafu(backtrace)]
+        source: frontend::error::Error,
+    },
+
     #[snafu(display("Failed to start frontend, source: {}", source))]
     StartFrontend {
         #[snafu(backtrace)]
@@ -38,6 +58,9 @@ pub enum Error {
 
     #[snafu(display("Missing config, msg: {}", msg))]
     MissingConfig { msg: String, backtrace: Backtrace },
+
+    #[snafu(display("Illegal config: {}", msg))]
+    IllegalConfig { msg: String, backtrace: Backtrace },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -51,6 +74,8 @@ impl ErrorExt for Error {
             Error::ReadConfig { .. } | Error::ParseConfig { .. } | Error::MissingConfig { .. } => {
                 StatusCode::InvalidArguments
             }
+            Error::IllegalConfig { .. } => StatusCode::InvalidArguments,
+            Error::BuildFrontend { source, .. } => source.status_code(),
         }
     }
 

src/cmd/src/frontend.rs

@@ -1,5 +1,19 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use clap::Parser;
-use frontend::frontend::{Frontend, FrontendOptions};
+use frontend::frontend::{Frontend, FrontendOptions, Mode};
 use frontend::grpc::GrpcOptions;
 use frontend::influxdb::InfluxdbOptions;
 use frontend::instance::Instance;
@@ -52,12 +66,19 @@ pub struct StartCommand {
     config_file: Option<String>,
     #[clap(short, long)]
     influxdb_enable: Option<bool>,
+    #[clap(long)]
+    metasrv_addr: Option<String>,
 }
 
 impl StartCommand {
     async fn run(self) -> Result<()> {
-        let opts = self.try_into()?;
-        let mut frontend = Frontend::new(opts, Instance::new());
+        let opts: FrontendOptions = self.try_into()?;
+        let mut frontend = Frontend::new(
+            opts.clone(),
+            Instance::try_new(&opts)
+                .await
+                .context(error::StartFrontendSnafu)?,
+        );
         frontend.start().await.context(error::StartFrontendSnafu)
     }
 }
@@ -102,6 +123,16 @@ impl TryFrom<StartCommand> for FrontendOptions {
         if let Some(enable) = cmd.influxdb_enable {
             opts.influxdb_options = Some(InfluxdbOptions { enable });
         }
+        if let Some(metasrv_addr) = cmd.metasrv_addr {
+            opts.metasrv_addr = Some(
+                metasrv_addr
+                    .split(',')
+                    .into_iter()
+                    .map(|x| x.trim().to_string())
+                    .collect::<Vec<String>>(),
+            );
+            opts.mode = Mode::Distributed;
+        }
         Ok(opts)
     }
 }
@@ -120,6 +151,7 @@ mod tests {
             opentsdb_addr: Some("127.0.0.1:4321".to_string()),
             influxdb_enable: Some(false),
             config_file: None,
+            metasrv_addr: None,
         };
 
         let opts: FrontendOptions = command.try_into().unwrap();

src/cmd/src/lib.rs

@@ -1,5 +1,22 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(assert_matches)]
+
 pub mod datanode;
 pub mod error;
 pub mod frontend;
 pub mod metasrv;
+pub mod standalone;
 mod toml_loader;

src/cmd/src/metasrv.rs

@@ -1,13 +1,25 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use clap::Parser;
 use common_telemetry::logging;
 use meta_srv::bootstrap;
 use meta_srv::metasrv::MetaSrvOptions;
 use snafu::ResultExt;
 
-use crate::error;
-use crate::error::Error;
-use crate::error::Result;
-use crate::toml_loader;
+use crate::error::{Error, Result};
+use crate::{error, toml_loader};
 
 #[derive(Parser)]
 pub struct Command {
@@ -92,13 +104,13 @@ mod tests {
     fn test_read_from_cmd() {
         let cmd = StartCommand {
             bind_addr: Some("127.0.0.1:3002".to_string()),
-            server_addr: Some("0.0.0.0:3002".to_string()),
+            server_addr: Some("127.0.0.1:3002".to_string()),
             store_addr: Some("127.0.0.1:2380".to_string()),
             config_file: None,
         };
         let options: MetaSrvOptions = cmd.try_into().unwrap();
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
-        assert_eq!("0.0.0.0:3002".to_string(), options.server_addr);
+        assert_eq!("127.0.0.1:3002".to_string(), options.server_addr);
         assert_eq!("127.0.0.1:2380".to_string(), options.store_addr);
     }
 
@@ -115,8 +127,8 @@ mod tests {
         };
         let options: MetaSrvOptions = cmd.try_into().unwrap();
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
-        assert_eq!("0.0.0.0:3002".to_string(), options.server_addr);
-        assert_eq!("127.0.0.1:2380".to_string(), options.store_addr);
-        assert_eq!(30, options.datanode_lease_secs);
+        assert_eq!("127.0.0.1:3002".to_string(), options.server_addr);
+        assert_eq!("127.0.0.1:2379".to_string(), options.store_addr);
+        assert_eq!(15, options.datanode_lease_secs);
     }
 }

src/cmd/src/standalone.rs

@@ -0,0 +1,289 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use clap::Parser;
+use common_telemetry::info;
+use datanode::datanode::{Datanode, DatanodeOptions, ObjectStoreConfig};
+use datanode::instance::InstanceRef;
+use frontend::frontend::{Frontend, FrontendOptions, Mode};
+use frontend::grpc::GrpcOptions;
+use frontend::influxdb::InfluxdbOptions;
+use frontend::instance::Instance as FeInstance;
+use frontend::mysql::MysqlOptions;
+use frontend::opentsdb::OpentsdbOptions;
+use frontend::postgres::PostgresOptions;
+use frontend::prometheus::PrometheusOptions;
+use serde::{Deserialize, Serialize};
+use snafu::ResultExt;
+use tokio::try_join;
+
+use crate::error::{
+    BuildFrontendSnafu, Error, IllegalConfigSnafu, Result, StartDatanodeSnafu, StartFrontendSnafu,
+};
+use crate::toml_loader;
+
+#[derive(Parser)]
+pub struct Command {
+    #[clap(subcommand)]
+    subcmd: SubCommand,
+}
+
+impl Command {
+    pub async fn run(self) -> Result<()> {
+        self.subcmd.run().await
+    }
+}
+
+#[derive(Parser)]
+enum SubCommand {
+    Start(StartCommand),
+}
+
+impl SubCommand {
+    async fn run(self) -> Result<()> {
+        match self {
+            SubCommand::Start(cmd) => cmd.run().await,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct StandaloneOptions {
+    pub http_addr: Option<String>,
+    pub grpc_options: Option<GrpcOptions>,
+    pub mysql_options: Option<MysqlOptions>,
+    pub postgres_options: Option<PostgresOptions>,
+    pub opentsdb_options: Option<OpentsdbOptions>,
+    pub influxdb_options: Option<InfluxdbOptions>,
+    pub prometheus_options: Option<PrometheusOptions>,
+    pub mode: Mode,
+    pub wal_dir: String,
+    pub storage: ObjectStoreConfig,
+    pub datanode_mysql_addr: String,
+    pub datanode_mysql_runtime_size: usize,
+}
+
+impl Default for StandaloneOptions {
+    fn default() -> Self {
+        Self {
+            http_addr: Some("127.0.0.1:4000".to_string()),
+            grpc_options: Some(GrpcOptions::default()),
+            mysql_options: Some(MysqlOptions::default()),
+            postgres_options: Some(PostgresOptions::default()),
+            opentsdb_options: Some(OpentsdbOptions::default()),
+            influxdb_options: Some(InfluxdbOptions::default()),
+            prometheus_options: Some(PrometheusOptions::default()),
+            mode: Mode::Standalone,
+            wal_dir: "/tmp/greptimedb/wal".to_string(),
+            storage: ObjectStoreConfig::default(),
+            datanode_mysql_addr: "127.0.0.1:3306".to_string(),
+            datanode_mysql_runtime_size: 4,
+        }
+    }
+}
+
+impl StandaloneOptions {
+    fn frontend_options(self) -> FrontendOptions {
+        FrontendOptions {
+            http_addr: self.http_addr,
+            grpc_options: self.grpc_options,
+            mysql_options: self.mysql_options,
+            postgres_options: self.postgres_options,
+            opentsdb_options: self.opentsdb_options,
+            influxdb_options: self.influxdb_options,
+            prometheus_options: self.prometheus_options,
+            mode: self.mode,
+            datanode_rpc_addr: "127.0.0.1:3001".to_string(),
+            metasrv_addr: None,
+        }
+    }
+
+    fn datanode_options(self) -> DatanodeOptions {
+        DatanodeOptions {
+            wal_dir: self.wal_dir,
+            storage: self.storage,
+            mysql_addr: self.datanode_mysql_addr,
+            mysql_runtime_size: self.datanode_mysql_runtime_size,
+            ..Default::default()
+        }
+    }
+}
+
+#[derive(Debug, Parser)]
+struct StartCommand {
+    #[clap(long)]
+    http_addr: Option<String>,
+    #[clap(long)]
+    rpc_addr: Option<String>,
+    #[clap(long)]
+    mysql_addr: Option<String>,
+    #[clap(long)]
+    postgres_addr: Option<String>,
+    #[clap(long)]
+    opentsdb_addr: Option<String>,
+    #[clap(short, long)]
+    influxdb_enable: bool,
+    #[clap(short, long)]
+    config_file: Option<String>,
+}
+
+impl StartCommand {
+    async fn run(self) -> Result<()> {
+        let config_file = self.config_file.clone();
+        let fe_opts = FrontendOptions::try_from(self)?;
+        let dn_opts: DatanodeOptions = {
+            let opts: StandaloneOptions = if let Some(path) = config_file {
+                toml_loader::from_file!(&path)?
+            } else {
+                StandaloneOptions::default()
+            };
+            opts.datanode_options()
+        };
+
+        info!(
+            "Standalone frontend options: {:#?}, datanode options: {:#?}",
+            fe_opts, dn_opts
+        );
+
+        let mut datanode = Datanode::new(dn_opts.clone())
+            .await
+            .context(StartDatanodeSnafu)?;
+        let mut frontend = build_frontend(fe_opts, &dn_opts, datanode.get_instance()).await?;
+
+        try_join!(
+            async { datanode.start().await.context(StartDatanodeSnafu) },
+            async { frontend.start().await.context(StartFrontendSnafu) }
+        )?;
+
+        Ok(())
+    }
+}
+
+/// Build frontend instance in standalone mode
+async fn build_frontend(
+    fe_opts: FrontendOptions,
+    dn_opts: &DatanodeOptions,
+    datanode_instance: InstanceRef,
+) -> Result<Frontend<FeInstance>> {
+    let grpc_server_addr = &dn_opts.rpc_addr;
+    info!(
+        "Build frontend with datanode gRPC addr: {}",
+        grpc_server_addr
+    );
+    let mut frontend_instance = FeInstance::try_new(&fe_opts)
+        .await
+        .context(BuildFrontendSnafu)?;
+    frontend_instance.set_catalog_manager(datanode_instance.catalog_manager().clone());
+    frontend_instance.set_script_handler(datanode_instance);
+    Ok(Frontend::new(fe_opts, frontend_instance))
+}
+
+impl TryFrom<StartCommand> for FrontendOptions {
+    type Error = Error;
+
+    fn try_from(cmd: StartCommand) -> std::result::Result<Self, Self::Error> {
+        let opts: StandaloneOptions = if let Some(path) = cmd.config_file {
+            toml_loader::from_file!(&path)?
+        } else {
+            StandaloneOptions::default()
+        };
+
+        let mut opts = opts.frontend_options();
+
+        opts.mode = Mode::Standalone;
+
+        if let Some(addr) = cmd.http_addr {
+            opts.http_addr = Some(addr);
+        }
+        if let Some(addr) = cmd.rpc_addr {
+            // frontend grpc addr conflict with datanode default grpc addr
+            let datanode_grpc_addr = DatanodeOptions::default().rpc_addr;
+            if addr == datanode_grpc_addr {
+                return IllegalConfigSnafu {
+                    msg: format!(
+                        "gRPC listen address conflicts with datanode reserved gRPC addr: {}",
+                        datanode_grpc_addr
+                    ),
+                }
+                .fail();
+            }
+            opts.grpc_options = Some(GrpcOptions {
+                addr,
+                ..Default::default()
+            });
+        }
+
+        if let Some(addr) = cmd.mysql_addr {
+            opts.mysql_options = Some(MysqlOptions {
+                addr,
+                ..Default::default()
+            })
+        }
+        if let Some(addr) = cmd.postgres_addr {
+            opts.postgres_options = Some(PostgresOptions {
+                addr,
+                ..Default::default()
+            })
+        }
+
+        if let Some(addr) = cmd.opentsdb_addr {
+            opts.opentsdb_options = Some(OpentsdbOptions {
+                addr,
+                ..Default::default()
+            });
+        }
+
+        if cmd.influxdb_enable {
+            opts.influxdb_options = Some(InfluxdbOptions { enable: true });
+        }
+
+        Ok(opts)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_read_config_file() {
+        let cmd = StartCommand {
+            http_addr: None,
+            rpc_addr: None,
+            mysql_addr: None,
+            postgres_addr: None,
+            opentsdb_addr: None,
+            config_file: Some(format!(
+                "{}/../../config/standalone.example.toml",
+                std::env::current_dir().unwrap().as_path().to_str().unwrap()
+            )),
+            influxdb_enable: false,
+        };
+
+        let fe_opts = FrontendOptions::try_from(cmd).unwrap();
+        assert_eq!(Mode::Standalone, fe_opts.mode);
+        assert_eq!("127.0.0.1:3001".to_string(), fe_opts.datanode_rpc_addr);
+        assert_eq!(Some("127.0.0.1:4000".to_string()), fe_opts.http_addr);
+        assert_eq!(
+            "127.0.0.1:4001".to_string(),
+            fe_opts.grpc_options.unwrap().addr
+        );
+        assert_eq!(
+            "127.0.0.1:4002",
+            fe_opts.mysql_options.as_ref().unwrap().addr
+        );
+        assert_eq!(2, fe_opts.mysql_options.as_ref().unwrap().runtime_size);
+        assert!(fe_opts.influxdb_options.as_ref().unwrap().enable);
+    }
+}

src/cmd/src/toml_loader.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 macro_rules! from_file {
     ($path: expr) => {
         toml::from_str(

src/common/base/Cargo.toml

@@ -2,6 +2,7 @@
 name = "common-base"
 version = "0.1.0"
 edition = "2021"
+license = "Apache-2.0"
 
 [dependencies]
 bitvec = "1.0"

src/common/base/src/bitset.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use bitvec::prelude as bv;
 
 // `Lsb0` provides the best codegen for bit manipulation,

src/common/base/src/buffer.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 use std::io::{Read, Write};
 

src/common/base/src/bytes.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::ops::Deref;
 
 use serde::{Deserialize, Deserializer, Serialize, Serializer};

src/common/base/src/lib.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 pub mod bitset;
 pub mod buffer;
 pub mod bytes;

src/common/base/tests/buffer_tests.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #![feature(assert_matches)]
 
 #[cfg(test)]

src/common/catalog/Cargo.toml

@@ -2,7 +2,7 @@
 name = "common-catalog"
 version = "0.1.0"
 edition = "2021"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+license = "Apache-2.0"
 
 [dependencies]
 async-trait = "0.1"

src/common/catalog/src/consts.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 pub const SYSTEM_CATALOG_NAME: &str = "system";
 pub const INFORMATION_SCHEMA_NAME: &str = "information_schema";
 pub const SYSTEM_CATALOG_TABLE_NAME: &str = "system_catalog";

src/common/catalog/src/error.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 
 use common_error::ext::ErrorExt;

src/common/catalog/src/helper.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::collections::HashMap;
 use std::fmt::{Display, Formatter};
 
@@ -29,7 +43,7 @@ lazy_static! {
 
 lazy_static! {
     static ref TABLE_GLOBAL_KEY_PATTERN: Regex = Regex::new(&format!(
-        "^{}-([a-zA-Z_]+)-([a-zA-Z_]+)-([a-zA-Z_]+)$",
+        "^{}-([a-zA-Z_]+)-([a-zA-Z_]+)-([a-zA-Z0-9_]+)$",
         TABLE_GLOBAL_KEY_PREFIX
     ))
     .unwrap();
@@ -37,7 +51,7 @@ lazy_static! {
 
 lazy_static! {
     static ref TABLE_REGIONAL_KEY_PATTERN: Regex = Regex::new(&format!(
-        "^{}-([a-zA-Z_]+)-([a-zA-Z_]+)-([a-zA-Z_]+)-([0-9]+)$",
+        "^{}-([a-zA-Z_]+)-([a-zA-Z_]+)-([a-zA-Z0-9_]+)-([0-9]+)$",
         TABLE_REGIONAL_KEY_PREFIX
     ))
     .unwrap();
@@ -118,12 +132,11 @@ pub struct TableGlobalValue {
     pub id: TableId,
     /// Id of datanode that created the global table info kv. only for debugging.
     pub node_id: u64,
+    // TODO(LFC): Maybe remove it?
     /// Allocation of region ids across all datanodes.
     pub regions_id_map: HashMap<u64, Vec<u32>>,
-    /// Node id -> region ids
+    // TODO(LFC): Too much for assembling the table schema that DistTable needs, find another way.
     pub meta: RawTableMeta,
-    /// Partition rules for table
-    pub partition_rules: String,
 }
 
 /// Table regional info that varies between datanode, so it contains a `node_id` field.
@@ -331,7 +344,6 @@ mod tests {
             node_id: 0,
             regions_id_map: HashMap::from([(0, vec![1, 2, 3])]),
             meta,
-            partition_rules: "{}".to_string(),
         };
         let serialized = serde_json::to_string(&value).unwrap();
         let deserialized = TableGlobalValue::parse(&serialized).unwrap();

src/common/catalog/src/lib.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 pub mod consts;
 pub mod error;
 mod helper;

src/common/error/Cargo.toml

@@ -2,7 +2,7 @@
 name = "common-error"
 version = "0.1.0"
 edition = "2021"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+license = "Apache-2.0"
 
 [dependencies]
 snafu = { version = "0.7", features = ["backtraces"] }

src/common/error/src/ext.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::any::Any;
 
 use crate::status_code::StatusCode;

src/common/error/src/format.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::fmt;
 
 use crate::ext::ErrorExt;
@@ -20,7 +34,7 @@ impl<'a, E: ErrorExt + ?Sized> fmt::Debug for DebugFormat<'a, E> {
             write!(f, " Caused by: {:?}", source)?;
         }
         if let Some(backtrace) = self.0.backtrace_opt() {
-            // Add a newline to seperate causes and backtrace.
+            // Add a newline to separate causes and backtrace.
             write!(f, "\nBacktrace:\n{}", backtrace)?;
         }
 
@@ -32,7 +46,8 @@ impl<'a, E: ErrorExt + ?Sized> fmt::Debug for DebugFormat<'a, E> {
 mod tests {
     use std::any::Any;
 
-    use snafu::{prelude::*, Backtrace, GenerateImplicitData};
+    use snafu::prelude::*;
+    use snafu::{Backtrace, GenerateImplicitData};
 
     use super::*;
 

src/common/error/src/lib.rs

@@ -1,10 +1,25 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 pub mod ext;
 pub mod format;
 pub mod mock;
 pub mod status_code;
 
 pub mod prelude {
-    pub use snafu::{prelude::*, Backtrace, ErrorCompat};
+    pub use snafu::prelude::*;
+    pub use snafu::{Backtrace, ErrorCompat};
 
     pub use crate::ext::{BoxedError, ErrorExt};
     pub use crate::format::DebugFormat;

src/common/error/src/mock.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //! Utils for mock.
 
 use std::any::Any;

src/common/error/src/status_code.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::fmt;
 
 /// Common status code for public API.

src/common/function-macro/Cargo.toml

@@ -2,6 +2,7 @@
 name = "common-function-macro"
 version = "0.1.0"
 edition = "2021"
+license = "Apache-2.0"
 
 [lib]
 proc-macro = true

src/common/function-macro/src/lib.rs

@@ -1,3 +1,17 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use proc_macro::TokenStream;
 use quote::{quote, quote_spanned};
 use syn::parse::Parser;