test(storage): wait task before checking scheduled task num (#1811)

* fix: prevent filter pushdown in distributed planner

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix metadata

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

.cargo/config.toml

@@ -3,3 +3,14 @@ linker = "aarch64-linux-gnu-gcc"
 
 [alias]
 sqlness = "run --bin sqlness-runner --"
+
+
+[build]
+rustflags = [
+    # lints
+    # TODO: use lint configuration in cargo https://github.com/rust-lang/cargo/issues/5034
+    "-Wclippy::print_stdout",
+    "-Wclippy::print_stderr",
+    "-Wclippy::implicit_clone",
+    "-Aclippy::items_after_test_module",
+]

.env.example

@@ -3,8 +3,15 @@ GT_S3_BUCKET=S3 bucket
 GT_S3_ACCESS_KEY_ID=S3 access key id
 GT_S3_ACCESS_KEY=S3 secret access key
 GT_S3_ENDPOINT_URL=S3 endpoint url
+GT_S3_REGION=S3 region
 # Settings for oss test
 GT_OSS_BUCKET=OSS bucket
 GT_OSS_ACCESS_KEY_ID=OSS access key id
 GT_OSS_ACCESS_KEY=OSS access key
 GT_OSS_ENDPOINT=OSS endpoint
+# Settings for azblob test
+GT_AZBLOB_CONTAINER=AZBLOB container
+GT_AZBLOB_ACCOUNT_NAME=AZBLOB account name
+GT_AZBLOB_ACCOUNT_KEY=AZBLOB account key
+GT_AZBLOB_ENDPOINT=AZBLOB endpoint
+

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -81,6 +81,5 @@ body:
         Please walk us through and provide steps and details on how
         to reproduce the issue. If possible, provide scripts that we
         can run to trigger the bug.
-      render: bash
     validations:
       required: true

.github/workflows/apidoc.yml

@@ -13,7 +13,7 @@ on:
 name: Build API docs
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-02-26
+  RUST_TOOLCHAIN: nightly-2023-05-03
 
 jobs:
   apidoc:

.github/workflows/develop.yml

@@ -24,7 +24,7 @@ on:
 name: CI
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-02-26
+  RUST_TOOLCHAIN: nightly-2023-05-03
 
 jobs:
   typos:
@@ -141,6 +141,7 @@ jobs:
       - name: Run sqlness
         run: cargo sqlness && ls /tmp
       - name: Upload sqlness logs
+        if: always()
         uses: actions/upload-artifact@v3
         with:
           name: sqlness-logs
@@ -183,7 +184,7 @@ jobs:
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
       - name: Run cargo clippy
-        run: cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr
+        run: cargo clippy --workspace --all-targets -- -D warnings
 
   coverage:
     if: github.event.pull_request.draft == false
@@ -216,7 +217,7 @@ jobs:
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
       - name: Collect coverage data
-        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend
+        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
         env:
           CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
           RUST_BACKTRACE: 1
@@ -224,6 +225,7 @@ jobs:
           GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
           GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
           GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          GT_S3_REGION: ${{ secrets.S3_REGION }}
           UNITTEST_LOG_DIR: "__unittest_logs"
       - name: Codecov upload
         uses: codecov/codecov-action@v2

.github/workflows/release.yml

@@ -5,26 +5,132 @@ on:
   schedule:
     # At 00:00 on Monday.
     - cron: '0 0 * * 1'
+  # Mannually trigger only builds binaries.
   workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Skip docker push and release steps'
+        type: boolean
+        default: true
+      skip_test:
+        description: 'Do not run tests during build'
+        type: boolean
+        default: false
 
 name: Release
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-02-26
+  RUST_TOOLCHAIN: nightly-2023-05-03
 
-  SCHEDULED_BUILD_VERSION_PREFIX: v0.2.0
+  SCHEDULED_BUILD_VERSION_PREFIX: v0.4.0
 
   SCHEDULED_PERIOD: nightly
 
   CARGO_PROFILE: nightly
 
-  ## FIXME(zyy17): Enable it after the tests are stabled.
-  DISABLE_RUN_TESTS: true
+  # Controls whether to run tests, include unit-test, integration-test and sqlness.
+  DISABLE_RUN_TESTS: ${{ inputs.skip_test || false }}
 
 jobs:
-  build:
-    name: Build binary
+  build-macos:
+    name: Build macOS binary
     strategy:
+      fail-fast: false
+      matrix:
+        # The file format is greptime-<os>-<arch>
+        include:
+          - arch: aarch64-apple-darwin
+            os: macos-latest
+            file: greptime-darwin-arm64
+            continue-on-error: false
+            opts: "-F servers/dashboard"
+          - arch: x86_64-apple-darwin
+            os: macos-latest
+            file: greptime-darwin-amd64
+            continue-on-error: false
+            opts: "-F servers/dashboard"
+          - arch: aarch64-apple-darwin
+            os: macos-latest
+            file: greptime-darwin-arm64-pyo3
+            continue-on-error: false
+            opts: "-F pyo3_backend,servers/dashboard"
+          - arch: x86_64-apple-darwin
+            os: macos-latest
+            file: greptime-darwin-amd64-pyo3
+            continue-on-error: false
+            opts: "-F pyo3_backend,servers/dashboard"
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.continue-on-error }}
+    if: github.repository == 'GreptimeTeam/greptimedb'
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Cache cargo assets
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ matrix.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install Protoc for macos
+        if: contains(matrix.arch, 'darwin')
+        run: |
+          brew install protobuf
+
+      - name: Install etcd for macos
+        if: contains(matrix.arch, 'darwin')
+        run: |
+          brew install etcd
+          brew services start etcd
+
+      - name: Install rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          targets: ${{ matrix.arch }}
+      - name: Install latest nextest release
+        uses: taiki-e/install-action@nextest
+      - name: Output package versions
+        run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version
+
+      - name: Run tests
+        if: env.DISABLE_RUN_TESTS == 'false'
+        run: make test sqlness-test
+
+      - name: Run cargo build
+        if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false
+        run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
+
+      - name: Calculate checksum and rename binary
+        shell: bash
+        run: |
+          cd target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}
+          chmod +x greptime
+          tar -zcvf ${{ matrix.file }}.tgz greptime
+          echo $(shasum -a 256 ${{ matrix.file }}.tgz | cut -f1 -d' ') > ${{ matrix.file }}.sha256sum
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.file }}
+          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.tgz
+
+      - name: Upload checksum of artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.file }}.sha256sum
+          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.sha256sum
+
+  build-linux:
+    name: Build linux binary
+    strategy:
+      fail-fast: false
       matrix:
         # The file format is greptime-<os>-<arch>
         include:
@@ -32,22 +138,22 @@ jobs:
             os: ubuntu-2004-16-cores
             file: greptime-linux-amd64
             continue-on-error: false
-            opts: "-F pyo3_backend"
+            opts: "-F servers/dashboard"
           - arch: aarch64-unknown-linux-gnu
             os: ubuntu-2004-16-cores
             file: greptime-linux-arm64
             continue-on-error: false
-            opts: "-F pyo3_backend"
-          - arch: aarch64-apple-darwin
-            os: macos-latest
-            file: greptime-darwin-arm64
+            opts: "-F servers/dashboard"
+          - arch: x86_64-unknown-linux-gnu
+            os: ubuntu-2004-16-cores
+            file: greptime-linux-amd64-pyo3
             continue-on-error: false
-            opts: "-F pyo3_backend"
-          - arch: x86_64-apple-darwin
-            os: macos-latest
-            file: greptime-darwin-amd64
+            opts: "-F pyo3_backend,servers/dashboard"
+          - arch: aarch64-unknown-linux-gnu
+            os: ubuntu-2004-16-cores
+            file: greptime-linux-arm64-pyo3
             continue-on-error: false
-            opts: "-F pyo3_backend"
+            opts: "-F pyo3_backend,servers/dashboard"
     runs-on: ${{ matrix.os }}
     continue-on-error: ${{ matrix.continue-on-error }}
     if: github.repository == 'GreptimeTeam/greptimedb'
@@ -75,11 +181,6 @@ jobs:
           sudo cp protoc/bin/protoc /usr/local/bin/
           sudo cp -r protoc/include/google /usr/local/include/
 
-      - name: Install Protoc for macos
-        if: contains(matrix.arch, 'darwin')
-        run: |
-          brew install protobuf
-
       - name: Install etcd for linux
         if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
         run: |
@@ -93,50 +194,80 @@ jobs:
           sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/
           nohup etcd >/tmp/etcd.log 2>&1 &
 
-      - name: Install etcd for macos
-        if: contains(matrix.arch, 'darwin')
-        run: |
-          brew install etcd
-          brew services start etcd
-
       - name: Install dependencies for linux
         if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
         run: |
           sudo apt-get -y update
           sudo apt-get -y install libssl-dev pkg-config g++-aarch64-linux-gnu gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu wget
 
-      - name: Compile Python 3.10.10 from source for Aarch64
-        if: contains(matrix.arch, 'aarch64-unknown-linux-gnu')
+      # FIXME(zyy17): Should we specify the version of python when building binary for darwin?
+      - name: Compile Python 3.10.10 from source for linux
+        if: contains(matrix.arch, 'linux') && contains(matrix.opts, 'pyo3_backend')
         run: |
           sudo chmod +x ./docker/aarch64/compile-python.sh
-          sudo ./docker/aarch64/compile-python.sh
+          sudo ./docker/aarch64/compile-python.sh ${{ matrix.arch }}
 
       - name: Install rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN }}
           targets: ${{ matrix.arch }}
-
+      - name: Install latest nextest release
+        uses: taiki-e/install-action@nextest
       - name: Output package versions
         run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version
 
       - name: Run tests
         if: env.DISABLE_RUN_TESTS == 'false'
-        run: make unit-test integration-test sqlness-test
-
-      - name: Run cargo build for aarch64-linux
-        if: contains(matrix.arch, 'aarch64-unknown-linux-gnu')
-        run: |
-          # TODO(zyy17): We should make PYO3_CROSS_LIB_DIR configurable.
-          export PYO3_CROSS_LIB_DIR=$(pwd)/python_arm64_build/lib
-          echo "PYO3_CROSS_LIB_DIR: $PYO3_CROSS_LIB_DIR"
-          alias python=python3
-          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
+        run: make test sqlness-test
 
       - name: Run cargo build
-        if: contains(matrix.arch, 'aarch64-unknown-linux-gnu') == false
+        if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false
         run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
 
+      - name: Run cargo build with pyo3 for aarch64-linux
+        if: contains(matrix.arch, 'aarch64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend')
+        run: |
+          # TODO(zyy17): We should make PYO3_CROSS_LIB_DIR configurable.
+          export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64
+          export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
+          export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
+          export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
+
+          export PYO3_CROSS_LIB_DIR=${PWD}/python-3.10.10/aarch64
+          echo "PYO3_CROSS_LIB_DIR: $PYO3_CROSS_LIB_DIR"
+          alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3
+          alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip
+
+          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
+
+      - name: Run cargo build with pyo3 for amd64-linux
+        if: contains(matrix.arch, 'x86_64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend')
+        run: |
+          export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64
+          export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
+          export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
+          export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
+
+          echo "implementation=CPython" >> pyo3.config
+          echo "version=3.10" >> pyo3.config
+          echo "implementation=CPython" >> pyo3.config
+          echo "shared=true" >> pyo3.config
+          echo "abi3=true" >> pyo3.config
+          echo "lib_name=python3.10" >> pyo3.config
+          echo "lib_dir=$PYTHON_INSTALL_PATH_AMD64/lib" >> pyo3.config
+          echo "executable=$PYTHON_INSTALL_PATH_AMD64/bin/python3" >> pyo3.config
+          echo "pointer_width=64" >> pyo3.config
+          echo "build_flags=" >> pyo3.config
+          echo "suppress_build_script_link_lines=false" >> pyo3.config
+
+          cat pyo3.config
+          export PYO3_CONFIG_FILE=${PWD}/pyo3.config
+          alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3
+          alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip
+
+          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
+
       - name: Calculate checksum and rename binary
         shell: bash
         run: |
@@ -159,9 +290,9 @@ jobs:
 
   docker:
     name: Build docker image
-    needs: [build]
+    needs: [build-linux, build-macos]
     runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb'
+    if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false)
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -174,7 +305,7 @@ jobs:
 
       - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
         shell: bash
-        if: github.event_name == 'schedule'
+        if: github.event_name != 'push'
         run: |
           buildTime=`date "+%Y%m%d"`
           SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
@@ -182,7 +313,7 @@ jobs:
 
       - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
         shell: bash
-        if: github.event_name != 'schedule'
+        if: github.event_name == 'push'
         run: |
           VERSION=${{ github.ref_name }}
           echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV
@@ -196,35 +327,33 @@ jobs:
       - name: Download amd64 binary
         uses: actions/download-artifact@v3
         with:
-          name: greptime-linux-amd64
+          name: greptime-linux-amd64-pyo3
           path: amd64
 
       - name: Unzip the amd64 artifacts
         run: |
-          cd amd64
-          tar xvf greptime-linux-amd64.tgz
-          rm greptime-linux-amd64.tgz
+          tar xvf amd64/greptime-linux-amd64-pyo3.tgz -C amd64/ && rm amd64/greptime-linux-amd64-pyo3.tgz
+          cp -r amd64 docker/ci
 
       - name: Download arm64 binary
         id: download-arm64
         uses: actions/download-artifact@v3
         with:
-          name: greptime-linux-arm64
+          name: greptime-linux-arm64-pyo3
           path: arm64
 
       - name: Unzip the arm64 artifacts
         id: unzip-arm64
         if: success() || steps.download-arm64.conclusion == 'success'
         run: |
-          cd arm64
-          tar xvf greptime-linux-arm64.tgz
-          rm greptime-linux-arm64.tgz
+          tar xvf arm64/greptime-linux-arm64-pyo3.tgz -C arm64/ && rm arm64/greptime-linux-arm64-pyo3.tgz
+          cp -r arm64 docker/ci
 
       - name: Build and push all
         uses: docker/build-push-action@v3
         if: success() || steps.unzip-arm64.conclusion == 'success' # Build and push all platform if unzip-arm64 succeeds
         with:
-          context: .
+          context: ./docker/ci/
           file: ./docker/ci/Dockerfile
           push: true
           platforms: linux/amd64,linux/arm64
@@ -236,7 +365,7 @@ jobs:
         uses: docker/build-push-action@v3
         if: success() || steps.download-arm64.conclusion == 'failure' # Only build and push amd64 platform if download-arm64 fails
         with:
-          context: .
+          context: ./docker/ci/
           file: ./docker/ci/Dockerfile
           push: true
           platforms: linux/amd64
@@ -247,9 +376,9 @@ jobs:
   release:
     name: Release artifacts
     # Release artifacts only when all the artifacts are built successfully.
-    needs: [build,docker]
+    needs: [build-linux, build-macos, docker]
     runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb'
+    if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false)
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -259,41 +388,56 @@ jobs:
 
       - name: Configure scheduled build version # the version would be ${SCHEDULED_BUILD_VERSION_PREFIX}-${SCHEDULED_PERIOD}-YYYYMMDD, like v0.2.0-nigthly-20230313.
         shell: bash
-        if: github.event_name == 'schedule'
+        if: github.event_name != 'push'
         run: |
           buildTime=`date "+%Y%m%d"`
           SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-${{ env.SCHEDULED_PERIOD }}-$buildTime
           echo "SCHEDULED_BUILD_VERSION=${SCHEDULED_BUILD_VERSION}" >> $GITHUB_ENV
 
+      # Only publish release when the release tag is like v1.0.0, v1.0.1, v1.0.2, etc.
+      - name: Set whether it is the latest release
+        run: |
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "prerelease=false" >> $GITHUB_ENV
+            echo "makeLatest=true" >> $GITHUB_ENV
+          else
+            echo "prerelease=true" >> $GITHUB_ENV
+            echo "makeLatest=false" >> $GITHUB_ENV
+          fi
+
       - name: Create scheduled build git tag
-        if: github.event_name == 'schedule'
+        if: github.event_name != 'push'
         run: |
           git tag ${{ env.SCHEDULED_BUILD_VERSION }}
 
       - name: Publish scheduled release # configure the different release title and tags.
-        uses: softprops/action-gh-release@v1
-        if: github.event_name == 'schedule'
+        uses: ncipollo/release-action@v1
+        if: github.event_name != 'push'
         with:
           name: "Release ${{ env.SCHEDULED_BUILD_VERSION }}"
-          tag_name: ${{ env.SCHEDULED_BUILD_VERSION }}
-          generate_release_notes: true
-          files: |
+          prerelease: ${{ env.prerelease }}
+          makeLatest: ${{ env.makeLatest }}
+          tag: ${{ env.SCHEDULED_BUILD_VERSION }}
+          generateReleaseNotes: true
+          artifacts: |
             **/greptime-*
 
       - name: Publish release
-        uses: softprops/action-gh-release@v1
-        if: github.event_name != 'schedule'
+        uses: ncipollo/release-action@v1
+        if: github.event_name == 'push'
         with:
-          name: "Release ${{ github.ref_name }}"
-          files: |
+          name: "${{ github.ref_name }}"
+          prerelease: ${{ env.prerelease }}
+          makeLatest: ${{ env.makeLatest }}
+          generateReleaseNotes: true
+          artifacts: |
             **/greptime-*
 
-  docker-push-uhub:
-    name: Push docker image to UCloud Container Registry
+  docker-push-acr:
+    name: Push docker image to alibaba cloud container registry
     needs: [docker]
     runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb'
-    # Push to uhub may fail(500 error), but we don't want to block the release process. The failed job will be retried manually.
+    if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false)
     continue-on-error: true
     steps:
       - name: Checkout sources
@@ -305,16 +449,16 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Login to UCloud Container Registry
+      - name: Login to alibaba cloud container registry
         uses: docker/login-action@v2
         with:
-          registry: uhub.service.ucloud.cn
-          username: ${{ secrets.UCLOUD_USERNAME }}
-          password: ${{ secrets.UCLOUD_PASSWORD }}
+          registry: registry.cn-hangzhou.aliyuncs.com
+          username: ${{ secrets.ALICLOUD_USERNAME }}
+          password: ${{ secrets.ALICLOUD_PASSWORD }}
 
       - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
         shell: bash
-        if: github.event_name == 'schedule'
+        if: github.event_name != 'push'
         run: |
           buildTime=`date "+%Y%m%d"`
           SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
@@ -322,14 +466,14 @@ jobs:
 
       - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
         shell: bash
-        if: github.event_name != 'schedule'
+        if: github.event_name == 'push'
         run: |
           VERSION=${{ github.ref_name }}
           echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV
 
-      - name: Push image to uhub # Use 'docker buildx imagetools create' to create a new image base on source image.
+      - name: Push image to alibaba cloud container registry # Use 'docker buildx imagetools create' to create a new image base on source image.
         run: |
           docker buildx imagetools create \
-            --tag uhub.service.ucloud.cn/greptime/greptimedb:latest \
-            --tag uhub.service.ucloud.cn/greptime/greptimedb:${{ env.IMAGE_TAG }} \
+            --tag registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb:latest \
+            --tag registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb:${{ env.IMAGE_TAG }} \
             greptime/greptimedb:${{ env.IMAGE_TAG }}

.gitignore

@@ -1,6 +1,8 @@
 # Generated by Cargo
 # will have compiled files and executables
 /target/
+# also ignore if it's a symbolic link
+/target
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
@@ -35,3 +37,10 @@ benchmarks/data
 
 # dotenv
 .env
+
+# dashboard files
+!/src/servers/dashboard/VERSION
+/src/servers/dashboard/*
+
+# Vscode workspace
+*.code-workspace

CONTRIBUTING.md

@@ -51,7 +51,7 @@ GreptimeDB uses the [Apache 2.0 license](https://github.com/GreptimeTeam/greptim
 - To ensure that community is free and confident in its ability to use your contributions, please sign the Contributor License Agreement (CLA) which will be incorporated in the pull request process.
 - Make sure all your codes are formatted and follow the [coding style](https://pingcap.github.io/style-guide/rust/).
 - Make sure all unit tests are passed (using `cargo test --workspace` or [nextest](https://nexte.st/index.html) `cargo nextest run`).
-- Make sure all clippy warnings are fixed (you can check it locally by running `cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr`).
+- Make sure all clippy warnings are fixed (you can check it locally by running `cargo clippy --workspace --all-targets -- -D warnings`).
 
 #### `pre-commit` Hooks
 
@@ -107,6 +107,6 @@ The core team will be thrilled if you participate in any way you like. When you
 
 Also, see some extra GreptimeDB content:
 
-- [GreptimeDB Docs](https://greptime.com/docs)
-- [Learn GreptimeDB](https://greptime.com/products/db)
+- [GreptimeDB Docs](https://docs.greptime.com/)
+- [Learn GreptimeDB](https://greptime.com/product/db)
 - [Greptime Inc. Website](https://greptime.com)

Cargo.toml

@@ -7,13 +7,17 @@ members = [
     "src/cmd",
     "src/common/base",
     "src/common/catalog",
+    "src/common/datasource",
     "src/common/error",
     "src/common/function",
     "src/common/function-macro",
     "src/common/grpc",
     "src/common/grpc-expr",
     "src/common/mem-prof",
+    "src/common/meta",
     "src/common/procedure",
+    "src/common/procedure-test",
+    "src/common/pprof",
     "src/common/query",
     "src/common/recordbatch",
     "src/common/runtime",
@@ -23,6 +27,7 @@ members = [
     "src/common/time",
     "src/datanode",
     "src/datatypes",
+    "src/file-table-engine",
     "src/frontend",
     "src/log-store",
     "src/meta-client",
@@ -45,38 +50,49 @@ members = [
 ]
 
 [workspace.package]
-version = "0.1.1"
+version = "0.4.0"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.dependencies]
-arrow = { version = "34.0" }
-arrow-array = "34.0"
-arrow-flight = "34.0"
-arrow-schema = { version = "34.0", features = ["serde"] }
+arrow = { version = "40.0" }
+arrow-array = "40.0"
+arrow-flight = "40.0"
+arrow-schema = { version = "40.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
-datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
-datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
-datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
-datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
-datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
-datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
+# TODO(ruihang): use arrow-datafusion when it contains https://github.com/apache/arrow-datafusion/pull/6032
+datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
+datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
+datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
+datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
+datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
+datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
+datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
 futures = "0.3"
 futures-util = "0.3"
-parquet = "34.0"
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "7aeaeaba1e0ca6a5c736b6ab2eb63144ae3d284b" }
+itertools = "0.10"
+parquet = "40.0"
 paste = "1.0"
 prost = "0.11"
+rand = "0.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }
-sqlparser = "0.32"
+sqlparser = "0.34"
 tempfile = "3"
-tokio = { version = "1.24.2", features = ["full"] }
-tokio-util = "0.7"
-tonic = { version = "0.8", features = ["tls"] }
+tokio = { version = "1.28", features = ["full"] }
+tokio-util = { version = "0.7", features = ["io-util", "compat"] }
+tonic = { version = "0.9", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
+metrics = "0.20"
+meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "f0798c4c648d89f51abe63e870919c75dd463199" }
+
+[workspace.dependencies.meter-macros]
+git = "https://github.com/GreptimeTeam/greptime-meter.git"
+rev = "f0798c4c648d89f51abe63e870919c75dd463199"
 
 [profile.release]
 debug = true

Cross.toml

@@ -0,0 +1,7 @@
+[build]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt update && apt install -y unzip zlib1g-dev:$CROSS_DEB_ARCH",
+    "curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip && unzip protoc-3.15.8-linux-x86_64.zip -d /usr/",
+    "chmod a+x /usr/bin/protoc && chmod -R a+rx /usr/include/google",
+]

Makefile

@@ -21,6 +21,10 @@ fmt: ## Format all the Rust code.
 
 .PHONY: fmt-toml
 fmt-toml: ## Format all TOML files.
+	taplo format --option "indent_string=    "
+
+.PHONY: check-toml
+check-toml: ## Check all TOML files.
 	taplo format --check --option "indent_string=    "
 
 .PHONY: docker-image
@@ -29,13 +33,12 @@ docker-image: ## Build docker image.
 
 ##@ Test
 
-.PHONY: unit-test
-unit-test: ## Run unit test.
-	cargo test --workspace
+test: nextest ## Run unit and integration tests.
+	cargo nextest run --retries 3
 
-.PHONY: integration-test
-integration-test: ## Run integation test.
-	cargo test integration
+.PHONY: nextest ## Install nextest tools.
+nextest:
+	cargo --list | grep nextest || cargo install cargo-nextest --locked
 
 .PHONY: sqlness-test
 sqlness-test: ## Run sqlness test.
@@ -47,7 +50,7 @@ check: ## Cargo check all the targets.
 
 .PHONY: clippy
 clippy: ## Check clippy rules.
-	cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr
+	cargo clippy --workspace --all-targets -- -D warnings
 
 .PHONY: fmt-check
 fmt-check: ## Check code format.

README.md

@@ -1,14 +1,14 @@
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: light)" srcset="/docs/logo-text-padding.png">
-    <source media="(prefers-color-scheme: dark)" srcset="/docs/logo-text-padding-dark.png">
-    <img alt="GreptimeDB Logo" src="/docs/logo-text-padding.png" width="400px">
+    <source media="(prefers-color-scheme: light)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding.png">
+    <source media="(prefers-color-scheme: dark)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding-dark.png">
+    <img alt="GreptimeDB Logo" src="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding.png" width="400px">
   </picture>
 </p>
 
 
 <h3 align="center">
-    The next-generation hybrid timeseries/analytics processing database in the cloud
+    The next-generation hybrid time-series/analytics processing database in the cloud
 </h3>
 
 <p align="center">
@@ -23,6 +23,8 @@
     <a href="https://twitter.com/greptime"><img src="https://img.shields.io/badge/twitter-follow_us-1d9bf0.svg"></a>
     &nbsp;
     <a href="https://www.linkedin.com/company/greptime/"><img src="https://img.shields.io/badge/linkedin-connect_with_us-0a66c2.svg"></a>
+    &nbsp;
+    <a href="https://greptime.com/slack"><img src="https://img.shields.io/badge/slack-GreptimeDB-0abd59?logo=slack" alt="slack" /></a>
 </p>
 
 ## What is GreptimeDB
@@ -36,15 +38,23 @@ Our core developers have been building time-series data platform
 for years. Based on their best-practices, GreptimeDB is born to give you:
 
 - A standalone binary that scales to highly-available distributed cluster, providing a transparent experience for cluster users
-- Optimized columnar layout for handling time-series data; compacted, compressed, stored on various storage backends
-- Flexible index options, tackling high cardinality issues down
+- Optimized columnar layout for handling time-series data; compacted, compressed, and stored on various storage backends
+- Flexible indexes, tackling high cardinality issues down
 - Distributed, parallel query execution, leveraging elastic computing resource
 - Native SQL, and Python scripting for advanced analytical scenarios
-- Widely adopted database protocols and APIs
+- Widely adopted database protocols and APIs, native PromQL supports
 - Extensible table engine architecture for extensive workloads
 
 ## Quick Start
 
+### GreptimePlay
+
+Try out the features of GreptimeDB right from your browser.
+
+<a href="https://greptime.com/playground" target="_blank"><img
+src="https://www.greptime.com/assets/greptime_play_button_colorful.1bbe2746.png"
+alt="GreptimePlay" width="200px" /></a>
+
 ### Build
 
 #### Build from Source
@@ -61,12 +71,12 @@ To compile GreptimeDB from source, you'll need:
   find an installation instructions [here](https://grpc.io/docs/protoc-installation/).
   **Note that `protoc` version needs to be >= 3.15** because we have used the `optional`
   keyword. You can check it with `protoc --version`.
-- python3-dev or python3-devel(Optional, only needed if you want to run scripts
-  in cpython): this install a Python shared library required for running python
+- python3-dev or python3-devel(Optional feature, only needed if you want to run scripts
+  in CPython, and also need to enable `pyo3_backend` feature when compiling(by `cargo run -F pyo3_backend` or add `pyo3_backend` to src/script/Cargo.toml 's `features.default` like `default = ["python", "pyo3_backend]`)): this install a Python shared library required for running Python
   scripting engine(In CPython Mode). This is available as `python3-dev` on
   ubuntu, you can install it with `sudo apt install python3-dev`, or
   `python3-devel` on RPM based distributions (e.g. Fedora, Red Hat, SuSE). Mac's
-  `Python3` package should have this shared library by default.
+  `Python3` package should have this shared library by default. More detail for compiling with PyO3 can be found in [PyO3](https://pyo3.rs/v0.18.1/building_and_distribution#configuring-the-python-version)'s documentation.
 
 #### Build with Docker
 
@@ -90,74 +100,33 @@ Or if you built from docker:
 docker run -p 4002:4002 -v "$(pwd):/tmp/greptimedb" greptime/greptimedb standalone start
 ```
 
-For more startup options, greptimedb's **distributed mode** and information
-about Kubernetes deployment, check our [docs](https://docs.greptime.com/).
+Please see [the online document site](https://docs.greptime.com/getting-started/overview#install-greptimedb) for more installation options and [operations info](https://docs.greptime.com/user-guide/operations/overview).
 
-### Connect
+### Get started
 
-1. Connect to GreptimeDB via standard [MySQL
-   client](https://dev.mysql.com/downloads/mysql/):
+Read the [complete getting started guide](https://docs.greptime.com/getting-started/overview#connect) on our [official document site](https://docs.greptime.com/).
 
-   ```
-   # The standalone instance listen on port 4002 by default.
-   mysql -h 127.0.0.1 -P 4002
-   ```
-
-2. Create table:
-
-   ```SQL
-   CREATE TABLE monitor (
-     host STRING,
-     ts TIMESTAMP,
-     cpu DOUBLE DEFAULT 0,
-     memory DOUBLE,
-     TIME INDEX (ts),
-     PRIMARY KEY(host)) ENGINE=mito WITH(regions=1);
-   ```
-
-3. Insert some data:
-
-   ```SQL
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host1', 66.6, 1024, 1660897955000);
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host2', 77.7, 2048, 1660897956000);
-   INSERT INTO monitor(host, cpu, memory, ts) VALUES ('host3', 88.8, 4096, 1660897957000);
-   ```
-
-4. Query the data:
-
-   ```SQL
-   SELECT * FROM monitor;
-   ```
-
-   ```TEXT
-   +-------+---------------------+------+--------+
-   | host  | ts                  | cpu  | memory |
-   +-------+---------------------+------+--------+
-   | host1 | 2022-08-19 08:32:35 | 66.6 |   1024 |
-   | host2 | 2022-08-19 08:32:36 | 77.7 |   2048 |
-   | host3 | 2022-08-19 08:32:37 | 88.8 |   4096 |
-   +-------+---------------------+------+--------+
-   3 rows in set (0.01 sec)
-   ```
-
-You can always cleanup test database by removing `/tmp/greptimedb`.
+To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/client/overview).
 
 ## Resources
 
 ### Installation
 
-- [Pre-built Binaries](https://github.com/GreptimeTeam/greptimedb/releases):
-  downloadable pre-built binaries for Linux and MacOS
-- [Docker Images](https://hub.docker.com/r/greptime/greptimedb): pre-built
-  Docker images
+- [Pre-built Binaries](https://greptime.com/download):
+  For Linux and macOS, you can easily download pre-built binaries including official releases and nightly builds that are ready to use. 
+  In most cases, downloading the version without PyO3 is sufficient. However, if you plan to run scripts in CPython (and use Python packages like NumPy and Pandas), you will need to download the version with PyO3 and install a Python with the same version as the Python in the PyO3 version.
+  We recommend using virtualenv for the installation process to manage multiple Python versions.
+- [Docker Images](https://hub.docker.com/r/greptime/greptimedb)(**recommended**): pre-built
+  Docker images, this is the easiest way to try GreptimeDB. By default it runs CPython script with `pyo3_backend` enabled.
 - [`gtctl`](https://github.com/GreptimeTeam/gtctl): the command-line tool for
   Kubernetes deployment
 
 ### Documentation
 
-- GreptimeDB [User Guide](https://docs.greptime.com/user-guide/concepts.html)
+- GreptimeDB [User Guide](https://docs.greptime.com/user-guide/concepts/overview)
 - GreptimeDB [Developer
   Guide](https://docs.greptime.com/developer-guide/overview.html)
+- GreptimeDB [internal code document](https://greptimedb.rs)
 
 ### Dashboard
 - [The dashboard UI for GreptimeDB](https://github.com/GreptimeTeam/dashboard)

SECURITY.md

@@ -0,0 +1,19 @@
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported          |
+| ------- | ------------------ |
+| >= v0.1.0 | :white_check_mark: |
+| < v0.1.0   | :x:                |
+
+## Reporting a Vulnerability
+
+We place great importance on the security of GreptimeDB code, software, 
+and cloud platform. If you come across a security vulnerability in GreptimeDB, 
+we kindly request that you inform us immediately. We will thoroughly investigate
+all valid reports and make every effort to resolve the issue promptly.
+
+To report any issues or vulnerabilities, please email us at info@greptime.com, rather than 
+posting publicly on GitHub. Be sure to provide us with the version identifier as well as details
+on how the vulnerability can be exploited.

benchmarks/Cargo.toml

@@ -9,6 +9,6 @@ arrow.workspace = true
 clap = { version = "4.0", features = ["derive"] }
 client = { path = "../src/client" }
 indicatif = "0.17.1"
-itertools = "0.10.5"
+itertools.workspace = true
 parquet.workspace = true
 tokio.workspace = true

benchmarks/src/bin/nyc-taxi.rs

@@ -21,12 +21,14 @@ use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::time::Instant;
 
-use arrow::array::{ArrayRef, PrimitiveArray, StringArray, TimestampNanosecondArray};
+use arrow::array::{ArrayRef, PrimitiveArray, StringArray, TimestampMicrosecondArray};
 use arrow::datatypes::{DataType, Float64Type, Int64Type};
 use arrow::record_batch::RecordBatch;
 use clap::Parser;
 use client::api::v1::column::Values;
-use client::api::v1::{Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, TableId};
+use client::api::v1::{
+    Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests,
+};
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
@@ -61,7 +63,7 @@ struct Args {
     #[arg(long = "skip-read")]
     skip_read: bool,
 
-    #[arg(short, long, default_value_t = String::from("127.0.0.1:3001"))]
+    #[arg(short, long, default_value_t = String::from("127.0.0.1:4001"))]
     endpoint: String,
 }
 
@@ -97,6 +99,9 @@ async fn write_data(
 
     for record_batch in record_batch_reader {
         let record_batch = record_batch.unwrap();
+        if !is_record_batch_full(&record_batch) {
+            continue;
+        }
         let (columns, row_count) = convert_record_batch(record_batch);
         let request = InsertRequest {
             table_name: TABLE_NAME.to_string(),
@@ -104,8 +109,12 @@ async fn write_data(
             columns,
             row_count,
         };
+        let requests = InsertRequests {
+            inserts: vec![request],
+        };
+
         let now = Instant::now();
-        db.insert(request).await.unwrap();
+        db.insert(requests).await.unwrap();
         let elapsed = now.elapsed();
         total_rpc_elapsed_ms += elapsed.as_millis();
         progress_bar.inc(row_count as _);
@@ -122,11 +131,17 @@ fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
     let mut columns = vec![];
 
     for (array, field) in record_batch.columns().iter().zip(fields.iter()) {
-        let values = build_values(array);
+        let (values, datatype) = build_values(array);
+
         let column = Column {
-            column_name: field.name().to_owned(),
+            column_name: field.name().clone(),
             values: Some(values),
-            null_mask: vec![],
+            null_mask: array
+                .to_data()
+                .nulls()
+                .map(|bitmap| bitmap.buffer().as_slice().to_vec())
+                .unwrap_or_default(),
+            datatype: datatype.into(),
             // datatype and semantic_type are set to default
             ..Default::default()
         };
@@ -136,7 +151,7 @@ fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
     (columns, row_count as _)
 }
 
-fn build_values(column: &ArrayRef) -> Values {
+fn build_values(column: &ArrayRef) -> (Values, ColumnDataType) {
     match column.data_type() {
         DataType::Int64 => {
             let array = column
@@ -144,10 +159,13 @@ fn build_values(column: &ArrayRef) -> Values {
                 .downcast_ref::<PrimitiveArray<Int64Type>>()
                 .unwrap();
             let values = array.values();
-            Values {
-                i64_values: values.to_vec(),
-                ..Default::default()
-            }
+            (
+                Values {
+                    i64_values: values.to_vec(),
+                    ..Default::default()
+                },
+                ColumnDataType::Int64,
+            )
         }
         DataType::Float64 => {
             let array = column
@@ -155,29 +173,38 @@ fn build_values(column: &ArrayRef) -> Values {
                 .downcast_ref::<PrimitiveArray<Float64Type>>()
                 .unwrap();
             let values = array.values();
-            Values {
-                f64_values: values.to_vec(),
-                ..Default::default()
-            }
+            (
+                Values {
+                    f64_values: values.to_vec(),
+                    ..Default::default()
+                },
+                ColumnDataType::Float64,
+            )
         }
         DataType::Timestamp(_, _) => {
             let array = column
                 .as_any()
-                .downcast_ref::<TimestampNanosecondArray>()
+                .downcast_ref::<TimestampMicrosecondArray>()
                 .unwrap();
             let values = array.values();
-            Values {
-                i64_values: values.to_vec(),
-                ..Default::default()
-            }
+            (
+                Values {
+                    ts_microsecond_values: values.to_vec(),
+                    ..Default::default()
+                },
+                ColumnDataType::TimestampMicrosecond,
+            )
         }
         DataType::Utf8 => {
             let array = column.as_any().downcast_ref::<StringArray>().unwrap();
             let values = array.iter().filter_map(|s| s.map(String::from)).collect();
-            Values {
-                string_values: values,
-                ..Default::default()
-            }
+            (
+                Values {
+                    string_values: values,
+                    ..Default::default()
+                },
+                ColumnDataType::String,
+            )
         }
         DataType::Null
         | DataType::Boolean
@@ -204,7 +231,7 @@ fn build_values(column: &ArrayRef) -> Values {
         | DataType::FixedSizeList(_, _)
         | DataType::LargeList(_)
         | DataType::Struct(_)
-        | DataType::Union(_, _, _)
+        | DataType::Union(_, _)
         | DataType::Dictionary(_, _)
         | DataType::Decimal128(_, _)
         | DataType::Decimal256(_, _)
@@ -213,6 +240,10 @@ fn build_values(column: &ArrayRef) -> Values {
     }
 }
 
+fn is_record_batch_full(batch: &RecordBatch) -> bool {
+    batch.columns().iter().all(|col| col.null_count() == 0)
+}
+
 fn create_table_expr() -> CreateTableExpr {
     CreateTableExpr {
         catalog_name: CATALOG_NAME.to_string(),
@@ -228,13 +259,13 @@ fn create_table_expr() -> CreateTableExpr {
             },
             ColumnDef {
                 name: "tpep_pickup_datetime".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::TimestampMicrosecond as i32,
                 is_nullable: true,
                 default_constraint: vec![],
             },
             ColumnDef {
                 name: "tpep_dropoff_datetime".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::TimestampMicrosecond as i32,
                 is_nullable: true,
                 default_constraint: vec![],
             },
@@ -339,8 +370,9 @@ fn create_table_expr() -> CreateTableExpr {
         primary_keys: vec!["VendorID".to_string()],
         create_if_not_exists: false,
         table_options: Default::default(),
-        region_ids: vec![0],
-        table_id: Some(TableId { id: 0 }),
+        region_numbers: vec![0],
+        table_id: None,
+        engine: "mito".to_string(),
     }
 }
 

config/datanode.example.toml

@@ -24,9 +24,10 @@ tcp_nodelay = true
 
 # WAL options, see `standalone.example.toml`.
 [wal]
-dir = "/tmp/greptimedb/wal"
-file_size = "1GB"
-purge_threshold = "50GB"
+# WAL data directory
+# dir = "/tmp/greptimedb/wal"
+file_size = "256MB"
+purge_threshold = "4GB"
 purge_interval = "10m"
 read_batch_size = 128
 sync_write = false
@@ -34,17 +35,47 @@ sync_write = false
 # Storage options, see `standalone.example.toml`.
 [storage]
 type = "File"
-data_dir = "/tmp/greptimedb/data/"
+data_home = "/tmp/greptimedb/"
+# TTL for all tables. Disabled by default.
+# global_ttl = "7d"
 
 # Compaction options, see `standalone.example.toml`.
-[compaction]
+[storage.compaction]
 max_inflight_tasks = 4
 max_files_in_level0 = 8
 max_purge_tasks = 32
 
+# Storage manifest options
+[storage.manifest]
+# Region checkpoint actions margin.
+# Create a checkpoint every <checkpoint_margin> actions.
+checkpoint_margin = 10
+# Region manifest logs and checkpoints gc execution duration
+gc_duration = '10m'
+# Whether to try creating a manifest checkpoint on region opening
+checkpoint_on_startup = false
+
+# Storage flush options
+[storage.flush]
+# Max inflight flush tasks.
+max_flush_tasks = 8
+# Default write buffer size for a region.
+region_write_buffer_size = "32MB"
+# Interval to check whether a region needs flush.
+picker_schedule_interval = "5m"
+# Interval to auto flush a region if it has not flushed yet.
+auto_flush_interval = "1h"
+# Global write buffer size for all regions.
+global_write_buffer_size = "1GB"
+
 # Procedure storage options, see `standalone.example.toml`.
-# [procedure.store]
-# type = "File"
-# data_dir = "/tmp/greptimedb/procedure/"
-# max_retry_times = 3
-# retry_delay = "500ms"
+[procedure]
+max_retry_times = 3
+retry_delay = "500ms"
+
+# Log options
+# [logging]
+# Specify logs directory.
+# dir = "/tmp/greptimedb/logs"
+# Specify the log level [info | debug | error | warn]
+# level = "info"

config/frontend.example.toml

@@ -56,3 +56,8 @@ metasrv_addrs = ["127.0.0.1:3002"]
 timeout_millis = 3000
 connect_timeout_millis = 5000
 tcp_nodelay = true
+
+# Log options, see `standalone.example.toml`
+# [logging]
+# dir = "/tmp/greptimedb/logs"
+# level = "info"

config/metasrv.example.toml

@@ -13,3 +13,8 @@ datanode_lease_secs = 15
 selector = "LeaseBased"
 # Store data in memory, false by default.
 use_memory_store = false
+
+# Log options, see `standalone.example.toml`
+# [logging]
+# dir = "/tmp/greptimedb/logs"
+# level = "info"

config/standalone.example.toml

@@ -78,12 +78,12 @@ addr = "127.0.0.1:4004"
 
 # WAL options.
 [wal]
-# WAL data directory.
-dir = "/tmp/greptimedb/wal"
+# WAL data directory
+# dir = "/tmp/greptimedb/wal"
 # WAL file size in bytes.
-file_size = "1GB"
-# WAL purge threshold in bytes.
-purge_threshold = "50GB"
+file_size = "256MB"
+# WAL purge threshold.
+purge_threshold = "4GB"
 # WAL purge interval in seconds.
 purge_interval = "10m"
 # WAL read batch size.
@@ -96,10 +96,12 @@ sync_write = false
 # Storage type.
 type = "File"
 # Data directory, "/tmp/greptimedb/data" by default.
-data_dir = "/tmp/greptimedb/data/"
+data_home = "/tmp/greptimedb/"
+# TTL for all tables. Disabled by default.
+# global_ttl = "7d"
 
 # Compaction options.
-[compaction]
+[storage.compaction]
 # Max task number that can concurrently run.
 max_inflight_tasks = 4
 # Max files in level 0 to trigger compaction.
@@ -107,14 +109,39 @@ max_files_in_level0 = 8
 # Max task number for SST purge task after compaction.
 max_purge_tasks = 32
 
+# Storage manifest options
+[storage.manifest]
+# Region checkpoint actions margin.
+# Create a checkpoint every <checkpoint_margin> actions.
+checkpoint_margin = 10
+# Region manifest logs and checkpoints gc execution duration
+gc_duration = '10m'
+# Whether to try creating a manifest checkpoint on region opening
+checkpoint_on_startup = false
+
+# Storage flush options
+[storage.flush]
+# Max inflight flush tasks.
+max_flush_tasks = 8
+# Default write buffer size for a region.
+region_write_buffer_size = "32MB"
+# Interval to check whether a region needs flush.
+picker_schedule_interval = "5m"
+# Interval to auto flush a region if it has not flushed yet.
+auto_flush_interval = "1h"
+# Global write buffer size for all regions.
+global_write_buffer_size = "1GB"
+
 # Procedure storage options.
-# Uncomment to enable.
-# [procedure.store]
-# # Storage type.
-# type = "File"
-# # Procedure data path.
-# data_dir = "/tmp/greptimedb/procedure/"
-# # Procedure max retry time.
-# max_retry_times = 3
-# # Initial retry delay of procedures, increases exponentially
-# retry_delay = "500ms"
+[procedure]
+# Procedure max retry time.
+max_retry_times = 3
+# Initial retry delay of procedures, increases exponentially
+retry_delay = "500ms"
+
+# Log options
+# [logging]
+# Specify logs directory.
+# dir = "/tmp/greptimedb/logs"
+# Specify the log level [info | debug | error | warn]
+# level = "info"

docker/Dockerfile

@@ -12,7 +12,9 @@ RUN apt-get update && apt-get install -y \
     pkg-config \
     python3 \
     python3-dev \
-    && pip install pyarrow
+    python3-pip \
+    && pip3 install --upgrade pip \
+    && pip3 install pyarrow
 
 # Install Rust.
 SHELL ["/bin/bash", "-c"]

docker/aarch64/compile-python.sh

@@ -1,9 +1,36 @@
+#!/usr/bin/env bash
+
+set -e
+
 # this script will download Python source code, compile it, and install it to /usr/local/lib
 # then use this python to compile cross-compiled python for aarch64
+ARCH=$1
+PYTHON_VERSION=3.10.10
+PYTHON_SOURCE_DIR=Python-${PYTHON_VERSION}
+PYTHON_INSTALL_PATH_AMD64=${PWD}/python-${PYTHON_VERSION}/amd64
+PYTHON_INSTALL_PATH_AARCH64=${PWD}/python-${PYTHON_VERSION}/aarch64
+
+function download_python_source_code() {
+  wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz
+  tar -xvf Python-$PYTHON_VERSION.tgz
+}
+
+function compile_for_amd64_platform() {
+  mkdir -p "$PYTHON_INSTALL_PATH_AMD64"
+
+  echo "Compiling for amd64 platform..."
+
+  ./configure \
+    --prefix="$PYTHON_INSTALL_PATH_AMD64" \
+    --enable-shared \
+    ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \
+    ac_cv_have_long_long_format=yes \
+    --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no
+
+  make
+  make install
+}
 
-wget https://www.python.org/ftp/python/3.10.10/Python-3.10.10.tgz
-tar -xvf Python-3.10.10.tgz
-cd Python-3.10.10
 # explain Python compile options here a bit:s
 # --enable-shared: enable building a shared Python library (default is no) but we do need it for calling from rust
 # CC, CXX, AR, LD, RANLIB: set the compiler, archiver, linker, and ranlib programs to use
@@ -14,33 +41,47 @@ cd Python-3.10.10
 # ac_cv_have_long_long_format=yes: target platform supports long long type
 # disable-ipv6: disable ipv6 support, we don't need it in here
 # ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no: disable pty support, we don't need it in here
+function compile_for_aarch64_platform() {
+  export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
+  export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
+  export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
+
+  mkdir -p "$PYTHON_INSTALL_PATH_AARCH64"
+
+  echo "Compiling for aarch64 platform..."
+  echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
+  echo "LIBRARY_PATH: $LIBRARY_PATH"
+  echo "PATH: $PATH"
+
+  ./configure --build=x86_64-linux-gnu --host=aarch64-linux-gnu \
+    --prefix="$PYTHON_INSTALL_PATH_AARCH64" --enable-optimizations \
+    CC=aarch64-linux-gnu-gcc \
+    CXX=aarch64-linux-gnu-g++ \
+    AR=aarch64-linux-gnu-ar \
+    LD=aarch64-linux-gnu-ld \
+    RANLIB=aarch64-linux-gnu-ranlib \
+    --enable-shared \
+    ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \
+    ac_cv_have_long_long_format=yes \
+    --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no
+
+  make
+  make altinstall
+}
+
+# Main script starts here.
+download_python_source_code
+
+# Enter the python source code directory.
+cd $PYTHON_SOURCE_DIR || exit 1
 
 # Build local python first, then build cross-compiled python.
-./configure \
---enable-shared \
-ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \
-ac_cv_have_long_long_format=yes \
---disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no && \
-make
-make install
-cd ..
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
-export LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib/
-export PY_INSTALL_PATH=$(pwd)/python_arm64_build
-cd Python-3.10.10 && \
-make clean && \
-make distclean && \
-alias python=python3 && \
-./configure --build=x86_64-linux-gnu --host=aarch64-linux-gnu \
---prefix=$PY_INSTALL_PATH --enable-optimizations \
-CC=aarch64-linux-gnu-gcc \
-CXX=aarch64-linux-gnu-g++ \
-AR=aarch64-linux-gnu-ar \
-LD=aarch64-linux-gnu-ld \
-RANLIB=aarch64-linux-gnu-ranlib \
---enable-shared \
-ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \
-ac_cv_have_long_long_format=yes \
---disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no && \
-make && make altinstall && \
-cd ..
+compile_for_amd64_platform
+
+# Clean the build directory.
+make clean && make distclean
+
+# Cross compile python for aarch64.
+if [ "$ARCH" = "aarch64-unknown-linux-gnu" ]; then
+  compile_for_aarch64_platform
+fi

docker/ci/Dockerfile

@@ -1,6 +1,14 @@
 FROM ubuntu:22.04
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    ca-certificates \
+    python3.10 \
+    python3.10-dev \
+    python3-pip
+
+COPY requirements.txt /etc/greptime/requirements.txt
+
+RUN python3 -m pip install -r /etc/greptime/requirements.txt
 
 ARG TARGETARCH
 

docker/ci/requirements.txt

@@ -0,0 +1,5 @@
+numpy>=1.24.2
+pandas>=1.5.3
+pyarrow>=11.0.0
+requests>=2.28.2
+scipy>=1.10.1

docs/how-to/how-to-implement-sql-statement.md

@@ -0,0 +1,74 @@
+This document introduces how to implement SQL statements in GreptimeDB.
+
+The execution entry point for SQL statements locates at Frontend Instance. You can see it has
+implemented `SqlQueryHandler`:
+
+```rust
+impl SqlQueryHandler for Instance {
+    type Error = Error;
+
+    async fn do_query(&self, query: &str, query_ctx: QueryContextRef) -> Vec<Result<Output>> {
+        // ...
+    }
+}
+```
+
+Normally, when a SQL query arrives at GreptimeDB, the `do_query` method will be called. After some parsing work, the SQL
+will be feed into `StatementExecutor`:
+
+```rust
+// in Frontend Instance:
+self.statement_executor.execute_sql(stmt, query_ctx).await
+```
+
+That's where we handle our SQL statements. You can just create a new match arm for your statement there, then the
+statement is implemented for both GreptimeDB Standalone and Cluster. You can see how `DESCRIBE TABLE` is implemented as
+an example.
+
+Now, what if the statements should be handled differently for GreptimeDB Standalone and Cluster? You can see there's
+a `SqlStatementExecutor` field in `StatementExecutor`. Each GreptimeDB Standalone and Cluster has its own implementation
+of `SqlStatementExecutor`. If you are going to implement the statements differently in the two mode (
+like `CREATE TABLE`), you have to implement them in their own `SqlStatementExecutor`s.
+
+Summarize as the diagram below:
+
+```text
+                             SQL query                            
+                                |                                
+                                v                                
+                  +---------------------------+                  
+                  | SqlQueryHandler::do_query |                  
+                  +---------------------------+                  
+                                |                                
+                                | SQL parsing                    
+                                v                                
+               +--------------------------------+                
+               | StatementExecutor::execute_sql |                
+               +--------------------------------+                
+                                |                                
+                                | SQL execution                    
+                                v                                
+               +----------------------------------+                
+               | commonly handled statements like |
+               | "plan_exec" for selection or     |
+               +----------------------------------+                
+                       |                |                        
+        For Standalone |                | For Cluster          
+                       v                v                        
++---------------------------+      +---------------------------+ 
+| SqlStatementExecutor impl |      | SqlStatementExecutor impl | 
+| in Datanode Instance      |      | in Frontend DistInstance  | 
++---------------------------+      +---------------------------+ 
+```
+
+Note that some SQL statements can be executed in our QueryEngine, in the form of `LogicalPlan`. You can follow the
+invocation path down to the `QueryEngine` implementation from `StatementExecutor::plan_exec`. For now, there's only
+one `DatafusionQueryEngine` for both GreptimeDB Standalone and Cluster. That lone query engine works for both modes is
+because GreptimeDB read/write data through `Table` trait, and each mode has its own `Table` implementation.
+
+We don't have any bias towards whether statements should be handled in query engine or `StatementExecutor`. You can
+implement one kind of statement in both places. For example, `Insert` with selection is handled in query engine, because
+we can easily do the query part there. However, `Insert` without selection is not, for the cost of parsing statement
+to `LogicalPlan` is not neglectable. So generally if the SQL query is simple enough, you can handle it
+in `StatementExecutor`; otherwise if it is complex or has some part of selection, it should be parsed to `LogicalPlan`
+and handled in query engine.  

docs/rfcs/2023-03-29-file-external-table/rfc.md

@@ -0,0 +1,174 @@
+---
+Feature Name: "File external table"
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/1041
+Date: 2023-03-08
+Author: "Xu Wenkang <wenymedia@gmail.com>"
+---
+
+File external table
+
+---
+
+# Summary
+
+Allows users to perform SQL queries on files
+
+# Motivation
+
+User data may already exist in other storages, i.g., file systems/s3, etc. in CSV, parquet, JSON format, etc. We can provide users the ability to perform SQL queries on these files.
+
+
+# Details
+
+## Overview
+
+The file external table providers users ability to perform SQL queries on these files.
+
+For example, a user has a CSV file on the local file system `/var/data/city.csv`:
+
+```
+Rank , Name , State , 2023 Population , 2020 Census , Annual Change , Density (mi²)
+1 , New York City , New York , 8,992,908 , 8,804,190 , 0.7% , 29,938
+2 , Los Angeles , California , 3,930,586 , 3,898,747 , 0.27% , 8,382
+3 , Chicago , Illinois , 2,761,625 , 2,746,388 , 0.18% , 12,146
+.....
+```
+
+Then user can create a file external table with:
+
+```sql
+CREATE EXTERNAL TABLE city with(location='/var/data/city.csv', format="CSV", field_delimiter = ',', record_delimiter = '\n', skip_header = 1);
+```
+
+Then query the external table with:
+
+```bash
+MySQL> select * from city;
+```
+
+| Rank | Name          | State      | 2023 Population | 2020 Census | Annual Change | Density (mi²) |
+| :--- | :------------ | :--------- | :-------------- | :---------- | :------------ | :------------ |
+| 1    | New York City | New York   | 8,992,908       | 8,804,190   | 0.7%          | 29,938        |
+| 2    | Los Angeles   | California | 3,930,586       | 3,898,747   | 0.27%         | 8,382         |
+| 3    | Chicago       | Illinois   | 2,761,625       | 2,746,388   | 0.18%         | 12,146        |
+
+Drop the external table, if needs with:
+
+```sql
+DROP EXTERNAL TABLE city
+```
+
+
+### Syntax
+
+```
+CREATE EXTERNAL [<database>.]<table_name>
+[
+ (
+    <col_name> <col_type> [NULL | NOT NULL] [COMMENT "<comment>"]
+ )
+]
+[ WITH
+ (
+     LOCATION = 'url'
+   [,FIELD_DELIMITER =  'delimiter' ]
+   [,RECORD_DELIMITER =  'delimiter' ]
+   [,SKIP_HEADER =  '<number>' ]
+   [,FORMAT =  { csv | json | parquet } ]
+   [,PATTERN = '<regex_pattern>' ]
+   [,ENDPOINT = '<uri>' ]
+   [,ACCESS_KEY_ID = '<key_id>' ]
+   [,SECRET_ACCESS_KEY = '<access_key>' ]
+   [,SESSION_TOKEN = '<token>' ]
+   [,REGION = '<region>' ]
+   [,ENABLE_VIRTUAL_HOST_STYLE = '<boolean>']
+   ..
+ )
+]
+```
+
+### Supported File Format
+
+The external file table supports multiple formats; We divide formats into row format and columnar format.
+
+Row formats:
+
+- CSV, JSON
+
+Columnar formats:
+
+- Parquet
+
+Some of these formats support filter pushdown, and others don't. If users use very large files, that format doesn't support pushdown, which might consume a lot of IO for scanning full files and cause a long running query.
+
+### File Table Engine
+
+![overview](external-table-engine-overview.png)
+
+We implement a file table engine that creates an external table by accepting user-specified file paths and treating all records as immutable.
+
+1. File Format Decoder: decode files to the `RecordBatch` stream.
+2. File Table Engine: implement the `TableProvider` trait, store necessary metadata in memory, and provide scan ability.
+
+Our implementation is better for small files. For large files(i.g., a GB-level CSV file), suggests our users import data to the database.
+
+## Drawbacks
+
+- Some formats don't support filter pushdown
+- Hard to support indexing
+
+## Life cycle
+
+### Register a table
+
+1. Write metadata to manifest.
+2. Create the table via file table engine.
+3. Register table to `CatalogProvider` and register table to `SystemCatalog`(persist tables to disk).
+
+### Deregister a table (Drop a table)
+
+1. Fetch the target table info (figure out table engine type).
+2. Deregister the target table in `CatalogProvider` and `SystemCatalog`.
+3. Find the target table engine.
+4. Drop the target table.
+
+### Recover a table when restarting
+
+1. Collect tables name and engine type info.
+2. Find the target tables in different engines.
+3. Open and register tables.
+
+# Alternatives
+
+## Using DataFusion API
+
+We can use datafusion API to register a file table:
+
+```rust
+let ctx = SessionContext::new();
+
+ctx.register_csv("example", "tests/data/example.csv", CsvReadOptions::new()).await?;
+
+// create a plan
+let df = ctx.sql("SELECT a, MIN(b) FROM example WHERE a <= b GROUP BY a LIMIT 100").await?;
+```
+
+### Drawbacks
+
+The DataFusion implements its own `Object Store` abstraction and supports parsing the partitioned directories, which can push down the filter and skips some directories. However, this makes it impossible to use our's `LruCacheLayer`(The parsing of the partitioned directories required paths as input). If we want to manage memory entirely, we should implement our own `TableProvider` or `Table`.
+
+- Impossible to use `CacheLayer`
+
+## Introduce an intermediate representation layer
+
+![overview](external-table-engine-way-2.png)
+
+We convert all files into `parquet` as an intermediate representation. Then we only need to implement a `parquet` file table engine, and we already have a similar one. Also, it supports limited filter pushdown via the `parquet` row group stats.
+
+### Drawbacks
+
+- Computing overhead
+- Storage overhead
+
+
+

docs/rfcs/2023-05-09-distributed-planner.md

@@ -0,0 +1,137 @@
+---
+Feature Name: distributed-planner
+Tracking Issue: TBD
+Date: 2023-05-09
+Author: "Ruihang Xia <waynestxia@gmail.com>"
+---
+
+Distributed Planner
+-------------------
+# Summary
+Enhance the logical planner with aware of distributed, multi-region table topology. To achieve "push computation down" execution rather than the current "pull data up" manner.
+
+# Motivation
+Query distributively can leverage the advantage of GreptimeDB's architecture to process large dataset that exceeds the capacity of a single node, or accelerate the query execution by executing it in parallel. This task includes two sub-tasks
+  - Be able to transform the plan that can push as much as possible computation down to data source.
+  - Be able to handle pipeline breaker (like `Join` or `Sort`) on multiple computation nodes.
+This is a relatively complex topic. To keep this RFC concentrated I'll focus on the first one.
+
+# Details
+## Background: Partition and Region
+GreptimeDB supports table partitioning, where the partition rule is set during table creation. Each partition can be further divided into one or more physical storage units known as "regions". Both partitions and regions are divided based on rows:
+``` text
+┌────────────────────────────────────┐
+│                                    │
+│               Table                │
+│                                    │
+└─────┬────────────┬────────────┬────┘
+      │            │            │
+      │            │            │
+┌─────▼────┐ ┌─────▼────┐ ┌─────▼────┐
+│ Region 1 │ │ Region 2 │ │ Region 3 │
+└──────────┘ └──────────┘ └──────────┘
+  Row 1~10     Row 11~20    Row 21~30
+```
+General speaking, region is the minimum element of data distribution, and we can also use it as the unit to distribute computation. This can greatly simplify the routing logic of this distributed planner, by always schedule the computation to the node that currently opening the corresponding region. And is also easy to scale more node for computing since GreptimeDB's data is persisted on shared storage backend like S3. But this is a bit beyond the scope of this specific topic.
+## Background: Commutativity
+Commutativity is an attribute that describes whether two operation can exchange their apply order: $P1(P2(R)) \Leftrightarrow P2(P1(R))$. If the equation keeps, we can transform one expression into another form without changing its result. This is useful on rewriting SQL expression, and is the theoretical basis of this RFC.
+
+Take this SQL as an example
+
+``` sql
+SELECT a FROM t WHERE a > 10;
+```
+
+As we know projection and filter are commutative (todo: latex), it can be translated to the following two identical plan trees:
+
+```text
+┌─────────────┐       ┌─────────────┐
+│Projection(a)│       │Filter(a>10) │
+└──────▲──────┘       └──────▲──────┘
+       │                     │
+┌──────┴──────┐       ┌──────┴──────┐
+│Filter(a>10) │       │Projection(a)│
+└──────▲──────┘       └──────▲──────┘
+       │                     │
+┌──────┴──────┐       ┌──────┴──────┐
+│  TableScan  │       │  TableScan  │
+└─────────────┘       └─────────────┘
+```
+
+## Merge Operation
+
+This RFC proposes to add a new expression node `MergeScan` to merge result from several regions in the frontend. It wrap the abstraction of remote data and execution, and expose a `TableScan` interface to upper level.
+
+``` text
+        ▲
+        │
+┌───────┼───────┐
+│       │       │
+│    ┌──┴──┐    │
+│    └──▲──┘    │
+│       │       │
+│    ┌──┴──┐    │
+│    └──▲──┘    │    ┌─────────────────────────────┐
+│       │       │    │                             │
+│  ┌────┴────┐  │    │ ┌──────────┐ ┌───┐    ┌───┐ │
+│  │MergeScan◄──┼────┤ │ Region 1 │ │   │ .. │   │ │
+│  └─────────┘  │    │ └──────────┘ └───┘    └───┘ │
+│               │    │                             │
+└─Frontend──────┘    └─Remote-Sources──────────────┘
+```
+This merge operation simply chains all the the underlying remote data sources and return `RecordBatch`, just like a coalesce op. And each remote sources is a gRPC query to datanode via the substrait logical plan interface. The plan is transformed and divided from the original query that comes to frontend.
+
+## Commutativity of MergeScan
+
+Obviously, The position of `MergeScan` is the key of the distributed plan. The more closer to the underlying `TableScan`, the less computation is taken by datanodes. Thus the goal is to pull the `MergeScan` up as more as possible. The word "pull up" means exchange `MergeScan` with its parent node in the plan tree, which means we should check the commutativity between the existing expression nodes and the `MergeScan`. Here I classify all the possibility into five categories:
+
+- Commutative: $P1(P2(R)) \Leftrightarrow P2(P1(R))$
+  - filter
+  - projection
+  - operations that match the partition key
+- Partial Commutative: $P1(P2(R)) \Leftrightarrow P1(P2(P1(R)))$
+  - $min(R) \rightarrow min(MERGE(min(R)))$
+  - $max(R) \rightarrow max(MERGE(max(R)))$
+- Conditional Commutative: $P1(P2(R)) \Leftrightarrow P3(P2(P1(R)))$
+  - $count(R) \rightarrow sum(count(R))$
+- Transformed Commutative: $P1(P2(R)) \Leftrightarrow P1(P3(R)) \Leftrightarrow P3(P1(R))$
+  - $avg(R) \rightarrow sum(R)/count(R)$
+- Non-commutative
+  - sort
+  - join
+  - percentile
+## Steps to plan
+After establishing the set of commutative relations for all expressions, we can begin transforming the logical plan. There are four steps:
+
+  - Add a merge node before table scan
+  - Evaluate commutativity in a bottom-up way, stop at the first non-commutative node
+  - Divide the TableScan to scan over partitions
+  - Execute
+
+First insert the `MergeScan` on top of the bottom `TableScan` node. Then examine the commutativity start from the `MergeScan` node transform the plan tree based on the result. Stop this process on the first non-commutative node.
+``` text
+                  ┌─────────────┐   ┌─────────────┐
+                  │    Sort     │   │    Sort     │
+                  └──────▲──────┘   └──────▲──────┘
+                         │                 │
+┌─────────────┐   ┌──────┴──────┐   ┌──────┴──────┐
+│    Sort     │   │Projection(a)│   │  MergeScan  │
+└──────▲──────┘   └──────▲──────┘   └──────▲──────┘
+       │                 │                 │
+┌──────┴──────┐   ┌──────┴──────┐   ┌──────┴──────┐
+│Projection(a)│   │  MergeScan  │   │Projection(a)│
+└──────▲──────┘   └──────▲──────┘   └──────▲──────┘
+       │                 │                 │
+┌──────┴──────┐   ┌──────┴──────┐   ┌──────┴──────┐
+│  TableScan  │   │  TableScan  │   │  TableScan  │
+└─────────────┘   └─────────────┘   └─────────────┘
+      (a)               (b)               (c)
+```
+Then in the physical planning phase, convert the sub-tree below `MergeScan` into a remote query request and dispatch to all the regions. And let the `MergeScan` to receive the results and feed to it parent node.
+
+To simplify the overall complexity, any error in the procedure will lead to a failure to the entire query, and cancel all other parts.
+# Alternatives
+## Spill
+If only consider the ability of processing large dataset, we can enable DataFusion's spill ability to temporary persist intermediate data into disk, like the "swap" memory. But this will lead to a super slow performance and very large write amplification.
+# Future Work
+As described in the `Motivation` section we can further explore the distributed planner on the physical execution level, by introducing mechanism like Spark's shuffle to improve parallelism and reduce intermediate pipeline breaker's stage.

docs/schema-structs.md

@@ -0,0 +1,527 @@
+# Schema Structs
+
+# Common Schemas
+The `datatypes` crate defines the elementary schema struct to describe the metadata.
+
+## ColumnSchema
+[ColumnSchema](https://github.com/GreptimeTeam/greptimedb/blob/9fa871a3fad07f583dc1863a509414da393747f8/src/datatypes/src/schema/column_schema.rs#L36) represents the metadata of a column. It is equivalent to arrow's [Field](https://docs.rs/arrow/latest/arrow/datatypes/struct.Field.html) with additional metadata such as default constraint and whether the column is a time index. The time index is the column with a `TIME INDEX` constraint of a table. We can convert the `ColumnSchema` into an arrow `Field` and convert the `Field` back to the `ColumnSchema` without losing metadata.
+
+```rust
+pub struct ColumnSchema {
+    pub name: String,
+    pub data_type: ConcreteDataType,
+    is_nullable: bool,
+    is_time_index: bool,
+    default_constraint: Option<ColumnDefaultConstraint>,
+    metadata: Metadata,
+}
+```
+
+## Schema
+[Schema](https://github.com/GreptimeTeam/greptimedb/blob/9fa871a3fad07f583dc1863a509414da393747f8/src/datatypes/src/schema.rs#L38) is an ordered sequence of `ColumnSchema`. It is equivalent to arrow's [Schema](https://docs.rs/arrow/latest/arrow/datatypes/struct.Schema.html) with additional metadata including the index of the time index column and the version of this schema. Same as `ColumnSchema`, we can convert our `Schema` from/to arrow's `Schema`.
+
+```rust
+use arrow::datatypes::Schema as ArrowSchema;
+
+pub struct Schema {
+    column_schemas: Vec<ColumnSchema>,
+    name_to_index: HashMap<String, usize>,
+    arrow_schema: Arc<ArrowSchema>,
+    timestamp_index: Option<usize>,
+    version: u32,
+}
+
+pub type SchemaRef = Arc<Schema>;
+```
+
+We alias `Arc<Schema>` as `SchemaRef` since it is used frequently. Mostly, we use our `ColumnSchema` and `Schema` structs instead of Arrow's `Field` and `Schema` unless we need to invoke third-party libraries (like DataFusion or ArrowFlight) that rely on Arrow.
+
+## RawSchema
+`Schema` contains fields like a map from column names to their indices in the `ColumnSchema` sequences and a cached arrow `Schema`. We can construct these fields from the `ColumnSchema` sequences thus we don't want to serialize them. This is why we don't derive `Serialize` and `Deserialize` for `Schema`. We introduce a new struct [RawSchema](https://github.com/GreptimeTeam/greptimedb/blob/9fa871a3fad07f583dc1863a509414da393747f8/src/datatypes/src/schema/raw.rs#L24) which keeps all required fields of a `Schema` and derives the serialization traits. To serialize a `Schema`, we need to convert it into a `RawSchema` first and serialize the `RawSchema`.
+
+```rust
+pub struct RawSchema {
+    pub column_schemas: Vec<ColumnSchema>,
+    pub timestamp_index: Option<usize>,
+    pub version: u32,
+}
+```
+
+We want to keep the `Schema` simple and avoid putting too much business-related metadata in it as many different structs or traits rely on it.
+
+# Schema of the Table
+A table maintains its schema in [TableMeta](https://github.com/GreptimeTeam/greptimedb/blob/9fa871a3fad07f583dc1863a509414da393747f8/src/table/src/metadata.rs#L97).
+```rust
+pub struct TableMeta {
+    pub schema: SchemaRef,
+    pub primary_key_indices: Vec<usize>,
+    pub value_indices: Vec<usize>,
+    // ...
+}
+```
+
+The order of columns in `TableMeta::schema` is the same as the order specified in the `CREATE TABLE` statement which users use to create this table.
+
+The field `primary_key_indices` stores indices of primary key columns. The field `value_indices` records the indices of value columns (non-primary key and time index, we sometimes call them field columns).
+
+Suppose we create a table with the following SQL
+```sql
+CREATE TABLE cpu (
+    ts TIMESTAMP,
+    host STRING,
+    usage_user DOUBLE,
+    usage_system DOUBLE,
+    datacenter STRING,
+    TIME INDEX (ts),
+    PRIMARY KEY(datacenter, host)) ENGINE=mito WITH(regions=1);
+```
+
+Then the table's `TableMeta` may look like this:
+```json
+{
+    "schema":{
+        "column_schemas":[
+            "ts",
+            "host",
+            "usage_user",
+            "usage_system",
+            "datacenter"
+        ],
+        "time_index":0,
+        "version":0
+    },
+    "primary_key_indices":[
+        4,
+        1
+    ],
+    "value_indices":[
+        2,
+        3
+    ]
+}
+```
+
+
+# Schemas of the storage engine
+We split a table into one or more units with the same schema and then store these units in the storage engine. Each unit is a region in the storage engine.
+
+The storage engine maintains schemas of regions in more complicated ways because it
+- adds internal columns that are invisible to users to store additional metadata for each row
+- provides a data model similar to the key-value model so it organizes columns in a different order
+- maintains additional metadata like column id or column family
+
+So the storage engine defines several schema structs:
+- RegionSchema
+- StoreSchema
+- ProjectedSchema
+
+## RegionSchema
+A [RegionSchema](https://github.com/GreptimeTeam/greptimedb/blob/9fa871a3fad07f583dc1863a509414da393747f8/src/storage/src/schema/region.rs#L37) describes the schema of a region.
+
+```rust
+pub struct RegionSchema {
+    user_schema: SchemaRef,
+    store_schema: StoreSchemaRef,
+    columns: ColumnsMetadataRef,
+}
+```
+
+Each region reserves some columns called `internal columns` for internal usage:
+- `__sequence`, sequence number of a row
+- `__op_type`, operation type of a row, such as `PUT` or `DELETE`
+- `__version`, user-specified version of a row, reserved but not used. We might remove this in the future
+
+The table engine can't see the `__sequence` and `__op_type` columns, so the `RegionSchema` itself maintains two internal schemas:
+- User schema, a `Schema` struct that doesn't have internal columns
+- Store schema, a `StoreSchema` struct that has internal columns
+
+The `ColumnsMetadata` struct keeps metadata about all columns but most time we only need to use metadata in user schema and store schema, so we just ignore it. We may remove this struct in the future.
+
+`RegionSchema` organizes columns in the following order:
+```
+key columns, timestamp, [__version,] value columns, __sequence, __op_type
+```
+
+We can ignore the `__version` column because it is disabled now:
+
+```
+key columns, timestamp, value columns, __sequence, __op_type
+```
+
+Key columns are columns of a table's primary key. Timestamp is the time index column. A region sorts all rows by key columns, timestamp, sequence, and op type.
+
+So the `RegionSchema` of our `cpu` table above looks like this:
+```json
+{
+    "user_schema":[
+        "datacenter",
+        "host",
+        "ts",
+        "usage_user",
+        "usage_system"
+    ],
+    "store_schema":[
+        "datacenter",
+        "host",
+        "ts",
+        "usage_user",
+        "usage_system",
+        "__sequence",
+        "__op_type"
+    ]
+}
+```
+
+## StoreSchema
+As described above, a [StoreSchema](https://github.com/GreptimeTeam/greptimedb/blob/9fa871a3fad07f583dc1863a509414da393747f8/src/storage/src/schema/store.rs#L36) is a schema that knows all internal columns.
+```rust
+struct StoreSchema {
+    columns: Vec<ColumnMetadata>,
+    schema: SchemaRef,
+    row_key_end: usize,
+    user_column_end: usize,
+}
+```
+
+The columns in the `columns` and `schema` fields have the same order. The `ColumnMetadata` has metadata like column id, column family id, and comment. The `StoreSchema` also stores this metadata in `StoreSchema::schema`, so we can convert the `StoreSchema` between arrow's `Schema`. We use this feature to persist the `StoreSchema` in the SST since our SST format is `Parquet`, which can take arrow's `Schema` as its schema.
+
+The `StoreSchema` of the region above is similar to this:
+```json
+{
+    "schema":{
+        "column_schemas":[
+            "datacenter",
+            "host",
+            "ts",
+            "usage_user",
+            "usage_system",
+            "__sequence",
+            "__op_type"
+        ],
+        "time_index":2,
+        "version":0
+    },
+    "row_key_end":3,
+    "user_column_end":5
+}
+```
+
+The key and timestamp columns form row keys of rows. We put them together so we can use `row_key_end` to get indices of all row key columns. Similarly, we can use the `user_column_end` to get indices of all user columns (non-internal columns).
+```rust
+impl StoreSchema {
+    #[inline]
+    pub(crate) fn row_key_indices(&self) -> impl Iterator<Item = usize> {
+        0..self.row_key_end
+    }
+
+    #[inline]
+    pub(crate) fn value_indices(&self) -> impl Iterator<Item = usize> {
+        self.row_key_end..self.user_column_end
+    }
+}
+```
+
+Another useful feature of `StoreSchema` is that we ensure it always contains key columns, a timestamp column, and internal columns because we need them to perform merge, deduplication, and delete. Projection on `StoreSchema` only projects value columns.
+
+## ProjectedSchema
+To support arbitrary projection, we introduce the [ProjectedSchema](https://github.com/GreptimeTeam/greptimedb/blob/9fa871a3fad07f583dc1863a509414da393747f8/src/storage/src/schema/projected.rs#L106).
+```rust
+pub struct ProjectedSchema {
+    projection: Option<Projection>,
+    schema_to_read: StoreSchemaRef,
+    projected_user_schema: SchemaRef,
+}
+```
+
+We need to handle many cases while doing projection:
+- The columns' order of table and region is different
+- The projection can be in arbitrary order, e.g. `select usage_user, host from cpu` and `select host, usage_user from cpu` have different projection order
+- We support `ALTER TABLE` so data files may have different schemas.
+
+### Projection
+Let's take an example to see how projection works. Suppose we want to select `ts`, `usage_system` from the `cpu` table.
+
+```sql
+CREATE TABLE cpu (
+    ts TIMESTAMP,
+    host STRING,
+    usage_user DOUBLE,
+    usage_system DOUBLE,
+    datacenter STRING,
+    TIME INDEX (ts),
+    PRIMARY KEY(datacenter, host)) ENGINE=mito WITH(regions=1);
+
+select ts, usage_system from cpu;
+```
+
+The query engine uses the projection `[0, 3]` to scan the table. However, columns in the region have a different order, so the table engine adjusts the projection to `2, 4`.
+```json
+{
+    "user_schema":[
+        "datacenter",
+        "host",
+        "ts",
+        "usage_user",
+        "usage_system"
+    ],
+}
+```
+
+As you can see, the output order is still `[ts, usage_system]`. This is the schema users can see after projection so we call it `projected user schema`.
+
+But the storage engine also needs to read key columns, a timestamp column, and internal columns. So we maintain a `StoreSchema` after projection in the `ProjectedSchema`.
+
+The `Projection` struct is a helper struct to help compute the projected user schema and store schema.
+
+So we can construct the following `ProjectedSchema`:
+```json
+{
+    "schema_to_read":{
+        "schema":{
+            "column_schemas":[
+                "datacenter",
+                "host",
+                "ts",
+                "usage_system",
+                "__sequence",
+                "__op_type"
+            ],
+            "time_index":2,
+            "version":0
+        },
+        "row_key_end":3,
+        "user_column_end":4
+    },
+    "projected_user_schema":{
+        "column_schemas":[
+            "ts",
+            "usage_system"
+        ],
+        "time_index":0
+    }
+}
+```
+
+As you can see, `schema_to_read` doesn't contain the column `usage_user` that is not intended to be read (not in projection).
+
+### ReadAdapter
+As mentioned above, we can alter a table so the underlying files (SSTs) and memtables in the storage engine may have different schemas.
+
+To simplify the logic of `ProjectedSchema`, we handle the difference between schemas before projection (constructing the `ProjectedSchema`). We introduce [ReadAdapter](https://github.com/GreptimeTeam/greptimedb/blob/9fa871a3fad07f583dc1863a509414da393747f8/src/storage/src/schema/compat.rs#L90) that adapts rows with different source schemas to the same expected schema.
+
+So we can always use the current `RegionSchema` of the region to construct the `ProjectedSchema`, and then create a `ReadAdapter` for each memtable or SST.
+```rust
+#[derive(Debug)]
+pub struct ReadAdapter {
+    source_schema: StoreSchemaRef,
+    dest_schema: ProjectedSchemaRef,
+    indices_in_result: Vec<Option<usize>>,
+    is_source_needed: Vec<bool>,
+}
+```
+
+For each column required by `dest_schema`, `indices_in_result` stores the index of that column in the row read from the source memtable or SST. If the source row doesn't contain that column, the index is `None`.
+
+The field `is_source_needed` stores whether a column in the source memtable or SST is needed.
+
+Suppose we add a new column `usage_idle` to the table `cpu`.
+```sql
+ALTER TABLE cpu ADD COLUMN usage_idle DOUBLE;
+```
+
+The new `StoreSchema` becomes:
+```json
+{
+    "schema":{
+        "column_schemas":[
+            "datacenter",
+            "host",
+            "ts",
+            "usage_user",
+            "usage_system",
+            "usage_idle",
+            "__sequence",
+            "__op_type"
+        ],
+        "time_index":2,
+        "version":1
+    },
+    "row_key_end":3,
+    "user_column_end":6
+}
+```
+
+Note that we bump the version of the schema to 1.
+
+If we want to select `ts`, `usage_system`, and `usage_idle`. While reading from the old schema, the storage engine creates a `ReadAdapter` like this:
+```json
+{
+    "source_schema":{
+        "schema":{
+            "column_schemas":[
+                "datacenter",
+                "host",
+                "ts",
+                "usage_user",
+                "usage_system",
+                "__sequence",
+                "__op_type"
+            ],
+            "time_index":2,
+            "version":0
+        },
+        "row_key_end":3,
+        "user_column_end":5
+    },
+    "dest_schema":{
+        "schema_to_read":{
+            "schema":{
+                "column_schemas":[
+                    "datacenter",
+                    "host",
+                    "ts",
+                    "usage_system",
+                    "usage_idle",
+                    "__sequence",
+                    "__op_type"
+                ],
+                "time_index":2,
+                "version":1
+            },
+            "row_key_end":3,
+            "user_column_end":5
+        },
+        "projected_user_schema":{
+            "column_schemas":[
+                "ts",
+                "usage_system",
+                "usage_idle"
+            ],
+            "time_index":0
+        }
+    },
+    "indices_in_result":[
+        0,
+        1,
+        2,
+        3,
+        null,
+        4,
+        5
+    ],
+    "is_source_needed":[
+        true,
+        true,
+        true,
+        false,
+        true,
+        true,
+        true
+    ]
+}
+```
+
+We don't need to read `usage_user` so `is_source_needed[3]` is false. The old schema doesn't have column `usage_idle` so `indices_in_result[4]` is `null` and the `ReadAdapter` needs to insert a null column to the output row so the output schema still contains `usage_idle`.
+
+The figure below shows the relationship between `RegionSchema`, `StoreSchema`, `ProjectedSchema`, and `ReadAdapter`.
+
+```text
+                   ┌──────────────────────────────┐
+                   │                              │
+                   │    ┌────────────────────┐    │
+                   │    │    store_schema    │    │
+                   │    │                    │    │
+                   │    │     StoreSchema    │    │
+                   │    │      version 1     │    │
+                   │    └────────────────────┘    │
+                   │                              │
+                   │    ┌────────────────────┐    │
+                   │    │     user_schema    │    │
+                   │    └────────────────────┘    │
+                   │                              │
+                   │         RegionSchema         │
+                   │                              │
+                   └──────────────┬───────────────┘
+                                  │
+                                  │
+                                  │
+                   ┌──────────────▼───────────────┐
+                   │                              │
+                   │ ┌──────────────────────────┐ │
+                   │ │     schema_to_read       │ │
+                   │ │                          │ │
+                   │ │  StoreSchema (projected) │ │
+                   │ │       version 1          │ │
+                   │ └──────────────────────────┘ │
+               ┌───┤                              ├───┐
+               │   │ ┌──────────────────────────┐ │   │
+               │   │ │  projected_user_schema   │ │   │
+               │   │ └──────────────────────────┘ │   │
+               │   │                              │   │
+               │   │       ProjectedSchema        │   │
+  dest schema  │   └──────────────────────────────┘   │   dest schema
+               │                                      │
+               │                                      │
+        ┌──────▼───────┐                      ┌───────▼──────┐
+        │              │                      │              │
+        │  ReadAdapter │                      │  ReadAdapter │
+        │              │                      │              │
+        └──────▲───────┘                      └───────▲──────┘
+               │                                      │
+               │                                      │
+source schema  │                                      │  source schema
+               │                                      │
+       ┌───────┴─────────┐                   ┌────────┴────────┐
+       │                 │                   │                 │
+       │ ┌─────────────┐ │                   │ ┌─────────────┐ │
+       │ │             │ │                   │ │             │ │
+       │ │ StoreSchema │ │                   │ │ StoreSchema │ │
+       │ │             │ │                   │ │             │ │
+       │ │  version 0  │ │                   │ │  version 1  │ │
+       │ │             │ │                   │ │             │ │
+       │ └─────────────┘ │                   │ └─────────────┘ │
+       │                 │                   │                 │
+       │      SST 0      │                   │      SST 1      │
+       │                 │                   │                 │
+       └─────────────────┘                   └─────────────────┘
+```
+
+# Conversion
+This figure shows the conversion between schemas:
+```text
+              ┌─────────────┐     schema                      From             ┌─────────────┐
+              │             ├──────────────────┐  ┌────────────────────────────►             │
+              │  TableMeta  │                  │  │                            │  RawSchema  │
+              │             │                  │  │  ┌─────────────────────────┤             │
+              └─────────────┘                  │  │  │        TryFrom          └─────────────┘
+                                               │  │  │
+                                               │  │  │
+                                               │  │  │
+                                               │  │  │
+                                               │  │  │
+    ┌───────────────────┐                ┌─────▼──┴──▼──┐   arrow_schema()    ┌─────────────────┐
+    │                   │                │              ├─────────────────────►                 │
+    │  ColumnsMetadata  │          ┌─────►    Schema    │                     │   ArrowSchema   ├──┐
+    │                   │          │     │              ◄─────────────────────┤                 │  │
+    └────┬───────────▲──┘          │     └───▲───▲──────┘       TryFrom       └─────────────────┘  │
+         │           │             │         │   │                                                 │
+         │           │             │         │   └────────────────────────────────────────┐        │
+         │           │             │         │                                            │        │
+         │   columns │    user_schema()      │                                            │        │
+         │           │             │         │ projected_user_schema()                 schema()    │
+         │           │             │         │                                            │        │
+         │       ┌───┴─────────────┴─┐       │                 ┌────────────────────┐     │        │
+columns  │       │                   │       └─────────────────┤                    │     │        │  TryFrom
+         │       │    RegionSchema   │                         │   ProjectedSchema  │     │        │
+         │       │                   ├─────────────────────────►                    │     │        │
+         │       └─────────────────┬─┘  ProjectedSchema::new() └──────────────────┬─┘     │        │
+         │                         │                                              │       │        │
+         │                         │                                              │       │        │
+         │                         │                                              │       │        │
+         │                         │                                              │       │        │
+    ┌────▼────────────────────┐    │               store_schema()            ┌────▼───────┴──┐     │
+    │                         │    └─────────────────────────────────────────►               │     │
+    │   Vec<ColumnMetadata>   │                                              │  StoreSchema  ◄─────┘
+    │                         ◄──────────────────────────────────────────────┤               │
+    └─────────────────────────┘                     columns                  └───────────────┘
+```

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-02-26"
+channel = "nightly-2023-05-03"

scripts/fetch-dashboard-assets.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# This script is used to download built dashboard assets from the "GreptimeTeam/dashboard" repository.
+
+set -e
+
+declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
+declare -r ROOT_DIR=$(dirname ${SCRIPT_DIR})
+declare -r STATIC_DIR="$ROOT_DIR/src/servers/dashboard"
+OUT_DIR="${1:-$SCRIPT_DIR}"
+
+RELEASE_VERSION="$(cat $STATIC_DIR/VERSION)"
+
+echo "Downloading assets to dir: $OUT_DIR"
+cd $OUT_DIR
+# Download the SHA256 checksum attached to the release. To verify the integrity
+# of the download, this checksum will be used to check the download tar file
+# containing the built dashboard assets.
+curl -Ls https://github.com/GreptimeTeam/dashboard/releases/download/$RELEASE_VERSION/sha256.txt --output sha256.txt
+
+# Download the tar file containing the built dashboard assets.
+curl -L https://github.com/GreptimeTeam/dashboard/releases/download/$RELEASE_VERSION/build.tar.gz --output build.tar.gz
+
+# Verify the checksums match; exit if they don't.
+case "$(uname -s)" in
+    FreeBSD | Darwin)
+        echo "$(cat sha256.txt)" | shasum --algorithm 256 --check \
+            || { echo "Checksums did not match for downloaded dashboard assets!"; exit 1; } ;;
+    Linux)
+        echo "$(cat sha256.txt)" | sha256sum --check -- \
+            || { echo "Checksums did not match for downloaded dashboard assets!"; exit 1; } ;;
+    *)
+        echo "The '$(uname -s)' operating system is not supported as a build host for the dashboard" >&2
+        exit 1
+esac
+
+# Extract the assets and clean up.
+tar -xzf build.tar.gz -C "$STATIC_DIR"
+rm sha256.txt
+rm build.tar.gz
+
+echo "Successfully download dashboard assets to $STATIC_DIR"

scripts/install.sh

@@ -51,13 +51,17 @@ get_os_type
 get_arch_type
 
 if [ -n "${OS_TYPE}" ] && [ -n "${ARCH_TYPE}" ]; then
-    echo "Downloading ${BIN}, OS: ${OS_TYPE}, Arch: ${ARCH_TYPE}, Version: ${VERSION}"
-
+    # Use the latest nightly version.
     if [ "${VERSION}" = "latest" ]; then
-        wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/latest/download/${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz"
-    else
-        wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz"
+        VERSION=$(curl -s -XGET "https://api.github.com/repos/${GITHUB_ORG}/${GITHUB_REPO}/releases" | grep tag_name | grep nightly | cut -d: -f 2 | sed 's/.*"\(.*\)".*/\1/' | uniq | sort -r | head -n 1)
+        if [ -z "${VERSION}" ]; then
+            echo "Failed to get the latest version."
+            exit 1
+        fi
     fi
 
+    echo "Downloading ${BIN}, OS: ${OS_TYPE}, Arch: ${ARCH_TYPE}, Version: ${VERSION}"
+
+    wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz"
     tar xvf ${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz && rm ${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz && echo "Run './${BIN} --help' to get started"
 fi

src/api/Cargo.toml

@@ -10,10 +10,10 @@ common-base = { path = "../common/base" }
 common-error = { path = "../common/error" }
 common-time = { path = "../common/time" }
 datatypes = { path = "../datatypes" }
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "3a715150563b89d5dfc81a5838eac1f66a5658a1" }
+greptime-proto.workspace = true
 prost.workspace = true
 snafu = { version = "0.7", features = ["backtraces"] }
 tonic.workspace = true
 
 [build-dependencies]
-tonic-build = "0.8"
+tonic-build = "0.9"

src/api/src/error.rs

@@ -18,7 +18,7 @@ use common_error::ext::ErrorExt;
 use common_error::prelude::StatusCode;
 use datatypes::prelude::ConcreteDataType;
 use snafu::prelude::*;
-use snafu::{Backtrace, ErrorCompat};
+use snafu::Location;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -26,12 +26,12 @@ pub type Result<T> = std::result::Result<T, Error>;
 #[snafu(visibility(pub))]
 pub enum Error {
     #[snafu(display("Unknown proto column datatype: {}", datatype))]
-    UnknownColumnDataType { datatype: i32, backtrace: Backtrace },
+    UnknownColumnDataType { datatype: i32, location: Location },
 
     #[snafu(display("Failed to create column datatype from {:?}", from))]
     IntoColumnDataType {
         from: ConcreteDataType,
-        backtrace: Backtrace,
+        location: Location,
     },
 
     #[snafu(display(
@@ -41,7 +41,7 @@ pub enum Error {
     ))]
     ConvertColumnDefaultConstraint {
         column: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
@@ -52,7 +52,7 @@ pub enum Error {
     ))]
     InvalidColumnDefaultConstraint {
         column: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 }
@@ -66,9 +66,6 @@ impl ErrorExt for Error {
             | Error::InvalidColumnDefaultConstraint { source, .. } => source.status_code(),
         }
     }
-    fn backtrace_opt(&self) -> Option<&Backtrace> {
-        ErrorCompat::backtrace(self)
-    }
 
     fn as_any(&self) -> &dyn Any {
         self

src/api/src/helper.rs

@@ -18,6 +18,10 @@ use datatypes::prelude::ConcreteDataType;
 use datatypes::types::TimestampType;
 use datatypes::value::Value;
 use datatypes::vectors::VectorRef;
+use greptime_proto::v1::ddl_request::Expr;
+use greptime_proto::v1::greptime_request::Request;
+use greptime_proto::v1::query_request::Query;
+use greptime_proto::v1::{DdlRequest, QueryRequest};
 use snafu::prelude::*;
 
 use crate::error::{self, Result};
@@ -224,6 +228,38 @@ pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
     column.null_mask = null_mask.into_vec();
 }
 
+/// Returns the type name of the [Request].
+pub fn request_type(request: &Request) -> &'static str {
+    match request {
+        Request::Inserts(_) => "inserts",
+        Request::Query(query_req) => query_request_type(query_req),
+        Request::Ddl(ddl_req) => ddl_request_type(ddl_req),
+        Request::Delete(_) => "delete",
+    }
+}
+
+/// Returns the type name of the [QueryRequest].
+fn query_request_type(request: &QueryRequest) -> &'static str {
+    match request.query {
+        Some(Query::Sql(_)) => "query.sql",
+        Some(Query::LogicalPlan(_)) => "query.logical_plan",
+        Some(Query::PromRangeQuery(_)) => "query.prom_range",
+        None => "query.empty",
+    }
+}
+
+/// Returns the type name of the [DdlRequest].
+fn ddl_request_type(request: &DdlRequest) -> &'static str {
+    match request.expr {
+        Some(Expr::CreateDatabase(_)) => "ddl.create_database",
+        Some(Expr::CreateTable(_)) => "ddl.create_table",
+        Some(Expr::Alter(_)) => "ddl.alter",
+        Some(Expr::DropTable(_)) => "ddl.drop_table",
+        Some(Expr::FlushTable(_)) => "ddl.flush_table",
+        None => "ddl.empty",
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;

src/api/src/lib.rs

@@ -23,4 +23,5 @@ pub mod prometheus {
 
 pub mod v1;
 
+pub use greptime_proto;
 pub use prost::DecodeError;

src/catalog/Cargo.toml

@@ -4,15 +4,20 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
+[features]
+testing = []
+
 [dependencies]
 api = { path = "../api" }
 arc-swap = "1.0"
+arrow-schema.workspace = true
 async-stream.workspace = true
 async-trait = "0.1"
 backoff = { version = "0.4", features = ["tokio"] }
 common-catalog = { path = "../common/catalog" }
 common-error = { path = "../common/error" }
 common-grpc = { path = "../common/grpc" }
+common-meta = { path = "../common/meta" }
 common-query = { path = "../common/query" }
 common-recordbatch = { path = "../common/recordbatch" }
 common-runtime = { path = "../common/runtime" }
@@ -23,8 +28,11 @@ datafusion.workspace = true
 datatypes = { path = "../datatypes" }
 futures = "0.3"
 futures-util.workspace = true
+key-lock = "0.1"
 lazy_static = "1.4"
 meta-client = { path = "../meta-client" }
+metrics.workspace = true
+moka = { version = "0.11", features = ["future"] }
 parking_lot = "0.12"
 regex = "1.6"
 serde = "1.0"
@@ -32,10 +40,12 @@ serde_json = "1.0"
 session = { path = "../session" }
 snafu = { version = "0.7", features = ["backtraces"] }
 storage = { path = "../storage" }
+store-api = { path = "../store-api" }
 table = { path = "../table" }
 tokio.workspace = true
 
 [dev-dependencies]
+catalog = { path = ".", features = ["testing"] }
 common-test-util = { path = "../common/test-util" }
 chrono.workspace = true
 log-store = { path = "../log-store" }

src/catalog/src/error.rs

@@ -19,22 +19,31 @@ use common_error::ext::{BoxedError, ErrorExt};
 use common_error::prelude::{Snafu, StatusCode};
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
-use snafu::{Backtrace, ErrorCompat};
+use snafu::Location;
+use tokio::task::JoinError;
 
 use crate::DeregisterTableRequest;
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
+    #[snafu(display(
+        "Failed to re-compile script due to internal error, source: {}",
+        source
+    ))]
+    CompileScriptInternal {
+        location: Location,
+        source: BoxedError,
+    },
     #[snafu(display("Failed to open system catalog table, source: {}", source))]
     OpenSystemCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("Failed to create system catalog table, source: {}", source))]
     CreateSystemCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -45,12 +54,12 @@ pub enum Error {
     ))]
     CreateTable {
         table_info: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("System catalog is not valid: {}", msg))]
-    SystemCatalog { msg: String, backtrace: Backtrace },
+    SystemCatalog { msg: String, location: Location },
 
     #[snafu(display(
         "System catalog table type mismatch, expected: binary, found: {:?}",
@@ -58,77 +67,93 @@ pub enum Error {
     ))]
     SystemCatalogTypeMismatch {
         data_type: ConcreteDataType,
-        backtrace: Backtrace,
+        location: Location,
     },
 
     #[snafu(display("Invalid system catalog entry type: {:?}", entry_type))]
     InvalidEntryType {
         entry_type: Option<u8>,
-        backtrace: Backtrace,
+        location: Location,
     },
 
     #[snafu(display("Invalid system catalog key: {:?}", key))]
     InvalidKey {
         key: Option<String>,
-        backtrace: Backtrace,
+        location: Location,
     },
 
     #[snafu(display("Catalog value is not present"))]
-    EmptyValue { backtrace: Backtrace },
+    EmptyValue { location: Location },
 
     #[snafu(display("Failed to deserialize value, source: {}", source))]
     ValueDeserialize {
         source: serde_json::error::Error,
-        backtrace: Backtrace,
+        location: Location,
+    },
+
+    #[snafu(display("Table engine not found: {}, source: {}", engine_name, source))]
+    TableEngineNotFound {
+        engine_name: String,
+        location: Location,
+        source: table::error::Error,
     },
 
     #[snafu(display("Cannot find catalog by name: {}", catalog_name))]
     CatalogNotFound {
         catalog_name: String,
-        backtrace: Backtrace,
+        location: Location,
     },
 
     #[snafu(display("Cannot find schema {} in catalog {}", schema, catalog))]
     SchemaNotFound {
         catalog: String,
         schema: String,
-        backtrace: Backtrace,
+        location: Location,
     },
 
     #[snafu(display("Table `{}` already exists", table))]
-    TableExists { table: String, backtrace: Backtrace },
+    TableExists { table: String, location: Location },
 
-    #[snafu(display("Table `{}` not exist", table))]
-    TableNotExist { table: String, backtrace: Backtrace },
+    #[snafu(display("Table not found: {}", table))]
+    TableNotExist { table: String, location: Location },
 
     #[snafu(display("Schema {} already exists", schema))]
-    SchemaExists {
-        schema: String,
-        backtrace: Backtrace,
-    },
+    SchemaExists { schema: String, location: Location },
 
     #[snafu(display("Operation {} not implemented yet", operation))]
     Unimplemented {
         operation: String,
-        backtrace: Backtrace,
+        location: Location,
     },
 
+    #[snafu(display("Operation {} not supported", op))]
+    NotSupported { op: String, location: Location },
+
     #[snafu(display("Failed to open table, table info: {}, source: {}", table_info, source))]
     OpenTable {
         table_info: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
+    #[snafu(display("Failed to open table in parallel, source: {}", source))]
+    ParallelOpenTable { source: JoinError },
+
     #[snafu(display("Table not found while opening table, table info: {}", table_info))]
     TableNotFound {
         table_info: String,
-        backtrace: Backtrace,
+        location: Location,
     },
 
     #[snafu(display("Failed to read system catalog table records"))]
     ReadSystemCatalog {
-        #[snafu(backtrace)]
+        location: Location,
+        source: common_recordbatch::error::Error,
+    },
+
+    #[snafu(display("Failed to create recordbatch, source: {}", source))]
+    CreateRecordBatch {
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
@@ -137,7 +162,7 @@ pub enum Error {
         source
     ))]
     InsertCatalogRecord {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -148,77 +173,69 @@ pub enum Error {
     ))]
     DeregisterTable {
         request: DeregisterTableRequest,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("Illegal catalog manager state: {}", msg))]
-    IllegalManagerState { backtrace: Backtrace, msg: String },
+    IllegalManagerState { location: Location, msg: String },
 
     #[snafu(display("Failed to scan system catalog table, source: {}", source))]
     SystemCatalogTableScan {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failure during SchemaProvider operation, source: {}", source))]
-    SchemaProviderOperation {
-        #[snafu(backtrace)]
-        source: BoxedError,
-    },
-
     #[snafu(display("{source}"))]
     Internal {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
     #[snafu(display("Failed to execute system catalog table scan, source: {}", source))]
     SystemCatalogTableScanExec {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_query::error::Error,
     },
     #[snafu(display("Cannot parse catalog value, source: {}", source))]
     InvalidCatalogValue {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_catalog::error::Error,
     },
 
     #[snafu(display("Failed to perform metasrv operation, source: {}", source))]
     MetaSrv {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_client::error::Error,
     },
 
     #[snafu(display("Invalid table info in catalog, source: {}", source))]
     InvalidTableInfoInCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
-    #[snafu(display("Failed to serialize or deserialize catalog entry: {}", source))]
-    CatalogEntrySerde {
-        #[snafu(backtrace)]
-        source: common_catalog::error::Error,
-    },
-
     #[snafu(display("Illegal access to catalog: {} and schema: {}", catalog, schema))]
     QueryAccessDenied { catalog: String, schema: String },
 
-    #[snafu(display(
-        "Failed to get region stats, catalog: {}, schema: {}, table: {}, source: {}",
-        catalog,
-        schema,
-        table,
-        source
-    ))]
-    RegionStats {
-        catalog: String,
-        schema: String,
-        table: String,
-        #[snafu(backtrace)]
+    #[snafu(display("Invalid system table definition: {err_msg}"))]
+    InvalidSystemTableDef { err_msg: String, location: Location },
+
+    #[snafu(display("{}: {}", msg, source))]
+    Datafusion {
+        msg: String,
+        source: DataFusionError,
+        location: Location,
+    },
+
+    #[snafu(display("Table schema mismatch, source: {}", source))]
+    TableSchemaMismatch {
+        location: Location,
         source: table::error::Error,
     },
+
+    #[snafu(display("A generic error has occurred, msg: {}", msg))]
+    Generic { msg: String, location: Location },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -231,22 +248,26 @@ impl ErrorExt for Error {
             | Error::TableNotFound { .. }
             | Error::IllegalManagerState { .. }
             | Error::CatalogNotFound { .. }
-            | Error::InvalidEntryType { .. } => StatusCode::Unexpected,
+            | Error::InvalidEntryType { .. }
+            | Error::InvalidSystemTableDef { .. }
+            | Error::ParallelOpenTable { .. } => StatusCode::Unexpected,
 
             Error::SystemCatalog { .. }
             | Error::EmptyValue { .. }
             | Error::ValueDeserialize { .. } => StatusCode::StorageUnavailable,
 
-            Error::SystemCatalogTypeMismatch { .. } => StatusCode::Internal,
+            Error::Generic { .. } | Error::SystemCatalogTypeMismatch { .. } => StatusCode::Internal,
 
-            Error::ReadSystemCatalog { source, .. } => source.status_code(),
-            Error::InvalidCatalogValue { source, .. } | Error::CatalogEntrySerde { source } => {
+            Error::ReadSystemCatalog { source, .. } | Error::CreateRecordBatch { source, .. } => {
                 source.status_code()
             }
+            Error::InvalidCatalogValue { source, .. } => source.status_code(),
 
             Error::TableExists { .. } => StatusCode::TableAlreadyExists,
             Error::TableNotExist { .. } => StatusCode::TableNotFound,
-            Error::SchemaExists { .. } => StatusCode::InvalidArguments,
+            Error::SchemaExists { .. } | Error::TableEngineNotFound { .. } => {
+                StatusCode::InvalidArguments
+            }
 
             Error::OpenSystemCatalog { source, .. }
             | Error::CreateSystemCatalog { source, .. }
@@ -254,25 +275,23 @@ impl ErrorExt for Error {
             | Error::OpenTable { source, .. }
             | Error::CreateTable { source, .. }
             | Error::DeregisterTable { source, .. }
-            | Error::RegionStats { source, .. } => source.status_code(),
+            | Error::TableSchemaMismatch { source, .. } => source.status_code(),
 
             Error::MetaSrv { source, .. } => source.status_code(),
-            Error::SystemCatalogTableScan { source } => source.status_code(),
-            Error::SystemCatalogTableScanExec { source } => source.status_code(),
-            Error::InvalidTableInfoInCatalog { source } => source.status_code(),
-            Error::SchemaProviderOperation { source } | Error::Internal { source } => {
+            Error::SystemCatalogTableScan { source, .. } => source.status_code(),
+            Error::SystemCatalogTableScanExec { source, .. } => source.status_code(),
+            Error::InvalidTableInfoInCatalog { source, .. } => source.status_code(),
+
+            Error::CompileScriptInternal { source, .. } | Error::Internal { source, .. } => {
                 source.status_code()
             }
 
-            Error::Unimplemented { .. } => StatusCode::Unsupported,
+            Error::Unimplemented { .. } | Error::NotSupported { .. } => StatusCode::Unsupported,
             Error::QueryAccessDenied { .. } => StatusCode::AccessDenied,
+            Error::Datafusion { .. } => StatusCode::EngineExecuteQuery,
         }
     }
 
-    fn backtrace_opt(&self) -> Option<&Backtrace> {
-        ErrorCompat::backtrace(self)
-    }
-
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -296,7 +315,7 @@ mod tests {
             StatusCode::TableAlreadyExists,
             Error::TableExists {
                 table: "some_table".to_string(),
-                backtrace: Backtrace::generate(),
+                location: Location::generate(),
             }
             .status_code()
         );
@@ -310,7 +329,7 @@ mod tests {
             StatusCode::StorageUnavailable,
             Error::SystemCatalog {
                 msg: "".to_string(),
-                backtrace: Backtrace::generate(),
+                location: Location::generate(),
             }
             .status_code()
         );
@@ -319,7 +338,7 @@ mod tests {
             StatusCode::Internal,
             Error::SystemCatalogTypeMismatch {
                 data_type: ConcreteDataType::binary_datatype(),
-                backtrace: Backtrace::generate(),
+                location: Location::generate(),
             }
             .status_code()
         );
@@ -333,7 +352,7 @@ mod tests {
     pub fn test_errors_to_datafusion_error() {
         let e: DataFusionError = Error::TableExists {
             table: "test_table".to_string(),
-            backtrace: Backtrace::generate(),
+            location: Location::generate(),
         }
         .into();
         match e {

src/catalog/src/helper.rs

@@ -91,7 +91,7 @@ pub fn build_table_regional_prefix(
 }
 
 /// Table global info has only one key across all datanodes so it does not have `node_id` field.
-#[derive(Clone)]
+#[derive(Clone, Hash, Eq, PartialEq)]
 pub struct TableGlobalKey {
     pub catalog_name: String,
     pub schema_name: String,
@@ -124,6 +124,14 @@ impl TableGlobalKey {
             table_name: captures[3].to_string(),
         })
     }
+
+    pub fn to_raw_key(&self) -> Vec<u8> {
+        self.to_string().into_bytes()
+    }
+
+    pub fn try_from_raw_key(key: &[u8]) -> Result<Self, Error> {
+        Self::parse(String::from_utf8_lossy(key))
+    }
 }
 
 /// Table global info contains necessary info for a datanode to create table regions, including
@@ -141,6 +149,10 @@ impl TableGlobalValue {
     pub fn table_id(&self) -> TableId {
         self.table_info.ident.table_id
     }
+
+    pub fn engine(&self) -> &str {
+        &self.table_info.meta.engine
+    }
 }
 
 /// Table regional info that varies between datanode, so it contains a `node_id` field.
@@ -191,6 +203,7 @@ impl TableRegionalKey {
 pub struct TableRegionalValue {
     pub version: TableVersion,
     pub regions_ids: Vec<u32>,
+    pub engine_name: Option<String>,
 }
 
 pub struct CatalogKey {

src/catalog/src/information_schema.rs

@@ -0,0 +1,169 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod columns;
+mod tables;
+
+use std::any::Any;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_error::prelude::BoxedError;
+use common_query::physical_plan::PhysicalPlanRef;
+use common_query::prelude::Expr;
+use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
+use datatypes::schema::SchemaRef;
+use futures_util::StreamExt;
+use snafu::ResultExt;
+use store_api::storage::ScanRequest;
+use table::error::{SchemaConversionSnafu, TablesRecordBatchSnafu};
+use table::{Result as TableResult, Table, TableRef};
+
+use self::columns::InformationSchemaColumns;
+use crate::error::Result;
+use crate::information_schema::tables::InformationSchemaTables;
+use crate::{CatalogProviderRef, SchemaProvider};
+
+const TABLES: &str = "tables";
+const COLUMNS: &str = "columns";
+
+pub(crate) struct InformationSchemaProvider {
+    catalog_name: String,
+    catalog_provider: CatalogProviderRef,
+    tables: Vec<String>,
+}
+
+impl InformationSchemaProvider {
+    pub(crate) fn new(catalog_name: String, catalog_provider: CatalogProviderRef) -> Self {
+        Self {
+            catalog_name,
+            catalog_provider,
+            tables: vec![TABLES.to_string(), COLUMNS.to_string()],
+        }
+    }
+}
+
+#[async_trait]
+impl SchemaProvider for InformationSchemaProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    async fn table_names(&self) -> Result<Vec<String>> {
+        Ok(self.tables.clone())
+    }
+
+    async fn table(&self, name: &str) -> Result<Option<TableRef>> {
+        let stream_builder = match name.to_ascii_lowercase().as_ref() {
+            TABLES => Arc::new(InformationSchemaTables::new(
+                self.catalog_name.clone(),
+                self.catalog_provider.clone(),
+            )) as _,
+            COLUMNS => Arc::new(InformationSchemaColumns::new(
+                self.catalog_name.clone(),
+                self.catalog_provider.clone(),
+            )) as _,
+            _ => {
+                return Ok(None);
+            }
+        };
+
+        Ok(Some(Arc::new(InformationTable::new(stream_builder))))
+    }
+
+    async fn table_exist(&self, name: &str) -> Result<bool> {
+        let normalized_name = name.to_ascii_lowercase();
+        Ok(self.tables.contains(&normalized_name))
+    }
+}
+
+// TODO(ruihang): make it a more generic trait:
+// https://github.com/GreptimeTeam/greptimedb/pull/1639#discussion_r1205001903
+pub trait InformationStreamBuilder: Send + Sync {
+    fn to_stream(&self) -> Result<SendableRecordBatchStream>;
+
+    fn schema(&self) -> SchemaRef;
+}
+
+pub struct InformationTable {
+    stream_builder: Arc<dyn InformationStreamBuilder>,
+}
+
+impl InformationTable {
+    pub fn new(stream_builder: Arc<dyn InformationStreamBuilder>) -> Self {
+        Self { stream_builder }
+    }
+}
+
+#[async_trait]
+impl Table for InformationTable {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.stream_builder.schema()
+    }
+
+    fn table_info(&self) -> table::metadata::TableInfoRef {
+        unreachable!("Should not call table_info() of InformationTable directly")
+    }
+
+    /// Scan the table and returns a SendableRecordBatchStream.
+    async fn scan(
+        &self,
+        _projection: Option<&Vec<usize>>,
+        _filters: &[Expr],
+        // limit can be used to reduce the amount scanned
+        // from the datasource as a performance optimization.
+        // If set, it contains the amount of rows needed by the `LogicalPlan`,
+        // The datasource should return *at least* this number of rows if available.
+        _limit: Option<usize>,
+    ) -> TableResult<PhysicalPlanRef> {
+        unimplemented!()
+    }
+
+    async fn scan_to_stream(&self, request: ScanRequest) -> TableResult<SendableRecordBatchStream> {
+        let projection = request.projection;
+        let projected_schema = if let Some(projection) = &projection {
+            Arc::new(
+                self.schema()
+                    .try_project(projection)
+                    .context(SchemaConversionSnafu)?,
+            )
+        } else {
+            self.schema()
+        };
+        let stream = self
+            .stream_builder
+            .to_stream()
+            .map_err(BoxedError::new)
+            .context(TablesRecordBatchSnafu)?
+            .map(move |batch| {
+                batch.and_then(|batch| {
+                    if let Some(projection) = &projection {
+                        batch.try_project(projection)
+                    } else {
+                        Ok(batch)
+                    }
+                })
+            });
+        let stream = RecordBatchStreamAdaptor {
+            schema: projected_schema,
+            stream: Box::pin(stream),
+            output_ordering: None,
+        };
+        Ok(Box::pin(stream))
+    }
+}

src/catalog/src/information_schema/columns.rs

@@ -0,0 +1,213 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use arrow_schema::SchemaRef as ArrowSchemaRef;
+use common_catalog::consts::{
+    SEMANTIC_TYPE_FIELD, SEMANTIC_TYPE_PRIMARY_KEY, SEMANTIC_TYPE_TIME_INDEX,
+};
+use common_error::prelude::BoxedError;
+use common_query::physical_plan::TaskContext;
+use common_recordbatch::adapter::RecordBatchStreamAdapter;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use datafusion::datasource::streaming::PartitionStream as DfPartitionStream;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
+use datatypes::prelude::{ConcreteDataType, DataType};
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::vectors::{StringVectorBuilder, VectorRef};
+use snafu::ResultExt;
+
+use super::InformationStreamBuilder;
+use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
+use crate::CatalogProviderRef;
+
+pub(super) struct InformationSchemaColumns {
+    schema: SchemaRef,
+    catalog_name: String,
+    catalog_provider: CatalogProviderRef,
+}
+
+const TABLE_CATALOG: &str = "table_catalog";
+const TABLE_SCHEMA: &str = "table_schema";
+const TABLE_NAME: &str = "table_name";
+const COLUMN_NAME: &str = "column_name";
+const DATA_TYPE: &str = "data_type";
+const SEMANTIC_TYPE: &str = "semantic_type";
+
+impl InformationSchemaColumns {
+    pub(super) fn new(catalog_name: String, catalog_provider: CatalogProviderRef) -> Self {
+        let schema = Arc::new(Schema::new(vec![
+            ColumnSchema::new(TABLE_CATALOG, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(TABLE_SCHEMA, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(TABLE_NAME, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(COLUMN_NAME, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(DATA_TYPE, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(SEMANTIC_TYPE, ConcreteDataType::string_datatype(), false),
+        ]));
+        Self {
+            schema,
+            catalog_name,
+            catalog_provider,
+        }
+    }
+
+    fn builder(&self) -> InformationSchemaColumnsBuilder {
+        InformationSchemaColumnsBuilder::new(
+            self.schema.clone(),
+            self.catalog_name.clone(),
+            self.catalog_provider.clone(),
+        )
+    }
+}
+
+impl InformationStreamBuilder for InformationSchemaColumns {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn to_stream(&self) -> Result<SendableRecordBatchStream> {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_tables()
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ));
+        Ok(Box::pin(
+            RecordBatchStreamAdapter::try_new(stream)
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?,
+        ))
+    }
+}
+
+struct InformationSchemaColumnsBuilder {
+    schema: SchemaRef,
+    catalog_name: String,
+    catalog_provider: CatalogProviderRef,
+
+    catalog_names: StringVectorBuilder,
+    schema_names: StringVectorBuilder,
+    table_names: StringVectorBuilder,
+    column_names: StringVectorBuilder,
+    data_types: StringVectorBuilder,
+    semantic_types: StringVectorBuilder,
+}
+
+impl InformationSchemaColumnsBuilder {
+    fn new(schema: SchemaRef, catalog_name: String, catalog_provider: CatalogProviderRef) -> Self {
+        Self {
+            schema,
+            catalog_name,
+            catalog_provider,
+            catalog_names: StringVectorBuilder::with_capacity(42),
+            schema_names: StringVectorBuilder::with_capacity(42),
+            table_names: StringVectorBuilder::with_capacity(42),
+            column_names: StringVectorBuilder::with_capacity(42),
+            data_types: StringVectorBuilder::with_capacity(42),
+            semantic_types: StringVectorBuilder::with_capacity(42),
+        }
+    }
+
+    /// Construct the `information_schema.tables` virtual table
+    async fn make_tables(&mut self) -> Result<RecordBatch> {
+        let catalog_name = self.catalog_name.clone();
+
+        for schema_name in self.catalog_provider.schema_names().await? {
+            let Some(schema) = self.catalog_provider.schema(&schema_name).await? else { continue };
+            for table_name in schema.table_names().await? {
+                let Some(table) = schema.table(&table_name).await? else { continue };
+                let keys = &table.table_info().meta.primary_key_indices;
+                let schema = table.schema();
+                for (idx, column) in schema.column_schemas().iter().enumerate() {
+                    let semantic_type = if column.is_time_index() {
+                        SEMANTIC_TYPE_TIME_INDEX
+                    } else if keys.contains(&idx) {
+                        SEMANTIC_TYPE_PRIMARY_KEY
+                    } else {
+                        SEMANTIC_TYPE_FIELD
+                    };
+                    self.add_column(
+                        &catalog_name,
+                        &schema_name,
+                        &table_name,
+                        &column.name,
+                        column.data_type.name(),
+                        semantic_type,
+                    );
+                }
+            }
+        }
+
+        self.finish()
+    }
+
+    fn add_column(
+        &mut self,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+        column_name: &str,
+        data_type: &str,
+        semantic_type: &str,
+    ) {
+        self.catalog_names.push(Some(catalog_name));
+        self.schema_names.push(Some(schema_name));
+        self.table_names.push(Some(table_name));
+        self.column_names.push(Some(column_name));
+        self.data_types.push(Some(data_type));
+        self.semantic_types.push(Some(semantic_type));
+    }
+
+    fn finish(&mut self) -> Result<RecordBatch> {
+        let columns: Vec<VectorRef> = vec![
+            Arc::new(self.catalog_names.finish()),
+            Arc::new(self.schema_names.finish()),
+            Arc::new(self.table_names.finish()),
+            Arc::new(self.column_names.finish()),
+            Arc::new(self.data_types.finish()),
+            Arc::new(self.semantic_types.finish()),
+        ];
+        RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
+    }
+}
+
+impl DfPartitionStream for InformationSchemaColumns {
+    fn schema(&self) -> &ArrowSchemaRef {
+        self.schema.arrow_schema()
+    }
+
+    fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_tables()
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ))
+    }
+}

src/catalog/src/information_schema/tables.rs

@@ -0,0 +1,205 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use arrow_schema::SchemaRef as ArrowSchemaRef;
+use common_catalog::consts::INFORMATION_SCHEMA_NAME;
+use common_error::prelude::BoxedError;
+use common_query::physical_plan::TaskContext;
+use common_recordbatch::adapter::RecordBatchStreamAdapter;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use datafusion::datasource::streaming::PartitionStream as DfPartitionStream;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
+use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
+use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::vectors::{StringVectorBuilder, UInt32VectorBuilder};
+use snafu::ResultExt;
+use table::metadata::TableType;
+
+use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
+use crate::information_schema::InformationStreamBuilder;
+use crate::CatalogProviderRef;
+
+pub(super) struct InformationSchemaTables {
+    schema: SchemaRef,
+    catalog_name: String,
+    catalog_provider: CatalogProviderRef,
+}
+
+impl InformationSchemaTables {
+    pub(super) fn new(catalog_name: String, catalog_provider: CatalogProviderRef) -> Self {
+        let schema = Arc::new(Schema::new(vec![
+            ColumnSchema::new("table_catalog", ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new("table_schema", ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new("table_name", ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new("table_type", ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new("table_id", ConcreteDataType::uint32_datatype(), true),
+            ColumnSchema::new("engine", ConcreteDataType::string_datatype(), true),
+        ]));
+        Self {
+            schema,
+            catalog_name,
+            catalog_provider,
+        }
+    }
+
+    fn builder(&self) -> InformationSchemaTablesBuilder {
+        InformationSchemaTablesBuilder::new(
+            self.schema.clone(),
+            self.catalog_name.clone(),
+            self.catalog_provider.clone(),
+        )
+    }
+}
+
+impl InformationStreamBuilder for InformationSchemaTables {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn to_stream(&self) -> Result<SendableRecordBatchStream> {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_tables()
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ));
+        Ok(Box::pin(
+            RecordBatchStreamAdapter::try_new(stream)
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?,
+        ))
+    }
+}
+
+/// Builds the `information_schema.TABLE` table row by row
+///
+/// Columns are based on <https://www.postgresql.org/docs/current/infoschema-columns.html>
+struct InformationSchemaTablesBuilder {
+    schema: SchemaRef,
+    catalog_name: String,
+    catalog_provider: CatalogProviderRef,
+
+    catalog_names: StringVectorBuilder,
+    schema_names: StringVectorBuilder,
+    table_names: StringVectorBuilder,
+    table_types: StringVectorBuilder,
+    table_ids: UInt32VectorBuilder,
+    engines: StringVectorBuilder,
+}
+
+impl InformationSchemaTablesBuilder {
+    fn new(schema: SchemaRef, catalog_name: String, catalog_provider: CatalogProviderRef) -> Self {
+        Self {
+            schema,
+            catalog_name,
+            catalog_provider,
+            catalog_names: StringVectorBuilder::with_capacity(42),
+            schema_names: StringVectorBuilder::with_capacity(42),
+            table_names: StringVectorBuilder::with_capacity(42),
+            table_types: StringVectorBuilder::with_capacity(42),
+            table_ids: UInt32VectorBuilder::with_capacity(42),
+            engines: StringVectorBuilder::with_capacity(42),
+        }
+    }
+
+    /// Construct the `information_schema.tables` virtual table
+    async fn make_tables(&mut self) -> Result<RecordBatch> {
+        let catalog_name = self.catalog_name.clone();
+
+        for schema_name in self.catalog_provider.schema_names().await? {
+            if schema_name == INFORMATION_SCHEMA_NAME {
+                continue;
+            }
+
+            let Some(schema) = self.catalog_provider.schema(&schema_name).await? else { continue };
+            for table_name in schema.table_names().await? {
+                let Some(table) = schema.table(&table_name).await? else { continue };
+                let table_info = table.table_info();
+                self.add_table(
+                    &catalog_name,
+                    &schema_name,
+                    &table_name,
+                    table.table_type(),
+                    Some(table_info.ident.table_id),
+                    Some(&table_info.meta.engine),
+                );
+            }
+        }
+
+        self.finish()
+    }
+
+    fn add_table(
+        &mut self,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+        table_type: TableType,
+        table_id: Option<u32>,
+        engine: Option<&str>,
+    ) {
+        self.catalog_names.push(Some(catalog_name));
+        self.schema_names.push(Some(schema_name));
+        self.table_names.push(Some(table_name));
+        self.table_types.push(Some(match table_type {
+            TableType::Base => "BASE TABLE",
+            TableType::View => "VIEW",
+            TableType::Temporary => "LOCAL TEMPORARY",
+        }));
+        self.table_ids.push(table_id);
+        self.engines.push(engine);
+    }
+
+    fn finish(&mut self) -> Result<RecordBatch> {
+        let columns: Vec<VectorRef> = vec![
+            Arc::new(self.catalog_names.finish()),
+            Arc::new(self.schema_names.finish()),
+            Arc::new(self.table_names.finish()),
+            Arc::new(self.table_types.finish()),
+            Arc::new(self.table_ids.finish()),
+            Arc::new(self.engines.finish()),
+        ];
+        RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
+    }
+}
+
+impl DfPartitionStream for InformationSchemaTables {
+    fn schema(&self) -> &ArrowSchemaRef {
+        self.schema.arrow_schema()
+    }
+
+    fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_tables()
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ))
+    }
+}

src/catalog/src/lib.rs

@@ -12,15 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![feature(trait_upcasting)]
 #![feature(assert_matches)]
 
 use std::any::Any;
+use std::collections::HashMap;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
 use api::v1::meta::{RegionStat, TableName};
-use common_telemetry::info;
-use snafu::{OptionExt, ResultExt};
+use common_telemetry::{info, warn};
+use snafu::ResultExt;
 use table::engine::{EngineContext, TableEngineRef};
 use table::metadata::TableId;
 use table::requests::CreateTableRequest;
@@ -31,62 +33,49 @@ pub use crate::schema::{SchemaProvider, SchemaProviderRef};
 
 pub mod error;
 pub mod helper;
+pub(crate) mod information_schema;
 pub mod local;
+mod metrics;
 pub mod remote;
 pub mod schema;
 pub mod system;
 pub mod table_source;
 pub mod tables;
 
-/// Represent a list of named catalogs
-pub trait CatalogList: Sync + Send {
-    /// Returns the catalog list as [`Any`](std::any::Any)
-    /// so that it can be downcast to a specific implementation.
-    fn as_any(&self) -> &dyn Any;
-
-    /// Adds a new catalog to this catalog list
-    /// If a catalog of the same name existed before, it is replaced in the list and returned.
-    fn register_catalog(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Result<Option<CatalogProviderRef>>;
-
-    /// Retrieves the list of available catalog names
-    fn catalog_names(&self) -> Result<Vec<String>>;
-
-    /// Retrieves a specific catalog by name, provided it exists.
-    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>>;
-}
-
 /// Represents a catalog, comprising a number of named schemas.
+#[async_trait::async_trait]
 pub trait CatalogProvider: Sync + Send {
     /// Returns the catalog provider as [`Any`](std::any::Any)
     /// so that it can be downcast to a specific implementation.
     fn as_any(&self) -> &dyn Any;
 
     /// Retrieves the list of available schema names in this catalog.
-    fn schema_names(&self) -> Result<Vec<String>>;
+    async fn schema_names(&self) -> Result<Vec<String>>;
 
     /// Registers schema to this catalog.
-    fn register_schema(
+    async fn register_schema(
         &self,
         name: String,
         schema: SchemaProviderRef,
     ) -> Result<Option<SchemaProviderRef>>;
 
     /// Retrieves a specific schema from the catalog by name, provided it exists.
-    fn schema(&self, name: &str) -> Result<Option<SchemaProviderRef>>;
+    async fn schema(&self, name: &str) -> Result<Option<SchemaProviderRef>>;
 }
 
-pub type CatalogListRef = Arc<dyn CatalogList>;
 pub type CatalogProviderRef = Arc<dyn CatalogProvider>;
 
 #[async_trait::async_trait]
-pub trait CatalogManager: CatalogList {
+pub trait CatalogManager: Send + Sync {
     /// Starts a catalog manager.
     async fn start(&self) -> Result<()>;
 
+    async fn register_catalog(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Result<Option<CatalogProviderRef>>;
+
     /// Registers a table within given catalog/schema to catalog manager,
     /// returns whether the table registered.
     async fn register_table(&self, request: RegisterTableRequest) -> Result<bool>;
@@ -106,7 +95,11 @@ pub trait CatalogManager: CatalogList {
     async fn register_system_table(&self, request: RegisterSystemTableRequest)
         -> error::Result<()>;
 
-    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>>;
+    async fn catalog_names(&self) -> Result<Vec<String>>;
+
+    async fn catalog(&self, catalog: &str) -> Result<Option<CatalogProviderRef>>;
+
+    async fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>>;
 
     /// Returns the table by catalog, schema and table name.
     async fn table(
@@ -115,6 +108,8 @@ pub trait CatalogManager: CatalogList {
         schema: &str,
         table_name: &str,
     ) -> Result<Option<TableRef>>;
+
+    fn as_any(&self) -> &dyn Any;
 }
 
 pub type CatalogManagerRef = Arc<dyn CatalogManager>;
@@ -228,43 +223,34 @@ pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
 
 /// The stat of regions in the datanode node.
 /// The number of regions can be got from len of vec.
-pub async fn region_stats(catalog_manager: &CatalogManagerRef) -> Result<Vec<RegionStat>> {
+///
+/// Ignores any errors occurred during iterating regions. The intention of this method is to
+/// collect region stats that will be carried in Datanode's heartbeat to Metasrv, so it's a
+/// "try our best" job.
+pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<RegionStat>) {
+    let mut region_number: u64 = 0;
     let mut region_stats = Vec::new();
-    for catalog_name in catalog_manager.catalog_names()? {
-        let catalog =
-            catalog_manager
-                .catalog(&catalog_name)?
-                .context(error::CatalogNotFoundSnafu {
-                    catalog_name: &catalog_name,
-                })?;
 
-        for schema_name in catalog.schema_names()? {
-            let schema = catalog
-                .schema(&schema_name)?
-                .context(error::SchemaNotFoundSnafu {
-                    catalog: &catalog_name,
-                    schema: &schema_name,
-                })?;
+    let Ok(catalog_names) = catalog_manager.catalog_names().await else { return (region_number, region_stats) };
+    for catalog_name in catalog_names {
+        let Ok(Some(catalog)) = catalog_manager.catalog(&catalog_name).await else { continue };
 
-            for table_name in schema.table_names()? {
-                let table =
-                    schema
-                        .table(&table_name)
-                        .await?
-                        .context(error::TableNotFoundSnafu {
-                            table_info: &table_name,
-                        })?;
+        let Ok(schema_names) = catalog.schema_names().await else { continue };
+        for schema_name in schema_names {
+            let Ok(Some(schema)) = catalog.schema(&schema_name).await else { continue };
 
-                region_stats.extend(
-                    table
-                        .region_stats()
-                        .context(error::RegionStatsSnafu {
-                            catalog: &catalog_name,
-                            schema: &schema_name,
-                            table: &table_name,
-                        })?
-                        .into_iter()
-                        .map(|stat| RegionStat {
+            let Ok(table_names) = schema.table_names().await else { continue };
+            for table_name in table_names {
+                let Ok(Some(table)) = schema.table(&table_name).await else { continue };
+
+                let region_numbers = &table.table_info().meta.region_numbers;
+                region_number += region_numbers.len() as u64;
+
+                let engine = &table.table_info().meta.engine;
+
+                match table.region_stats() {
+                    Ok(stats) => {
+                        let stats = stats.into_iter().map(|stat| RegionStat {
                             region_id: stat.region_id,
                             table_name: Some(TableName {
                                 catalog_name: catalog_name.clone(),
@@ -272,11 +258,18 @@ pub async fn region_stats(catalog_manager: &CatalogManagerRef) -> Result<Vec<Reg
                                 table_name: table_name.clone(),
                             }),
                             approximate_bytes: stat.disk_usage_bytes as i64,
+                            attrs: HashMap::from([("engine_name".to_owned(), engine.clone())]),
                             ..Default::default()
-                        }),
-                );
+                        });
+
+                        region_stats.extend(stats);
+                    }
+                    Err(e) => {
+                        warn!("Failed to get region status, err: {:?}", e);
+                    }
+                };
             }
         }
     }
-    Ok(region_stats)
+    (region_number, region_stats)
 }

src/catalog/src/local/manager.rs

@@ -18,7 +18,7 @@ use std::sync::Arc;
 
 use common_catalog::consts::{
     DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
-    SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_NAME,
+    MITO_ENGINE, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_NAME,
 };
 use common_catalog::format_full_table_name;
 use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
@@ -26,8 +26,10 @@ use common_telemetry::{error, info};
 use datatypes::prelude::ScalarVector;
 use datatypes::vectors::{BinaryVector, UInt8Vector};
 use futures_util::lock::Mutex;
+use metrics::increment_gauge;
 use snafu::{ensure, OptionExt, ResultExt};
-use table::engine::{EngineContext, TableEngineRef};
+use table::engine::manager::TableEngineManagerRef;
+use table::engine::EngineContext;
 use table::metadata::TableId;
 use table::requests::OpenTableRequest;
 use table::table::numbers::NumbersTable;
@@ -37,7 +39,8 @@ use table::TableRef;
 use crate::error::{
     self, CatalogNotFoundSnafu, IllegalManagerStateSnafu, OpenTableSnafu, ReadSystemCatalogSnafu,
     Result, SchemaExistsSnafu, SchemaNotFoundSnafu, SystemCatalogSnafu,
-    SystemCatalogTypeMismatchSnafu, TableExistsSnafu, TableNotFoundSnafu,
+    SystemCatalogTypeMismatchSnafu, TableEngineNotFoundSnafu, TableExistsSnafu, TableNotExistSnafu,
+    TableNotFoundSnafu,
 };
 use crate::local::memory::{MemoryCatalogManager, MemoryCatalogProvider, MemorySchemaProvider};
 use crate::system::{
@@ -46,16 +49,16 @@ use crate::system::{
 };
 use crate::tables::SystemCatalog;
 use crate::{
-    handle_system_table_request, CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef,
-    DeregisterTableRequest, RegisterSchemaRequest, RegisterSystemTableRequest,
-    RegisterTableRequest, RenameTableRequest, SchemaProvider, SchemaProviderRef,
+    handle_system_table_request, CatalogManager, CatalogProviderRef, DeregisterTableRequest,
+    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
+    SchemaProviderRef,
 };
 
 /// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
 pub struct LocalCatalogManager {
     system: Arc<SystemCatalog>,
     catalogs: Arc<MemoryCatalogManager>,
-    engine: TableEngineRef,
+    engine_manager: TableEngineManagerRef,
     next_table_id: AtomicU32,
     init_lock: Mutex<bool>,
     register_lock: Mutex<()>,
@@ -63,19 +66,20 @@ pub struct LocalCatalogManager {
 }
 
 impl LocalCatalogManager {
-    /// Create a new [CatalogManager] with given user catalogs and table engine
-    pub async fn try_new(engine: TableEngineRef) -> Result<Self> {
+    /// Create a new [CatalogManager] with given user catalogs and mito engine
+    pub async fn try_new(engine_manager: TableEngineManagerRef) -> Result<Self> {
+        let engine = engine_manager
+            .engine(MITO_ENGINE)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: MITO_ENGINE,
+            })?;
         let table = SystemCatalogTable::new(engine.clone()).await?;
         let memory_catalog_list = crate::local::memory::new_memory_catalog_list()?;
-        let system_catalog = Arc::new(SystemCatalog::new(
-            table,
-            memory_catalog_list.clone(),
-            engine.clone(),
-        ));
+        let system_catalog = Arc::new(SystemCatalog::new(table));
         Ok(Self {
             system: system_catalog,
             catalogs: memory_catalog_list,
-            engine,
+            engine_manager,
             next_table_id: AtomicU32::new(MIN_USER_TABLE_ID),
             init_lock: Mutex::new(false),
             register_lock: Mutex::new(()),
@@ -85,7 +89,7 @@ impl LocalCatalogManager {
 
     /// Scan all entries from system catalog table
     pub async fn init(&self) -> Result<()> {
-        self.init_system_catalog()?;
+        self.init_system_catalog().await?;
         let system_records = self.system.information_schema.system.records().await?;
         let entries = self.collect_system_catalog_entries(system_records).await?;
         let max_table_id = self.handle_system_catalog_entries(entries).await?;
@@ -100,31 +104,38 @@ impl LocalCatalogManager {
 
         // Processing system table hooks
         let mut sys_table_requests = self.system_table_requests.lock().await;
-        handle_system_table_request(self, self.engine.clone(), &mut sys_table_requests).await?;
+        let engine = self
+            .engine_manager
+            .engine(MITO_ENGINE)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: MITO_ENGINE,
+            })?;
+
+        handle_system_table_request(self, engine, &mut sys_table_requests).await?;
         Ok(())
     }
 
-    fn init_system_catalog(&self) -> Result<()> {
+    async fn init_system_catalog(&self) -> Result<()> {
         let system_schema = Arc::new(MemorySchemaProvider::new());
-        system_schema.register_table(
+        system_schema.register_table_sync(
             SYSTEM_CATALOG_TABLE_NAME.to_string(),
             self.system.information_schema.system.clone(),
         )?;
         let system_catalog = Arc::new(MemoryCatalogProvider::new());
-        system_catalog.register_schema(INFORMATION_SCHEMA_NAME.to_string(), system_schema)?;
+        system_catalog.register_schema_sync(INFORMATION_SCHEMA_NAME.to_string(), system_schema)?;
         self.catalogs
-            .register_catalog(SYSTEM_CATALOG_NAME.to_string(), system_catalog)?;
+            .register_catalog_sync(SYSTEM_CATALOG_NAME.to_string(), system_catalog)?;
 
         let default_catalog = Arc::new(MemoryCatalogProvider::new());
         let default_schema = Arc::new(MemorySchemaProvider::new());
 
         // Add numbers table for test
         let table = Arc::new(NumbersTable::default());
-        default_schema.register_table("numbers".to_string(), table)?;
+        default_schema.register_table_sync("numbers".to_string(), table)?;
 
-        default_catalog.register_schema(DEFAULT_SCHEMA_NAME.to_string(), default_schema)?;
+        default_catalog.register_schema_sync(DEFAULT_SCHEMA_NAME.to_string(), default_schema)?;
         self.catalogs
-            .register_catalog(DEFAULT_CATALOG_NAME.to_string(), default_catalog)?;
+            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string(), default_catalog)?;
         Ok(())
     }
 
@@ -203,16 +214,17 @@ impl LocalCatalogManager {
                     info!("Register catalog: {}", c.catalog_name);
                 }
                 Entry::Schema(s) => {
-                    let catalog =
-                        self.catalogs
-                            .catalog(&s.catalog_name)?
-                            .context(CatalogNotFoundSnafu {
-                                catalog_name: &s.catalog_name,
-                            })?;
-                    catalog.register_schema(
-                        s.schema_name.clone(),
-                        Arc::new(MemorySchemaProvider::new()),
-                    )?;
+                    self.catalogs
+                        .catalog(&s.catalog_name)
+                        .await?
+                        .context(CatalogNotFoundSnafu {
+                            catalog_name: &s.catalog_name,
+                        })?
+                        .register_schema(
+                            s.schema_name.clone(),
+                            Arc::new(MemorySchemaProvider::new()),
+                        )
+                        .await?;
                     info!("Registered schema: {:?}", s);
                 }
                 Entry::Table(t) => {
@@ -233,14 +245,16 @@ impl LocalCatalogManager {
     }
 
     async fn open_and_register_table(&self, t: &TableEntry) -> Result<()> {
-        let catalog = self
-            .catalogs
-            .catalog(&t.catalog_name)?
-            .context(CatalogNotFoundSnafu {
-                catalog_name: &t.catalog_name,
-            })?;
+        let catalog =
+            self.catalogs
+                .catalog(&t.catalog_name)
+                .await?
+                .context(CatalogNotFoundSnafu {
+                    catalog_name: &t.catalog_name,
+                })?;
         let schema = catalog
-            .schema(&t.schema_name)?
+            .schema(&t.schema_name)
+            .await?
             .context(SchemaNotFoundSnafu {
                 catalog: &t.catalog_name,
                 schema: &t.schema_name,
@@ -252,10 +266,16 @@ impl LocalCatalogManager {
             schema_name: t.schema_name.clone(),
             table_name: t.table_name.clone(),
             table_id: t.table_id,
+            region_numbers: vec![0],
         };
+        let engine = self
+            .engine_manager
+            .engine(&t.engine)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: &t.engine,
+            })?;
 
-        let option = self
-            .engine
+        let option = engine
             .open_table(&context, request)
             .await
             .with_context(|_| OpenTableSnafu {
@@ -271,39 +291,11 @@ impl LocalCatalogManager {
                 ),
             })?;
 
-        schema.register_table(t.table_name.clone(), option)?;
+        schema.register_table(t.table_name.clone(), option).await?;
         Ok(())
     }
 }
 
-impl CatalogList for LocalCatalogManager {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn register_catalog(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Result<Option<CatalogProviderRef>> {
-        self.catalogs.register_catalog(name, catalog)
-    }
-
-    fn catalog_names(&self) -> Result<Vec<String>> {
-        let mut res = self.catalogs.catalog_names()?;
-        res.push(SYSTEM_CATALOG_NAME.to_string());
-        Ok(res)
-    }
-
-    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>> {
-        if name.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
-            Ok(Some(self.system.clone()))
-        } else {
-            self.catalogs.catalog(name)
-        }
-    }
-}
-
 #[async_trait::async_trait]
 impl TableIdProvider for LocalCatalogManager {
     async fn next_table_id(&self) -> table::Result<TableId> {
@@ -334,10 +326,12 @@ impl CatalogManager for LocalCatalogManager {
 
         let catalog = self
             .catalogs
-            .catalog(catalog_name)?
+            .catalog(catalog_name)
+            .await?
             .context(CatalogNotFoundSnafu { catalog_name })?;
         let schema = catalog
-            .schema(schema_name)?
+            .schema(schema_name)
+            .await?
             .with_context(|| SchemaNotFoundSnafu {
                 catalog: catalog_name,
                 schema: schema_name,
@@ -364,6 +358,7 @@ impl CatalogManager for LocalCatalogManager {
                 // Try to register table with same table id, just ignore.
                 Ok(false)
             } else {
+                let engine = request.table.table_info().meta.engine.to_string();
                 // table does not exist
                 self.system
                     .register_table(
@@ -371,9 +366,17 @@ impl CatalogManager for LocalCatalogManager {
                         schema_name.clone(),
                         request.table_name.clone(),
                         request.table_id,
+                        engine,
                     )
                     .await?;
-                schema.register_table(request.table_name, request.table)?;
+                schema
+                    .register_table(request.table_name, request.table)
+                    .await?;
+                increment_gauge!(
+                    crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+                    1.0,
+                    &[crate::metrics::db_label(catalog_name, schema_name)],
+                );
                 Ok(true)
             }
         }
@@ -394,16 +397,33 @@ impl CatalogManager for LocalCatalogManager {
 
         let catalog = self
             .catalogs
-            .catalog(catalog_name)?
+            .catalog(catalog_name)
+            .await?
             .context(CatalogNotFoundSnafu { catalog_name })?;
 
         let schema = catalog
-            .schema(schema_name)?
+            .schema(schema_name)
+            .await?
             .with_context(|| SchemaNotFoundSnafu {
                 catalog: catalog_name,
                 schema: schema_name,
             })?;
 
+        let _lock = self.register_lock.lock().await;
+        ensure!(
+            !schema.table_exist(&request.new_table_name).await?,
+            TableExistsSnafu {
+                table: &request.new_table_name
+            }
+        );
+        let old_table = schema
+            .table(&request.table_name)
+            .await?
+            .context(TableNotExistSnafu {
+                table: &request.table_name,
+            })?;
+
+        let engine = old_table.table_info().meta.engine.to_string();
         // rename table in system catalog
         self.system
             .register_table(
@@ -411,11 +431,15 @@ impl CatalogManager for LocalCatalogManager {
                 schema_name.clone(),
                 request.new_table_name.clone(),
                 request.table_id,
+                engine,
             )
             .await?;
-        Ok(schema
-            .rename_table(&request.table_name, request.new_table_name)
-            .is_ok())
+
+        let renamed = schema
+            .rename_table(&request.table_name, request.new_table_name.clone())
+            .await
+            .is_ok();
+        Ok(renamed)
     }
 
     async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool> {
@@ -443,10 +467,7 @@ impl CatalogManager for LocalCatalogManager {
                 .ident
                 .table_id;
 
-            if !self.system.deregister_table(&request, table_id).await? {
-                return Ok(false);
-            }
-
+            self.system.deregister_table(&request, table_id).await?;
             self.catalogs.deregister_table(request).await
         }
     }
@@ -464,13 +485,14 @@ impl CatalogManager for LocalCatalogManager {
 
         let catalog = self
             .catalogs
-            .catalog(catalog_name)?
+            .catalog(catalog_name)
+            .await?
             .context(CatalogNotFoundSnafu { catalog_name })?;
 
         {
             let _lock = self.register_lock.lock().await;
             ensure!(
-                catalog.schema(schema_name)?.is_none(),
+                catalog.schema(schema_name).await?.is_none(),
                 SchemaExistsSnafu {
                     schema: schema_name,
                 }
@@ -478,12 +500,18 @@ impl CatalogManager for LocalCatalogManager {
             self.system
                 .register_schema(request.catalog, schema_name.clone())
                 .await?;
-            catalog.register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))?;
+            catalog
+                .register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))
+                .await?;
+
             Ok(true)
         }
     }
 
     async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
+        let catalog_name = request.create_table_request.catalog_name.clone();
+        let schema_name = request.create_table_request.schema_name.clone();
+
         ensure!(
             !*self.init_lock.lock().await,
             IllegalManagerStateSnafu {
@@ -493,17 +521,23 @@ impl CatalogManager for LocalCatalogManager {
 
         let mut sys_table_requests = self.system_table_requests.lock().await;
         sys_table_requests.push(request);
-
+        increment_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&catalog_name, &schema_name)],
+        );
         Ok(())
     }
 
-    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+    async fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
         self.catalogs
-            .catalog(catalog)?
+            .catalog(catalog)
+            .await?
             .context(CatalogNotFoundSnafu {
                 catalog_name: catalog,
             })?
             .schema(schema)
+            .await
     }
 
     async fn table(
@@ -514,22 +548,50 @@ impl CatalogManager for LocalCatalogManager {
     ) -> Result<Option<TableRef>> {
         let catalog = self
             .catalogs
-            .catalog(catalog_name)?
+            .catalog(catalog_name)
+            .await?
             .context(CatalogNotFoundSnafu { catalog_name })?;
         let schema = catalog
-            .schema(schema_name)?
+            .schema(schema_name)
+            .await?
             .with_context(|| SchemaNotFoundSnafu {
                 catalog: catalog_name,
                 schema: schema_name,
             })?;
         schema.table(table_name).await
     }
+
+    async fn catalog(&self, catalog: &str) -> Result<Option<CatalogProviderRef>> {
+        if catalog.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
+            Ok(Some(self.system.clone()))
+        } else {
+            self.catalogs.catalog(catalog).await
+        }
+    }
+
+    async fn catalog_names(&self) -> Result<Vec<String>> {
+        self.catalogs.catalog_names().await
+    }
+
+    async fn register_catalog(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Result<Option<CatalogProviderRef>> {
+        self.catalogs.register_catalog(name, catalog).await
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use std::assert_matches::assert_matches;
 
+    use mito::engine::MITO_ENGINE;
+
     use super::*;
     use crate::system::{CatalogEntry, SchemaEntry};
 
@@ -541,6 +603,7 @@ mod tests {
                 schema_name: "S1".to_string(),
                 table_name: "T1".to_string(),
                 table_id: 1,
+                engine: MITO_ENGINE.to_string(),
             }),
             Entry::Catalog(CatalogEntry {
                 catalog_name: "C2".to_string(),
@@ -561,6 +624,7 @@ mod tests {
                 schema_name: "S1".to_string(),
                 table_name: "T2".to_string(),
                 table_id: 2,
+                engine: MITO_ENGINE.to_string(),
             }),
         ];
         let res = LocalCatalogManager::sort_entries(vec);

src/catalog/src/local/memory.rs

@@ -21,6 +21,7 @@ use std::sync::{Arc, RwLock};
 use async_trait::async_trait;
 use common_catalog::consts::MIN_USER_TABLE_ID;
 use common_telemetry::error;
+use metrics::{decrement_gauge, increment_gauge};
 use snafu::{ensure, OptionExt};
 use table::metadata::TableId;
 use table::table::TableIdProvider;
@@ -31,7 +32,7 @@ use crate::error::{
 };
 use crate::schema::SchemaProvider;
 use crate::{
-    CatalogList, CatalogManager, CatalogProvider, CatalogProviderRef, DeregisterTableRequest,
+    CatalogManager, CatalogProvider, CatalogProviderRef, DeregisterTableRequest,
     RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
     SchemaProviderRef,
 };
@@ -51,10 +52,10 @@ impl Default for MemoryCatalogManager {
         };
         let default_catalog = Arc::new(MemoryCatalogProvider::new());
         manager
-            .register_catalog("greptime".to_string(), default_catalog.clone())
+            .register_catalog_sync("greptime".to_string(), default_catalog.clone())
             .unwrap();
         default_catalog
-            .register_schema("public".to_string(), Arc::new(MemorySchemaProvider::new()))
+            .register_schema_sync("public".to_string(), Arc::new(MemorySchemaProvider::new()))
             .unwrap();
         manager
     }
@@ -75,70 +76,81 @@ impl CatalogManager for MemoryCatalogManager {
     }
 
     async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        let catalogs = self.catalogs.write().unwrap();
-        let catalog = catalogs
-            .get(&request.catalog)
+        let schema = self
+            .catalog(&request.catalog)
             .context(CatalogNotFoundSnafu {
                 catalog_name: &request.catalog,
             })?
-            .clone();
-        let schema = catalog
-            .schema(&request.schema)?
-            .with_context(|| SchemaNotFoundSnafu {
+            .schema(&request.schema)
+            .await?
+            .context(SchemaNotFoundSnafu {
                 catalog: &request.catalog,
                 schema: &request.schema,
             })?;
+        increment_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
         schema
             .register_table(request.table_name, request.table)
+            .await
             .map(|v| v.is_none())
     }
 
     async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
-        let catalogs = self.catalogs.write().unwrap();
-        let catalog = catalogs
-            .get(&request.catalog)
+        let catalog = self
+            .catalog(&request.catalog)
             .context(CatalogNotFoundSnafu {
                 catalog_name: &request.catalog,
-            })?
-            .clone();
-        let schema = catalog
-            .schema(&request.schema)?
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: &request.catalog,
-                schema: &request.schema,
             })?;
+        let schema =
+            catalog
+                .schema(&request.schema)
+                .await?
+                .with_context(|| SchemaNotFoundSnafu {
+                    catalog: &request.catalog,
+                    schema: &request.schema,
+                })?;
         Ok(schema
             .rename_table(&request.table_name, request.new_table_name)
+            .await
             .is_ok())
     }
 
     async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool> {
-        let catalogs = self.catalogs.write().unwrap();
-        let catalog = catalogs
-            .get(&request.catalog)
+        let schema = self
+            .catalog(&request.catalog)
             .context(CatalogNotFoundSnafu {
                 catalog_name: &request.catalog,
             })?
-            .clone();
-        let schema = catalog
-            .schema(&request.schema)?
+            .schema(&request.schema)
+            .await?
             .with_context(|| SchemaNotFoundSnafu {
                 catalog: &request.catalog,
                 schema: &request.schema,
             })?;
+        decrement_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
         schema
             .deregister_table(&request.table_name)
+            .await
             .map(|v| v.is_some())
     }
 
     async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        let catalogs = self.catalogs.write().unwrap();
-        let catalog = catalogs
-            .get(&request.catalog)
+        let catalog = self
+            .catalog(&request.catalog)
             .context(CatalogNotFoundSnafu {
                 catalog_name: &request.catalog,
             })?;
-        catalog.register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))?;
+        catalog
+            .register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))
+            .await?;
+        increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
         Ok(true)
     }
 
@@ -147,10 +159,9 @@ impl CatalogManager for MemoryCatalogManager {
         Ok(())
     }
 
-    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
-        let catalogs = self.catalogs.read().unwrap();
-        if let Some(c) = catalogs.get(catalog) {
-            c.schema(schema)
+    async fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
+        if let Some(c) = self.catalog(catalog) {
+            c.schema(schema).await
         } else {
             Ok(None)
         }
@@ -162,15 +173,31 @@ impl CatalogManager for MemoryCatalogManager {
         schema: &str,
         table_name: &str,
     ) -> Result<Option<TableRef>> {
-        let catalog = {
-            let c = self.catalogs.read().unwrap();
-            let Some(c) = c.get(catalog) else { return Ok(None) };
-            c.clone()
-        };
-        match catalog.schema(schema)? {
-            None => Ok(None),
-            Some(s) => s.table(table_name).await,
-        }
+        let Some(catalog) = self
+            .catalog(catalog) else { return Ok(None)};
+        let Some(s) = catalog.schema(schema).await? else { return Ok(None) };
+        s.table(table_name).await
+    }
+
+    async fn catalog(&self, catalog: &str) -> Result<Option<CatalogProviderRef>> {
+        Ok(self.catalogs.read().unwrap().get(catalog).cloned())
+    }
+
+    async fn catalog_names(&self) -> Result<Vec<String>> {
+        Ok(self.catalogs.read().unwrap().keys().cloned().collect())
+    }
+
+    async fn register_catalog(
+        &self,
+        name: String,
+        catalog: CatalogProviderRef,
+    ) -> Result<Option<CatalogProviderRef>> {
+        increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
+        self.register_catalog_sync(name, catalog)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
     }
 }
 
@@ -192,14 +219,8 @@ impl MemoryCatalogManager {
             }
         }
     }
-}
 
-impl CatalogList for MemoryCatalogManager {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn register_catalog(
+    pub fn register_catalog_sync(
         &self,
         name: String,
         catalog: CatalogProviderRef,
@@ -208,14 +229,8 @@ impl CatalogList for MemoryCatalogManager {
         Ok(catalogs.insert(name, catalog))
     }
 
-    fn catalog_names(&self) -> Result<Vec<String>> {
-        let catalogs = self.catalogs.read().unwrap();
-        Ok(catalogs.keys().map(|s| s.to_string()).collect())
-    }
-
-    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>> {
-        let catalogs = self.catalogs.read().unwrap();
-        Ok(catalogs.get(name).cloned())
+    fn catalog(&self, catalog_name: &str) -> Option<CatalogProviderRef> {
+        self.catalogs.read().unwrap().get(catalog_name).cloned()
     }
 }
 
@@ -237,19 +252,13 @@ impl MemoryCatalogProvider {
             schemas: RwLock::new(HashMap::new()),
         }
     }
-}
 
-impl CatalogProvider for MemoryCatalogProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema_names(&self) -> Result<Vec<String>> {
+    pub fn schema_names_sync(&self) -> Result<Vec<String>> {
         let schemas = self.schemas.read().unwrap();
         Ok(schemas.keys().cloned().collect())
     }
 
-    fn register_schema(
+    pub fn register_schema_sync(
         &self,
         name: String,
         schema: SchemaProviderRef,
@@ -259,15 +268,39 @@ impl CatalogProvider for MemoryCatalogProvider {
             !schemas.contains_key(&name),
             error::SchemaExistsSnafu { schema: &name }
         );
+        increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
         Ok(schemas.insert(name, schema))
     }
 
-    fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>> {
+    pub fn schema_sync(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>> {
         let schemas = self.schemas.read().unwrap();
         Ok(schemas.get(name).cloned())
     }
 }
 
+#[async_trait::async_trait]
+impl CatalogProvider for MemoryCatalogProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    async fn schema_names(&self) -> Result<Vec<String>> {
+        self.schema_names_sync()
+    }
+
+    async fn register_schema(
+        &self,
+        name: String,
+        schema: SchemaProviderRef,
+    ) -> Result<Option<SchemaProviderRef>> {
+        self.register_schema_sync(name, schema)
+    }
+
+    async fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>> {
+        self.schema_sync(name)
+    }
+}
+
 /// Simple in-memory implementation of a schema.
 pub struct MemorySchemaProvider {
     tables: RwLock<HashMap<String, TableRef>>,
@@ -280,31 +313,8 @@ impl MemorySchemaProvider {
             tables: RwLock::new(HashMap::new()),
         }
     }
-}
 
-impl Default for MemorySchemaProvider {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-#[async_trait]
-impl SchemaProvider for MemorySchemaProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn table_names(&self) -> Result<Vec<String>> {
-        let tables = self.tables.read().unwrap();
-        Ok(tables.keys().cloned().collect())
-    }
-
-    async fn table(&self, name: &str) -> Result<Option<TableRef>> {
-        let tables = self.tables.read().unwrap();
-        Ok(tables.get(name).cloned())
-    }
-
-    fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
+    pub fn register_table_sync(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
         let mut tables = self.tables.write().unwrap();
         if let Some(existing) = tables.get(name.as_str()) {
             // if table with the same name but different table id exists, then it's a fatal bug
@@ -322,28 +332,71 @@ impl SchemaProvider for MemorySchemaProvider {
         }
     }
 
-    fn rename_table(&self, name: &str, new_name: String) -> Result<TableRef> {
+    pub fn rename_table_sync(&self, name: &str, new_name: String) -> Result<TableRef> {
         let mut tables = self.tables.write().unwrap();
-        if tables.get(name).is_some() {
-            let table = tables.remove(name).unwrap();
-            tables.insert(new_name, table.clone());
-            Ok(table)
-        } else {
-            TableNotFoundSnafu {
+        let Some(table) = tables.remove(name) else {
+            return TableNotFoundSnafu {
                 table_info: name.to_string(),
             }
-            .fail()?
-        }
+                .fail()?;
+        };
+        let e = match tables.entry(new_name) {
+            Entry::Vacant(e) => e,
+            Entry::Occupied(e) => {
+                return TableExistsSnafu { table: e.key() }.fail();
+            }
+        };
+        e.insert(table.clone());
+        Ok(table)
     }
 
-    fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
+    pub fn table_exist_sync(&self, name: &str) -> Result<bool> {
+        let tables = self.tables.read().unwrap();
+        Ok(tables.contains_key(name))
+    }
+
+    pub fn deregister_table_sync(&self, name: &str) -> Result<Option<TableRef>> {
         let mut tables = self.tables.write().unwrap();
         Ok(tables.remove(name))
     }
+}
 
-    fn table_exist(&self, name: &str) -> Result<bool> {
+impl Default for MemorySchemaProvider {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[async_trait]
+impl SchemaProvider for MemorySchemaProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    async fn table_names(&self) -> Result<Vec<String>> {
         let tables = self.tables.read().unwrap();
-        Ok(tables.contains_key(name))
+        Ok(tables.keys().cloned().collect())
+    }
+
+    async fn table(&self, name: &str) -> Result<Option<TableRef>> {
+        let tables = self.tables.read().unwrap();
+        Ok(tables.get(name).cloned())
+    }
+
+    async fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
+        self.register_table_sync(name, table)
+    }
+
+    async fn rename_table(&self, name: &str, new_name: String) -> Result<TableRef> {
+        self.rename_table_sync(name, new_name)
+    }
+
+    async fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
+        self.deregister_table_sync(name)
+    }
+
+    async fn table_exist(&self, name: &str) -> Result<bool> {
+        self.table_exist_sync(name)
     }
 }
 
@@ -364,15 +417,20 @@ mod tests {
     #[tokio::test]
     async fn test_new_memory_catalog_list() {
         let catalog_list = new_memory_catalog_list().unwrap();
-        let default_catalog = catalog_list.catalog(DEFAULT_CATALOG_NAME).unwrap().unwrap();
+        let default_catalog = CatalogManager::catalog(&*catalog_list, DEFAULT_CATALOG_NAME)
+            .await
+            .unwrap()
+            .unwrap();
 
         let default_schema = default_catalog
             .schema(DEFAULT_SCHEMA_NAME)
+            .await
             .unwrap()
             .unwrap();
 
         default_schema
             .register_table("numbers".to_string(), Arc::new(NumbersTable::default()))
+            .await
             .unwrap();
 
         let table = default_schema.table("numbers").await.unwrap();
@@ -384,19 +442,18 @@ mod tests {
     async fn test_mem_provider() {
         let provider = MemorySchemaProvider::new();
         let table_name = "numbers";
-        assert!(!provider.table_exist(table_name).unwrap());
-        assert!(provider.deregister_table(table_name).unwrap().is_none());
+        assert!(!provider.table_exist_sync(table_name).unwrap());
+        provider.deregister_table_sync(table_name).unwrap();
         let test_table = NumbersTable::default();
         // register table successfully
         assert!(provider
-            .register_table(table_name.to_string(), Arc::new(test_table))
+            .register_table_sync(table_name.to_string(), Arc::new(test_table))
             .unwrap()
             .is_none());
-        assert!(provider.table_exist(table_name).unwrap());
+        assert!(provider.table_exist_sync(table_name).unwrap());
         let other_table = NumbersTable::new(12);
-        let result = provider.register_table(table_name.to_string(), Arc::new(other_table));
+        let result = provider.register_table_sync(table_name.to_string(), Arc::new(other_table));
         let err = result.err().unwrap();
-        assert!(err.backtrace_opt().is_some());
         assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
     }
 
@@ -404,27 +461,27 @@ mod tests {
     async fn test_mem_provider_rename_table() {
         let provider = MemorySchemaProvider::new();
         let table_name = "num";
-        assert!(!provider.table_exist(table_name).unwrap());
+        assert!(!provider.table_exist_sync(table_name).unwrap());
         let test_table: TableRef = Arc::new(NumbersTable::default());
         // register test table
         assert!(provider
-            .register_table(table_name.to_string(), test_table.clone())
+            .register_table_sync(table_name.to_string(), test_table.clone())
             .unwrap()
             .is_none());
-        assert!(provider.table_exist(table_name).unwrap());
+        assert!(provider.table_exist_sync(table_name).unwrap());
 
         // rename test table
         let new_table_name = "numbers";
         provider
-            .rename_table(table_name, new_table_name.to_string())
+            .rename_table_sync(table_name, new_table_name.to_string())
             .unwrap();
 
         // test old table name not exist
-        assert!(!provider.table_exist(table_name).unwrap());
-        assert!(provider.deregister_table(table_name).unwrap().is_none());
+        assert!(!provider.table_exist_sync(table_name).unwrap());
+        provider.deregister_table_sync(table_name).unwrap();
 
         // test new table name exists
-        assert!(provider.table_exist(new_table_name).unwrap());
+        assert!(provider.table_exist_sync(new_table_name).unwrap());
         let registered_table = provider.table(new_table_name).await.unwrap().unwrap();
         assert_eq!(
             registered_table.table_info().ident.table_id,
@@ -432,7 +489,9 @@ mod tests {
         );
 
         let other_table = Arc::new(NumbersTable::new(2));
-        let result = provider.register_table(new_table_name.to_string(), other_table);
+        let result = provider
+            .register_table(new_table_name.to_string(), other_table)
+            .await;
         let err = result.err().unwrap();
         assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
     }
@@ -442,6 +501,7 @@ mod tests {
         let catalog = MemoryCatalogManager::default();
         let schema = catalog
             .schema(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
+            .await
             .unwrap()
             .unwrap();
 
@@ -457,10 +517,10 @@ mod tests {
             table,
         };
         assert!(catalog.register_table(register_table_req).await.unwrap());
-        assert!(schema.table_exist(table_name).unwrap());
+        assert!(schema.table_exist(table_name).await.unwrap());
 
         // rename table
-        let new_table_name = "numbers";
+        let new_table_name = "numbers_new";
         let rename_table_req = RenameTableRequest {
             catalog: DEFAULT_CATALOG_NAME.to_string(),
             schema: DEFAULT_SCHEMA_NAME.to_string(),
@@ -469,8 +529,8 @@ mod tests {
             table_id,
         };
         assert!(catalog.rename_table(rename_table_req).await.unwrap());
-        assert!(!schema.table_exist(table_name).unwrap());
-        assert!(schema.table_exist(new_table_name).unwrap());
+        assert!(!schema.table_exist(table_name).await.unwrap());
+        assert!(schema.table_exist(new_table_name).await.unwrap());
 
         let registered_table = catalog
             .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
@@ -504,6 +564,7 @@ mod tests {
         let catalog = MemoryCatalogManager::default();
         let schema = catalog
             .schema(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
+            .await
             .unwrap()
             .unwrap();
 
@@ -515,7 +576,7 @@ mod tests {
             table: Arc::new(NumbersTable::default()),
         };
         catalog.register_table(register_table_req).await.unwrap();
-        assert!(schema.table_exist("numbers").unwrap());
+        assert!(schema.table_exist("numbers").await.unwrap());
 
         let deregister_table_req = DeregisterTableRequest {
             catalog: DEFAULT_CATALOG_NAME.to_string(),
@@ -526,6 +587,6 @@ mod tests {
             .deregister_table(deregister_table_req)
             .await
             .unwrap();
-        assert!(!schema.table_exist("numbers").unwrap());
+        assert!(!schema.table_exist("numbers").await.unwrap());
     }
 }

src/catalog/src/metrics.rs

@@ -0,0 +1,29 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_catalog::build_db_string;
+
+pub(crate) const METRIC_DB_LABEL: &str = "db";
+
+pub(crate) const METRIC_CATALOG_MANAGER_CATALOG_COUNT: &str = "catalog.catalog_count";
+pub(crate) const METRIC_CATALOG_MANAGER_SCHEMA_COUNT: &str = "catalog.schema_count";
+pub(crate) const METRIC_CATALOG_MANAGER_TABLE_COUNT: &str = "catalog.table_count";
+
+pub(crate) const METRIC_CATALOG_KV_REMOTE_GET: &str = "catalog.kv.get.remote";
+pub(crate) const METRIC_CATALOG_KV_GET: &str = "catalog.kv.get";
+
+#[inline]
+pub(crate) fn db_label(catalog: &str, schema: &str) -> (&'static str, String) {
+    (METRIC_DB_LABEL, build_db_string(catalog, schema))
+}

src/catalog/src/remote.rs

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::any::Any;
 use std::fmt::Debug;
 use std::pin::Pin;
 use std::sync::Arc;
 
-pub use client::MetaKvBackend;
+pub use client::{CachedMetaKvBackend, MetaKvBackend};
 use futures::Stream;
 use futures_util::StreamExt;
 pub use manager::{RemoteCatalogManager, RemoteCatalogProvider, RemoteSchemaProvider};
@@ -26,6 +27,10 @@ use crate::error::Error;
 mod client;
 mod manager;
 
+#[cfg(feature = "testing")]
+pub mod mock;
+pub mod region_alive_keeper;
+
 #[derive(Debug, Clone)]
 pub struct Kv(pub Vec<u8>, pub Vec<u8>);
 
@@ -70,10 +75,22 @@ pub trait KvBackend: Send + Sync {
         }
         return Ok(None);
     }
+
+    /// MoveValue atomically renames the key to the given updated key.
+    async fn move_value(&self, from_key: &[u8], to_key: &[u8]) -> Result<(), Error>;
+
+    fn as_any(&self) -> &dyn Any;
 }
 
 pub type KvBackendRef = Arc<dyn KvBackend>;
 
+#[async_trait::async_trait]
+pub trait KvCacheInvalidator: Send + Sync {
+    async fn invalidate_key(&self, key: &[u8]);
+}
+
+pub type KvCacheInvalidatorRef = Arc<dyn KvCacheInvalidator>;
+
 #[cfg(test)]
 mod tests {
     use async_stream::stream;
@@ -114,17 +131,29 @@ mod tests {
         async fn delete_range(&self, _key: &[u8], _end: &[u8]) -> Result<(), Error> {
             unimplemented!()
         }
+
+        async fn move_value(&self, _from_key: &[u8], _to_key: &[u8]) -> Result<(), Error> {
+            unimplemented!()
+        }
+
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
     }
 
     #[tokio::test]
     async fn test_get() {
         let backend = MockKvBackend {};
+
         let result = backend.get(0.to_string().as_bytes()).await;
         assert_eq!(0.to_string().as_bytes(), result.unwrap().unwrap().0);
+
         let result = backend.get(1.to_string().as_bytes()).await;
         assert_eq!(1.to_string().as_bytes(), result.unwrap().unwrap().0);
+
         let result = backend.get(2.to_string().as_bytes()).await;
         assert_eq!(2.to_string().as_bytes(), result.unwrap().unwrap().0);
+
         let result = backend.get(3.to_string().as_bytes()).await;
         assert!(result.unwrap().is_none());
     }

src/catalog/src/remote/client.rs

@@ -12,17 +12,149 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::any::Any;
 use std::fmt::Debug;
 use std::sync::Arc;
+use std::time::Duration;
 
 use async_stream::stream;
-use common_telemetry::info;
+use common_meta::rpc::store::{
+    CompareAndPutRequest, DeleteRangeRequest, MoveValueRequest, PutRequest, RangeRequest,
+};
+use common_telemetry::{info, timer};
 use meta_client::client::MetaClient;
-use meta_client::rpc::{CompareAndPutRequest, DeleteRangeRequest, PutRequest, RangeRequest};
+use moka::future::{Cache, CacheBuilder};
 use snafu::ResultExt;
 
-use crate::error::{Error, MetaSrvSnafu};
-use crate::remote::{Kv, KvBackend, ValueIter};
+use super::KvCacheInvalidator;
+use crate::error::{Error, GenericSnafu, MetaSrvSnafu, Result};
+use crate::metrics::{METRIC_CATALOG_KV_GET, METRIC_CATALOG_KV_REMOTE_GET};
+use crate::remote::{Kv, KvBackend, KvBackendRef, ValueIter};
+
+const CACHE_MAX_CAPACITY: u64 = 10000;
+const CACHE_TTL_SECOND: u64 = 10 * 60;
+const CACHE_TTI_SECOND: u64 = 5 * 60;
+
+pub type CacheBackendRef = Arc<Cache<Vec<u8>, Option<Kv>>>;
+pub struct CachedMetaKvBackend {
+    kv_backend: KvBackendRef,
+    cache: CacheBackendRef,
+}
+
+#[async_trait::async_trait]
+impl KvBackend for CachedMetaKvBackend {
+    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
+    where
+        'a: 'b,
+    {
+        self.kv_backend.range(key)
+    }
+
+    async fn get(&self, key: &[u8]) -> Result<Option<Kv>> {
+        let _timer = timer!(METRIC_CATALOG_KV_GET);
+
+        let init = async {
+            let _timer = timer!(METRIC_CATALOG_KV_REMOTE_GET);
+
+            self.kv_backend.get(key).await
+        };
+
+        let schema_provider = self.cache.try_get_with_by_ref(key, init).await;
+        schema_provider.map_err(|e| GenericSnafu { msg: e.to_string() }.build())
+    }
+
+    async fn set(&self, key: &[u8], val: &[u8]) -> Result<()> {
+        let ret = self.kv_backend.set(key, val).await;
+
+        if ret.is_ok() {
+            self.invalidate_key(key).await;
+        }
+
+        ret
+    }
+
+    async fn delete(&self, key: &[u8]) -> Result<()> {
+        let ret = self.kv_backend.delete_range(key, &[]).await;
+
+        if ret.is_ok() {
+            self.invalidate_key(key).await;
+        }
+
+        ret
+    }
+
+    async fn delete_range(&self, _key: &[u8], _end: &[u8]) -> Result<()> {
+        // TODO(fys): implement it
+        unimplemented!()
+    }
+
+    async fn compare_and_set(
+        &self,
+        key: &[u8],
+        expect: &[u8],
+        val: &[u8],
+    ) -> Result<std::result::Result<(), Option<Vec<u8>>>> {
+        let ret = self.kv_backend.compare_and_set(key, expect, val).await;
+
+        if ret.is_ok() {
+            self.invalidate_key(key).await;
+        }
+
+        ret
+    }
+
+    async fn move_value(&self, from_key: &[u8], to_key: &[u8]) -> Result<()> {
+        let ret = self.kv_backend.move_value(from_key, to_key).await;
+
+        if ret.is_ok() {
+            self.invalidate_key(from_key).await;
+            self.invalidate_key(to_key).await;
+        }
+
+        ret
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+#[async_trait::async_trait]
+impl KvCacheInvalidator for CachedMetaKvBackend {
+    async fn invalidate_key(&self, key: &[u8]) {
+        self.cache.invalidate(key).await
+    }
+}
+
+impl CachedMetaKvBackend {
+    pub fn new(client: Arc<MetaClient>) -> Self {
+        let cache = Arc::new(
+            CacheBuilder::new(CACHE_MAX_CAPACITY)
+                .time_to_live(Duration::from_secs(CACHE_TTL_SECOND))
+                .time_to_idle(Duration::from_secs(CACHE_TTI_SECOND))
+                .build(),
+        );
+        let kv_backend = Arc::new(MetaKvBackend { client });
+
+        Self { kv_backend, cache }
+    }
+
+    pub fn wrap(kv_backend: KvBackendRef) -> Self {
+        let cache = Arc::new(
+            CacheBuilder::new(CACHE_MAX_CAPACITY)
+                .time_to_live(Duration::from_secs(CACHE_TTL_SECOND))
+                .time_to_idle(Duration::from_secs(CACHE_TTI_SECOND))
+                .build(),
+        );
+
+        Self { kv_backend, cache }
+    }
+
+    pub fn cache(&self) -> &CacheBackendRef {
+        &self.cache
+    }
+}
+
 #[derive(Debug)]
 pub struct MetaKvBackend {
     pub client: Arc<MetaClient>,
@@ -51,7 +183,7 @@ impl KvBackend for MetaKvBackend {
         }))
     }
 
-    async fn get(&self, key: &[u8]) -> Result<Option<Kv>, Error> {
+    async fn get(&self, key: &[u8]) -> Result<Option<Kv>> {
         let mut response = self
             .client
             .range(RangeRequest::new().with_key(key))
@@ -63,7 +195,7 @@ impl KvBackend for MetaKvBackend {
             .map(|kv| Kv(kv.take_key(), kv.take_value())))
     }
 
-    async fn set(&self, key: &[u8], val: &[u8]) -> Result<(), Error> {
+    async fn set(&self, key: &[u8], val: &[u8]) -> Result<()> {
         let req = PutRequest::new()
             .with_key(key.to_vec())
             .with_value(val.to_vec());
@@ -71,7 +203,7 @@ impl KvBackend for MetaKvBackend {
         Ok(())
     }
 
-    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Error> {
+    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<()> {
         let req = DeleteRangeRequest::new().with_range(key.to_vec(), end.to_vec());
         let resp = self.client.delete_range(req).await.context(MetaSrvSnafu)?;
         info!(
@@ -89,7 +221,7 @@ impl KvBackend for MetaKvBackend {
         key: &[u8],
         expect: &[u8],
         val: &[u8],
-    ) -> Result<Result<(), Option<Vec<u8>>>, Error> {
+    ) -> Result<std::result::Result<(), Option<Vec<u8>>>> {
         let request = CompareAndPutRequest::new()
             .with_key(key.to_vec())
             .with_expect(expect.to_vec())
@@ -105,4 +237,14 @@ impl KvBackend for MetaKvBackend {
             Ok(Err(response.take_prev_kv().map(|v| v.value().to_vec())))
         }
     }
+
+    async fn move_value(&self, from_key: &[u8], to_key: &[u8]) -> Result<()> {
+        let req = MoveValueRequest::new(from_key, to_key);
+        self.client.move_value(req).await.context(MetaSrvSnafu)?;
+        Ok(())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }

src/catalog/src/remote/mock.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::any::Any;
 use std::collections::btree_map::Entry;
 use std::collections::{BTreeMap, HashMap};
 use std::fmt::{Display, Formatter};
@@ -19,26 +20,56 @@ use std::str::FromStr;
 use std::sync::Arc;
 
 use async_stream::stream;
-use catalog::error::Error;
-use catalog::remote::{Kv, KvBackend, ValueIter};
+use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_recordbatch::RecordBatch;
 use common_telemetry::logging::info;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, Schema};
 use datatypes::vectors::StringVector;
 use serde::Serializer;
-use table::engine::{EngineContext, TableEngine, TableReference};
+use table::engine::{CloseTableResult, EngineContext, TableEngine, TableReference};
 use table::metadata::TableId;
-use table::requests::{AlterTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest};
+use table::requests::{
+    AlterTableRequest, CloseTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest,
+};
 use table::test_util::MemTable;
 use table::TableRef;
 use tokio::sync::RwLock;
 
-#[derive(Default)]
+use crate::error::Error;
+use crate::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
+use crate::remote::{Kv, KvBackend, ValueIter};
+
 pub struct MockKvBackend {
     map: RwLock<BTreeMap<Vec<u8>, Vec<u8>>>,
 }
 
+impl Default for MockKvBackend {
+    fn default() -> Self {
+        let mut map = BTreeMap::default();
+        let catalog_value = CatalogValue {}.as_bytes().unwrap();
+        let schema_value = SchemaValue {}.as_bytes().unwrap();
+
+        let default_catalog_key = CatalogKey {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        }
+        .to_string();
+
+        let default_schema_key = SchemaKey {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+        }
+        .to_string();
+
+        // create default catalog and schema
+        map.insert(default_catalog_key.into(), catalog_value);
+        map.insert(default_schema_key.into(), schema_value);
+
+        let map = RwLock::new(map);
+        Self { map }
+    }
+}
+
 impl Display for MockKvBackend {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         futures::executor::block_on(async {
@@ -112,14 +143,26 @@ impl KvBackend for MockKvBackend {
     }
 
     async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Error> {
-        let start = key.to_vec();
-        let end = end.to_vec();
-        let range = start..end;
-
         let mut map = self.map.write().await;
-        map.retain(|k, _| !range.contains(k));
+        if end.is_empty() {
+            let _ = map.remove(key);
+        } else {
+            let start = key.to_vec();
+            let end = end.to_vec();
+            let range = start..end;
+
+            map.retain(|k, _| !range.contains(k));
+        }
         Ok(())
     }
+
+    async fn move_value(&self, _from_key: &[u8], _to_key: &[u8]) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }
 
 #[derive(Default)]
@@ -142,6 +185,8 @@ impl TableEngine for MockTableEngine {
         let table_name = request.table_name.clone();
         let catalog_name = request.catalog_name.clone();
         let schema_name = request.schema_name.clone();
+        let table_full_name =
+            TableReference::full(&catalog_name, &schema_name, &table_name).to_string();
 
         let default_table_id = "0".to_owned();
         let table_id = TableId::from_str(
@@ -170,7 +215,7 @@ impl TableEngine for MockTableEngine {
         )) as Arc<_>;
 
         let mut tables = self.tables.write().await;
-        tables.insert(table_name, table.clone() as TableRef);
+        tables.insert(table_full_name, table.clone() as TableRef);
         Ok(table)
     }
 
@@ -222,6 +267,19 @@ impl TableEngine for MockTableEngine {
         unimplemented!()
     }
 
+    async fn close_table(
+        &self,
+        _ctx: &EngineContext,
+        request: CloseTableRequest,
+    ) -> table::Result<CloseTableResult> {
+        let _ = self
+            .tables
+            .write()
+            .await
+            .remove(&request.table_ref().to_string());
+        Ok(CloseTableResult::Released(vec![]))
+    }
+
     async fn close(&self) -> table::Result<()> {
         Ok(())
     }

src/catalog/src/remote/region_alive_keeper.rs

@@ -0,0 +1,821 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_meta::error::InvalidProtoMsgSnafu;
+use common_meta::heartbeat::handler::{
+    HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
+};
+use common_meta::ident::TableIdent;
+use common_meta::RegionIdent;
+use common_telemetry::{debug, error, info, warn};
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::RegionNumber;
+use table::engine::manager::TableEngineManagerRef;
+use table::engine::{CloseTableResult, EngineContext, TableEngineRef};
+use table::requests::CloseTableRequest;
+use table::TableRef;
+use tokio::sync::{mpsc, oneshot, Mutex};
+use tokio::task::JoinHandle;
+use tokio::time::{Duration, Instant};
+
+use crate::error::{Result, TableEngineNotFoundSnafu};
+
+/// [RegionAliveKeepers] manages all [RegionAliveKeeper] in a scope of tables.
+pub struct RegionAliveKeepers {
+    table_engine_manager: TableEngineManagerRef,
+    keepers: Arc<Mutex<HashMap<TableIdent, Arc<RegionAliveKeeper>>>>,
+    heartbeat_interval_millis: u64,
+    started: AtomicBool,
+
+    /// The epoch when [RegionAliveKeepers] is created. It's used to get a monotonically non-decreasing
+    /// elapsed time when submitting heartbeats to Metasrv (because [Instant] is monotonically
+    /// non-decreasing). The heartbeat request will carry the duration since this epoch, and the
+    /// duration acts like an "invariant point" for region's keep alive lease.
+    epoch: Instant,
+}
+
+impl RegionAliveKeepers {
+    pub fn new(
+        table_engine_manager: TableEngineManagerRef,
+        heartbeat_interval_millis: u64,
+    ) -> Self {
+        Self {
+            table_engine_manager,
+            keepers: Arc::new(Mutex::new(HashMap::new())),
+            heartbeat_interval_millis,
+            started: AtomicBool::new(false),
+            epoch: Instant::now(),
+        }
+    }
+
+    pub async fn find_keeper(&self, table_ident: &TableIdent) -> Option<Arc<RegionAliveKeeper>> {
+        self.keepers.lock().await.get(table_ident).cloned()
+    }
+
+    pub async fn register_table(&self, table_ident: TableIdent, table: TableRef) -> Result<()> {
+        let keeper = self.find_keeper(&table_ident).await;
+        if keeper.is_some() {
+            return Ok(());
+        }
+
+        let table_engine = self
+            .table_engine_manager
+            .engine(&table_ident.engine)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: &table_ident.engine,
+            })?;
+
+        let keeper = Arc::new(RegionAliveKeeper::new(
+            table_engine,
+            table_ident.clone(),
+            self.heartbeat_interval_millis,
+        ));
+        for r in table.table_info().meta.region_numbers.iter() {
+            keeper.register_region(*r).await;
+        }
+
+        let mut keepers = self.keepers.lock().await;
+        keepers.insert(table_ident.clone(), keeper.clone());
+
+        if self.started.load(Ordering::Relaxed) {
+            keeper.start().await;
+
+            info!("RegionAliveKeeper for table {table_ident} is started!");
+        } else {
+            info!("RegionAliveKeeper for table {table_ident} is registered but not started yet!");
+        }
+        Ok(())
+    }
+
+    pub async fn deregister_table(
+        &self,
+        table_ident: &TableIdent,
+    ) -> Option<Arc<RegionAliveKeeper>> {
+        self.keepers.lock().await.remove(table_ident).map(|x| {
+            info!("Deregister RegionAliveKeeper for table {table_ident}");
+            x
+        })
+    }
+
+    pub async fn register_region(&self, region_ident: &RegionIdent) {
+        let table_ident = &region_ident.table_ident;
+        let Some(keeper) = self.find_keeper(table_ident).await else {
+            // Alive keeper could be affected by lagging msg, just warn and ignore.
+            warn!("Alive keeper for region {region_ident} is not found!");
+            return;
+        };
+        keeper.register_region(region_ident.region_number).await
+    }
+
+    pub async fn deregister_region(&self, region_ident: &RegionIdent) {
+        let table_ident = &region_ident.table_ident;
+        let Some(keeper) = self.find_keeper(table_ident).await else {
+            // Alive keeper could be affected by lagging msg, just warn and ignore.
+            warn!("Alive keeper for region {region_ident} is not found!");
+            return;
+        };
+        let _ = keeper.deregister_region(region_ident.region_number).await;
+    }
+
+    pub async fn start(&self) {
+        let keepers = self.keepers.lock().await;
+        for keeper in keepers.values() {
+            keeper.start().await;
+        }
+        self.started.store(true, Ordering::Relaxed);
+
+        info!(
+            "RegionAliveKeepers for tables {:?} are started!",
+            keepers.keys().map(|x| x.to_string()).collect::<Vec<_>>(),
+        );
+    }
+
+    pub fn epoch(&self) -> Instant {
+        self.epoch
+    }
+}
+
+#[async_trait]
+impl HeartbeatResponseHandler for RegionAliveKeepers {
+    fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
+        !ctx.response.region_leases.is_empty()
+    }
+
+    async fn handle(
+        &self,
+        ctx: &mut HeartbeatResponseHandlerContext,
+    ) -> common_meta::error::Result<HandleControl> {
+        let leases = ctx.response.region_leases.drain(..).collect::<Vec<_>>();
+        for lease in leases {
+            let table_ident: TableIdent = match lease
+                .table_ident
+                .context(InvalidProtoMsgSnafu {
+                    err_msg: "'table_ident' is missing in RegionLease",
+                })
+                .and_then(|x| x.try_into())
+            {
+                Ok(x) => x,
+                Err(e) => {
+                    error!(e; "");
+                    continue;
+                }
+            };
+
+            let Some(keeper) = self.keepers.lock().await.get(&table_ident).cloned() else {
+                // Alive keeper could be affected by lagging msg, just warn and ignore.
+                warn!("Alive keeper for table {table_ident} is not found!");
+                continue;
+            };
+
+            let start_instant = self.epoch + Duration::from_millis(lease.duration_since_epoch);
+            let deadline = start_instant + Duration::from_secs(lease.lease_seconds);
+            keeper.keep_lived(lease.regions, deadline).await;
+        }
+        Ok(HandleControl::Continue)
+    }
+}
+
+/// [RegionAliveKeeper] starts a countdown for each region in a table. When deadline is reached,
+/// the region will be closed.
+/// The deadline is controlled by Metasrv. It works like "lease" for regions: a Datanode submits its
+/// opened regions to Metasrv, in heartbeats. If Metasrv decides some region could be resided in this
+/// Datanode, it will "extend" the region's "lease", with a deadline for [RegionAliveKeeper] to
+/// countdown.
+pub struct RegionAliveKeeper {
+    table_engine: TableEngineRef,
+    table_ident: TableIdent,
+    countdown_task_handles: Arc<Mutex<HashMap<RegionNumber, Arc<CountdownTaskHandle>>>>,
+    heartbeat_interval_millis: u64,
+    started: AtomicBool,
+}
+
+impl RegionAliveKeeper {
+    fn new(
+        table_engine: TableEngineRef,
+        table_ident: TableIdent,
+        heartbeat_interval_millis: u64,
+    ) -> Self {
+        Self {
+            table_engine,
+            table_ident,
+            countdown_task_handles: Arc::new(Mutex::new(HashMap::new())),
+            heartbeat_interval_millis,
+            started: AtomicBool::new(false),
+        }
+    }
+
+    async fn find_handle(&self, region: &RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
+        self.countdown_task_handles
+            .lock()
+            .await
+            .get(region)
+            .cloned()
+    }
+
+    async fn register_region(&self, region: RegionNumber) {
+        if self.find_handle(&region).await.is_some() {
+            return;
+        }
+
+        let countdown_task_handles = Arc::downgrade(&self.countdown_task_handles);
+        let on_task_finished = async move {
+            if let Some(x) = countdown_task_handles.upgrade() {
+                x.lock().await.remove(&region);
+            } // Else the countdown task handles map could be dropped because the keeper is dropped.
+        };
+        let handle = Arc::new(CountdownTaskHandle::new(
+            self.table_engine.clone(),
+            self.table_ident.clone(),
+            region,
+            || on_task_finished,
+        ));
+
+        let mut handles = self.countdown_task_handles.lock().await;
+        handles.insert(region, handle.clone());
+
+        if self.started.load(Ordering::Relaxed) {
+            handle.start(self.heartbeat_interval_millis).await;
+
+            info!(
+                "Region alive countdown for region {region} in table {} is started!",
+                self.table_ident
+            );
+        } else {
+            info!(
+                "Region alive countdown for region {region} in table {} is registered but not started yet!",
+                self.table_ident
+            );
+        }
+    }
+
+    async fn deregister_region(&self, region: RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
+        self.countdown_task_handles
+            .lock()
+            .await
+            .remove(&region)
+            .map(|x| {
+                info!(
+                    "Deregister alive countdown for region {region} in table {}",
+                    self.table_ident
+                );
+                x
+            })
+    }
+
+    async fn start(&self) {
+        let handles = self.countdown_task_handles.lock().await;
+        for handle in handles.values() {
+            handle.start(self.heartbeat_interval_millis).await;
+        }
+
+        self.started.store(true, Ordering::Relaxed);
+        info!(
+            "Region alive countdowns for regions {:?} in table {} are started!",
+            handles.keys().copied().collect::<Vec<_>>(),
+            self.table_ident
+        );
+    }
+
+    async fn keep_lived(&self, designated_regions: Vec<RegionNumber>, deadline: Instant) {
+        for region in designated_regions {
+            if let Some(handle) = self.find_handle(&region).await {
+                handle.reset_deadline(deadline).await;
+            }
+            // Else the region alive keeper might be triggered by lagging messages, we can safely ignore it.
+        }
+    }
+
+    pub async fn deadline(&self, region: RegionNumber) -> Option<Instant> {
+        let mut deadline = None;
+        if let Some(handle) = self.find_handle(&region).await {
+            let (s, r) = oneshot::channel();
+            if handle.tx.send(CountdownCommand::Deadline(s)).await.is_ok() {
+                deadline = r.await.ok()
+            }
+        }
+        deadline
+    }
+}
+
+#[derive(Debug)]
+enum CountdownCommand {
+    Start(u64),
+    Reset(Instant),
+    Deadline(oneshot::Sender<Instant>),
+}
+
+struct CountdownTaskHandle {
+    tx: mpsc::Sender<CountdownCommand>,
+    handler: JoinHandle<()>,
+    table_ident: TableIdent,
+    region: RegionNumber,
+}
+
+impl CountdownTaskHandle {
+    /// Creates a new [CountdownTaskHandle] and starts the countdown task.
+    /// # Params
+    /// - `on_task_finished`: a callback to be invoked when the task is finished. Note that it will not
+    ///   be invoked if the task is cancelled (by dropping the handle). This is because we want something
+    ///   meaningful to be done when the task is finished, e.g. deregister the handle from the map.
+    ///   While dropping the handle does not necessarily mean the task is finished.
+    fn new<Fut>(
+        table_engine: TableEngineRef,
+        table_ident: TableIdent,
+        region: RegionNumber,
+        on_task_finished: impl FnOnce() -> Fut + Send + 'static,
+    ) -> Self
+    where
+        Fut: Future<Output = ()> + Send,
+    {
+        let (tx, rx) = mpsc::channel(1024);
+
+        let mut countdown_task = CountdownTask {
+            table_engine,
+            table_ident: table_ident.clone(),
+            region,
+            rx,
+        };
+        let handler = common_runtime::spawn_bg(async move {
+            countdown_task.run().await;
+            on_task_finished().await;
+        });
+
+        Self {
+            tx,
+            handler,
+            table_ident,
+            region,
+        }
+    }
+
+    async fn start(&self, heartbeat_interval_millis: u64) {
+        if let Err(e) = self
+            .tx
+            .send(CountdownCommand::Start(heartbeat_interval_millis))
+            .await
+        {
+            warn!(
+                "Failed to start region alive keeper countdown: {e}. \
+                Maybe the task is stopped due to region been closed."
+            );
+        }
+    }
+
+    async fn reset_deadline(&self, deadline: Instant) {
+        if let Err(e) = self.tx.send(CountdownCommand::Reset(deadline)).await {
+            warn!(
+                "Failed to reset region alive keeper deadline: {e}. \
+                Maybe the task is stopped due to region been closed."
+            );
+        }
+    }
+}
+
+impl Drop for CountdownTaskHandle {
+    fn drop(&mut self) {
+        debug!(
+            "Aborting region alive countdown task for region {} in table {}",
+            self.region, self.table_ident,
+        );
+        self.handler.abort();
+    }
+}
+
+struct CountdownTask {
+    table_engine: TableEngineRef,
+    table_ident: TableIdent,
+    region: RegionNumber,
+    rx: mpsc::Receiver<CountdownCommand>,
+}
+
+impl CountdownTask {
+    async fn run(&mut self) {
+        // 30 years. See `Instant::far_future`.
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 30);
+
+        // Make sure the alive countdown is not gonna happen before heartbeat task is started (the
+        // "start countdown" command will be sent from heartbeat task).
+        let countdown = tokio::time::sleep_until(far_future);
+        tokio::pin!(countdown);
+
+        let region = &self.region;
+        let table_ident = &self.table_ident;
+        loop {
+            tokio::select! {
+                command = self.rx.recv() => {
+                    match command {
+                        Some(CountdownCommand::Start(heartbeat_interval_millis)) => {
+                            // Set first deadline in 4 heartbeats (roughly after 20 seconds from now if heartbeat
+                            // interval is set to default 5 seconds), to make Datanode and Metasrv more tolerable to
+                            // network or other jitters during startup.
+                            let first_deadline = Instant::now() + Duration::from_millis(heartbeat_interval_millis) * 4;
+                            countdown.set(tokio::time::sleep_until(first_deadline));
+                        },
+                        Some(CountdownCommand::Reset(deadline)) => {
+                            if countdown.deadline() < deadline {
+                                debug!(
+                                    "Reset deadline of region {region} of table {table_ident} to approximately {} seconds later",
+                                    (deadline - Instant::now()).as_secs_f32(),
+                                );
+                                countdown.set(tokio::time::sleep_until(deadline));
+                            }
+                            // Else the countdown could be either:
+                            // - not started yet;
+                            // - during startup protection;
+                            // - received a lagging heartbeat message.
+                            // All can be safely ignored.
+                        },
+                        None => {
+                            info!(
+                                "The handle of countdown task for region {region} of table {table_ident} \
+                                is dropped, RegionAliveKeeper out."
+                            );
+                            break;
+                        },
+                        Some(CountdownCommand::Deadline(tx)) => {
+                            let _ = tx.send(countdown.deadline());
+                        }
+                    }
+                }
+                () = &mut countdown => {
+                    let result = self.close_region().await;
+                    warn!(
+                        "Region {region} of table {table_ident} is closed, result: {result:?}. \
+                        RegionAliveKeeper out.",
+                    );
+                    break;
+                }
+            }
+        }
+    }
+
+    async fn close_region(&self) -> CloseTableResult {
+        let ctx = EngineContext::default();
+        let region = self.region;
+        let table_ident = &self.table_ident;
+        loop {
+            let request = CloseTableRequest {
+                catalog_name: table_ident.catalog.clone(),
+                schema_name: table_ident.schema.clone(),
+                table_name: table_ident.table.clone(),
+                region_numbers: vec![region],
+                flush: true,
+            };
+            match self.table_engine.close_table(&ctx, request).await {
+                Ok(result) => return result,
+                // If region is failed to close, immediately retry. Maybe we should panic instead?
+                Err(e) => error!(e;
+                    "Failed to close region {region} of table {table_ident}. \
+                    For the integrity of data, retry closing and retry without wait.",
+                ),
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::Arc;
+
+    use api::v1::meta::{HeartbeatResponse, RegionLease};
+    use common_meta::heartbeat::mailbox::HeartbeatMailbox;
+    use datatypes::schema::RawSchema;
+    use table::engine::manager::MemoryTableEngineManager;
+    use table::engine::{TableEngine, TableReference};
+    use table::requests::{CreateTableRequest, TableOptions};
+    use table::test_util::EmptyTable;
+
+    use super::*;
+    use crate::remote::mock::MockTableEngine;
+
+    async fn prepare_keepers() -> (TableIdent, RegionAliveKeepers) {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_engine_manager = Arc::new(MemoryTableEngineManager::new(table_engine));
+        let keepers = RegionAliveKeepers::new(table_engine_manager, 5000);
+
+        let catalog = "my_catalog";
+        let schema = "my_schema";
+        let table = "my_table";
+        let table_ident = TableIdent {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table: table.to_string(),
+            table_id: 1,
+            engine: "MockTableEngine".to_string(),
+        };
+        let table = Arc::new(EmptyTable::new(CreateTableRequest {
+            id: 1,
+            catalog_name: catalog.to_string(),
+            schema_name: schema.to_string(),
+            table_name: table.to_string(),
+            desc: None,
+            schema: RawSchema {
+                column_schemas: vec![],
+                timestamp_index: None,
+                version: 0,
+            },
+            region_numbers: vec![1, 2, 3],
+            primary_key_indices: vec![],
+            create_if_not_exists: false,
+            table_options: TableOptions::default(),
+            engine: "MockTableEngine".to_string(),
+        }));
+        keepers
+            .register_table(table_ident.clone(), table)
+            .await
+            .unwrap();
+        assert!(keepers.keepers.lock().await.contains_key(&table_ident));
+
+        (table_ident, keepers)
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_handle_heartbeat_response() {
+        let (table_ident, keepers) = prepare_keepers().await;
+
+        keepers.start().await;
+        let startup_protection_until = Instant::now() + Duration::from_secs(21);
+
+        let duration_since_epoch = (Instant::now() - keepers.epoch).as_millis() as _;
+        let lease_seconds = 100;
+        let response = HeartbeatResponse {
+            region_leases: vec![RegionLease {
+                table_ident: Some(table_ident.clone().into()),
+                regions: vec![1, 3], // Not extending region 2's lease time.
+                duration_since_epoch,
+                lease_seconds,
+            }],
+            ..Default::default()
+        };
+        let keep_alive_until = keepers.epoch
+            + Duration::from_millis(duration_since_epoch)
+            + Duration::from_secs(lease_seconds);
+
+        let (tx, _) = mpsc::channel(8);
+        let mailbox = Arc::new(HeartbeatMailbox::new(tx));
+        let mut ctx = HeartbeatResponseHandlerContext::new(mailbox, response);
+
+        assert!(keepers.handle(&mut ctx).await.unwrap() == HandleControl::Continue);
+
+        // sleep to wait for background task spawned in `handle`
+        tokio::time::sleep(Duration::from_secs(1)).await;
+
+        async fn test(
+            keeper: &Arc<RegionAliveKeeper>,
+            region_number: RegionNumber,
+            startup_protection_until: Instant,
+            keep_alive_until: Instant,
+            is_kept_live: bool,
+        ) {
+            let deadline = keeper.deadline(region_number).await.unwrap();
+            if is_kept_live {
+                assert!(deadline > startup_protection_until && deadline == keep_alive_until);
+            } else {
+                assert!(deadline <= startup_protection_until);
+            }
+        }
+
+        let keeper = &keepers
+            .keepers
+            .lock()
+            .await
+            .get(&table_ident)
+            .cloned()
+            .unwrap();
+
+        // Test region 1 and 3 is kept lived. Their deadlines are updated to desired instant.
+        test(keeper, 1, startup_protection_until, keep_alive_until, true).await;
+        test(keeper, 3, startup_protection_until, keep_alive_until, true).await;
+
+        // Test region 2 is not kept lived. It's deadline is not updated: still during startup protection period.
+        test(keeper, 2, startup_protection_until, keep_alive_until, false).await;
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_region_alive_keepers() {
+        let (table_ident, keepers) = prepare_keepers().await;
+
+        keepers
+            .register_region(&RegionIdent {
+                cluster_id: 1,
+                datanode_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 4,
+            })
+            .await;
+
+        keepers.start().await;
+        for keeper in keepers.keepers.lock().await.values() {
+            let regions = {
+                let handles = keeper.countdown_task_handles.lock().await;
+                handles.keys().copied().collect::<Vec<_>>()
+            };
+            for region in regions {
+                // assert countdown tasks are started
+                let deadline = keeper.deadline(region).await.unwrap();
+                assert!(deadline <= Instant::now() + Duration::from_secs(20));
+            }
+        }
+
+        keepers
+            .deregister_region(&RegionIdent {
+                cluster_id: 1,
+                datanode_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 1,
+            })
+            .await;
+        let mut regions = keepers
+            .find_keeper(&table_ident)
+            .await
+            .unwrap()
+            .countdown_task_handles
+            .lock()
+            .await
+            .keys()
+            .copied()
+            .collect::<Vec<_>>();
+        regions.sort();
+        assert_eq!(regions, vec![2, 3, 4]);
+
+        let keeper = keepers.deregister_table(&table_ident).await.unwrap();
+        assert!(Arc::try_unwrap(keeper).is_ok(), "keeper is not dropped");
+        assert!(keepers.keepers.lock().await.is_empty());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_region_alive_keeper() {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_ident = TableIdent {
+            catalog: "my_catalog".to_string(),
+            schema: "my_schema".to_string(),
+            table: "my_table".to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let keeper = RegionAliveKeeper::new(table_engine, table_ident, 1000);
+
+        let region = 1;
+        assert!(keeper.find_handle(&region).await.is_none());
+        keeper.register_region(region).await;
+        assert!(keeper.find_handle(&region).await.is_some());
+
+        let ten_seconds_later = || Instant::now() + Duration::from_secs(10);
+
+        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
+        assert!(keeper.find_handle(&2).await.is_none());
+        assert!(keeper.find_handle(&3).await.is_none());
+
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
+        // assert if keeper is not started, keep_lived is of no use
+        assert!(keeper.deadline(region).await.unwrap() > far_future);
+
+        keeper.start().await;
+        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
+        // assert keep_lived works if keeper is started
+        assert!(keeper.deadline(region).await.unwrap() <= ten_seconds_later());
+
+        let handle = keeper.deregister_region(region).await.unwrap();
+        assert!(Arc::try_unwrap(handle).is_ok(), "handle is not dropped");
+        assert!(keeper.find_handle(&region).await.is_none());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_countdown_task_handle() {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_ident = TableIdent {
+            catalog: "my_catalog".to_string(),
+            schema: "my_schema".to_string(),
+            table: "my_table".to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let finished = Arc::new(AtomicBool::new(false));
+        let finished_clone = finished.clone();
+        let handle = CountdownTaskHandle::new(
+            table_engine.clone(),
+            table_ident.clone(),
+            1,
+            || async move { finished_clone.store(true, Ordering::Relaxed) },
+        );
+        let tx = handle.tx.clone();
+
+        // assert countdown task is running
+        assert!(tx.send(CountdownCommand::Start(5000)).await.is_ok());
+        assert!(!finished.load(Ordering::Relaxed));
+
+        drop(handle);
+        tokio::time::sleep(Duration::from_secs(1)).await;
+
+        // assert countdown task is stopped
+        assert!(tx
+            .try_send(CountdownCommand::Reset(
+                Instant::now() + Duration::from_secs(10)
+            ))
+            .is_err());
+        // assert `on_task_finished` is not called (because the task is aborted by the handle's drop)
+        assert!(!finished.load(Ordering::Relaxed));
+
+        let finished = Arc::new(AtomicBool::new(false));
+        let finished_clone = finished.clone();
+        let handle = CountdownTaskHandle::new(table_engine, table_ident, 1, || async move {
+            finished_clone.store(true, Ordering::Relaxed)
+        });
+        handle.tx.send(CountdownCommand::Start(100)).await.unwrap();
+        tokio::time::sleep(Duration::from_secs(1)).await;
+        // assert `on_task_finished` is called when task is finished normally
+        assert!(finished.load(Ordering::Relaxed));
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_countdown_task_run() {
+        let ctx = &EngineContext::default();
+        let catalog = "my_catalog";
+        let schema = "my_schema";
+        let table = "my_table";
+        let request = CreateTableRequest {
+            id: 1,
+            catalog_name: catalog.to_string(),
+            schema_name: schema.to_string(),
+            table_name: table.to_string(),
+            desc: None,
+            schema: RawSchema {
+                column_schemas: vec![],
+                timestamp_index: None,
+                version: 0,
+            },
+            region_numbers: vec![],
+            primary_key_indices: vec![],
+            create_if_not_exists: false,
+            table_options: TableOptions::default(),
+            engine: "mito".to_string(),
+        };
+        let table_ref = TableReference::full(catalog, schema, table);
+
+        let table_engine = Arc::new(MockTableEngine::default());
+        table_engine.create_table(ctx, request).await.unwrap();
+
+        let table_ident = TableIdent {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table: table.to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let (tx, rx) = mpsc::channel(10);
+        let mut task = CountdownTask {
+            table_engine: table_engine.clone(),
+            table_ident,
+            region: 1,
+            rx,
+        };
+        common_runtime::spawn_bg(async move {
+            task.run().await;
+        });
+
+        async fn deadline(tx: &mpsc::Sender<CountdownCommand>) -> Instant {
+            let (s, r) = oneshot::channel();
+            tx.send(CountdownCommand::Deadline(s)).await.unwrap();
+            r.await.unwrap()
+        }
+
+        // if countdown task is not started, its deadline is set to far future
+        assert!(deadline(&tx).await > Instant::now() + Duration::from_secs(86400 * 365 * 29));
+
+        // start countdown in 250ms * 4 = 1s
+        tx.send(CountdownCommand::Start(250)).await.unwrap();
+        // assert deadline is correctly set
+        assert!(deadline(&tx).await <= Instant::now() + Duration::from_secs(1));
+
+        // reset countdown in 1.5s
+        tx.send(CountdownCommand::Reset(
+            Instant::now() + Duration::from_millis(1500),
+        ))
+        .await
+        .unwrap();
+
+        // assert the table is closed after deadline is reached
+        assert!(table_engine.table_exists(ctx, &table_ref));
+        // spare 500ms for the task to close the table
+        tokio::time::sleep(Duration::from_millis(2000)).await;
+        assert!(!table_engine.table_exists(ctx, &table_ref));
+    }
+}

src/catalog/src/schema.rs

@@ -18,7 +18,7 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use table::TableRef;
 
-use crate::error::Result;
+use crate::error::{NotSupportedSnafu, Result};
 
 /// Represents a schema, comprising a number of named tables.
 #[async_trait]
@@ -28,27 +28,42 @@ pub trait SchemaProvider: Sync + Send {
     fn as_any(&self) -> &dyn Any;
 
     /// Retrieves the list of available table names in this schema.
-    fn table_names(&self) -> Result<Vec<String>>;
+    async fn table_names(&self) -> Result<Vec<String>>;
 
     /// Retrieves a specific table from the schema by name, provided it exists.
     async fn table(&self, name: &str) -> Result<Option<TableRef>>;
 
     /// If supported by the implementation, adds a new table to this schema.
     /// If a table of the same name existed before, it returns "Table already exists" error.
-    fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>>;
+    async fn register_table(&self, name: String, _table: TableRef) -> Result<Option<TableRef>> {
+        NotSupportedSnafu {
+            op: format!("register_table({name}, <table>)"),
+        }
+        .fail()
+    }
 
     /// If supported by the implementation, renames an existing table from this schema and returns it.
     /// If no table of that name exists, returns "Table not found" error.
-    fn rename_table(&self, name: &str, new_name: String) -> Result<TableRef>;
+    async fn rename_table(&self, name: &str, new_name: String) -> Result<TableRef> {
+        NotSupportedSnafu {
+            op: format!("rename_table({name}, {new_name})"),
+        }
+        .fail()
+    }
 
     /// If supported by the implementation, removes an existing table from this schema and returns it.
     /// If no table of that name exists, returns Ok(None).
-    fn deregister_table(&self, name: &str) -> Result<Option<TableRef>>;
+    async fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
+        NotSupportedSnafu {
+            op: format!("deregister_table({name})"),
+        }
+        .fail()
+    }
 
     /// If supported by the implementation, checks the table exist in the schema provider or not.
     /// If no matched table in the schema provider, return false.
     /// Otherwise, return true.
-    fn table_exist(&self, name: &str) -> Result<bool>;
+    async fn table_exist(&self, name: &str) -> Result<bool>;
 }
 
 pub type SchemaProviderRef = Arc<dyn SchemaProvider>;

src/catalog/src/system.rs

@@ -17,8 +17,8 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use common_catalog::consts::{
-    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_NAME,
-    SYSTEM_CATALOG_TABLE_ID, SYSTEM_CATALOG_TABLE_NAME,
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MITO_ENGINE,
+    SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID, SYSTEM_CATALOG_TABLE_NAME,
 };
 use common_query::logical_plan::Expr;
 use common_query::physical_plan::{PhysicalPlanRef, SessionContext};
@@ -30,12 +30,13 @@ use datatypes::schema::{ColumnSchema, RawSchema, SchemaRef};
 use datatypes::vectors::{BinaryVector, TimestampMillisecondVector, UInt8Vector};
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
+use store_api::storage::ScanRequest;
 use table::engine::{EngineContext, TableEngineRef};
 use table::metadata::{TableId, TableInfoRef};
 use table::requests::{
     CreateTableRequest, DeleteRequest, InsertRequest, OpenTableRequest, TableOptions,
 };
-use table::{Table, TableRef};
+use table::{Result as TableResult, Table, TableRef};
 
 use crate::error::{
     self, CreateSystemCatalogSnafu, EmptyValueSnafu, Error, InvalidEntryTypeSnafu, InvalidKeySnafu,
@@ -68,8 +69,12 @@ impl Table for SystemCatalogTable {
         self.0.scan(projection, filters, limit).await
     }
 
+    async fn scan_to_stream(&self, request: ScanRequest) -> TableResult<SendableRecordBatchStream> {
+        self.0.scan_to_stream(request).await
+    }
+
     /// Insert values into table.
-    async fn insert(&self, request: InsertRequest) -> table::error::Result<usize> {
+    async fn insert(&self, request: InsertRequest) -> TableResult<usize> {
         self.0.insert(request).await
     }
 
@@ -77,9 +82,13 @@ impl Table for SystemCatalogTable {
         self.0.table_info()
     }
 
-    async fn delete(&self, request: DeleteRequest) -> table::Result<usize> {
+    async fn delete(&self, request: DeleteRequest) -> TableResult<usize> {
         self.0.delete(request).await
     }
+
+    fn statistics(&self) -> Option<table::stats::TableStatistics> {
+        self.0.statistics()
+    }
 }
 
 impl SystemCatalogTable {
@@ -89,6 +98,7 @@ impl SystemCatalogTable {
             schema_name: INFORMATION_SCHEMA_NAME.to_string(),
             table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
             table_id: SYSTEM_CATALOG_TABLE_ID,
+            region_numbers: vec![0],
         };
         let schema = build_system_catalog_schema();
         let ctx = EngineContext::default();
@@ -112,6 +122,7 @@ impl SystemCatalogTable {
                 primary_key_indices: vec![ENTRY_TYPE_INDEX, KEY_INDEX],
                 create_if_not_exists: true,
                 table_options: TableOptions::default(),
+                engine: engine.name().to_string(),
             };
 
             let table = engine
@@ -194,12 +205,13 @@ pub fn build_table_insert_request(
     schema: String,
     table_name: String,
     table_id: TableId,
+    engine: String,
 ) -> InsertRequest {
     let entry_key = format_table_entry_key(&catalog, &schema, table_id);
     build_insert_request(
         EntryType::Table,
         entry_key.as_bytes(),
-        serde_json::to_string(&TableEntryValue { table_name })
+        serde_json::to_string(&TableEntryValue { table_name, engine })
             .unwrap()
             .as_bytes(),
     )
@@ -330,6 +342,7 @@ pub fn decode_system_catalog(
                 schema_name: table_parts[1].to_string(),
                 table_name: table_meta.table_name,
                 table_id,
+                engine: table_meta.engine,
             }))
         }
     }
@@ -385,11 +398,19 @@ pub struct TableEntry {
     pub schema_name: String,
     pub table_name: String,
     pub table_id: TableId,
+    pub engine: String,
 }
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct TableEntryValue {
     pub table_name: String,
+
+    #[serde(default = "mito_engine")]
+    pub engine: String,
+}
+
+fn mito_engine() -> String {
+    MITO_ENGINE.to_string()
 }
 
 #[cfg(test)]
@@ -399,8 +420,8 @@ mod tests {
     use datatypes::value::Value;
     use log_store::NoopLogStore;
     use mito::config::EngineConfig;
-    use mito::engine::MitoEngine;
-    use object_store::{ObjectStore, ObjectStoreBuilder};
+    use mito::engine::{MitoEngine, MITO_ENGINE};
+    use object_store::ObjectStore;
     use storage::compaction::noop::NoopCompactionScheduler;
     use storage::config::EngineConfig as StorageEngineConfig;
     use storage::EngineImpl;
@@ -482,11 +503,9 @@ mod tests {
     pub async fn prepare_table_engine() -> (TempDir, TableEngineRef) {
         let dir = create_temp_dir("system-table-test");
         let store_dir = dir.path().to_string_lossy();
-        let accessor = object_store::services::Fs::default()
-            .root(&store_dir)
-            .build()
-            .unwrap();
-        let object_store = ObjectStore::new(accessor).finish();
+        let mut builder = object_store::services::Fs::default();
+        builder.root(&store_dir);
+        let object_store = ObjectStore::new(builder).unwrap().finish();
         let noop_compaction_scheduler = Arc::new(NoopCompactionScheduler::default());
         let table_engine = Arc::new(MitoEngine::new(
             EngineConfig::default(),
@@ -495,7 +514,8 @@ mod tests {
                 Arc::new(NoopLogStore::default()),
                 object_store.clone(),
                 noop_compaction_scheduler,
-            ),
+            )
+            .unwrap(),
             object_store,
         ));
         (dir, table_engine)
@@ -530,6 +550,7 @@ mod tests {
             DEFAULT_SCHEMA_NAME.to_string(),
             "my_table".to_string(),
             1,
+            MITO_ENGINE.to_string(),
         );
         let result = catalog_table.insert(table_insertion).await.unwrap();
         assert_eq!(result, 1);
@@ -550,6 +571,7 @@ mod tests {
             schema_name: DEFAULT_SCHEMA_NAME.to_string(),
             table_name: "my_table".to_string(),
             table_id: 1,
+            engine: MITO_ENGINE.to_string(),
         });
         assert_eq!(entry, expected);
 

src/catalog/src/table_source.rs

@@ -15,8 +15,9 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use common_catalog::consts::INFORMATION_SCHEMA_NAME;
 use common_catalog::format_full_table_name;
-use datafusion::common::{OwnedTableReference, ResolvedTableReference, TableReference};
+use datafusion::common::{ResolvedTableReference, TableReference};
 use datafusion::datasource::provider_as_source;
 use datafusion::logical_expr::TableSource;
 use session::context::QueryContext;
@@ -26,10 +27,11 @@ use table::table::adapter::DfTableProviderAdapter;
 use crate::error::{
     CatalogNotFoundSnafu, QueryAccessDeniedSnafu, Result, SchemaNotFoundSnafu, TableNotExistSnafu,
 };
-use crate::CatalogListRef;
+use crate::information_schema::InformationSchemaProvider;
+use crate::CatalogManagerRef;
 
 pub struct DfTableSourceProvider {
-    catalog_list: CatalogListRef,
+    catalog_manager: CatalogManagerRef,
     resolved_tables: HashMap<String, Arc<dyn TableSource>>,
     disallow_cross_schema_query: bool,
     default_catalog: String,
@@ -38,12 +40,12 @@ pub struct DfTableSourceProvider {
 
 impl DfTableSourceProvider {
     pub fn new(
-        catalog_list: CatalogListRef,
+        catalog_manager: CatalogManagerRef,
         disallow_cross_schema_query: bool,
         query_ctx: &QueryContext,
     ) -> Self {
         Self {
-            catalog_list,
+            catalog_manager,
             disallow_cross_schema_query,
             resolved_tables: HashMap::new(),
             default_catalog: query_ctx.current_catalog(),
@@ -60,7 +62,8 @@ impl DfTableSourceProvider {
                 TableReference::Bare { .. } => (),
                 TableReference::Partial { schema, .. } => {
                     ensure!(
-                        schema.as_ref() == self.default_schema,
+                        schema.as_ref() == self.default_schema
+                            || schema.as_ref() == INFORMATION_SCHEMA_NAME,
                         QueryAccessDeniedSnafu {
                             catalog: &self.default_catalog,
                             schema: schema.as_ref(),
@@ -72,7 +75,8 @@ impl DfTableSourceProvider {
                 } => {
                     ensure!(
                         catalog.as_ref() == self.default_catalog
-                            && schema.as_ref() == self.default_schema,
+                            && (schema.as_ref() == self.default_schema
+                                || schema.as_ref() == INFORMATION_SCHEMA_NAME),
                         QueryAccessDeniedSnafu {
                             catalog: catalog.as_ref(),
                             schema: schema.as_ref()
@@ -87,9 +91,8 @@ impl DfTableSourceProvider {
 
     pub async fn resolve_table(
         &mut self,
-        table_ref: OwnedTableReference,
+        table_ref: TableReference<'_>,
     ) -> Result<Arc<dyn TableSource>> {
-        let table_ref = table_ref.as_table_reference();
         let table_ref = self.resolve_table_ref(table_ref)?;
 
         let resolved_name = table_ref.to_string();
@@ -101,14 +104,30 @@ impl DfTableSourceProvider {
         let schema_name = table_ref.schema.as_ref();
         let table_name = table_ref.table.as_ref();
 
-        let catalog = self
-            .catalog_list
-            .catalog(catalog_name)?
-            .context(CatalogNotFoundSnafu { catalog_name })?;
-        let schema = catalog.schema(schema_name)?.context(SchemaNotFoundSnafu {
-            catalog: catalog_name,
-            schema: schema_name,
-        })?;
+        let schema = if schema_name != INFORMATION_SCHEMA_NAME {
+            let catalog = self
+                .catalog_manager
+                .catalog(catalog_name)
+                .await?
+                .context(CatalogNotFoundSnafu { catalog_name })?;
+            catalog
+                .schema(schema_name)
+                .await?
+                .context(SchemaNotFoundSnafu {
+                    catalog: catalog_name,
+                    schema: schema_name,
+                })?
+        } else {
+            let catalog_provider = self
+                .catalog_manager
+                .catalog(catalog_name)
+                .await?
+                .context(CatalogNotFoundSnafu { catalog_name })?;
+            Arc::new(InformationSchemaProvider::new(
+                catalog_name.to_string(),
+                catalog_provider,
+            ))
+        };
         let table = schema
             .table(table_name)
             .await?
@@ -174,5 +193,25 @@ mod tests {
         };
         let result = table_provider.resolve_table_ref(table_ref);
         assert!(result.is_err());
+
+        let table_ref = TableReference::Partial {
+            schema: Cow::Borrowed("information_schema"),
+            table: Cow::Borrowed("columns"),
+        };
+        assert!(table_provider.resolve_table_ref(table_ref).is_ok());
+
+        let table_ref = TableReference::Full {
+            catalog: Cow::Borrowed("greptime"),
+            schema: Cow::Borrowed("information_schema"),
+            table: Cow::Borrowed("columns"),
+        };
+        assert!(table_provider.resolve_table_ref(table_ref).is_ok());
+
+        let table_ref = TableReference::Full {
+            catalog: Cow::Borrowed("dummy"),
+            schema: Cow::Borrowed("information_schema"),
+            table: Cow::Borrowed("columns"),
+        };
+        assert!(table_provider.resolve_table_ref(table_ref).is_err());
     }
 }

src/catalog/src/tables.rs

@@ -15,28 +15,13 @@
 // The `tables` table in system catalog keeps a record of all tables created by user.
 
 use std::any::Any;
-use std::pin::Pin;
 use std::sync::Arc;
-use std::task::{Context, Poll};
 
-use async_stream::stream;
 use async_trait::async_trait;
 use common_catalog::consts::{INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_TABLE_NAME};
-use common_error::ext::BoxedError;
-use common_query::logical_plan::Expr;
-use common_query::physical_plan::PhysicalPlanRef;
-use common_recordbatch::error::Result as RecordBatchResult;
-use common_recordbatch::{RecordBatch, RecordBatchStream};
-use datatypes::prelude::{ConcreteDataType, DataType};
-use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
-use datatypes::value::ValueRef;
-use datatypes::vectors::VectorRef;
-use futures::Stream;
+use common_telemetry::logging;
 use snafu::ResultExt;
-use table::engine::TableEngineRef;
-use table::error::TablesRecordBatchSnafu;
-use table::metadata::{TableId, TableInfoRef};
-use table::table::scan::SimpleTableScan;
+use table::metadata::TableId;
 use table::{Table, TableRef};
 
 use crate::error::{self, Error, InsertCatalogRecordSnafu, Result as CatalogResult};
@@ -44,160 +29,9 @@ use crate::system::{
     build_schema_insert_request, build_table_deletion_request, build_table_insert_request,
     SystemCatalogTable,
 };
-use crate::{
-    CatalogListRef, CatalogProvider, DeregisterTableRequest, SchemaProvider, SchemaProviderRef,
-};
-
-/// Tables holds all tables created by user.
-pub struct Tables {
-    schema: SchemaRef,
-    catalogs: CatalogListRef,
-    engine_name: String,
-}
-
-impl Tables {
-    pub fn new(catalogs: CatalogListRef, engine_name: String) -> Self {
-        Self {
-            schema: Arc::new(build_schema_for_tables()),
-            catalogs,
-            engine_name,
-        }
-    }
-}
-
-#[async_trait::async_trait]
-impl Table for Tables {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn table_info(&self) -> TableInfoRef {
-        unreachable!("Tables does not support table_info method")
-    }
-
-    async fn scan(
-        &self,
-        _projection: Option<&Vec<usize>>,
-        _filters: &[Expr],
-        _limit: Option<usize>,
-    ) -> table::error::Result<PhysicalPlanRef> {
-        let catalogs = self.catalogs.clone();
-        let schema_ref = self.schema.clone();
-        let engine_name = self.engine_name.clone();
-
-        let stream = stream!({
-            for catalog_name in catalogs
-                .catalog_names()
-                .map_err(BoxedError::new)
-                .context(TablesRecordBatchSnafu)?
-            {
-                let catalog = catalogs
-                    .catalog(&catalog_name)
-                    .map_err(BoxedError::new)
-                    .context(TablesRecordBatchSnafu)?
-                    .unwrap();
-                for schema_name in catalog
-                    .schema_names()
-                    .map_err(BoxedError::new)
-                    .context(TablesRecordBatchSnafu)?
-                {
-                    let mut tables_in_schema = Vec::with_capacity(
-                        catalog
-                            .schema_names()
-                            .map_err(BoxedError::new)
-                            .context(TablesRecordBatchSnafu)?
-                            .len(),
-                    );
-                    let schema = catalog
-                        .schema(&schema_name)
-                        .map_err(BoxedError::new)
-                        .context(TablesRecordBatchSnafu)?
-                        .unwrap();
-                    for table_name in schema
-                        .table_names()
-                        .map_err(BoxedError::new)
-                        .context(TablesRecordBatchSnafu)?
-                    {
-                        tables_in_schema.push(table_name);
-                    }
-
-                    let vec = tables_to_record_batch(
-                        &catalog_name,
-                        &schema_name,
-                        tables_in_schema,
-                        &engine_name,
-                    );
-                    let record_batch_res = RecordBatch::new(schema_ref.clone(), vec);
-                    yield record_batch_res;
-                }
-            }
-        });
-
-        let stream = Box::pin(TablesRecordBatchStream {
-            schema: self.schema.clone(),
-            stream: Box::pin(stream),
-        });
-        Ok(Arc::new(SimpleTableScan::new(stream)))
-    }
-}
-
-/// Convert tables info to `RecordBatch`.
-fn tables_to_record_batch(
-    catalog_name: &str,
-    schema_name: &str,
-    table_names: Vec<String>,
-    engine: &str,
-) -> Vec<VectorRef> {
-    let mut catalog_vec =
-        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
-    let mut schema_vec =
-        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
-    let mut table_name_vec =
-        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
-    let mut engine_vec =
-        ConcreteDataType::string_datatype().create_mutable_vector(table_names.len());
-
-    for table_name in table_names {
-        // Safety: All these vectors are string type.
-        catalog_vec.push_value_ref(ValueRef::String(catalog_name));
-        schema_vec.push_value_ref(ValueRef::String(schema_name));
-        table_name_vec.push_value_ref(ValueRef::String(&table_name));
-        engine_vec.push_value_ref(ValueRef::String(engine));
-    }
-
-    vec![
-        catalog_vec.to_vector(),
-        schema_vec.to_vector(),
-        table_name_vec.to_vector(),
-        engine_vec.to_vector(),
-    ]
-}
-
-pub struct TablesRecordBatchStream {
-    schema: SchemaRef,
-    stream: Pin<Box<dyn Stream<Item = RecordBatchResult<RecordBatch>> + Send>>,
-}
-
-impl Stream for TablesRecordBatchStream {
-    type Item = RecordBatchResult<RecordBatch>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        Pin::new(&mut self.stream).poll_next(cx)
-    }
-}
-
-impl RecordBatchStream for TablesRecordBatchStream {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-}
+use crate::{CatalogProvider, DeregisterTableRequest, SchemaProvider, SchemaProviderRef};
 
 pub struct InformationSchema {
-    pub tables: Arc<Tables>,
     pub system: Arc<SystemCatalogTable>,
 }
 
@@ -207,42 +41,20 @@ impl SchemaProvider for InformationSchema {
         self
     }
 
-    fn table_names(&self) -> Result<Vec<String>, Error> {
-        Ok(vec![
-            "tables".to_string(),
-            SYSTEM_CATALOG_TABLE_NAME.to_string(),
-        ])
+    async fn table_names(&self) -> Result<Vec<String>, Error> {
+        Ok(vec![SYSTEM_CATALOG_TABLE_NAME.to_string()])
     }
 
     async fn table(&self, name: &str) -> Result<Option<TableRef>, Error> {
-        if name.eq_ignore_ascii_case("tables") {
-            Ok(Some(self.tables.clone()))
-        } else if name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME) {
+        if name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME) {
             Ok(Some(self.system.clone()))
         } else {
             Ok(None)
         }
     }
 
-    fn register_table(
-        &self,
-        _name: String,
-        _table: TableRef,
-    ) -> crate::error::Result<Option<TableRef>> {
-        panic!("System catalog & schema does not support register table")
-    }
-
-    fn rename_table(&self, _name: &str, _new_name: String) -> crate::error::Result<TableRef> {
-        unimplemented!("System catalog & schema does not support rename table")
-    }
-
-    fn deregister_table(&self, _name: &str) -> crate::error::Result<Option<TableRef>> {
-        panic!("System catalog & schema does not support deregister table")
-    }
-
-    fn table_exist(&self, name: &str) -> Result<bool, Error> {
-        Ok(name.eq_ignore_ascii_case("tables")
-            || name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME))
+    async fn table_exist(&self, name: &str) -> Result<bool, Error> {
+        Ok(name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME))
     }
 }
 
@@ -251,13 +63,8 @@ pub struct SystemCatalog {
 }
 
 impl SystemCatalog {
-    pub fn new(
-        system: SystemCatalogTable,
-        catalogs: CatalogListRef,
-        engine: TableEngineRef,
-    ) -> Self {
+    pub(crate) fn new(system: SystemCatalogTable) -> Self {
         let schema = InformationSchema {
-            tables: Arc::new(Tables::new(catalogs, engine.name().to_string())),
             system: Arc::new(system),
         };
         Self {
@@ -271,8 +78,9 @@ impl SystemCatalog {
         schema: String,
         table_name: String,
         table_id: TableId,
+        engine: String,
     ) -> crate::error::Result<usize> {
-        let request = build_table_insert_request(catalog, schema, table_name, table_id);
+        let request = build_table_insert_request(catalog, schema, table_name, table_id, engine);
         self.information_schema
             .system
             .insert(request)
@@ -284,12 +92,21 @@ impl SystemCatalog {
         &self,
         request: &DeregisterTableRequest,
         table_id: TableId,
-    ) -> CatalogResult<bool> {
+    ) -> CatalogResult<()> {
         self.information_schema
             .system
             .delete(build_table_deletion_request(request, table_id))
             .await
-            .map(|x| x == 1)
+            .map(|x| {
+                if x != 1 {
+                    let table = common_catalog::format_full_table_name(
+                        &request.catalog,
+                        &request.schema,
+                        &request.table_name
+                    );
+                    logging::warn!("Failed to delete table record from information_schema, unexpected returned result: {x}, table: {table}");
+                }
+            })
             .with_context(|_| error::DeregisterTableSnafu {
                 request: request.clone(),
             })
@@ -309,16 +126,17 @@ impl SystemCatalog {
     }
 }
 
+#[async_trait::async_trait]
 impl CatalogProvider for SystemCatalog {
     fn as_any(&self) -> &dyn Any {
         self
     }
 
-    fn schema_names(&self) -> Result<Vec<String>, Error> {
+    async fn schema_names(&self) -> Result<Vec<String>, Error> {
         Ok(vec![INFORMATION_SCHEMA_NAME.to_string()])
     }
 
-    fn register_schema(
+    async fn register_schema(
         &self,
         _name: String,
         _schema: SchemaProviderRef,
@@ -326,7 +144,7 @@ impl CatalogProvider for SystemCatalog {
         panic!("System catalog does not support registering schema!")
     }
 
-    fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>, Error> {
+    async fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>, Error> {
         if name.eq_ignore_ascii_case(INFORMATION_SCHEMA_NAME) {
             Ok(Some(self.information_schema.clone()))
         } else {
@@ -334,104 +152,3 @@ impl CatalogProvider for SystemCatalog {
         }
     }
 }
-
-fn build_schema_for_tables() -> Schema {
-    let cols = vec![
-        ColumnSchema::new(
-            "catalog".to_string(),
-            ConcreteDataType::string_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "schema".to_string(),
-            ConcreteDataType::string_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "table_name".to_string(),
-            ConcreteDataType::string_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "engine".to_string(),
-            ConcreteDataType::string_datatype(),
-            false,
-        ),
-    ];
-    Schema::new(cols)
-}
-
-#[cfg(test)]
-mod tests {
-    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-    use common_query::physical_plan::SessionContext;
-    use futures_util::StreamExt;
-    use table::table::numbers::NumbersTable;
-
-    use super::*;
-    use crate::local::memory::new_memory_catalog_list;
-    use crate::CatalogList;
-
-    #[tokio::test]
-    async fn test_tables() {
-        let catalog_list = new_memory_catalog_list().unwrap();
-        let schema = catalog_list
-            .catalog(DEFAULT_CATALOG_NAME)
-            .unwrap()
-            .unwrap()
-            .schema(DEFAULT_SCHEMA_NAME)
-            .unwrap()
-            .unwrap();
-        schema
-            .register_table("test_table".to_string(), Arc::new(NumbersTable::default()))
-            .unwrap();
-
-        let tables = Tables::new(catalog_list, "test_engine".to_string());
-        let tables_stream = tables.scan(None, &[], None).await.unwrap();
-        let session_ctx = SessionContext::new();
-        let mut tables_stream = tables_stream.execute(0, session_ctx.task_ctx()).unwrap();
-
-        if let Some(t) = tables_stream.next().await {
-            let batch = t.unwrap();
-            assert_eq!(1, batch.num_rows());
-            assert_eq!(4, batch.num_columns());
-            assert_eq!(
-                ConcreteDataType::string_datatype(),
-                batch.column(0).data_type()
-            );
-            assert_eq!(
-                ConcreteDataType::string_datatype(),
-                batch.column(1).data_type()
-            );
-            assert_eq!(
-                ConcreteDataType::string_datatype(),
-                batch.column(2).data_type()
-            );
-            assert_eq!(
-                ConcreteDataType::string_datatype(),
-                batch.column(3).data_type()
-            );
-            assert_eq!(
-                "greptime",
-                batch.column(0).get_ref(0).as_string().unwrap().unwrap()
-            );
-
-            assert_eq!(
-                "public",
-                batch.column(1).get_ref(0).as_string().unwrap().unwrap()
-            );
-
-            assert_eq!(
-                "test_table",
-                batch.column(2).get_ref(0).as_string().unwrap().unwrap()
-            );
-
-            assert_eq!(
-                "test_engine",
-                batch.column(3).get_ref(0).as_string().unwrap().unwrap()
-            );
-        } else {
-            panic!("Record batch should not be empty!")
-        }
-    }
-}

src/catalog/tests/local_catalog_tests.rs

@@ -20,28 +20,32 @@ mod tests {
     use catalog::{CatalogManager, RegisterTableRequest, RenameTableRequest};
     use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
     use common_telemetry::{error, info};
+    use common_test_util::temp_dir::TempDir;
     use mito::config::EngineConfig;
+    use table::engine::manager::MemoryTableEngineManager;
     use table::table::numbers::NumbersTable;
     use table::TableRef;
     use tokio::sync::Mutex;
 
-    async fn create_local_catalog_manager() -> Result<LocalCatalogManager, catalog::error::Error> {
-        let (_dir, object_store) =
+    async fn create_local_catalog_manager(
+    ) -> Result<(TempDir, LocalCatalogManager), catalog::error::Error> {
+        let (dir, object_store) =
             mito::table::test_util::new_test_object_store("setup_mock_engine_and_table").await;
         let mock_engine = Arc::new(mito::table::test_util::MockMitoEngine::new(
             EngineConfig::default(),
             mito::table::test_util::MockEngine::default(),
             object_store,
         ));
-        let catalog_manager = LocalCatalogManager::try_new(mock_engine).await.unwrap();
+        let engine_manager = Arc::new(MemoryTableEngineManager::new(mock_engine.clone()));
+        let catalog_manager = LocalCatalogManager::try_new(engine_manager).await.unwrap();
         catalog_manager.start().await?;
-        Ok(catalog_manager)
+        Ok((dir, catalog_manager))
     }
 
     #[tokio::test]
     async fn test_rename_table() {
         common_telemetry::init_default_ut_logging();
-        let catalog_manager = create_local_catalog_manager().await.unwrap();
+        let (_dir, catalog_manager) = create_local_catalog_manager().await.unwrap();
         // register table
         let table_name = "test_table";
         let table_id = 42;
@@ -79,7 +83,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_duplicate_register() {
-        let catalog_manager = create_local_catalog_manager().await.unwrap();
+        let (_dir, catalog_manager) = create_local_catalog_manager().await.unwrap();
         let request = RegisterTableRequest {
             catalog: DEFAULT_CATALOG_NAME.to_string(),
             schema: DEFAULT_SCHEMA_NAME.to_string(),
@@ -116,8 +120,9 @@ mod tests {
     fn test_concurrent_register() {
         common_telemetry::init_default_ut_logging();
         let rt = Arc::new(tokio::runtime::Builder::new_multi_thread().build().unwrap());
-        let catalog_manager =
-            Arc::new(rt.block_on(async { create_local_catalog_manager().await.unwrap() }));
+        let (_dir, catalog_manager) =
+            rt.block_on(async { create_local_catalog_manager().await.unwrap() });
+        let catalog_manager = Arc::new(catalog_manager);
 
         let succeed: Arc<Mutex<Option<TableRef>>> = Arc::new(Mutex::new(None));
 

src/catalog/tests/remote_catalog_tests.rs

@@ -14,26 +14,43 @@
 
 #![feature(assert_matches)]
 
-mod mock;
-
 #[cfg(test)]
 mod tests {
     use std::assert_matches::assert_matches;
     use std::collections::HashSet;
     use std::sync::Arc;
+    use std::time::Duration;
 
     use catalog::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
+    use catalog::remote::mock::{MockKvBackend, MockTableEngine};
+    use catalog::remote::region_alive_keeper::RegionAliveKeepers;
     use catalog::remote::{
-        KvBackend, KvBackendRef, RemoteCatalogManager, RemoteCatalogProvider, RemoteSchemaProvider,
+        CachedMetaKvBackend, KvBackend, KvBackendRef, RemoteCatalogManager, RemoteCatalogProvider,
+        RemoteSchemaProvider,
     };
-    use catalog::{CatalogList, CatalogManager, RegisterTableRequest};
-    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use catalog::{CatalogManager, RegisterTableRequest};
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
+    use common_meta::ident::TableIdent;
     use datatypes::schema::RawSchema;
     use futures_util::StreamExt;
+    use table::engine::manager::{MemoryTableEngineManager, TableEngineManagerRef};
     use table::engine::{EngineContext, TableEngineRef};
     use table::requests::CreateTableRequest;
+    use table::test_util::EmptyTable;
+    use tokio::time::Instant;
 
-    use crate::mock::{MockKvBackend, MockTableEngine};
+    struct TestingComponents {
+        kv_backend: KvBackendRef,
+        catalog_manager: Arc<RemoteCatalogManager>,
+        table_engine_manager: TableEngineManagerRef,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
+    }
+
+    impl TestingComponents {
+        fn table_engine(&self) -> TableEngineRef {
+            self.table_engine_manager.engine(MITO_ENGINE).unwrap()
+        }
+    }
 
     #[tokio::test]
     async fn test_backend() {
@@ -75,34 +92,101 @@ mod tests {
         );
     }
 
-    async fn prepare_components(
-        node_id: u64,
-    ) -> (KvBackendRef, TableEngineRef, Arc<RemoteCatalogManager>) {
-        let backend = Arc::new(MockKvBackend::default()) as KvBackendRef;
+    #[tokio::test]
+    async fn test_cached_backend() {
+        common_telemetry::init_default_ut_logging();
+        let backend = CachedMetaKvBackend::wrap(Arc::new(MockKvBackend::default()));
+
+        let default_catalog_key = CatalogKey {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        }
+        .to_string();
+
+        backend
+            .set(
+                default_catalog_key.as_bytes(),
+                &CatalogValue {}.as_bytes().unwrap(),
+            )
+            .await
+            .unwrap();
+
+        let ret = backend.get(b"__c-greptime").await.unwrap();
+        assert!(ret.is_some());
+
+        let _ = backend
+            .compare_and_set(
+                b"__c-greptime",
+                &CatalogValue {}.as_bytes().unwrap(),
+                b"123",
+            )
+            .await
+            .unwrap();
+
+        let ret = backend.get(b"__c-greptime").await.unwrap();
+        assert!(ret.is_some());
+        assert_eq!(&b"123"[..], &(ret.as_ref().unwrap().1));
+
+        let _ = backend.set(b"__c-greptime", b"1234").await;
+
+        let ret = backend.get(b"__c-greptime").await.unwrap();
+        assert!(ret.is_some());
+        assert_eq!(&b"1234"[..], &(ret.as_ref().unwrap().1));
+
+        backend.delete(b"__c-greptime").await.unwrap();
+
+        let ret = backend.get(b"__c-greptime").await.unwrap();
+        assert!(ret.is_none());
+    }
+
+    async fn prepare_components(node_id: u64) -> TestingComponents {
+        let cached_backend = Arc::new(CachedMetaKvBackend::wrap(
+            Arc::new(MockKvBackend::default()),
+        ));
+
         let table_engine = Arc::new(MockTableEngine::default());
-        let catalog_manager =
-            RemoteCatalogManager::new(table_engine.clone(), node_id, backend.clone());
+        let engine_manager = Arc::new(MemoryTableEngineManager::alias(
+            MITO_ENGINE.to_string(),
+            table_engine,
+        ));
+
+        let region_alive_keepers = Arc::new(RegionAliveKeepers::new(engine_manager.clone(), 5000));
+
+        let catalog_manager = RemoteCatalogManager::new(
+            engine_manager.clone(),
+            node_id,
+            cached_backend.clone(),
+            region_alive_keepers.clone(),
+        );
         catalog_manager.start().await.unwrap();
-        (backend, table_engine, Arc::new(catalog_manager))
+
+        TestingComponents {
+            kv_backend: cached_backend,
+            catalog_manager: Arc::new(catalog_manager),
+            table_engine_manager: engine_manager,
+            region_alive_keepers,
+        }
     }
 
     #[tokio::test]
     async fn test_remote_catalog_default() {
         common_telemetry::init_default_ut_logging();
         let node_id = 42;
-        let (_, _, catalog_manager) = prepare_components(node_id).await;
+        let TestingComponents {
+            catalog_manager, ..
+        } = prepare_components(node_id).await;
         assert_eq!(
             vec![DEFAULT_CATALOG_NAME.to_string()],
-            catalog_manager.catalog_names().unwrap()
+            catalog_manager.catalog_names().await.unwrap()
         );
 
         let default_catalog = catalog_manager
             .catalog(DEFAULT_CATALOG_NAME)
+            .await
             .unwrap()
             .unwrap();
         assert_eq!(
             vec![DEFAULT_SCHEMA_NAME.to_string()],
-            default_catalog.schema_names().unwrap()
+            default_catalog.schema_names().await.unwrap()
         );
     }
 
@@ -110,14 +194,16 @@ mod tests {
     async fn test_remote_catalog_register_nonexistent() {
         common_telemetry::init_default_ut_logging();
         let node_id = 42;
-        let (_, table_engine, catalog_manager) = prepare_components(node_id).await;
+        let components = prepare_components(node_id).await;
+
         // register a new table with an nonexistent catalog
         let catalog_name = "nonexistent_catalog".to_string();
         let schema_name = "nonexistent_schema".to_string();
         let table_name = "fail_table".to_string();
         // this schema has no effect
         let table_schema = RawSchema::new(vec![]);
-        let table = table_engine
+        let table = components
+            .table_engine()
             .create_table(
                 &EngineContext {},
                 CreateTableRequest {
@@ -131,6 +217,7 @@ mod tests {
                     primary_key_indices: vec![],
                     create_if_not_exists: false,
                     table_options: Default::default(),
+                    engine: MITO_ENGINE.to_string(),
                 },
             )
             .await
@@ -142,7 +229,7 @@ mod tests {
             table_id: 1,
             table,
         };
-        let res = catalog_manager.register_table(reg_req).await;
+        let res = components.catalog_manager.register_table(reg_req).await;
 
         // because nonexistent_catalog does not exist yet.
         assert_matches!(
@@ -154,21 +241,23 @@ mod tests {
     #[tokio::test]
     async fn test_register_table() {
         let node_id = 42;
-        let (_, table_engine, catalog_manager) = prepare_components(node_id).await;
+        let components = prepare_components(node_id).await;
+        let catalog_manager = &components.catalog_manager;
         let default_catalog = catalog_manager
             .catalog(DEFAULT_CATALOG_NAME)
+            .await
             .unwrap()
             .unwrap();
         assert_eq!(
             vec![DEFAULT_SCHEMA_NAME.to_string()],
-            default_catalog.schema_names().unwrap()
+            default_catalog.schema_names().await.unwrap()
         );
 
         let default_schema = default_catalog
             .schema(DEFAULT_SCHEMA_NAME)
+            .await
             .unwrap()
             .unwrap();
-        assert_eq!(vec!["numbers"], default_schema.table_names().unwrap());
 
         // register a new table with an nonexistent catalog
         let catalog_name = DEFAULT_CATALOG_NAME.to_string();
@@ -177,7 +266,8 @@ mod tests {
         let table_id = 1;
         // this schema has no effect
         let table_schema = RawSchema::new(vec![]);
-        let table = table_engine
+        let table = components
+            .table_engine()
             .create_table(
                 &EngineContext {},
                 CreateTableRequest {
@@ -191,6 +281,7 @@ mod tests {
                     primary_key_indices: vec![],
                     create_if_not_exists: false,
                     table_options: Default::default(),
+                    engine: MITO_ENGINE.to_string(),
                 },
             )
             .await
@@ -204,40 +295,42 @@ mod tests {
         };
         assert!(catalog_manager.register_table(reg_req).await.unwrap());
         assert_eq!(
-            HashSet::from([table_name, "numbers".to_string()]),
-            default_schema
-                .table_names()
-                .unwrap()
-                .into_iter()
-                .collect::<HashSet<_>>()
+            vec![table_name],
+            default_schema.table_names().await.unwrap()
         );
     }
 
     #[tokio::test]
     async fn test_register_catalog_schema_table() {
         let node_id = 42;
-        let (backend, table_engine, catalog_manager) = prepare_components(node_id).await;
+        let components = prepare_components(node_id).await;
+        let backend = &components.kv_backend;
+        let catalog_manager = components.catalog_manager.clone();
+        let engine_manager = components.table_engine_manager.clone();
 
         let catalog_name = "test_catalog".to_string();
         let schema_name = "nonexistent_schema".to_string();
         let catalog = Arc::new(RemoteCatalogProvider::new(
             catalog_name.clone(),
             backend.clone(),
+            engine_manager.clone(),
             node_id,
+            components.region_alive_keepers.clone(),
         ));
 
         // register catalog to catalog manager
-        catalog_manager
-            .register_catalog(catalog_name.clone(), catalog)
+        CatalogManager::register_catalog(&*catalog_manager, catalog_name.clone(), catalog)
+            .await
             .unwrap();
         assert_eq!(
             HashSet::<String>::from_iter(
                 vec![DEFAULT_CATALOG_NAME.to_string(), catalog_name.clone()].into_iter()
             ),
-            HashSet::from_iter(catalog_manager.catalog_names().unwrap().into_iter())
+            HashSet::from_iter(catalog_manager.catalog_names().await.unwrap().into_iter())
         );
 
-        let table_to_register = table_engine
+        let table_to_register = components
+            .table_engine()
             .create_table(
                 &EngineContext {},
                 CreateTableRequest {
@@ -251,6 +344,7 @@ mod tests {
                     primary_key_indices: vec![],
                     create_if_not_exists: false,
                     table_options: Default::default(),
+                    engine: MITO_ENGINE.to_string(),
                 },
             )
             .await
@@ -274,24 +368,123 @@ mod tests {
 
         let new_catalog = catalog_manager
             .catalog(&catalog_name)
+            .await
             .unwrap()
             .expect("catalog should exist since it's already registered");
         let schema = Arc::new(RemoteSchemaProvider::new(
             catalog_name.clone(),
             schema_name.clone(),
             node_id,
+            engine_manager,
             backend.clone(),
+            components.region_alive_keepers.clone(),
         ));
 
         let prev = new_catalog
             .register_schema(schema_name.clone(), schema.clone())
+            .await
             .expect("Register schema should not fail");
         assert!(prev.is_none());
         assert!(catalog_manager.register_table(reg_req).await.unwrap());
 
         assert_eq!(
             HashSet::from([schema_name.clone()]),
-            new_catalog.schema_names().unwrap().into_iter().collect()
+            new_catalog
+                .schema_names()
+                .await
+                .unwrap()
+                .into_iter()
+                .collect()
         )
     }
+
+    #[tokio::test]
+    async fn test_register_table_before_and_after_region_alive_keeper_started() {
+        let components = prepare_components(42).await;
+        let catalog_manager = &components.catalog_manager;
+        let region_alive_keepers = &components.region_alive_keepers;
+
+        let table_before = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "table_before".to_string(),
+            table_id: 1,
+            engine: MITO_ENGINE.to_string(),
+        };
+        let request = RegisterTableRequest {
+            catalog: table_before.catalog.clone(),
+            schema: table_before.schema.clone(),
+            table_name: table_before.table.clone(),
+            table_id: table_before.table_id,
+            table: Arc::new(EmptyTable::new(CreateTableRequest {
+                id: table_before.table_id,
+                catalog_name: table_before.catalog.clone(),
+                schema_name: table_before.schema.clone(),
+                table_name: table_before.table.clone(),
+                desc: None,
+                schema: RawSchema::new(vec![]),
+                region_numbers: vec![0],
+                primary_key_indices: vec![],
+                create_if_not_exists: false,
+                table_options: Default::default(),
+                engine: MITO_ENGINE.to_string(),
+            })),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        let keeper = region_alive_keepers
+            .find_keeper(&table_before)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
+        // assert region alive countdown is not started
+        assert!(deadline > far_future);
+
+        region_alive_keepers.start().await;
+
+        let table_after = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "table_after".to_string(),
+            table_id: 2,
+            engine: MITO_ENGINE.to_string(),
+        };
+        let request = RegisterTableRequest {
+            catalog: table_after.catalog.clone(),
+            schema: table_after.schema.clone(),
+            table_name: table_after.table.clone(),
+            table_id: table_after.table_id,
+            table: Arc::new(EmptyTable::new(CreateTableRequest {
+                id: table_after.table_id,
+                catalog_name: table_after.catalog.clone(),
+                schema_name: table_after.schema.clone(),
+                table_name: table_after.table.clone(),
+                desc: None,
+                schema: RawSchema::new(vec![]),
+                region_numbers: vec![0],
+                primary_key_indices: vec![],
+                create_if_not_exists: false,
+                table_options: Default::default(),
+                engine: MITO_ENGINE.to_string(),
+            })),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        let keeper = region_alive_keepers
+            .find_keeper(&table_after)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        // assert countdown is started for the table registered after [RegionAliveKeepers] started
+        assert!(deadline <= Instant::now() + Duration::from_secs(20));
+
+        let keeper = region_alive_keepers
+            .find_keeper(&table_before)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        // assert countdown is started for the table registered before [RegionAliveKeepers] started, too
+        assert!(deadline <= Instant::now() + Duration::from_secs(20));
+    }
 }

src/client/Cargo.toml

@@ -4,6 +4,9 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
+[features]
+testing = []
+
 [dependencies]
 api = { path = "../api" }
 arrow-flight.workspace = true
@@ -16,25 +19,28 @@ common-grpc-expr = { path = "../common/grpc-expr" }
 common-query = { path = "../common/query" }
 common-recordbatch = { path = "../common/recordbatch" }
 common-time = { path = "../common/time" }
+common-meta = { path = "../common/meta" }
 common-telemetry = { path = "../common/telemetry" }
 datafusion.workspace = true
 datatypes = { path = "../datatypes" }
 enum_dispatch = "0.3"
 futures-util.workspace = true
+moka = { version = "0.9", features = ["future"] }
 parking_lot = "0.12"
 prost.workspace = true
-rand = "0.8"
+rand.workspace = true
 snafu.workspace = true
+tokio-stream = { version = "0.1", features = ["net"] }
+tokio.workspace = true
 tonic.workspace = true
 
 [dev-dependencies]
 datanode = { path = "../datanode" }
 substrait = { path = "../common/substrait" }
-tokio.workspace = true
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 prost.workspace = true
 
 [dev-dependencies.substrait_proto]
 package = "substrait"
-version = "0.4"
+version = "0.7"

src/client/examples/logical.rs

@@ -14,7 +14,7 @@
 
 use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, TableId};
 use client::{Client, Database};
-use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
 use prost::Message;
 use substrait_proto::proto::plan_rel::RelType as PlanRelType;
 use substrait_proto::proto::read_rel::{NamedTable, ReadType};
@@ -63,7 +63,8 @@ async fn run() {
         create_if_not_exists: false,
         table_options: Default::default(),
         table_id: Some(TableId { id: 1024 }),
-        region_ids: vec![0],
+        region_numbers: vec![0],
+        engine: MITO_ENGINE.to_string(),
     };
 
     let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);

src/client/src/client.rs

@@ -14,6 +14,10 @@
 
 use std::sync::Arc;
 
+use api::v1::greptime_database_client::GreptimeDatabaseClient;
+use api::v1::health_check_client::HealthCheckClient;
+use api::v1::prometheus_gateway_client::PrometheusGatewayClient;
+use api::v1::HealthCheckRequest;
 use arrow_flight::flight_service_client::FlightServiceClient;
 use common_grpc::channel_manager::ChannelManager;
 use parking_lot::RwLock;
@@ -23,6 +27,10 @@ use tonic::transport::Channel;
 use crate::load_balance::{LoadBalance, Loadbalancer};
 use crate::{error, Result};
 
+pub(crate) struct DatabaseClient {
+    pub(crate) inner: GreptimeDatabaseClient<Channel>,
+}
+
 pub(crate) struct FlightClient {
     addr: String,
     client: FlightServiceClient<Channel>,
@@ -118,7 +126,7 @@ impl Client {
         self.inner.set_peers(urls);
     }
 
-    pub(crate) fn make_client(&self) -> Result<FlightClient> {
+    fn find_channel(&self) -> Result<(String, Channel)> {
         let addr = self
             .inner
             .get_peer()
@@ -131,11 +139,35 @@ impl Client {
             .channel_manager
             .get(&addr)
             .context(error::CreateChannelSnafu { addr: &addr })?;
+        Ok((addr, channel))
+    }
+
+    pub(crate) fn make_flight_client(&self) -> Result<FlightClient> {
+        let (addr, channel) = self.find_channel()?;
         Ok(FlightClient {
             addr,
             client: FlightServiceClient::new(channel),
         })
     }
+
+    pub(crate) fn make_database_client(&self) -> Result<DatabaseClient> {
+        let (_, channel) = self.find_channel()?;
+        Ok(DatabaseClient {
+            inner: GreptimeDatabaseClient::new(channel),
+        })
+    }
+
+    pub fn make_prometheus_gateway_client(&self) -> Result<PrometheusGatewayClient<Channel>> {
+        let (_, channel) = self.find_channel()?;
+        Ok(PrometheusGatewayClient::new(channel))
+    }
+
+    pub async fn health_check(&self) -> Result<()> {
+        let (_, channel) = self.find_channel()?;
+        let mut client = HealthCheckClient::new(channel);
+        client.health_check(HealthCheckRequest {}).await?;
+        Ok(())
+    }
 }
 
 #[cfg(test)]

src/client/src/client_manager.rs

@@ -12,32 +12,60 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt::{Debug, Formatter};
+use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use client::Client;
-use common_grpc::channel_manager::ChannelManager;
-use meta_client::rpc::Peer;
+use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
+use common_meta::peer::Peer;
+use common_telemetry::info;
 use moka::future::{Cache, CacheBuilder};
 
+use crate::Client;
+
 pub struct DatanodeClients {
     channel_manager: ChannelManager,
     clients: Cache<Peer, Client>,
+    started: Arc<Mutex<bool>>,
 }
 
 impl Default for DatanodeClients {
     fn default() -> Self {
+        let config = ChannelConfig::new().timeout(Duration::from_secs(8));
+
         Self {
-            channel_manager: ChannelManager::new(),
+            channel_manager: ChannelManager::with_config(config),
             clients: CacheBuilder::new(1024)
                 .time_to_live(Duration::from_secs(30 * 60))
                 .time_to_idle(Duration::from_secs(5 * 60))
                 .build(),
+            started: Arc::new(Mutex::new(false)),
         }
     }
 }
 
+impl Debug for DatanodeClients {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DatanodeClients")
+            .field("channel_manager", &self.channel_manager)
+            .finish()
+    }
+}
+
 impl DatanodeClients {
-    pub(crate) async fn get_client(&self, datanode: &Peer) -> Client {
+    pub fn start(&self) {
+        let mut started = self.started.lock().unwrap();
+        if *started {
+            return;
+        }
+
+        self.channel_manager.start_channel_recycle();
+
+        info!("Datanode clients manager is started!");
+        *started = true;
+    }
+
+    pub async fn get_client(&self, datanode: &Peer) -> Client {
         self.clients
             .get_with_by_ref(datanode, async move {
                 Client::with_manager_and_urls(
@@ -48,8 +76,8 @@ impl DatanodeClients {
             .await
     }
 
-    #[cfg(test)]
-    pub(crate) async fn insert_client(&self, datanode: Peer, client: Client) {
+    #[cfg(feature = "testing")]
+    pub async fn insert_client(&self, datanode: Peer, client: Client) {
         self.clients.insert(datanode, client).await
     }
 }

src/client/src/database.rs

@@ -12,46 +12,75 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::str::FromStr;
-
 use api::v1::auth_header::AuthScheme;
 use api::v1::ddl_request::Expr as DdlExpr;
 use api::v1::greptime_request::Request;
 use api::v1::query_request::Query;
 use api::v1::{
-    AlterExpr, AuthHeader, CreateTableExpr, DdlRequest, DropTableExpr, FlushTableExpr,
-    GreptimeRequest, InsertRequest, PromRangeQuery, QueryRequest, RequestHeader,
+    greptime_response, AffectedRows, AlterExpr, AuthHeader, CreateTableExpr, DdlRequest,
+    DeleteRequest, DropTableExpr, FlushTableExpr, GreptimeRequest, InsertRequests, PromRangeQuery,
+    QueryRequest, RequestHeader,
 };
 use arrow_flight::{FlightData, Ticket};
 use common_error::prelude::*;
 use common_grpc::flight::{flight_messages_to_recordbatches, FlightDecoder, FlightMessage};
 use common_query::Output;
-use common_telemetry::logging;
+use common_telemetry::{logging, timer};
 use futures_util::{TryFutureExt, TryStreamExt};
 use prost::Message;
 use snafu::{ensure, ResultExt};
+use tokio::sync::mpsc::Sender;
+use tokio::sync::{mpsc, OnceCell};
+use tokio_stream::wrappers::ReceiverStream;
 
-use crate::error::{ConvertFlightDataSnafu, IllegalFlightMessagesSnafu};
-use crate::{error, Client, Result};
+use crate::error::{
+    ConvertFlightDataSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
+};
+use crate::{error, metrics, Client, Result};
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default)]
 pub struct Database {
     // The "catalog" and "schema" to be used in processing the requests at the server side.
     // They are the "hint" or "context", just like how the "database" in "USE" statement is treated in MySQL.
     // They will be carried in the request header.
     catalog: String,
     schema: String,
+    // The dbname follows naming rule as out mysql, postgres and http
+    // protocol. The server treat dbname in priority of catalog/schema.
+    dbname: String,
 
     client: Client,
+    streaming_client: OnceCell<Sender<GreptimeRequest>>,
     ctx: FlightContext,
 }
 
 impl Database {
+    /// Create database service client using catalog and schema
     pub fn new(catalog: impl Into<String>, schema: impl Into<String>, client: Client) -> Self {
         Self {
             catalog: catalog.into(),
             schema: schema.into(),
+            dbname: "".to_string(),
             client,
+            streaming_client: OnceCell::new(),
+            ctx: FlightContext::default(),
+        }
+    }
+
+    /// Create database service client using dbname.
+    ///
+    /// This API is designed for external usage. `dbname` is:
+    ///
+    /// - the name of database when using GreptimeDB standalone or cluster
+    /// - the name provided by GreptimeCloud or other multi-tenant GreptimeDB
+    /// environment
+    pub fn new_with_dbname(dbname: impl Into<String>, client: Client) -> Self {
+        Self {
+            catalog: "".to_string(),
+            schema: "".to_string(),
+            dbname: dbname.into(),
+            client,
+            streaming_client: OnceCell::new(),
             ctx: FlightContext::default(),
         }
     }
@@ -72,17 +101,84 @@ impl Database {
         self.schema = schema.into();
     }
 
+    pub fn dbname(&self) -> &String {
+        &self.dbname
+    }
+
+    pub fn set_dbname(&mut self, dbname: impl Into<String>) {
+        self.dbname = dbname.into();
+    }
+
     pub fn set_auth(&mut self, auth: AuthScheme) {
         self.ctx.auth_header = Some(AuthHeader {
             auth_scheme: Some(auth),
         });
     }
 
-    pub async fn insert(&self, request: InsertRequest) -> Result<Output> {
-        self.do_get(Request::Insert(request)).await
+    pub async fn insert(&self, requests: InsertRequests) -> Result<u32> {
+        let _timer = timer!(metrics::METRIC_GRPC_INSERT);
+        self.handle(Request::Inserts(requests)).await
+    }
+
+    pub async fn insert_to_stream(&self, requests: InsertRequests) -> Result<()> {
+        let streaming_client = self
+            .streaming_client
+            .get_or_try_init(|| self.client_stream())
+            .await?;
+
+        let request = self.to_rpc_request(Request::Inserts(requests));
+
+        streaming_client.send(request).await.map_err(|e| {
+            error::ClientStreamingSnafu {
+                err_msg: e.to_string(),
+            }
+            .build()
+        })
+    }
+
+    pub async fn delete(&self, request: DeleteRequest) -> Result<u32> {
+        let _timer = timer!(metrics::METRIC_GRPC_DELETE);
+        self.handle(Request::Delete(request)).await
+    }
+
+    async fn handle(&self, request: Request) -> Result<u32> {
+        let mut client = self.client.make_database_client()?.inner;
+        let request = self.to_rpc_request(request);
+        let response = client
+            .handle(request)
+            .await?
+            .into_inner()
+            .response
+            .context(IllegalDatabaseResponseSnafu {
+                err_msg: "GreptimeResponse is empty",
+            })?;
+        let greptime_response::Response::AffectedRows(AffectedRows { value }) = response;
+        Ok(value)
+    }
+
+    #[inline]
+    fn to_rpc_request(&self, request: Request) -> GreptimeRequest {
+        GreptimeRequest {
+            header: Some(RequestHeader {
+                catalog: self.catalog.clone(),
+                schema: self.schema.clone(),
+                authorization: self.ctx.auth_header.clone(),
+                dbname: self.dbname.clone(),
+            }),
+            request: Some(request),
+        }
+    }
+
+    async fn client_stream(&self) -> Result<Sender<GreptimeRequest>> {
+        let mut client = self.client.make_database_client()?.inner;
+        let (sender, receiver) = mpsc::channel::<GreptimeRequest>(65536);
+        let receiver = ReceiverStream::new(receiver);
+        client.handle_requests(receiver).await?;
+        Ok(sender)
     }
 
     pub async fn sql(&self, sql: &str) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_SQL);
         self.do_get(Request::Query(QueryRequest {
             query: Some(Query::Sql(sql.to_string())),
         }))
@@ -90,6 +186,7 @@ impl Database {
     }
 
     pub async fn logical_plan(&self, logical_plan: Vec<u8>) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_LOGICAL_PLAN);
         self.do_get(Request::Query(QueryRequest {
             query: Some(Query::LogicalPlan(logical_plan)),
         }))
@@ -103,6 +200,7 @@ impl Database {
         end: &str,
         step: &str,
     ) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_PROMQL_RANGE_QUERY);
         self.do_get(Request::Query(QueryRequest {
             query: Some(Query::PromRangeQuery(PromRangeQuery {
                 query: promql.to_string(),
@@ -115,6 +213,7 @@ impl Database {
     }
 
     pub async fn create(&self, expr: CreateTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_CREATE_TABLE);
         self.do_get(Request::Ddl(DdlRequest {
             expr: Some(DdlExpr::CreateTable(expr)),
         }))
@@ -122,6 +221,7 @@ impl Database {
     }
 
     pub async fn alter(&self, expr: AlterExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_ALTER);
         self.do_get(Request::Ddl(DdlRequest {
             expr: Some(DdlExpr::Alter(expr)),
         }))
@@ -129,6 +229,7 @@ impl Database {
     }
 
     pub async fn drop_table(&self, expr: DropTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_DROP_TABLE);
         self.do_get(Request::Ddl(DdlRequest {
             expr: Some(DdlExpr::DropTable(expr)),
         }))
@@ -136,6 +237,7 @@ impl Database {
     }
 
     pub async fn flush_table(&self, expr: FlushTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_FLUSH_TABLE);
         self.do_get(Request::Ddl(DdlRequest {
             expr: Some(DdlExpr::FlushTable(expr)),
         }))
@@ -143,43 +245,37 @@ impl Database {
     }
 
     async fn do_get(&self, request: Request) -> Result<Output> {
-        let request = GreptimeRequest {
-            header: Some(RequestHeader {
-                catalog: self.catalog.clone(),
-                schema: self.schema.clone(),
-                authorization: self.ctx.auth_header.clone(),
-            }),
-            request: Some(request),
-        };
+        // FIXME(paomian): should be added some labels for metrics
+        let _timer = timer!(metrics::METRIC_GRPC_DO_GET);
+        let request = self.to_rpc_request(request);
         let request = Ticket {
             ticket: request.encode_to_vec().into(),
         };
 
-        let mut client = self.client.make_client()?;
+        let mut client = self.client.make_flight_client()?;
 
-        // TODO(LFC): Streaming get flight data.
         let flight_data: Vec<FlightData> = client
             .mut_inner()
             .do_get(request)
             .and_then(|response| response.into_inner().try_collect())
             .await
             .map_err(|e| {
-                let code = get_metadata_value(&e, INNER_ERROR_CODE)
-                    .and_then(|s| StatusCode::from_str(&s).ok())
-                    .unwrap_or(StatusCode::Unknown);
-                let msg = get_metadata_value(&e, INNER_ERROR_MSG).unwrap_or(e.to_string());
-                error::ExternalSnafu { code, msg }
+                let tonic_code = e.code();
+                let e: error::Error = e.into();
+                let code = e.status_code();
+                let msg = e.to_string();
+                error::ServerSnafu { code, msg }
                     .fail::<()>()
                     .map_err(BoxedError::new)
                     .context(error::FlightGetSnafu {
-                        tonic_code: e.code(),
+                        tonic_code,
                         addr: client.addr(),
                     })
                     .map_err(|error| {
                         logging::error!(
                             "Failed to do Flight get, addr: {}, code: {}, source: {}",
                             client.addr(),
-                            e.code(),
+                            tonic_code,
                             error
                         );
                         error
@@ -210,12 +306,6 @@ impl Database {
     }
 }
 
-fn get_metadata_value(e: &tonic::Status, key: &str) -> Option<String> {
-    e.metadata()
-        .get(key)
-        .and_then(|v| String::from_utf8(v.as_bytes().to_vec()).ok())
-}
-
 #[derive(Default, Debug, Clone)]
 pub struct FlightContext {
     auth_header: Option<AuthHeader>,

src/client/src/error.rs

@@ -13,18 +13,17 @@
 // limitations under the License.
 
 use std::any::Any;
+use std::str::FromStr;
 
 use common_error::prelude::*;
-use tonic::Code;
+use snafu::Location;
+use tonic::{Code, Status};
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
     #[snafu(display("Illegal Flight messages, reason: {}", reason))]
-    IllegalFlightMessages {
-        reason: String,
-        backtrace: Backtrace,
-    },
+    IllegalFlightMessages { reason: String, location: Location },
 
     #[snafu(display("Failed to do Flight get, code: {}, source: {}", tonic_code, source))]
     FlightGet {
@@ -35,24 +34,21 @@ pub enum Error {
 
     #[snafu(display("Failed to convert FlightData, source: {}", source))]
     ConvertFlightData {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc::Error,
     },
 
     #[snafu(display("Column datatype error, source: {}", source))]
     ColumnDataType {
-        #[snafu(backtrace)]
+        location: Location,
         source: api::error::Error,
     },
 
     #[snafu(display("Illegal GRPC client state: {}", err_msg))]
-    IllegalGrpcClientState {
-        err_msg: String,
-        backtrace: Backtrace,
-    },
+    IllegalGrpcClientState { err_msg: String, location: Location },
 
     #[snafu(display("Missing required field in protobuf, field: {}", field))]
-    MissingField { field: String, backtrace: Backtrace },
+    MissingField { field: String, location: Location },
 
     #[snafu(display(
         "Failed to create gRPC channel, peer address: {}, source: {}",
@@ -61,13 +57,19 @@ pub enum Error {
     ))]
     CreateChannel {
         addr: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc::error::Error,
     },
 
-    /// Error deserialized from gRPC metadata
+    // Server error carried in Tonic Status's metadata.
     #[snafu(display("{}", msg))]
-    ExternalError { code: StatusCode, msg: String },
+    Server { code: StatusCode, msg: String },
+
+    #[snafu(display("Illegal Database response: {err_msg}"))]
+    IllegalDatabaseResponse { err_msg: String },
+
+    #[snafu(display("Failed to send request with streaming: {}", err_msg))]
+    ClientStreaming { err_msg: String, location: Location },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -77,21 +79,38 @@ impl ErrorExt for Error {
         match self {
             Error::IllegalFlightMessages { .. }
             | Error::ColumnDataType { .. }
-            | Error::MissingField { .. } => StatusCode::Internal,
+            | Error::MissingField { .. }
+            | Error::IllegalDatabaseResponse { .. }
+            | Error::ClientStreaming { .. } => StatusCode::Internal,
+
+            Error::Server { code, .. } => *code,
             Error::FlightGet { source, .. } => source.status_code(),
-            Error::CreateChannel { source, .. } | Error::ConvertFlightData { source } => {
+            Error::CreateChannel { source, .. } | Error::ConvertFlightData { source, .. } => {
                 source.status_code()
             }
             Error::IllegalGrpcClientState { .. } => StatusCode::Unexpected,
-            Error::ExternalError { code, .. } => *code,
         }
     }
 
-    fn backtrace_opt(&self) -> Option<&Backtrace> {
-        ErrorCompat::backtrace(self)
-    }
-
     fn as_any(&self) -> &dyn Any {
         self
     }
 }
+
+impl From<Status> for Error {
+    fn from(e: Status) -> Self {
+        fn get_metadata_value(e: &Status, key: &str) -> Option<String> {
+            e.metadata()
+                .get(key)
+                .and_then(|v| String::from_utf8(v.as_bytes().to_vec()).ok())
+        }
+
+        let code = get_metadata_value(&e, INNER_ERROR_CODE)
+            .and_then(|s| StatusCode::from_str(&s).ok())
+            .unwrap_or(StatusCode::Unknown);
+
+        let msg = get_metadata_value(&e, INNER_ERROR_MSG).unwrap_or(e.to_string());
+
+        Self::Server { code, msg }
+    }
+}

src/client/src/lib.rs

@@ -13,9 +13,11 @@
 // limitations under the License.
 
 mod client;
+pub mod client_manager;
 mod database;
 mod error;
 pub mod load_balance;
+mod metrics;
 
 pub use api;
 pub use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};

src/client/src/metrics.rs

@@ -0,0 +1,25 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! client metrics
+pub const METRIC_GRPC_CREATE_TABLE: &str = "grpc.create_table";
+pub const METRIC_GRPC_PROMQL_RANGE_QUERY: &str = "grpc.promql.range_query";
+pub const METRIC_GRPC_INSERT: &str = "grpc.insert";
+pub const METRIC_GRPC_DELETE: &str = "grpc.delete";
+pub const METRIC_GRPC_SQL: &str = "grpc.sql";
+pub const METRIC_GRPC_LOGICAL_PLAN: &str = "grpc.logical_plan";
+pub const METRIC_GRPC_ALTER: &str = "grpc.alter";
+pub const METRIC_GRPC_DROP_TABLE: &str = "grpc.drop_table";
+pub const METRIC_GRPC_FLUSH_TABLE: &str = "grpc.flush_table";
+pub const METRIC_GRPC_DO_GET: &str = "grpc.do_get";

src/cmd/Cargo.toml

@@ -10,7 +10,7 @@ name = "greptime"
 path = "src/bin/greptime.rs"
 
 [features]
-mem-prof = ["tikv-jemallocator", "tikv-jemalloc-ctl"]
+tokio-console = ["common-telemetry/tokio-console"]
 
 [dependencies]
 anymap = "1.0.0-beta.2"
@@ -24,12 +24,14 @@ common-recordbatch = { path = "../common/recordbatch" }
 common-telemetry = { path = "../common/telemetry", features = [
     "deadlock_detection",
 ] }
+config = "0.13"
 datanode = { path = "../datanode" }
 either = "1.8"
 frontend = { path = "../frontend" }
 futures.workspace = true
 meta-client = { path = "../meta-client" }
 meta-srv = { path = "../meta-srv" }
+metrics.workspace = true
 nu-ansi-term = "0.46"
 partition = { path = "../partition" }
 query = { path = "../query" }
@@ -39,16 +41,15 @@ servers = { path = "../servers" }
 session = { path = "../session" }
 snafu.workspace = true
 substrait = { path = "../common/substrait" }
-tikv-jemalloc-ctl = { version = "0.5", optional = true }
-tikv-jemallocator = { version = "0.5", optional = true }
+tikv-jemallocator = "0.5"
 tokio.workspace = true
-toml = "0.5"
-
 
 [dev-dependencies]
 common-test-util = { path = "../common/test-util" }
 rexpect = "0.5"
+temp-env = "0.3"
 serde.workspace = true
+toml = "0.5"
 
 [build-dependencies]
-build-data = "0.1.3"
+build-data = "0.1.4"

src/cmd/build.rs

@@ -18,6 +18,10 @@ fn main() {
         "cargo:rustc-env=GIT_COMMIT={}",
         build_data::get_git_commit().unwrap_or_else(|_| DEFAULT_VALUE.to_string())
     );
+    println!(
+        "cargo:rustc-env=GIT_COMMIT_SHORT={}",
+        build_data::get_git_commit_short().unwrap_or_else(|_| DEFAULT_VALUE.to_string())
+    );
     println!(
         "cargo:rustc-env=GIT_BRANCH={}",
         build_data::get_git_branch().unwrap_or_else(|_| DEFAULT_VALUE.to_string())

src/cmd/src/bin/greptime.rs

@@ -12,22 +12,30 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![doc = include_str!("../../../../README.md")]
+
 use std::fmt;
 
 use clap::Parser;
 use cmd::error::Result;
+use cmd::options::{Options, TopLevelOptions};
 use cmd::{cli, datanode, frontend, metasrv, standalone};
-use common_telemetry::logging::{error, info};
+use common_telemetry::logging::{error, info, TracingOptions};
+use metrics::gauge;
 
 #[derive(Parser)]
 #[clap(name = "greptimedb", version = print_version())]
 struct Command {
-    #[clap(long, default_value = "/tmp/greptimedb/logs")]
-    log_dir: String,
-    #[clap(long, default_value = "info")]
-    log_level: String,
+    #[clap(long)]
+    log_dir: Option<String>,
+    #[clap(long)]
+    log_level: Option<String>,
     #[clap(subcommand)]
     subcmd: SubCommand,
+
+    #[cfg(feature = "tokio-console")]
+    #[clap(long)]
+    tokio_console_addr: Option<String>,
 }
 
 pub enum Application {
@@ -39,13 +47,13 @@ pub enum Application {
 }
 
 impl Application {
-    async fn run(&mut self) -> Result<()> {
+    async fn start(&mut self) -> Result<()> {
         match self {
-            Application::Datanode(instance) => instance.run().await,
-            Application::Frontend(instance) => instance.run().await,
-            Application::Metasrv(instance) => instance.run().await,
-            Application::Standalone(instance) => instance.run().await,
-            Application::Cli(instance) => instance.run().await,
+            Application::Datanode(instance) => instance.start().await,
+            Application::Frontend(instance) => instance.start().await,
+            Application::Metasrv(instance) => instance.start().await,
+            Application::Standalone(instance) => instance.start().await,
+            Application::Cli(instance) => instance.start().await,
         }
     }
 
@@ -61,8 +69,20 @@ impl Application {
 }
 
 impl Command {
-    async fn build(self) -> Result<Application> {
-        self.subcmd.build().await
+    async fn build(self, opts: Options) -> Result<Application> {
+        self.subcmd.build(opts).await
+    }
+
+    fn load_options(&self) -> Result<Options> {
+        let top_level_opts = self.top_level_options();
+        self.subcmd.load_options(top_level_opts)
+    }
+
+    fn top_level_options(&self) -> TopLevelOptions {
+        TopLevelOptions {
+            log_dir: self.log_dir.clone(),
+            log_level: self.log_level.clone(),
+        }
     }
 }
 
@@ -81,28 +101,40 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self) -> Result<Application> {
-        match self {
-            SubCommand::Datanode(cmd) => {
-                let app = cmd.build().await?;
+    async fn build(self, opts: Options) -> Result<Application> {
+        match (self, opts) {
+            (SubCommand::Datanode(cmd), Options::Datanode(dn_opts)) => {
+                let app = cmd.build(*dn_opts).await?;
                 Ok(Application::Datanode(app))
             }
-            SubCommand::Frontend(cmd) => {
-                let app = cmd.build().await?;
+            (SubCommand::Frontend(cmd), Options::Frontend(fe_opts)) => {
+                let app = cmd.build(*fe_opts).await?;
                 Ok(Application::Frontend(app))
             }
-            SubCommand::Metasrv(cmd) => {
-                let app = cmd.build().await?;
+            (SubCommand::Metasrv(cmd), Options::Metasrv(meta_opts)) => {
+                let app = cmd.build(*meta_opts).await?;
                 Ok(Application::Metasrv(app))
             }
-            SubCommand::Standalone(cmd) => {
-                let app = cmd.build().await?;
+            (SubCommand::Standalone(cmd), Options::Standalone(opts)) => {
+                let app = cmd.build(opts.fe_opts, opts.dn_opts).await?;
                 Ok(Application::Standalone(app))
             }
-            SubCommand::Cli(cmd) => {
+            (SubCommand::Cli(cmd), Options::Cli(_)) => {
                 let app = cmd.build().await?;
                 Ok(Application::Cli(app))
             }
+
+            _ => unreachable!(),
+        }
+    }
+
+    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        match self {
+            SubCommand::Datanode(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Frontend(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Metasrv(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Standalone(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Cli(cmd) => cmd.load_options(top_level_opts),
         }
     }
 }
@@ -132,27 +164,63 @@ fn print_version() -> &'static str {
     )
 }
 
-#[cfg(feature = "mem-prof")]
+fn short_version() -> &'static str {
+    env!("CARGO_PKG_VERSION")
+}
+
+// {app_name}-{branch_name}-{commit_short}
+// The branch name (tag) of a release build should already contain the short
+// version so the full version doesn't concat the short version explicitly.
+fn full_version() -> &'static str {
+    concat!(
+        "greptimedb-",
+        env!("GIT_BRANCH"),
+        "-",
+        env!("GIT_COMMIT_SHORT")
+    )
+}
+
+fn log_env_flags() {
+    info!("command line arguments");
+    for argument in std::env::args() {
+        info!("argument: {}", argument);
+    }
+}
+
 #[global_allocator]
 static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
 #[tokio::main]
 async fn main() -> Result<()> {
     let cmd = Command::parse();
-    // TODO(dennis):
-    // 1. adds ip/port to app
     let app_name = &cmd.subcmd.to_string();
-    let log_dir = &cmd.log_dir;
-    let log_level = &cmd.log_level;
+
+    let opts = cmd.load_options()?;
+    let logging_opts = opts.logging_options();
+    let tracing_opts = TracingOptions {
+        #[cfg(feature = "tokio-console")]
+        tokio_console_addr: cmd.tokio_console_addr.clone(),
+    };
 
     common_telemetry::set_panic_hook();
     common_telemetry::init_default_metrics_recorder();
-    let _guard = common_telemetry::init_global_logging(app_name, log_dir, log_level, false);
+    let _guard = common_telemetry::init_global_logging(app_name, logging_opts, tracing_opts);
 
-    let mut app = cmd.build().await?;
+    // Report app version as gauge.
+    gauge!("app_version", 1.0, "short_version" => short_version(), "version" => full_version());
+
+    // Log version and argument flags.
+    info!(
+        "short_version: {}, full_version: {}",
+        short_version(),
+        full_version()
+    );
+    log_env_flags();
+
+    let mut app = cmd.build(opts).await?;
 
     tokio::select! {
-        result = app.run() => {
+        result = app.start() => {
             if let Err(err) = result {
                 error!(err; "Fatal error occurs!");
             }

src/cmd/src/cli.rs

@@ -17,21 +17,22 @@ mod helper;
 mod repl;
 
 use clap::Parser;
+use common_telemetry::logging::LoggingOptions;
 pub use repl::Repl;
 
 use crate::error::Result;
+use crate::options::{Options, TopLevelOptions};
 
 pub struct Instance {
     repl: Repl,
 }
 
 impl Instance {
-    pub async fn run(&mut self) -> Result<()> {
+    pub async fn start(&mut self) -> Result<()> {
         self.repl.run().await
     }
 
     pub async fn stop(&self) -> Result<()> {
-        // TODO: handle cli shutdown
         Ok(())
     }
 }
@@ -46,6 +47,17 @@ impl Command {
     pub async fn build(self) -> Result<Instance> {
         self.cmd.build().await
     }
+
+    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        let mut logging_opts = LoggingOptions::default();
+        if let Some(dir) = top_level_opts.log_dir {
+            logging_opts.dir = dir;
+        }
+        if top_level_opts.log_level.is_some() {
+            logging_opts.level = top_level_opts.log_level;
+        }
+        Ok(Options::Cli(Box::new(logging_opts)))
+    }
 }
 
 #[derive(Parser)]
@@ -77,3 +89,46 @@ impl AttachCommand {
         Ok(Instance { repl })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_load_options() {
+        let cmd = Command {
+            cmd: SubCommand::Attach(AttachCommand {
+                grpc_addr: String::from(""),
+                meta_addr: None,
+                disable_helper: false,
+            }),
+        };
+
+        let opts = cmd.load_options(TopLevelOptions::default()).unwrap();
+        let logging_opts = opts.logging_options();
+        assert_eq!("/tmp/greptimedb/logs", logging_opts.dir);
+        assert!(logging_opts.level.is_none());
+        assert!(!logging_opts.enable_jaeger_tracing);
+    }
+
+    #[test]
+    fn test_top_level_options() {
+        let cmd = Command {
+            cmd: SubCommand::Attach(AttachCommand {
+                grpc_addr: String::from(""),
+                meta_addr: None,
+                disable_helper: false,
+            }),
+        };
+
+        let opts = cmd
+            .load_options(TopLevelOptions {
+                log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
+                log_level: Some("debug".to_string()),
+            })
+            .unwrap();
+        let logging_opts = opts.logging_options();
+        assert_eq!("/tmp/greptimedb/test/logs", logging_opts.dir);
+        assert_eq!("debug", logging_opts.level.as_ref().unwrap());
+    }
+}

src/cmd/src/cli/repl.rs

@@ -16,7 +16,8 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;
 
-use catalog::remote::MetaKvBackend;
+use catalog::remote::CachedMetaKvBackend;
+use client::client_manager::DatanodeClients;
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_error::prelude::ErrorExt;
 use common_query::Output;
@@ -24,7 +25,6 @@ use common_recordbatch::RecordBatches;
 use common_telemetry::logging;
 use either::Either;
 use frontend::catalog::FrontendCatalogManager;
-use frontend::datanode::DatanodeClients;
 use meta_client::client::MetaClientBuilder;
 use partition::manager::PartitionRuleManager;
 use partition::route::TableRoutes;
@@ -253,9 +253,7 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
         .context(StartMetaClientSnafu)?;
     let meta_client = Arc::new(meta_client);
 
-    let backend = Arc::new(MetaKvBackend {
-        client: meta_client.clone(),
-    });
+    let cached_meta_backend = Arc::new(CachedMetaKvBackend::new(meta_client.clone()));
 
     let table_routes = Arc::new(TableRoutes::new(meta_client));
     let partition_manager = Arc::new(PartitionRuleManager::new(table_routes));
@@ -263,11 +261,18 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
     let datanode_clients = Arc::new(DatanodeClients::default());
 
     let catalog_list = Arc::new(FrontendCatalogManager::new(
-        backend,
+        cached_meta_backend.clone(),
+        cached_meta_backend,
         partition_manager,
         datanode_clients,
     ));
-    let state = Arc::new(QueryEngineState::new(catalog_list, Default::default()));
+    let state = Arc::new(QueryEngineState::new(
+        catalog_list,
+        false,
+        None,
+        None,
+        Default::default(),
+    ));
 
     Ok(DatafusionQueryEngine::new(state))
 }

src/cmd/src/datanode.rs

@@ -12,30 +12,32 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::time::Duration;
+
 use clap::Parser;
 use common_telemetry::logging;
-use datanode::datanode::{
-    Datanode, DatanodeOptions, FileConfig, ObjectStoreConfig, ProcedureConfig,
-};
+use datanode::datanode::{Datanode, DatanodeOptions, FileConfig, ObjectStoreConfig};
 use meta_client::MetaClientOptions;
 use servers::Mode;
 use snafu::ResultExt;
 
-use crate::error::{Error, MissingConfigSnafu, Result, StartDatanodeSnafu};
-use crate::toml_loader;
+use crate::error::{MissingConfigSnafu, Result, ShutdownDatanodeSnafu, StartDatanodeSnafu};
+use crate::options::{Options, TopLevelOptions};
 
 pub struct Instance {
     datanode: Datanode,
 }
 
 impl Instance {
-    pub async fn run(&mut self) -> Result<()> {
+    pub async fn start(&mut self) -> Result<()> {
         self.datanode.start().await.context(StartDatanodeSnafu)
     }
 
     pub async fn stop(&self) -> Result<()> {
-        // TODO: handle datanode shutdown
-        Ok(())
+        self.datanode
+            .shutdown()
+            .await
+            .context(ShutdownDatanodeSnafu)
     }
 }
 
@@ -46,8 +48,12 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self) -> Result<Instance> {
-        self.subcmd.build().await
+    pub async fn build(self, opts: DatanodeOptions) -> Result<Instance> {
+        self.subcmd.build(opts).await
+    }
+
+    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        self.subcmd.load_options(top_level_opts)
     }
 }
 
@@ -57,9 +63,15 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self) -> Result<Instance> {
+    async fn build(self, opts: DatanodeOptions) -> Result<Instance> {
         match self {
-            SubCommand::Start(cmd) => cmd.build().await,
+            SubCommand::Start(cmd) => cmd.build(opts).await,
+        }
+    }
+
+    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        match self {
+            SubCommand::Start(cmd) => cmd.load_options(top_level_opts),
         }
     }
 }
@@ -72,67 +84,54 @@ struct StartCommand {
     rpc_addr: Option<String>,
     #[clap(long)]
     rpc_hostname: Option<String>,
-    #[clap(long)]
-    mysql_addr: Option<String>,
-    #[clap(long)]
-    metasrv_addr: Option<String>,
+    #[clap(long, multiple = true, value_delimiter = ',')]
+    metasrv_addr: Option<Vec<String>>,
     #[clap(short, long)]
     config_file: Option<String>,
     #[clap(long)]
-    data_dir: Option<String>,
+    data_home: Option<String>,
     #[clap(long)]
     wal_dir: Option<String>,
     #[clap(long)]
-    procedure_dir: Option<String>,
+    http_addr: Option<String>,
+    #[clap(long)]
+    http_timeout: Option<u64>,
+    #[clap(long, default_value = "GREPTIMEDB_DATANODE")]
+    env_prefix: String,
 }
 
 impl StartCommand {
-    async fn build(self) -> Result<Instance> {
-        logging::info!("Datanode start command: {:#?}", self);
+    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        let mut opts: DatanodeOptions = Options::load_layered_options(
+            self.config_file.as_deref(),
+            self.env_prefix.as_ref(),
+            DatanodeOptions::env_list_keys(),
+        )?;
 
-        let opts: DatanodeOptions = self.try_into()?;
-
-        logging::info!("Datanode options: {:#?}", opts);
-
-        let datanode = Datanode::new(opts).await.context(StartDatanodeSnafu)?;
-
-        Ok(Instance { datanode })
-    }
-}
-
-impl TryFrom<StartCommand> for DatanodeOptions {
-    type Error = Error;
-    fn try_from(cmd: StartCommand) -> Result<Self> {
-        let mut opts: DatanodeOptions = if let Some(path) = cmd.config_file {
-            toml_loader::from_file!(&path)?
-        } else {
-            DatanodeOptions::default()
-        };
-
-        if let Some(addr) = cmd.rpc_addr {
-            opts.rpc_addr = addr;
+        if let Some(dir) = top_level_opts.log_dir {
+            opts.logging.dir = dir;
         }
 
-        if cmd.rpc_hostname.is_some() {
-            opts.rpc_hostname = cmd.rpc_hostname;
+        if top_level_opts.log_level.is_some() {
+            opts.logging.level = top_level_opts.log_level;
         }
 
-        if let Some(addr) = cmd.mysql_addr {
-            opts.mysql_addr = addr;
+        if let Some(addr) = &self.rpc_addr {
+            opts.rpc_addr = addr.clone();
         }
 
-        if let Some(node_id) = cmd.node_id {
+        if self.rpc_hostname.is_some() {
+            opts.rpc_hostname = self.rpc_hostname.clone();
+        }
+
+        if let Some(node_id) = self.node_id {
             opts.node_id = Some(node_id);
         }
 
-        if let Some(meta_addr) = cmd.metasrv_addr {
+        if let Some(metasrv_addrs) = &self.metasrv_addr {
             opts.meta_client_options
                 .get_or_insert_with(MetaClientOptions::default)
-                .metasrv_addrs = meta_addr
-                .split(',')
-                .map(&str::trim)
-                .map(&str::to_string)
-                .collect::<_>();
+                .metasrv_addrs = metasrv_addrs.clone();
             opts.mode = Mode::Distributed;
         }
 
@@ -143,32 +142,52 @@ impl TryFrom<StartCommand> for DatanodeOptions {
             .fail();
         }
 
-        if let Some(data_dir) = cmd.data_dir {
-            opts.storage = ObjectStoreConfig::File(FileConfig { data_dir });
+        if let Some(data_home) = &self.data_home {
+            opts.storage.store = ObjectStoreConfig::File(FileConfig {
+                data_home: data_home.clone(),
+            });
         }
 
-        if let Some(wal_dir) = cmd.wal_dir {
-            opts.wal.dir = wal_dir;
-        }
-        if let Some(procedure_dir) = cmd.procedure_dir {
-            opts.procedure = Some(ProcedureConfig::from_file_path(procedure_dir));
+        if let Some(wal_dir) = &self.wal_dir {
+            opts.wal.dir = Some(wal_dir.clone());
         }
 
-        Ok(opts)
+        if let Some(http_addr) = &self.http_addr {
+            opts.http_opts.addr = http_addr.clone();
+        }
+
+        if let Some(http_timeout) = self.http_timeout {
+            opts.http_opts.timeout = Duration::from_secs(http_timeout)
+        }
+
+        // Disable dashboard in datanode.
+        opts.http_opts.disable_dashboard = true;
+
+        Ok(Options::Datanode(Box::new(opts)))
+    }
+
+    async fn build(self, opts: DatanodeOptions) -> Result<Instance> {
+        logging::info!("Datanode start command: {:#?}", self);
+        logging::info!("Datanode options: {:#?}", opts);
+
+        let datanode = Datanode::new(opts).await.context(StartDatanodeSnafu)?;
+
+        Ok(Instance { datanode })
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use std::assert_matches::assert_matches;
     use std::io::Write;
     use std::time::Duration;
 
+    use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::datanode::{CompactionConfig, ObjectStoreConfig};
+    use datanode::datanode::{CompactionConfig, ObjectStoreConfig, RegionManifestConfig};
     use servers::Mode;
 
     use super::*;
+    use crate::options::ENV_VAR_SEP;
 
     #[test]
     fn test_read_from_config_file() {
@@ -180,8 +199,6 @@ mod tests {
             rpc_addr = "127.0.0.1:3001"
             rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
-            mysql_addr = "127.0.0.1:4406"
-            mysql_runtime_size = 2
 
             [meta_client_options]
             metasrv_addrs = ["127.0.0.1:3002"]
@@ -190,7 +207,7 @@ mod tests {
             tcp_nodelay = true
 
             [wal]
-            dir = "/tmp/greptimedb/wal"
+            dir = "/other/wal"
             file_size = "1GB"
             purge_threshold = "50GB"
             purge_interval = "10m"
@@ -199,12 +216,22 @@ mod tests {
 
             [storage]
             type = "File"
-            data_dir = "/tmp/greptimedb/data/"
+            data_home = "/tmp/greptimedb/"
 
-            [compaction]
-            max_inflight_tasks = 4
-            max_files_in_level0 = 8
+            [storage.compaction]
+            max_inflight_tasks = 3
+            max_files_in_level0 = 7
             max_purge_tasks = 32
+
+            [storage.manifest]
+            checkpoint_margin = 9
+            gc_duration = '7s'
+            checkpoint_on_startup = true
+            compress = true
+
+            [logging]
+            level = "debug"
+            dir = "/tmp/greptimedb/test/logs"
         "#;
         write!(file, "{}", toml_str).unwrap();
 
@@ -212,12 +239,14 @@ mod tests {
             config_file: Some(file.path().to_str().unwrap().to_string()),
             ..Default::default()
         };
-        let options: DatanodeOptions = cmd.try_into().unwrap();
+
+        let Options::Datanode(options) =
+            cmd.load_options(TopLevelOptions::default()).unwrap() else { unreachable!() };
+
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
-        assert_eq!("127.0.0.1:4406".to_string(), options.mysql_addr);
-        assert_eq!(2, options.mysql_runtime_size);
         assert_eq!(Some(42), options.node_id);
 
+        assert_eq!("/other/wal", options.wal.dir.unwrap());
         assert_eq!(Duration::from_secs(600), options.wal.purge_interval);
         assert_eq!(1024 * 1024 * 1024, options.wal.file_size.0);
         assert_eq!(1024 * 1024 * 1024 * 50, options.wal.purge_threshold.0);
@@ -235,53 +264,207 @@ mod tests {
         assert_eq!(3000, timeout_millis);
         assert!(tcp_nodelay);
 
-        match options.storage {
-            ObjectStoreConfig::File(FileConfig { data_dir }) => {
-                assert_eq!("/tmp/greptimedb/data/".to_string(), data_dir)
+        match &options.storage.store {
+            ObjectStoreConfig::File(FileConfig { data_home, .. }) => {
+                assert_eq!("/tmp/greptimedb/", data_home)
             }
             ObjectStoreConfig::S3 { .. } => unreachable!(),
             ObjectStoreConfig::Oss { .. } => unreachable!(),
+            ObjectStoreConfig::Azblob { .. } => unreachable!(),
         };
 
         assert_eq!(
             CompactionConfig {
-                max_inflight_tasks: 4,
-                max_files_in_level0: 8,
+                max_inflight_tasks: 3,
+                max_files_in_level0: 7,
                 max_purge_tasks: 32,
+                sst_write_buffer_size: ReadableSize::mb(8),
             },
-            options.compaction
+            options.storage.compaction,
         );
+        assert_eq!(
+            RegionManifestConfig {
+                checkpoint_margin: Some(9),
+                gc_duration: Some(Duration::from_secs(7)),
+                checkpoint_on_startup: true,
+                compress: true
+            },
+            options.storage.manifest,
+        );
+
+        assert_eq!("debug", options.logging.level.unwrap());
+        assert_eq!("/tmp/greptimedb/test/logs".to_string(), options.logging.dir);
     }
 
     #[test]
     fn test_try_from_cmd() {
-        assert_eq!(
-            Mode::Standalone,
-            DatanodeOptions::try_from(StartCommand::default())
-                .unwrap()
-                .mode
-        );
+        if let Options::Datanode(opt) = StartCommand::default()
+            .load_options(TopLevelOptions::default())
+            .unwrap()
+        {
+            assert_eq!(Mode::Standalone, opt.mode)
+        }
 
-        let mode = DatanodeOptions::try_from(StartCommand {
+        if let Options::Datanode(opt) = (StartCommand {
             node_id: Some(42),
-            metasrv_addr: Some("127.0.0.1:3002".to_string()),
+            metasrv_addr: Some(vec!["127.0.0.1:3002".to_string()]),
             ..Default::default()
         })
+        .load_options(TopLevelOptions::default())
         .unwrap()
-        .mode;
-        assert_matches!(mode, Mode::Distributed);
+        {
+            assert_eq!(Mode::Distributed, opt.mode)
+        }
 
-        assert!(DatanodeOptions::try_from(StartCommand {
-            metasrv_addr: Some("127.0.0.1:3002".to_string()),
+        assert!((StartCommand {
+            metasrv_addr: Some(vec!["127.0.0.1:3002".to_string()]),
             ..Default::default()
         })
+        .load_options(TopLevelOptions::default())
         .is_err());
 
         // Providing node_id but leave metasrv_addr absent is ok since metasrv_addr has default value
-        DatanodeOptions::try_from(StartCommand {
+        (StartCommand {
             node_id: Some(42),
             ..Default::default()
         })
+        .load_options(TopLevelOptions::default())
         .unwrap();
     }
+
+    #[test]
+    fn test_top_level_options() {
+        let cmd = StartCommand::default();
+
+        let options = cmd
+            .load_options(TopLevelOptions {
+                log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
+                log_level: Some("debug".to_string()),
+            })
+            .unwrap();
+
+        let logging_opt = options.logging_options();
+        assert_eq!("/tmp/greptimedb/test/logs", logging_opt.dir);
+        assert_eq!("debug", logging_opt.level.as_ref().unwrap());
+    }
+
+    #[test]
+    fn test_config_precedence_order() {
+        let mut file = create_named_temp_file();
+        let toml_str = r#"
+            mode = "distributed"
+            enable_memory_catalog = false
+            node_id = 42
+            rpc_addr = "127.0.0.1:3001"
+            rpc_hostname = "127.0.0.1"
+            rpc_runtime_size = 8
+
+            [meta_client_options]
+            timeout_millis = 3000
+            connect_timeout_millis = 5000
+            tcp_nodelay = true
+
+            [wal]
+            file_size = "1GB"
+            purge_threshold = "50GB"
+            purge_interval = "10m"
+            read_batch_size = 128
+            sync_write = false
+
+            [storage]
+            type = "File"
+            data_home = "/tmp/greptimedb/"
+
+            [storage.compaction]
+            max_inflight_tasks = 3
+            max_files_in_level0 = 7
+            max_purge_tasks = 32
+
+            [storage.manifest]
+            checkpoint_on_startup = true
+
+            [logging]
+            level = "debug"
+            dir = "/tmp/greptimedb/test/logs"
+        "#;
+        write!(file, "{}", toml_str).unwrap();
+
+        let env_prefix = "DATANODE_UT";
+        temp_env::with_vars(
+            vec![
+                (
+                    // storage.manifest.gc_duration = 9s
+                    vec![
+                        env_prefix.to_string(),
+                        "storage".to_uppercase(),
+                        "manifest".to_uppercase(),
+                        "gc_duration".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("9s"),
+                ),
+                (
+                    // storage.compaction.max_purge_tasks = 99
+                    vec![
+                        env_prefix.to_string(),
+                        "storage".to_uppercase(),
+                        "compaction".to_uppercase(),
+                        "max_purge_tasks".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("99"),
+                ),
+                (
+                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    vec![
+                        env_prefix.to_string(),
+                        "meta_client_options".to_uppercase(),
+                        "metasrv_addrs".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003"),
+                ),
+            ],
+            || {
+                let command = StartCommand {
+                    config_file: Some(file.path().to_str().unwrap().to_string()),
+                    wal_dir: Some("/other/wal/dir".to_string()),
+                    env_prefix: env_prefix.to_string(),
+                    ..Default::default()
+                };
+
+                let Options::Datanode(opts) =
+                    command.load_options(TopLevelOptions::default()).unwrap() else {unreachable!()};
+
+                // Should be read from env, env > default values.
+                assert_eq!(
+                    opts.storage.manifest.gc_duration,
+                    Some(Duration::from_secs(9))
+                );
+                assert_eq!(
+                    opts.meta_client_options.unwrap().metasrv_addrs,
+                    vec![
+                        "127.0.0.1:3001".to_string(),
+                        "127.0.0.1:3002".to_string(),
+                        "127.0.0.1:3003".to_string()
+                    ]
+                );
+
+                // Should be read from config file, config file > env > default values.
+                assert_eq!(opts.storage.compaction.max_purge_tasks, 32);
+
+                // Should be read from cli, cli > config file > env > default values.
+                assert_eq!(opts.wal.dir.unwrap(), "/other/wal/dir");
+
+                // Should be default value.
+                assert_eq!(
+                    opts.storage.manifest.checkpoint_margin,
+                    DatanodeOptions::default()
+                        .storage
+                        .manifest
+                        .checkpoint_margin
+                );
+            },
+        );
+    }
 }

src/cmd/src/error.rs

@@ -15,64 +15,71 @@
 use std::any::Any;
 
 use common_error::prelude::*;
+use config::ConfigError;
 use rustyline::error::ReadlineError;
+use snafu::Location;
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
     #[snafu(display("Failed to start datanode, source: {}", source))]
     StartDatanode {
-        #[snafu(backtrace)]
+        location: Location,
+        source: datanode::error::Error,
+    },
+
+    #[snafu(display("Failed to shutdown datanode, source: {}", source))]
+    ShutdownDatanode {
+        location: Location,
         source: datanode::error::Error,
     },
 
     #[snafu(display("Failed to start frontend, source: {}", source))]
     StartFrontend {
-        #[snafu(backtrace)]
+        location: Location,
+        source: frontend::error::Error,
+    },
+
+    #[snafu(display("Failed to shutdown frontend, source: {}", source))]
+    ShutdownFrontend {
+        location: Location,
         source: frontend::error::Error,
     },
 
     #[snafu(display("Failed to build meta server, source: {}", source))]
     BuildMetaServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_srv::error::Error,
     },
 
     #[snafu(display("Failed to start meta server, source: {}", source))]
     StartMetaServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_srv::error::Error,
     },
 
-    #[snafu(display("Failed to read config file: {}, source: {}", path, source))]
-    ReadConfig {
-        path: String,
-        source: std::io::Error,
-        backtrace: Backtrace,
-    },
-
-    #[snafu(display("Failed to parse config, source: {}", source))]
-    ParseConfig {
-        source: toml::de::Error,
-        backtrace: Backtrace,
+    #[snafu(display("Failed to shutdown meta server, source: {}", source))]
+    ShutdownMetaServer {
+        location: Location,
+        source: meta_srv::error::Error,
     },
 
     #[snafu(display("Missing config, msg: {}", msg))]
-    MissingConfig { msg: String, backtrace: Backtrace },
+    MissingConfig { msg: String, location: Location },
 
     #[snafu(display("Illegal config: {}", msg))]
-    IllegalConfig { msg: String, backtrace: Backtrace },
+    IllegalConfig { msg: String, location: Location },
 
     #[snafu(display("Illegal auth config: {}", source))]
     IllegalAuthConfig {
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::auth::Error,
     },
 
     #[snafu(display("Unsupported selector type, {} source: {}", selector_type, source))]
     UnsupportedSelectorType {
         selector_type: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_srv::error::Error,
     },
 
@@ -82,58 +89,70 @@ pub enum Error {
     #[snafu(display("Cannot create REPL: {}", source))]
     ReplCreation {
         source: ReadlineError,
-        backtrace: Backtrace,
+        location: Location,
     },
 
     #[snafu(display("Error reading command: {}", source))]
     Readline {
         source: ReadlineError,
-        backtrace: Backtrace,
+        location: Location,
     },
 
     #[snafu(display("Failed to request database, sql: {sql}, source: {source}"))]
     RequestDatabase {
         sql: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: client::Error,
     },
 
     #[snafu(display("Failed to collect RecordBatches, source: {source}"))]
     CollectRecordBatches {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
     #[snafu(display("Failed to pretty print Recordbatches, source: {source}"))]
     PrettyPrintRecordBatches {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
     #[snafu(display("Failed to start Meta client, source: {}", source))]
     StartMetaClient {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_client::error::Error,
     },
 
     #[snafu(display("Failed to parse SQL: {}, source: {}", sql, source))]
     ParseSql {
         sql: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to plan statement, source: {}", source))]
     PlanStatement {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to encode logical plan in substrait, source: {}", source))]
     SubstraitEncodeLogicalPlan {
-        #[snafu(backtrace)]
+        location: Location,
         source: substrait::error::Error,
     },
+
+    #[snafu(display("Failed to load layered config, source: {}", source))]
+    LoadLayeredConfig {
+        source: ConfigError,
+        location: Location,
+    },
+
+    #[snafu(display("Failed to start catalog manager, source: {}", source))]
+    StartCatalogManager {
+        location: Location,
+        source: catalog::error::Error,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -141,103 +160,33 @@ pub type Result<T> = std::result::Result<T, Error>;
 impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         match self {
-            Error::StartDatanode { source } => source.status_code(),
-            Error::StartFrontend { source } => source.status_code(),
-            Error::StartMetaServer { source } => source.status_code(),
-            Error::BuildMetaServer { source } => source.status_code(),
+            Error::StartDatanode { source, .. } => source.status_code(),
+            Error::StartFrontend { source, .. } => source.status_code(),
+            Error::ShutdownDatanode { source, .. } => source.status_code(),
+            Error::ShutdownFrontend { source, .. } => source.status_code(),
+            Error::StartMetaServer { source, .. } => source.status_code(),
+            Error::ShutdownMetaServer { source, .. } => source.status_code(),
+            Error::BuildMetaServer { source, .. } => source.status_code(),
             Error::UnsupportedSelectorType { source, .. } => source.status_code(),
-            Error::ReadConfig { .. } | Error::ParseConfig { .. } | Error::MissingConfig { .. } => {
-                StatusCode::InvalidArguments
-            }
-            Error::IllegalConfig { .. } | Error::InvalidReplCommand { .. } => {
-                StatusCode::InvalidArguments
-            }
-            Error::IllegalAuthConfig { .. } => StatusCode::InvalidArguments,
+            Error::MissingConfig { .. }
+            | Error::LoadLayeredConfig { .. }
+            | Error::IllegalConfig { .. }
+            | Error::InvalidReplCommand { .. }
+            | Error::IllegalAuthConfig { .. } => StatusCode::InvalidArguments,
             Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,
             Error::RequestDatabase { source, .. } => source.status_code(),
-            Error::CollectRecordBatches { source } | Error::PrettyPrintRecordBatches { source } => {
+            Error::CollectRecordBatches { source, .. }
+            | Error::PrettyPrintRecordBatches { source, .. } => source.status_code(),
+            Error::StartMetaClient { source, .. } => source.status_code(),
+            Error::ParseSql { source, .. } | Error::PlanStatement { source, .. } => {
                 source.status_code()
             }
-            Error::StartMetaClient { source } => source.status_code(),
-            Error::ParseSql { source, .. } | Error::PlanStatement { source } => {
-                source.status_code()
-            }
-            Error::SubstraitEncodeLogicalPlan { source } => source.status_code(),
+            Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(),
+            Error::StartCatalogManager { source, .. } => source.status_code(),
         }
     }
 
-    fn backtrace_opt(&self) -> Option<&Backtrace> {
-        ErrorCompat::backtrace(self)
-    }
-
     fn as_any(&self) -> &dyn Any {
         self
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    type StdResult<E> = std::result::Result<(), E>;
-
-    #[test]
-    fn test_start_node_error() {
-        fn throw_datanode_error() -> StdResult<datanode::error::Error> {
-            datanode::error::MissingNodeIdSnafu {}.fail()
-        }
-
-        let e = throw_datanode_error()
-            .context(StartDatanodeSnafu)
-            .err()
-            .unwrap();
-
-        assert!(e.backtrace_opt().is_some());
-        assert_eq!(e.status_code(), StatusCode::InvalidArguments);
-    }
-
-    #[test]
-    fn test_start_frontend_error() {
-        fn throw_frontend_error() -> StdResult<frontend::error::Error> {
-            frontend::error::InvalidSqlSnafu { err_msg: "failed" }.fail()
-        }
-
-        let e = throw_frontend_error()
-            .context(StartFrontendSnafu)
-            .err()
-            .unwrap();
-
-        assert!(e.backtrace_opt().is_some());
-        assert_eq!(e.status_code(), StatusCode::InvalidArguments);
-    }
-
-    #[test]
-    fn test_start_metasrv_error() {
-        fn throw_metasrv_error() -> StdResult<meta_srv::error::Error> {
-            meta_srv::error::StreamNoneSnafu {}.fail()
-        }
-
-        let e = throw_metasrv_error()
-            .context(StartMetaServerSnafu)
-            .err()
-            .unwrap();
-
-        assert!(e.backtrace_opt().is_some());
-        assert_eq!(e.status_code(), StatusCode::Internal);
-    }
-
-    #[test]
-    fn test_read_config_error() {
-        fn throw_read_config_error() -> StdResult<std::io::Error> {
-            Err(std::io::ErrorKind::NotFound.into())
-        }
-
-        let e = throw_read_config_error()
-            .context(ReadConfigSnafu { path: "test" })
-            .err()
-            .unwrap();
-
-        assert!(e.backtrace_opt().is_some());
-        assert_eq!(e.status_code(), StatusCode::InvalidArguments);
-    }
-}

src/cmd/src/frontend.rs

@@ -16,30 +16,31 @@ use std::sync::Arc;
 
 use clap::Parser;
 use common_base::Plugins;
+use common_telemetry::logging;
 use frontend::frontend::FrontendOptions;
-use frontend::grpc::GrpcOptions;
-use frontend::influxdb::InfluxdbOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
-use frontend::mysql::MysqlOptions;
-use frontend::opentsdb::OpentsdbOptions;
-use frontend::postgres::PostgresOptions;
-use frontend::prom::PromOptions;
+use frontend::service_config::{InfluxdbOptions, PromOptions};
 use meta_client::MetaClientOptions;
 use servers::auth::UserProviderRef;
-use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::{auth, Mode};
 use snafu::ResultExt;
 
-use crate::error::{self, IllegalAuthConfigSnafu, Result};
-use crate::toml_loader;
+use crate::error::{self, IllegalAuthConfigSnafu, Result, StartCatalogManagerSnafu};
+use crate::options::{Options, TopLevelOptions};
 
 pub struct Instance {
     frontend: FeInstance,
 }
 
 impl Instance {
-    pub async fn run(&mut self) -> Result<()> {
+    pub async fn start(&mut self) -> Result<()> {
+        self.frontend
+            .catalog_manager()
+            .start()
+            .await
+            .context(StartCatalogManagerSnafu)?;
+
         self.frontend
             .start()
             .await
@@ -47,8 +48,10 @@ impl Instance {
     }
 
     pub async fn stop(&self) -> Result<()> {
-        // TODO: handle frontend shutdown
-        Ok(())
+        self.frontend
+            .shutdown()
+            .await
+            .context(error::ShutdownFrontendSnafu)
     }
 }
 
@@ -59,8 +62,12 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self) -> Result<Instance> {
-        self.subcmd.build().await
+    pub async fn build(self, opts: FrontendOptions) -> Result<Instance> {
+        self.subcmd.build(opts).await
+    }
+
+    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        self.subcmd.load_options(top_level_opts)
     }
 }
 
@@ -70,14 +77,20 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self) -> Result<Instance> {
+    async fn build(self, opts: FrontendOptions) -> Result<Instance> {
         match self {
-            SubCommand::Start(cmd) => cmd.build().await,
+            SubCommand::Start(cmd) => cmd.build(opts).await,
+        }
+    }
+
+    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        match self {
+            SubCommand::Start(cmd) => cmd.load_options(top_level_opts),
         }
     }
 }
 
-#[derive(Debug, Parser)]
+#[derive(Debug, Default, Parser)]
 pub struct StartCommand {
     #[clap(long)]
     http_addr: Option<String>,
@@ -95,8 +108,8 @@ pub struct StartCommand {
     config_file: Option<String>,
     #[clap(short, long)]
     influxdb_enable: Option<bool>,
-    #[clap(long)]
-    metasrv_addr: Option<String>,
+    #[clap(long, multiple = true, value_delimiter = ',')]
+    metasrv_addr: Option<Vec<String>>,
     #[clap(long)]
     tls_mode: Option<TlsMode>,
     #[clap(long)]
@@ -105,19 +118,102 @@ pub struct StartCommand {
     tls_key_path: Option<String>,
     #[clap(long)]
     user_provider: Option<String>,
+    #[clap(long)]
+    disable_dashboard: Option<bool>,
+    #[clap(long, default_value = "GREPTIMEDB_FRONTEND")]
+    env_prefix: String,
 }
 
 impl StartCommand {
-    async fn build(self) -> Result<Instance> {
+    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        let mut opts: FrontendOptions = Options::load_layered_options(
+            self.config_file.as_deref(),
+            self.env_prefix.as_ref(),
+            FrontendOptions::env_list_keys(),
+        )?;
+
+        if let Some(dir) = top_level_opts.log_dir {
+            opts.logging.dir = dir;
+        }
+
+        if top_level_opts.log_level.is_some() {
+            opts.logging.level = top_level_opts.log_level;
+        }
+
+        let tls_opts = TlsOption::new(
+            self.tls_mode.clone(),
+            self.tls_cert_path.clone(),
+            self.tls_key_path.clone(),
+        );
+
+        if let Some(addr) = &self.http_addr {
+            if let Some(http_opts) = &mut opts.http_options {
+                http_opts.addr = addr.clone()
+            }
+        }
+
+        if let Some(disable_dashboard) = self.disable_dashboard {
+            opts.http_options
+                .get_or_insert_with(Default::default)
+                .disable_dashboard = disable_dashboard;
+        }
+
+        if let Some(addr) = &self.grpc_addr {
+            if let Some(grpc_opts) = &mut opts.grpc_options {
+                grpc_opts.addr = addr.clone()
+            }
+        }
+
+        if let Some(addr) = &self.mysql_addr {
+            if let Some(mysql_opts) = &mut opts.mysql_options {
+                mysql_opts.addr = addr.clone();
+                mysql_opts.tls = tls_opts.clone();
+            }
+        }
+
+        if let Some(addr) = &self.prom_addr {
+            opts.prom_options = Some(PromOptions { addr: addr.clone() });
+        }
+
+        if let Some(addr) = &self.postgres_addr {
+            if let Some(postgres_opts) = &mut opts.postgres_options {
+                postgres_opts.addr = addr.clone();
+                postgres_opts.tls = tls_opts;
+            }
+        }
+
+        if let Some(addr) = &self.opentsdb_addr {
+            if let Some(opentsdb_addr) = &mut opts.opentsdb_options {
+                opentsdb_addr.addr = addr.clone();
+            }
+        }
+
+        if let Some(enable) = self.influxdb_enable {
+            opts.influxdb_options = Some(InfluxdbOptions { enable });
+        }
+
+        if let Some(metasrv_addrs) = &self.metasrv_addr {
+            opts.meta_client_options
+                .get_or_insert_with(MetaClientOptions::default)
+                .metasrv_addrs = metasrv_addrs.clone();
+            opts.mode = Mode::Distributed;
+        }
+
+        Ok(Options::Frontend(Box::new(opts)))
+    }
+
+    async fn build(self, opts: FrontendOptions) -> Result<Instance> {
+        logging::info!("Frontend start command: {:#?}", self);
+        logging::info!("Frontend options: {:#?}", opts);
+
         let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
-        let opts: FrontendOptions = self.try_into()?;
 
         let mut instance = FeInstance::try_new_distributed(&opts, plugins.clone())
             .await
             .context(error::StartFrontendSnafu)?;
 
         instance
-            .build_servers(&opts, plugins)
+            .build_servers(&opts)
             .await
             .context(error::StartFrontendSnafu)?;
 
@@ -126,7 +222,7 @@ impl StartCommand {
 }
 
 pub fn load_frontend_plugins(user_provider: &Option<String>) -> Result<Plugins> {
-    let mut plugins = Plugins::new();
+    let plugins = Plugins::new();
 
     if let Some(provider) = user_provider {
         let provider = auth::user_provider_from_option(provider).context(IllegalAuthConfigSnafu)?;
@@ -135,99 +231,34 @@ pub fn load_frontend_plugins(user_provider: &Option<String>) -> Result<Plugins>
     Ok(plugins)
 }
 
-impl TryFrom<StartCommand> for FrontendOptions {
-    type Error = error::Error;
-
-    fn try_from(cmd: StartCommand) -> Result<Self> {
-        let mut opts: FrontendOptions = if let Some(path) = cmd.config_file {
-            toml_loader::from_file!(&path)?
-        } else {
-            FrontendOptions::default()
-        };
-
-        let tls_option = TlsOption::new(cmd.tls_mode, cmd.tls_cert_path, cmd.tls_key_path);
-
-        if let Some(addr) = cmd.http_addr {
-            opts.http_options = Some(HttpOptions {
-                addr,
-                ..Default::default()
-            });
-        }
-        if let Some(addr) = cmd.grpc_addr {
-            opts.grpc_options = Some(GrpcOptions {
-                addr,
-                ..Default::default()
-            });
-        }
-        if let Some(addr) = cmd.mysql_addr {
-            opts.mysql_options = Some(MysqlOptions {
-                addr,
-                tls: tls_option.clone(),
-                ..Default::default()
-            });
-        }
-        if let Some(addr) = cmd.prom_addr {
-            opts.prom_options = Some(PromOptions { addr });
-        }
-        if let Some(addr) = cmd.postgres_addr {
-            opts.postgres_options = Some(PostgresOptions {
-                addr,
-                tls: tls_option,
-                ..Default::default()
-            });
-        }
-        if let Some(addr) = cmd.opentsdb_addr {
-            opts.opentsdb_options = Some(OpentsdbOptions {
-                addr,
-                ..Default::default()
-            });
-        }
-        if let Some(enable) = cmd.influxdb_enable {
-            opts.influxdb_options = Some(InfluxdbOptions { enable });
-        }
-        if let Some(metasrv_addr) = cmd.metasrv_addr {
-            opts.meta_client_options
-                .get_or_insert_with(MetaClientOptions::default)
-                .metasrv_addrs = metasrv_addr
-                .split(',')
-                .map(&str::trim)
-                .map(&str::to_string)
-                .collect::<Vec<_>>();
-            opts.mode = Mode::Distributed;
-        }
-        Ok(opts)
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use std::io::Write;
     use std::time::Duration;
 
     use common_test_util::temp_dir::create_named_temp_file;
+    use frontend::service_config::GrpcOptions;
     use servers::auth::{Identity, Password, UserProviderRef};
 
     use super::*;
+    use crate::options::ENV_VAR_SEP;
 
     #[test]
     fn test_try_from_start_command() {
         let command = StartCommand {
             http_addr: Some("127.0.0.1:1234".to_string()),
-            grpc_addr: None,
             prom_addr: Some("127.0.0.1:4444".to_string()),
             mysql_addr: Some("127.0.0.1:5678".to_string()),
             postgres_addr: Some("127.0.0.1:5432".to_string()),
             opentsdb_addr: Some("127.0.0.1:4321".to_string()),
             influxdb_enable: Some(false),
-            config_file: None,
-            metasrv_addr: None,
-            tls_mode: None,
-            tls_cert_path: None,
-            tls_key_path: None,
-            user_provider: None,
+            disable_dashboard: Some(false),
+            ..Default::default()
         };
 
-        let opts: FrontendOptions = command.try_into().unwrap();
+        let Options::Frontend(opts) =
+            command.load_options(TopLevelOptions::default()).unwrap() else { unreachable!() };
+
         assert_eq!(opts.http_options.as_ref().unwrap().addr, "127.0.0.1:1234");
         assert_eq!(opts.mysql_options.as_ref().unwrap().addr, "127.0.0.1:5678");
         assert_eq!(
@@ -270,26 +301,21 @@ mod tests {
             [http_options]
             addr = "127.0.0.1:4000"
             timeout = "30s"
+
+            [logging]
+            level = "debug"
+            dir = "/tmp/greptimedb/test/logs"
         "#;
         write!(file, "{}", toml_str).unwrap();
 
         let command = StartCommand {
-            http_addr: None,
-            grpc_addr: None,
-            mysql_addr: None,
-            prom_addr: None,
-            postgres_addr: None,
-            opentsdb_addr: None,
-            influxdb_enable: None,
             config_file: Some(file.path().to_str().unwrap().to_string()),
-            metasrv_addr: None,
-            tls_mode: None,
-            tls_cert_path: None,
-            tls_key_path: None,
-            user_provider: None,
+            disable_dashboard: Some(false),
+            ..Default::default()
         };
 
-        let fe_opts = FrontendOptions::try_from(command).unwrap();
+        let Options::Frontend(fe_opts) =
+            command.load_options(TopLevelOptions::default()).unwrap() else {unreachable!()};
         assert_eq!(Mode::Distributed, fe_opts.mode);
         assert_eq!(
             "127.0.0.1:4000".to_string(),
@@ -299,24 +325,17 @@ mod tests {
             Duration::from_secs(30),
             fe_opts.http_options.as_ref().unwrap().timeout
         );
+
+        assert_eq!("debug", fe_opts.logging.level.as_ref().unwrap());
+        assert_eq!("/tmp/greptimedb/test/logs".to_string(), fe_opts.logging.dir);
     }
 
     #[tokio::test]
     async fn test_try_from_start_command_to_anymap() {
         let command = StartCommand {
-            http_addr: None,
-            grpc_addr: None,
-            mysql_addr: None,
-            prom_addr: None,
-            postgres_addr: None,
-            opentsdb_addr: None,
-            influxdb_enable: None,
-            config_file: None,
-            metasrv_addr: None,
-            tls_mode: None,
-            tls_cert_path: None,
-            tls_key_path: None,
             user_provider: Some("static_user_provider:cmd:test=test".to_string()),
+            disable_dashboard: Some(false),
+            ..Default::default()
         };
 
         let plugins = load_frontend_plugins(&command.user_provider);
@@ -327,8 +346,140 @@ mod tests {
 
         let provider = provider.unwrap();
         let result = provider
-            .authenticate(Identity::UserId("test", None), Password::PlainText("test"))
+            .authenticate(
+                Identity::UserId("test", None),
+                Password::PlainText("test".to_string().into()),
+            )
             .await;
         assert!(result.is_ok());
     }
+
+    #[test]
+    fn test_top_level_options() {
+        let cmd = StartCommand {
+            disable_dashboard: Some(false),
+            ..Default::default()
+        };
+
+        let options = cmd
+            .load_options(TopLevelOptions {
+                log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
+                log_level: Some("debug".to_string()),
+            })
+            .unwrap();
+
+        let logging_opt = options.logging_options();
+        assert_eq!("/tmp/greptimedb/test/logs", logging_opt.dir);
+        assert_eq!("debug", logging_opt.level.as_ref().unwrap());
+    }
+
+    #[test]
+    fn test_config_precedence_order() {
+        let mut file = create_named_temp_file();
+        let toml_str = r#"
+            mode = "distributed"
+
+            [http_options]
+            addr = "127.0.0.1:4000"
+
+            [meta_client_options]
+            timeout_millis = 3000
+            connect_timeout_millis = 5000
+            tcp_nodelay = true
+
+            [mysql_options]
+            addr = "127.0.0.1:4002"
+        "#;
+        write!(file, "{}", toml_str).unwrap();
+
+        let env_prefix = "FRONTEND_UT";
+        temp_env::with_vars(
+            vec![
+                (
+                    // mysql_options.addr = 127.0.0.1:14002
+                    vec![
+                        env_prefix.to_string(),
+                        "mysql_options".to_uppercase(),
+                        "addr".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("127.0.0.1:14002"),
+                ),
+                (
+                    // mysql_options.runtime_size = 11
+                    vec![
+                        env_prefix.to_string(),
+                        "mysql_options".to_uppercase(),
+                        "runtime_size".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("11"),
+                ),
+                (
+                    // http_options.addr = 127.0.0.1:24000
+                    vec![
+                        env_prefix.to_string(),
+                        "http_options".to_uppercase(),
+                        "addr".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("127.0.0.1:24000"),
+                ),
+                (
+                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    vec![
+                        env_prefix.to_string(),
+                        "meta_client_options".to_uppercase(),
+                        "metasrv_addrs".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003"),
+                ),
+            ],
+            || {
+                let command = StartCommand {
+                    config_file: Some(file.path().to_str().unwrap().to_string()),
+                    http_addr: Some("127.0.0.1:14000".to_string()),
+                    env_prefix: env_prefix.to_string(),
+                    ..Default::default()
+                };
+
+                let top_level_opts = TopLevelOptions {
+                    log_dir: None,
+                    log_level: Some("error".to_string()),
+                };
+                let Options::Frontend(fe_opts) =
+                    command.load_options(top_level_opts).unwrap() else {unreachable!()};
+
+                // Should be read from env, env > default values.
+                assert_eq!(fe_opts.mysql_options.as_ref().unwrap().runtime_size, 11);
+                assert_eq!(
+                    fe_opts.meta_client_options.unwrap().metasrv_addrs,
+                    vec![
+                        "127.0.0.1:3001".to_string(),
+                        "127.0.0.1:3002".to_string(),
+                        "127.0.0.1:3003".to_string()
+                    ]
+                );
+
+                // Should be read from config file, config file > env > default values.
+                assert_eq!(
+                    fe_opts.mysql_options.as_ref().unwrap().addr,
+                    "127.0.0.1:4002"
+                );
+
+                // Should be read from cli, cli > config file > env > default values.
+                assert_eq!(
+                    fe_opts.http_options.as_ref().unwrap().addr,
+                    "127.0.0.1:14000"
+                );
+
+                // Should be default value.
+                assert_eq!(
+                    fe_opts.grpc_options.as_ref().unwrap().addr,
+                    GrpcOptions::default().addr
+                );
+            },
+        );
+    }
 }

src/cmd/src/lib.rs

@@ -19,5 +19,5 @@ pub mod datanode;
 pub mod error;
 pub mod frontend;
 pub mod metasrv;
+pub mod options;
 pub mod standalone;
-mod toml_loader;

src/cmd/src/metasrv.rs

@@ -12,31 +12,34 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::time::Duration;
+
 use clap::Parser;
-use common_telemetry::{info, logging, warn};
+use common_telemetry::logging;
 use meta_srv::bootstrap::MetaSrvInstance;
 use meta_srv::metasrv::MetaSrvOptions;
 use snafu::ResultExt;
 
-use crate::error::{Error, Result};
-use crate::{error, toml_loader};
+use crate::error::{self, Result};
+use crate::options::{Options, TopLevelOptions};
 
 pub struct Instance {
     instance: MetaSrvInstance,
 }
 
 impl Instance {
-    pub async fn run(&mut self) -> Result<()> {
+    pub async fn start(&mut self) -> Result<()> {
         self.instance
             .start()
             .await
-            .context(error::StartMetaServerSnafu)?;
-        Ok(())
+            .context(error::StartMetaServerSnafu)
     }
 
     pub async fn stop(&self) -> Result<()> {
-        // TODO: handle metasrv shutdown
-        Ok(())
+        self.instance
+            .shutdown()
+            .await
+            .context(error::ShutdownMetaServerSnafu)
     }
 }
 
@@ -47,8 +50,12 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self) -> Result<Instance> {
-        self.subcmd.build().await
+    pub async fn build(self, opts: MetaSrvOptions) -> Result<Instance> {
+        self.subcmd.build(opts).await
+    }
+
+    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        self.subcmd.load_options(top_level_opts)
     }
 }
 
@@ -58,14 +65,20 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self) -> Result<Instance> {
+    async fn build(self, opts: MetaSrvOptions) -> Result<Instance> {
         match self {
-            SubCommand::Start(cmd) => cmd.build().await,
+            SubCommand::Start(cmd) => cmd.build(opts).await,
+        }
+    }
+
+    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        match self {
+            SubCommand::Start(cmd) => cmd.load_options(top_level_opts),
         }
     }
 }
 
-#[derive(Debug, Parser)]
+#[derive(Debug, Default, Parser)]
 struct StartCommand {
     #[clap(long)]
     bind_addr: Option<String>,
@@ -79,15 +92,73 @@ struct StartCommand {
     selector: Option<String>,
     #[clap(long)]
     use_memory_store: bool,
+    #[clap(long)]
+    disable_region_failover: bool,
+    #[clap(long)]
+    http_addr: Option<String>,
+    #[clap(long)]
+    http_timeout: Option<u64>,
+    #[clap(long, default_value = "GREPTIMEDB_METASRV")]
+    env_prefix: String,
 }
 
 impl StartCommand {
-    async fn build(self) -> Result<Instance> {
+    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+        let mut opts: MetaSrvOptions = Options::load_layered_options(
+            self.config_file.as_deref(),
+            self.env_prefix.as_ref(),
+            None,
+        )?;
+
+        if let Some(dir) = top_level_opts.log_dir {
+            opts.logging.dir = dir;
+        }
+
+        if top_level_opts.log_level.is_some() {
+            opts.logging.level = top_level_opts.log_level;
+        }
+
+        if let Some(addr) = &self.bind_addr {
+            opts.bind_addr = addr.clone();
+        }
+
+        if let Some(addr) = &self.server_addr {
+            opts.server_addr = addr.clone();
+        }
+
+        if let Some(addr) = &self.store_addr {
+            opts.store_addr = addr.clone();
+        }
+
+        if let Some(selector_type) = &self.selector {
+            opts.selector = selector_type[..]
+                .try_into()
+                .context(error::UnsupportedSelectorTypeSnafu { selector_type })?;
+        }
+
+        opts.use_memory_store = self.use_memory_store;
+
+        opts.disable_region_failover = self.disable_region_failover;
+
+        if let Some(http_addr) = &self.http_addr {
+            opts.http_opts.addr = http_addr.clone();
+        }
+
+        if let Some(http_timeout) = self.http_timeout {
+            opts.http_opts.timeout = Duration::from_secs(http_timeout);
+        }
+
+        // Disable dashboard in metasrv.
+        opts.http_opts.disable_dashboard = true;
+
+        Ok(Options::Metasrv(Box::new(opts)))
+    }
+
+    async fn build(self, opts: MetaSrvOptions) -> Result<Instance> {
         logging::info!("MetaSrv start command: {:#?}", self);
 
-        let opts: MetaSrvOptions = self.try_into()?;
-
         logging::info!("MetaSrv options: {:#?}", opts);
+
         let instance = MetaSrvInstance::new(opts)
             .await
             .context(error::BuildMetaServerSnafu)?;
@@ -96,41 +167,6 @@ impl StartCommand {
     }
 }
 
-impl TryFrom<StartCommand> for MetaSrvOptions {
-    type Error = Error;
-
-    fn try_from(cmd: StartCommand) -> Result<Self> {
-        let mut opts: MetaSrvOptions = if let Some(path) = cmd.config_file {
-            toml_loader::from_file!(&path)?
-        } else {
-            MetaSrvOptions::default()
-        };
-
-        if let Some(addr) = cmd.bind_addr {
-            opts.bind_addr = addr;
-        }
-        if let Some(addr) = cmd.server_addr {
-            opts.server_addr = addr;
-        }
-        if let Some(addr) = cmd.store_addr {
-            opts.store_addr = addr;
-        }
-        if let Some(selector_type) = &cmd.selector {
-            opts.selector = selector_type[..]
-                .try_into()
-                .context(error::UnsupportedSelectorTypeSnafu { selector_type })?;
-            info!("Using {} selector", selector_type);
-        }
-
-        if cmd.use_memory_store {
-            warn!("Using memory store for Meta. Make sure you are in running tests.");
-            opts.use_memory_store = true;
-        }
-
-        Ok(opts)
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use std::io::Write;
@@ -139,6 +175,7 @@ mod tests {
     use meta_srv::selector::SelectorType;
 
     use super::*;
+    use crate::options::ENV_VAR_SEP;
 
     #[test]
     fn test_read_from_cmd() {
@@ -146,13 +183,13 @@ mod tests {
             bind_addr: Some("127.0.0.1:3002".to_string()),
             server_addr: Some("127.0.0.1:3002".to_string()),
             store_addr: Some("127.0.0.1:2380".to_string()),
-            config_file: None,
             selector: Some("LoadBased".to_string()),
-            use_memory_store: false,
+            ..Default::default()
         };
-        let options: MetaSrvOptions = cmd.try_into().unwrap();
+
+        let Options::Metasrv(options) =
+        cmd.load_options(TopLevelOptions::default()).unwrap() else { unreachable!() };
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
-        assert_eq!("127.0.0.1:3002".to_string(), options.server_addr);
         assert_eq!("127.0.0.1:2380".to_string(), options.store_addr);
         assert_eq!(SelectorType::LoadBased, options.selector);
     }
@@ -167,22 +204,116 @@ mod tests {
             datanode_lease_secs = 15
             selector = "LeaseBased"
             use_memory_store = false
+
+            [logging]
+            level = "debug"
+            dir = "/tmp/greptimedb/test/logs"
         "#;
         write!(file, "{}", toml_str).unwrap();
 
         let cmd = StartCommand {
-            bind_addr: None,
-            server_addr: None,
-            store_addr: None,
-            selector: None,
             config_file: Some(file.path().to_str().unwrap().to_string()),
-            use_memory_store: false,
+            ..Default::default()
         };
-        let options: MetaSrvOptions = cmd.try_into().unwrap();
+
+        let Options::Metasrv(options) =
+            cmd.load_options(TopLevelOptions::default()).unwrap() else { unreachable!() };
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
         assert_eq!("127.0.0.1:3002".to_string(), options.server_addr);
         assert_eq!("127.0.0.1:2379".to_string(), options.store_addr);
         assert_eq!(15, options.datanode_lease_secs);
         assert_eq!(SelectorType::LeaseBased, options.selector);
+        assert_eq!("debug", options.logging.level.as_ref().unwrap());
+        assert_eq!("/tmp/greptimedb/test/logs".to_string(), options.logging.dir);
+    }
+
+    #[test]
+    fn test_top_level_options() {
+        let cmd = StartCommand {
+            bind_addr: Some("127.0.0.1:3002".to_string()),
+            server_addr: Some("127.0.0.1:3002".to_string()),
+            store_addr: Some("127.0.0.1:2380".to_string()),
+            selector: Some("LoadBased".to_string()),
+            ..Default::default()
+        };
+
+        let options = cmd
+            .load_options(TopLevelOptions {
+                log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
+                log_level: Some("debug".to_string()),
+            })
+            .unwrap();
+
+        let logging_opt = options.logging_options();
+        assert_eq!("/tmp/greptimedb/test/logs", logging_opt.dir);
+        assert_eq!("debug", logging_opt.level.as_ref().unwrap());
+    }
+
+    #[test]
+    fn test_config_precedence_order() {
+        let mut file = create_named_temp_file();
+        let toml_str = r#"
+            server_addr = "127.0.0.1:3002"
+            datanode_lease_secs = 15
+            selector = "LeaseBased"
+            use_memory_store = false
+
+            [http_options]
+            addr = "127.0.0.1:4000"
+
+            [logging]
+            level = "debug"
+            dir = "/tmp/greptimedb/test/logs"
+        "#;
+        write!(file, "{}", toml_str).unwrap();
+
+        let env_prefix = "METASRV_UT";
+        temp_env::with_vars(
+            vec![
+                (
+                    // bind_addr = 127.0.0.1:14002
+                    vec![env_prefix.to_string(), "bind_addr".to_uppercase()].join(ENV_VAR_SEP),
+                    Some("127.0.0.1:14002"),
+                ),
+                (
+                    // server_addr = 127.0.0.1:13002
+                    vec![env_prefix.to_string(), "server_addr".to_uppercase()].join(ENV_VAR_SEP),
+                    Some("127.0.0.1:13002"),
+                ),
+                (
+                    // http_options.addr = 127.0.0.1:24000
+                    vec![
+                        env_prefix.to_string(),
+                        "http_options".to_uppercase(),
+                        "addr".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("127.0.0.1:24000"),
+                ),
+            ],
+            || {
+                let command = StartCommand {
+                    http_addr: Some("127.0.0.1:14000".to_string()),
+                    config_file: Some(file.path().to_str().unwrap().to_string()),
+                    env_prefix: env_prefix.to_string(),
+                    ..Default::default()
+                };
+
+                let Options::Metasrv(opts) =
+                    command.load_options(TopLevelOptions::default()).unwrap() else {unreachable!()};
+
+                // Should be read from env, env > default values.
+                assert_eq!(opts.bind_addr, "127.0.0.1:14002");
+
+                // Should be read from config file, config file > env > default values.
+                assert_eq!(opts.server_addr, "127.0.0.1:3002");
+
+                // Should be read from cli, cli > config file > env > default values.
+                assert_eq!(opts.http_opts.addr, "127.0.0.1:14000");
+
+                // Should be default value.
+                assert_eq!(opts.store_addr, "127.0.0.1:2379");
+            },
+        );
     }
 }

src/cmd/src/options.rs

@@ -0,0 +1,274 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_telemetry::logging::LoggingOptions;
+use config::{Config, Environment, File, FileFormat};
+use datanode::datanode::DatanodeOptions;
+use frontend::frontend::FrontendOptions;
+use meta_srv::metasrv::MetaSrvOptions;
+use serde::{Deserialize, Serialize};
+use snafu::ResultExt;
+
+use crate::error::{LoadLayeredConfigSnafu, Result};
+
+pub const ENV_VAR_SEP: &str = "__";
+pub const ENV_LIST_SEP: &str = ",";
+
+pub struct MixOptions {
+    pub fe_opts: FrontendOptions,
+    pub dn_opts: DatanodeOptions,
+    pub logging: LoggingOptions,
+}
+
+pub enum Options {
+    Datanode(Box<DatanodeOptions>),
+    Frontend(Box<FrontendOptions>),
+    Metasrv(Box<MetaSrvOptions>),
+    Standalone(Box<MixOptions>),
+    Cli(Box<LoggingOptions>),
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct TopLevelOptions {
+    pub log_dir: Option<String>,
+    pub log_level: Option<String>,
+}
+
+impl Options {
+    pub fn logging_options(&self) -> &LoggingOptions {
+        match self {
+            Options::Datanode(opts) => &opts.logging,
+            Options::Frontend(opts) => &opts.logging,
+            Options::Metasrv(opts) => &opts.logging,
+            Options::Standalone(opts) => &opts.logging,
+            Options::Cli(opts) => opts,
+        }
+    }
+
+    /// Load the configuration from multiple sources and merge them.
+    /// The precedence order is: config file > environment variables > default values.
+    /// `env_prefix` is the prefix of environment variables, e.g. "FRONTEND__xxx".
+    /// The function will use dunder(double underscore) `__` as the separator for environment variables, for example:
+    /// `DATANODE__STORAGE__MANIFEST__CHECKPOINT_MARGIN` will be mapped to `DatanodeOptions.storage.manifest.checkpoint_margin` field in the configuration.
+    /// `list_keys` is the list of keys that should be parsed as a list, for example, you can pass `Some(&["meta_client_options.metasrv_addrs"]` to parse `GREPTIMEDB_METASRV__META_CLIENT_OPTIONS__METASRV_ADDRS` as a list.
+    /// The function will use comma `,` as the separator for list values, for example: `127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003`.
+    pub fn load_layered_options<'de, T: Serialize + Deserialize<'de> + Default>(
+        config_file: Option<&str>,
+        env_prefix: &str,
+        list_keys: Option<&[&str]>,
+    ) -> Result<T> {
+        let default_opts = T::default();
+
+        let env_source = {
+            let mut env = Environment::default();
+
+            if !env_prefix.is_empty() {
+                env = env.prefix(env_prefix);
+            }
+
+            if let Some(list_keys) = list_keys {
+                env = env.list_separator(ENV_LIST_SEP);
+                for key in list_keys {
+                    env = env.with_list_parse_key(key);
+                }
+            }
+
+            env.try_parsing(true)
+                .separator(ENV_VAR_SEP)
+                .ignore_empty(true)
+        };
+
+        // Add default values and environment variables as the sources of the configuration.
+        let mut layered_config = Config::builder()
+            .add_source(Config::try_from(&default_opts).context(LoadLayeredConfigSnafu)?)
+            .add_source(env_source);
+
+        // Add config file as the source of the configuration if it is specified.
+        if let Some(config_file) = config_file {
+            layered_config = layered_config.add_source(File::new(config_file, FileFormat::Toml));
+        }
+
+        let opts = layered_config
+            .build()
+            .context(LoadLayeredConfigSnafu)?
+            .try_deserialize()
+            .context(LoadLayeredConfigSnafu)?;
+
+        Ok(opts)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::Write;
+    use std::time::Duration;
+
+    use common_test_util::temp_dir::create_named_temp_file;
+    use datanode::datanode::{DatanodeOptions, ObjectStoreConfig};
+
+    use super::*;
+
+    #[test]
+    fn test_load_layered_options() {
+        let mut file = create_named_temp_file();
+        let toml_str = r#"
+            mode = "distributed"
+            enable_memory_catalog = false
+            rpc_addr = "127.0.0.1:3001"
+            rpc_hostname = "127.0.0.1"
+            rpc_runtime_size = 8
+            mysql_addr = "127.0.0.1:4406"
+            mysql_runtime_size = 2
+
+            [meta_client_options]
+            timeout_millis = 3000
+            connect_timeout_millis = 5000
+            tcp_nodelay = true
+
+            [wal]
+            dir = "/tmp/greptimedb/wal"
+            file_size = "1GB"
+            purge_threshold = "50GB"
+            purge_interval = "10m"
+            read_batch_size = 128
+            sync_write = false
+
+            [storage.compaction]
+            max_inflight_tasks = 3
+            max_files_in_level0 = 7
+            max_purge_tasks = 32
+
+            [logging]
+            level = "debug"
+            dir = "/tmp/greptimedb/test/logs"
+        "#;
+        write!(file, "{}", toml_str).unwrap();
+
+        let env_prefix = "DATANODE_UT";
+        temp_env::with_vars(
+            // The following environment variables will be used to override the values in the config file.
+            vec![
+                (
+                    // storage.manifest.checkpoint_margin = 99
+                    vec![
+                        env_prefix.to_string(),
+                        "storage".to_uppercase(),
+                        "manifest".to_uppercase(),
+                        "checkpoint_margin".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("99"),
+                ),
+                (
+                    // storage.type = S3
+                    vec![
+                        env_prefix.to_string(),
+                        "storage".to_uppercase(),
+                        "type".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("S3"),
+                ),
+                (
+                    // storage.bucket = mybucket
+                    vec![
+                        env_prefix.to_string(),
+                        "storage".to_uppercase(),
+                        "bucket".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("mybucket"),
+                ),
+                (
+                    // storage.manifest.gc_duration = 42s
+                    vec![
+                        env_prefix.to_string(),
+                        "storage".to_uppercase(),
+                        "manifest".to_uppercase(),
+                        "gc_duration".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("42s"),
+                ),
+                (
+                    // storage.manifest.checkpoint_on_startup = true
+                    vec![
+                        env_prefix.to_string(),
+                        "storage".to_uppercase(),
+                        "manifest".to_uppercase(),
+                        "checkpoint_on_startup".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("true"),
+                ),
+                (
+                    // wal.dir = /other/wal/dir
+                    vec![
+                        env_prefix.to_string(),
+                        "wal".to_uppercase(),
+                        "dir".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("/other/wal/dir"),
+                ),
+                (
+                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    vec![
+                        env_prefix.to_string(),
+                        "meta_client_options".to_uppercase(),
+                        "metasrv_addrs".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003"),
+                ),
+            ],
+            || {
+                let opts: DatanodeOptions = Options::load_layered_options(
+                    Some(file.path().to_str().unwrap()),
+                    env_prefix,
+                    DatanodeOptions::env_list_keys(),
+                )
+                .unwrap();
+
+                // Check the configs from environment variables.
+                assert_eq!(opts.storage.manifest.checkpoint_margin, Some(99));
+                match opts.storage.store {
+                    ObjectStoreConfig::S3(s3_config) => {
+                        assert_eq!(s3_config.bucket, "mybucket".to_string());
+                    }
+                    _ => panic!("unexpected store type"),
+                }
+                assert_eq!(
+                    opts.storage.manifest.gc_duration,
+                    Some(Duration::from_secs(42))
+                );
+                assert!(opts.storage.manifest.checkpoint_on_startup);
+                assert_eq!(
+                    opts.meta_client_options.unwrap().metasrv_addrs,
+                    vec![
+                        "127.0.0.1:3001".to_string(),
+                        "127.0.0.1:3002".to_string(),
+                        "127.0.0.1:3003".to_string()
+                    ]
+                );
+
+                // Should be the values from config file, not environment variables.
+                assert_eq!(opts.wal.dir.unwrap(), "/tmp/greptimedb/wal");
+
+                // Should be default values.
+                assert_eq!(opts.node_id, None);
+            },
+        );
+    }
+}

src/cmd/src/standalone.rs

@@ -17,28 +17,27 @@ use std::sync::Arc;
 use clap::Parser;
 use common_base::Plugins;
 use common_telemetry::info;
-use datanode::datanode::{
-    CompactionConfig, Datanode, DatanodeOptions, ObjectStoreConfig, ProcedureConfig, WalConfig,
-};
+use common_telemetry::logging::LoggingOptions;
+use datanode::datanode::{Datanode, DatanodeOptions, ProcedureConfig, StorageConfig, WalConfig};
 use datanode::instance::InstanceRef;
 use frontend::frontend::FrontendOptions;
-use frontend::grpc::GrpcOptions;
-use frontend::influxdb::InfluxdbOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
-use frontend::mysql::MysqlOptions;
-use frontend::opentsdb::OpentsdbOptions;
-use frontend::postgres::PostgresOptions;
-use frontend::prom::PromOptions;
-use frontend::prometheus::PrometheusOptions;
+use frontend::service_config::{
+    GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromOptions,
+    PrometheusOptions,
+};
 use serde::{Deserialize, Serialize};
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
 use snafu::ResultExt;
 
-use crate::error::{Error, IllegalConfigSnafu, Result, StartDatanodeSnafu, StartFrontendSnafu};
+use crate::error::{
+    IllegalConfigSnafu, Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu,
+    StartFrontendSnafu,
+};
 use crate::frontend::load_frontend_plugins;
-use crate::toml_loader;
+use crate::options::{MixOptions, Options, TopLevelOptions};
 
 #[derive(Parser)]
 pub struct Command {
@@ -47,8 +46,16 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self) -> Result<Instance> {
-        self.subcmd.build().await
+    pub async fn build(
+        self,
+        fe_opts: FrontendOptions,
+        dn_opts: DatanodeOptions,
+    ) -> Result<Instance> {
+        self.subcmd.build(fe_opts, dn_opts).await
+    }
+
+    pub fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
+        self.subcmd.load_options(top_level_options)
     }
 }
 
@@ -58,9 +65,15 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self) -> Result<Instance> {
+    async fn build(self, fe_opts: FrontendOptions, dn_opts: DatanodeOptions) -> Result<Instance> {
         match self {
-            SubCommand::Start(cmd) => cmd.build().await,
+            SubCommand::Start(cmd) => cmd.build(fe_opts, dn_opts).await,
+        }
+    }
+
+    fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
+        match self {
+            SubCommand::Start(cmd) => cmd.load_options(top_level_options),
         }
     }
 }
@@ -79,9 +92,9 @@ pub struct StandaloneOptions {
     pub prometheus_options: Option<PrometheusOptions>,
     pub prom_options: Option<PromOptions>,
     pub wal: WalConfig,
-    pub storage: ObjectStoreConfig,
-    pub compaction: CompactionConfig,
-    pub procedure: Option<ProcedureConfig>,
+    pub storage: StorageConfig,
+    pub procedure: ProcedureConfig,
+    pub logging: LoggingOptions,
 }
 
 impl Default for StandaloneOptions {
@@ -98,9 +111,9 @@ impl Default for StandaloneOptions {
             prometheus_options: Some(PrometheusOptions::default()),
             prom_options: Some(PromOptions::default()),
             wal: WalConfig::default(),
-            storage: ObjectStoreConfig::default(),
-            compaction: CompactionConfig::default(),
-            procedure: None,
+            storage: StorageConfig::default(),
+            procedure: ProcedureConfig::default(),
+            logging: LoggingOptions::default(),
         }
     }
 }
@@ -118,6 +131,7 @@ impl StandaloneOptions {
             prometheus_options: self.prometheus_options,
             prom_options: self.prom_options,
             meta_client_options: None,
+            logging: self.logging,
         }
     }
 
@@ -126,7 +140,6 @@ impl StandaloneOptions {
             enable_memory_catalog: self.enable_memory_catalog,
             wal: self.wal,
             storage: self.storage,
-            compaction: self.compaction,
             procedure: self.procedure,
             ..Default::default()
         }
@@ -139,7 +152,7 @@ pub struct Instance {
 }
 
 impl Instance {
-    pub async fn run(&mut self) -> Result<()> {
+    pub async fn start(&mut self) -> Result<()> {
         // Start datanode instance before starting services, to avoid requests come in before internal components are started.
         self.datanode
             .start_instance()
@@ -152,12 +165,22 @@ impl Instance {
     }
 
     pub async fn stop(&self) -> Result<()> {
-        // TODO: handle standalone shutdown
+        self.frontend
+            .shutdown()
+            .await
+            .context(ShutdownFrontendSnafu)?;
+
+        self.datanode
+            .shutdown_instance()
+            .await
+            .context(ShutdownDatanodeSnafu)?;
+        info!("Datanode instance stopped.");
+
         Ok(())
     }
 }
 
-#[derive(Debug, Parser)]
+#[derive(Debug, Default, Parser)]
 struct StartCommand {
     #[clap(long)]
     http_addr: Option<String>,
@@ -185,24 +208,101 @@ struct StartCommand {
     tls_key_path: Option<String>,
     #[clap(long)]
     user_provider: Option<String>,
+    #[clap(long, default_value = "GREPTIMEDB_STANDALONE")]
+    env_prefix: String,
 }
 
 impl StartCommand {
-    async fn build(self) -> Result<Instance> {
-        let enable_memory_catalog = self.enable_memory_catalog;
-        let config_file = self.config_file.clone();
-        let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
-        let fe_opts = FrontendOptions::try_from(self)?;
-        let dn_opts: DatanodeOptions = {
-            let mut opts: StandaloneOptions = if let Some(path) = config_file {
-                toml_loader::from_file!(&path)?
-            } else {
-                StandaloneOptions::default()
-            };
-            opts.enable_memory_catalog = enable_memory_catalog;
-            opts.datanode_options()
-        };
+    fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
+        let mut opts: StandaloneOptions = Options::load_layered_options(
+            self.config_file.as_deref(),
+            self.env_prefix.as_ref(),
+            None,
+        )?;
 
+        opts.enable_memory_catalog = self.enable_memory_catalog;
+
+        opts.mode = Mode::Standalone;
+
+        if let Some(dir) = top_level_options.log_dir {
+            opts.logging.dir = dir;
+        }
+
+        if top_level_options.log_level.is_some() {
+            opts.logging.level = top_level_options.log_level;
+        }
+
+        let tls_opts = TlsOption::new(
+            self.tls_mode.clone(),
+            self.tls_cert_path.clone(),
+            self.tls_key_path.clone(),
+        );
+
+        if let Some(addr) = &self.http_addr {
+            if let Some(http_opts) = &mut opts.http_options {
+                http_opts.addr = addr.clone()
+            }
+        }
+
+        if let Some(addr) = &self.rpc_addr {
+            // frontend grpc addr conflict with datanode default grpc addr
+            let datanode_grpc_addr = DatanodeOptions::default().rpc_addr;
+            if addr.eq(&datanode_grpc_addr) {
+                return IllegalConfigSnafu {
+                    msg: format!(
+                        "gRPC listen address conflicts with datanode reserved gRPC addr: {datanode_grpc_addr}",
+                    ),
+                }
+                .fail();
+            }
+            if let Some(grpc_opts) = &mut opts.grpc_options {
+                grpc_opts.addr = addr.clone()
+            }
+        }
+
+        if let Some(addr) = &self.mysql_addr {
+            if let Some(mysql_opts) = &mut opts.mysql_options {
+                mysql_opts.addr = addr.clone();
+                mysql_opts.tls = tls_opts.clone();
+            }
+        }
+
+        if let Some(addr) = &self.prom_addr {
+            opts.prom_options = Some(PromOptions { addr: addr.clone() })
+        }
+
+        if let Some(addr) = &self.postgres_addr {
+            if let Some(postgres_opts) = &mut opts.postgres_options {
+                postgres_opts.addr = addr.clone();
+                postgres_opts.tls = tls_opts;
+            }
+        }
+
+        if let Some(addr) = &self.opentsdb_addr {
+            if let Some(opentsdb_addr) = &mut opts.opentsdb_options {
+                opentsdb_addr.addr = addr.clone();
+            }
+        }
+
+        if self.influxdb_enable {
+            opts.influxdb_options = Some(InfluxdbOptions { enable: true });
+        }
+
+        let fe_opts = opts.clone().frontend_options();
+        let logging = opts.logging.clone();
+        let dn_opts = opts.datanode_options();
+
+        Ok(Options::Standalone(Box::new(MixOptions {
+            fe_opts,
+            dn_opts,
+            logging,
+        })))
+    }
+
+    async fn build(self, fe_opts: FrontendOptions, dn_opts: DatanodeOptions) -> Result<Instance> {
+        let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
+
+        info!("Standalone start command: {:#?}", self);
         info!(
             "Standalone frontend options: {:#?}, datanode options: {:#?}",
             fe_opts, dn_opts
@@ -215,7 +315,7 @@ impl StartCommand {
         let mut frontend = build_frontend(plugins.clone(), datanode.get_instance()).await?;
 
         frontend
-            .build_servers(&fe_opts, plugins)
+            .build_servers(&fe_opts)
             .await
             .context(StartFrontendSnafu)?;
 
@@ -228,124 +328,105 @@ async fn build_frontend(
     plugins: Arc<Plugins>,
     datanode_instance: InstanceRef,
 ) -> Result<FeInstance> {
-    let mut frontend_instance = FeInstance::new_standalone(datanode_instance.clone());
-    frontend_instance.set_script_handler(datanode_instance);
+    let mut frontend_instance = FeInstance::try_new_standalone(datanode_instance.clone())
+        .await
+        .context(StartFrontendSnafu)?;
     frontend_instance.set_plugins(plugins.clone());
     Ok(frontend_instance)
 }
 
-impl TryFrom<StartCommand> for FrontendOptions {
-    type Error = Error;
-
-    fn try_from(cmd: StartCommand) -> std::result::Result<Self, Self::Error> {
-        let opts: StandaloneOptions = if let Some(path) = cmd.config_file {
-            toml_loader::from_file!(&path)?
-        } else {
-            StandaloneOptions::default()
-        };
-
-        let mut opts = opts.frontend_options();
-
-        opts.mode = Mode::Standalone;
-
-        if let Some(addr) = cmd.http_addr {
-            opts.http_options = Some(HttpOptions {
-                addr,
-                ..Default::default()
-            });
-        }
-        if let Some(addr) = cmd.rpc_addr {
-            // frontend grpc addr conflict with datanode default grpc addr
-            let datanode_grpc_addr = DatanodeOptions::default().rpc_addr;
-            if addr == datanode_grpc_addr {
-                return IllegalConfigSnafu {
-                    msg: format!(
-                        "gRPC listen address conflicts with datanode reserved gRPC addr: {datanode_grpc_addr}",
-                    ),
-                }
-                .fail();
-            }
-            opts.grpc_options = Some(GrpcOptions {
-                addr,
-                ..Default::default()
-            });
-        }
-
-        if let Some(addr) = cmd.mysql_addr {
-            opts.mysql_options = Some(MysqlOptions {
-                addr,
-                ..Default::default()
-            })
-        }
-
-        if let Some(addr) = cmd.prom_addr {
-            opts.prom_options = Some(PromOptions { addr })
-        }
-
-        if let Some(addr) = cmd.postgres_addr {
-            opts.postgres_options = Some(PostgresOptions {
-                addr,
-                ..Default::default()
-            })
-        }
-
-        if let Some(addr) = cmd.opentsdb_addr {
-            opts.opentsdb_options = Some(OpentsdbOptions {
-                addr,
-                ..Default::default()
-            });
-        }
-
-        if cmd.influxdb_enable {
-            opts.influxdb_options = Some(InfluxdbOptions { enable: true });
-        }
-
-        let tls_option = TlsOption::new(cmd.tls_mode, cmd.tls_cert_path, cmd.tls_key_path);
-
-        if let Some(mut mysql_options) = opts.mysql_options {
-            mysql_options.tls = tls_option.clone();
-            opts.mysql_options = Some(mysql_options);
-        }
-
-        if let Some(mut postgres_options) = opts.postgres_options {
-            postgres_options.tls = tls_option;
-            opts.postgres_options = Some(postgres_options);
-        }
-
-        Ok(opts)
-    }
-}
-
 #[cfg(test)]
 mod tests {
+    use std::default::Default;
+    use std::io::Write;
     use std::time::Duration;
 
+    use common_test_util::temp_dir::create_named_temp_file;
     use servers::auth::{Identity, Password, UserProviderRef};
+    use servers::Mode;
 
     use super::*;
+    use crate::options::ENV_VAR_SEP;
 
-    #[test]
-    fn test_read_config_file() {
-        let cmd = StartCommand {
-            http_addr: None,
-            rpc_addr: None,
-            mysql_addr: None,
-            prom_addr: None,
-            postgres_addr: None,
-            opentsdb_addr: None,
-            config_file: Some(format!(
-                "{}/../../config/standalone.example.toml",
-                std::env::current_dir().unwrap().as_path().to_str().unwrap()
-            )),
-            influxdb_enable: false,
-            enable_memory_catalog: false,
-            tls_mode: None,
-            tls_cert_path: None,
-            tls_key_path: None,
-            user_provider: None,
+    #[tokio::test]
+    async fn test_try_from_start_command_to_anymap() {
+        let command = StartCommand {
+            user_provider: Some("static_user_provider:cmd:test=test".to_string()),
+            ..Default::default()
         };
 
-        let fe_opts = FrontendOptions::try_from(cmd).unwrap();
+        let plugins = load_frontend_plugins(&command.user_provider);
+        assert!(plugins.is_ok());
+        let plugins = plugins.unwrap();
+        let provider = plugins.get::<UserProviderRef>();
+        assert!(provider.is_some());
+        let provider = provider.unwrap();
+        let result = provider
+            .authenticate(
+                Identity::UserId("test", None),
+                Password::PlainText("test".to_string().into()),
+            )
+            .await;
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_toml() {
+        let opts = StandaloneOptions::default();
+        let toml_string = toml::to_string(&opts).unwrap();
+        let _parsed: StandaloneOptions = toml::from_str(&toml_string).unwrap();
+    }
+
+    #[test]
+    fn test_read_from_config_file() {
+        let mut file = create_named_temp_file();
+        let toml_str = r#"
+            mode = "distributed"
+
+            enable_memory_catalog = true
+
+            [wal]
+            dir = "/tmp/greptimedb/test/wal"
+            file_size = "1GB"
+            purge_threshold = "50GB"
+            purge_interval = "10m"
+            read_batch_size = 128
+            sync_write = false
+
+            [storage]
+            type = "S3"
+            access_key_id = "access_key_id"
+            secret_access_key = "secret_access_key"
+
+            [storage.compaction]
+            max_inflight_tasks = 3
+            max_files_in_level0 = 7
+            max_purge_tasks = 32
+
+            [storage.manifest]
+            checkpoint_margin = 9
+            gc_duration = '7s'
+            checkpoint_on_startup = true
+
+            [http_options]
+            addr = "127.0.0.1:4000"
+            timeout = "30s"
+
+            [logging]
+            level = "debug"
+            dir = "/tmp/greptimedb/test/logs"
+        "#;
+        write!(file, "{}", toml_str).unwrap();
+        let cmd = StartCommand {
+            config_file: Some(file.path().to_str().unwrap().to_string()),
+            user_provider: Some("static_user_provider:cmd:test=test".to_string()),
+            ..Default::default()
+        };
+
+        let Options::Standalone(options) = cmd.load_options(TopLevelOptions::default()).unwrap() else {unreachable!()};
+        let fe_opts = options.fe_opts;
+        let dn_opts = options.dn_opts;
+        let logging_opts = options.logging;
         assert_eq!(Mode::Standalone, fe_opts.mode);
         assert_eq!(
             "127.0.0.1:4000".to_string(),
@@ -369,42 +450,125 @@ mod tests {
             fe_opts.mysql_options.as_ref().unwrap().reject_no_database
         );
         assert!(fe_opts.influxdb_options.as_ref().unwrap().enable);
-    }
 
-    #[tokio::test]
-    async fn test_try_from_start_command_to_anymap() {
-        let command = StartCommand {
-            http_addr: None,
-            rpc_addr: None,
-            prom_addr: None,
-            mysql_addr: None,
-            postgres_addr: None,
-            opentsdb_addr: None,
-            config_file: None,
-            influxdb_enable: false,
-            enable_memory_catalog: false,
-            tls_mode: None,
-            tls_cert_path: None,
-            tls_key_path: None,
-            user_provider: Some("static_user_provider:cmd:test=test".to_string()),
-        };
+        assert_eq!("/tmp/greptimedb/test/wal", dn_opts.wal.dir.unwrap());
+        match &dn_opts.storage.store {
+            datanode::datanode::ObjectStoreConfig::S3(s3_config) => {
+                assert_eq!(
+                    "Secret([REDACTED alloc::string::String])".to_string(),
+                    format!("{:?}", s3_config.access_key_id)
+                );
+            }
+            _ => {
+                unreachable!()
+            }
+        }
 
-        let plugins = load_frontend_plugins(&command.user_provider);
-        assert!(plugins.is_ok());
-        let plugins = plugins.unwrap();
-        let provider = plugins.get::<UserProviderRef>();
-        assert!(provider.is_some());
-        let provider = provider.unwrap();
-        let result = provider
-            .authenticate(Identity::UserId("test", None), Password::PlainText("test"))
-            .await;
-        assert!(result.is_ok());
+        assert_eq!("debug", logging_opts.level.as_ref().unwrap());
+        assert_eq!("/tmp/greptimedb/test/logs".to_string(), logging_opts.dir);
     }
 
     #[test]
-    fn test_toml() {
-        let opts = StandaloneOptions::default();
-        let toml_string = toml::to_string(&opts).unwrap();
-        let _parsed: StandaloneOptions = toml::from_str(&toml_string).unwrap();
+    fn test_top_level_options() {
+        let cmd = StartCommand {
+            user_provider: Some("static_user_provider:cmd:test=test".to_string()),
+            ..Default::default()
+        };
+
+        let Options::Standalone(opts) = cmd
+            .load_options(TopLevelOptions {
+                log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
+                log_level: Some("debug".to_string()),
+            })
+            .unwrap() else {
+            unreachable!()
+        };
+
+        assert_eq!("/tmp/greptimedb/test/logs", opts.logging.dir);
+        assert_eq!("debug", opts.logging.level.unwrap());
+    }
+
+    #[test]
+    fn test_config_precedence_order() {
+        let mut file = create_named_temp_file();
+        let toml_str = r#"
+            mode = "standalone"
+
+            [http_options]
+            addr = "127.0.0.1:4000"
+
+            [logging]
+            level = "debug"
+        "#;
+        write!(file, "{}", toml_str).unwrap();
+
+        let env_prefix = "STANDALONE_UT";
+        temp_env::with_vars(
+            vec![
+                (
+                    // logging.dir = /other/log/dir
+                    vec![
+                        env_prefix.to_string(),
+                        "logging".to_uppercase(),
+                        "dir".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("/other/log/dir"),
+                ),
+                (
+                    // logging.level = info
+                    vec![
+                        env_prefix.to_string(),
+                        "logging".to_uppercase(),
+                        "level".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("info"),
+                ),
+                (
+                    // http_options.addr = 127.0.0.1:24000
+                    vec![
+                        env_prefix.to_string(),
+                        "http_options".to_uppercase(),
+                        "addr".to_uppercase(),
+                    ]
+                    .join(ENV_VAR_SEP),
+                    Some("127.0.0.1:24000"),
+                ),
+            ],
+            || {
+                let command = StartCommand {
+                    config_file: Some(file.path().to_str().unwrap().to_string()),
+                    http_addr: Some("127.0.0.1:14000".to_string()),
+                    env_prefix: env_prefix.to_string(),
+                    ..Default::default()
+                };
+
+                let top_level_opts = TopLevelOptions {
+                    log_dir: None,
+                    log_level: None,
+                };
+                let Options::Standalone(opts) =
+                    command.load_options(top_level_opts).unwrap() else {unreachable!()};
+
+                // Should be read from env, env > default values.
+                assert_eq!(opts.logging.dir, "/other/log/dir");
+
+                // Should be read from config file, config file > env > default values.
+                assert_eq!(opts.logging.level.as_ref().unwrap(), "debug");
+
+                // Should be read from cli, cli > config file > env > default values.
+                assert_eq!(
+                    opts.fe_opts.http_options.as_ref().unwrap().addr,
+                    "127.0.0.1:14000"
+                );
+
+                // Should be default value.
+                assert_eq!(
+                    opts.fe_opts.grpc_options.unwrap().addr,
+                    GrpcOptions::default().addr
+                );
+            },
+        );
     }
 }

src/cmd/src/toml_loader.rs

@@ -1,94 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-macro_rules! from_file {
-    ($path: expr) => {
-        toml::from_str(
-            &std::fs::read_to_string($path)
-                .context(crate::error::ReadConfigSnafu { path: $path })?,
-        )
-        .context(crate::error::ParseConfigSnafu)
-    };
-}
-
-pub(crate) use from_file;
-
-#[cfg(test)]
-mod tests {
-    use std::fs::File;
-    use std::io::Write;
-
-    use common_test_util::temp_dir::create_temp_dir;
-    use serde::{Deserialize, Serialize};
-    use snafu::ResultExt;
-
-    use super::*;
-    use crate::error::Result;
-
-    #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)]
-    #[serde(default)]
-    struct MockConfig {
-        path: String,
-        port: u32,
-        host: String,
-    }
-
-    impl Default for MockConfig {
-        fn default() -> Self {
-            Self {
-                path: "test".to_string(),
-                port: 0,
-                host: "localhost".to_string(),
-            }
-        }
-    }
-
-    #[test]
-    fn test_from_file() -> Result<()> {
-        let config = MockConfig {
-            path: "/tmp".to_string(),
-            port: 999,
-            host: "greptime.test".to_string(),
-        };
-
-        let dir = create_temp_dir("test_from_file");
-        let test_file = format!("{}/test.toml", dir.path().to_str().unwrap());
-
-        let s = toml::to_string(&config).unwrap();
-        assert!(s.contains("host") && s.contains("path") && s.contains("port"));
-
-        let mut file = File::create(&test_file).unwrap();
-        file.write_all(s.as_bytes()).unwrap();
-
-        let loaded_config: MockConfig = from_file!(&test_file)?;
-        assert_eq!(loaded_config, config);
-
-        // Only host in file
-        let mut file = File::create(&test_file).unwrap();
-        file.write_all("host='greptime.test'\n".as_bytes()).unwrap();
-
-        let loaded_config: MockConfig = from_file!(&test_file)?;
-        assert_eq!(loaded_config.host, "greptime.test");
-        assert_eq!(loaded_config.port, 0);
-        assert_eq!(loaded_config.path, "test");
-
-        // Truncate the file.
-        let file = File::create(&test_file).unwrap();
-        file.set_len(0).unwrap();
-        let loaded_config: MockConfig = from_file!(&test_file)?;
-        assert_eq!(loaded_config, MockConfig::default());
-
-        Ok(())
-    }
-}

src/cmd/tests/cli.rs

@@ -51,7 +51,7 @@ mod tests {
     #[ignore]
     #[test]
     fn test_repl() {
-        let data_dir = create_temp_dir("data");
+        let data_home = create_temp_dir("data");
         let wal_dir = create_temp_dir("wal");
 
         let mut bin_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
@@ -65,7 +65,7 @@ mod tests {
                 "start",
                 "--rpc-addr=0.0.0.0:4321",
                 "--node-id=1",
-                &format!("--data-dir={}", data_dir.path().display()),
+                &format!("--data-home={}", data_home.path().display()),
                 &format!("--wal-dir={}", wal_dir.path().display()),
             ])
             .stdout(Stdio::null())

src/common/base/src/buffer.rs

@@ -18,7 +18,7 @@ use std::io::{Read, Write};
 use bytes::{Buf, BufMut, BytesMut};
 use common_error::prelude::ErrorExt;
 use paste::paste;
-use snafu::{ensure, Backtrace, ErrorCompat, ResultExt, Snafu};
+use snafu::{ensure, Location, ResultExt, Snafu};
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
@@ -31,29 +31,33 @@ pub enum Error {
     Overflow {
         src_len: usize,
         dst_len: usize,
-        backtrace: Backtrace,
+        location: Location,
     },
 
     #[snafu(display("Buffer underflow"))]
-    Underflow { backtrace: Backtrace },
+    Underflow { location: Location },
 
     #[snafu(display("IO operation reach EOF, source: {}", source))]
     Eof {
         source: std::io::Error,
-        backtrace: Backtrace,
+        location: Location,
     },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
 
 impl ErrorExt for Error {
-    fn backtrace_opt(&self) -> Option<&Backtrace> {
-        ErrorCompat::backtrace(self)
-    }
-
     fn as_any(&self) -> &dyn Any {
         self
     }
+
+    fn location_opt(&self) -> Option<common_error::snafu::Location> {
+        match self {
+            Error::Overflow { location, .. } => Some(*location),
+            Error::Underflow { location, .. } => Some(*location),
+            Error::Eof { location, .. } => Some(*location),
+        }
+    }
 }
 
 macro_rules! impl_read_le {

src/common/base/src/bytes.rs

@@ -92,6 +92,14 @@ impl StringBytes {
     pub fn as_utf8(&self) -> &str {
         unsafe { std::str::from_utf8_unchecked(&self.0) }
     }
+
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
 }
 
 impl From<String> for StringBytes {
@@ -178,6 +186,17 @@ mod tests {
         assert_eq!(world, &bytes);
     }
 
+    #[test]
+    fn test_bytes_len() {
+        let hello = b"hello".to_vec();
+        let bytes = Bytes::from(hello.clone());
+        assert_eq!(bytes.len(), hello.len());
+
+        let zero = b"".to_vec();
+        let bytes = Bytes::from(zero);
+        assert!(bytes.is_empty());
+    }
+
     #[test]
     fn test_string_bytes_from() {
         let hello = "hello".to_string();
@@ -191,6 +210,17 @@ mod tests {
         assert_eq!(&bytes, world);
     }
 
+    #[test]
+    fn test_string_bytes_len() {
+        let hello = "hello".to_string();
+        let bytes = StringBytes::from(hello.clone());
+        assert_eq!(bytes.len(), hello.len());
+
+        let zero = "".to_string();
+        let bytes = StringBytes::from(zero);
+        assert!(bytes.is_empty());
+    }
+
     fn check_str(expect: &str, given: &str) {
         assert_eq!(expect, given);
     }

src/common/base/src/lib.rs

@@ -15,9 +15,64 @@
 pub mod bit_vec;
 pub mod buffer;
 pub mod bytes;
+pub mod paths;
 #[allow(clippy::all)]
 pub mod readable_size;
 
+use core::any::Any;
+use std::sync::{Arc, Mutex, MutexGuard};
+
 pub use bit_vec::BitVec;
 
-pub type Plugins = anymap::Map<dyn core::any::Any + Send + Sync>;
+#[derive(Default, Clone)]
+pub struct Plugins {
+    inner: Arc<Mutex<anymap::Map<dyn Any + Send + Sync>>>,
+}
+
+impl Plugins {
+    pub fn new() -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(anymap::Map::new())),
+        }
+    }
+
+    fn lock(&self) -> MutexGuard<anymap::Map<dyn Any + Send + Sync>> {
+        self.inner.lock().unwrap()
+    }
+
+    pub fn insert<T: 'static + Send + Sync>(&self, value: T) {
+        self.lock().insert(value);
+    }
+
+    pub fn get<T: 'static + Send + Sync + Clone>(&self) -> Option<T> {
+        let binding = self.lock();
+        binding.get::<T>().cloned()
+    }
+
+    pub fn map_mut<T: 'static + Send + Sync, F, R>(&self, mapper: F) -> R
+    where
+        F: FnOnce(Option<&mut T>) -> R,
+    {
+        let mut binding = self.lock();
+        let opt = binding.get_mut::<T>();
+        mapper(opt)
+    }
+
+    pub fn map<T: 'static + Send + Sync, F, R>(&self, mapper: F) -> Option<R>
+    where
+        F: FnOnce(&T) -> R,
+    {
+        let binding = self.lock();
+        binding.get::<T>().map(mapper)
+    }
+
+    pub fn len(&self) -> usize {
+        let binding = self.lock();
+        binding.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        let binding = self.lock();
+        binding.is_empty()
+    }
+}

src/common/base/src/paths.rs

@@ -12,18 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+//! Path constants for table engines, cluster states and WAL
+/// All paths relative to data_home(file storage) or root path(S3, OSS etc).
 
-use promql_parser::parser::{Expr, Value};
+/// WAL dir for local file storage
+pub const WAL_DIR: &str = "wal/";
 
-use crate::engine::Context;
-use crate::error::Result;
+/// Data dir for table engines
+pub const DATA_DIR: &str = "data/";
 
-/// An evaluator evaluates given expressions over given fixed timestamps.
-pub struct Evaluator {}
-
-impl Evaluator {
-    pub fn eval(_ctx: &Context, _expr: &Expr) -> Result<Arc<dyn Value>> {
-        unimplemented!();
-    }
-}
+/// Cluster state dir
+pub const CLUSTER_DIR: &str = "cluster/";

src/common/base/src/readable_size.rs

@@ -53,6 +53,10 @@ impl ReadableSize {
     pub const fn as_mb(self) -> u64 {
         self.0 / MIB
     }
+
+    pub const fn as_bytes(self) -> u64 {
+        self.0
+    }
 }
 
 impl Div<u64> for ReadableSize {

src/common/catalog/src/consts.rs

@@ -21,7 +21,16 @@ pub const DEFAULT_SCHEMA_NAME: &str = "public";
 /// Reserves [0,MIN_USER_TABLE_ID) for internal usage.
 /// User defined table id starts from this value.
 pub const MIN_USER_TABLE_ID: u32 = 1024;
+/// the max system table id
+pub const MAX_SYS_TABLE_ID: u32 = MIN_USER_TABLE_ID - 1;
 /// system_catalog table id
 pub const SYSTEM_CATALOG_TABLE_ID: u32 = 0;
 /// scripts table id
 pub const SCRIPTS_TABLE_ID: u32 = 1;
+
+pub const MITO_ENGINE: &str = "mito";
+pub const IMMUTABLE_FILE_ENGINE: &str = "file";
+
+pub const SEMANTIC_TYPE_PRIMARY_KEY: &str = "PRIMARY KEY";
+pub const SEMANTIC_TYPE_FIELD: &str = "FIELD";
+pub const SEMANTIC_TYPE_TIME_INDEX: &str = "TIME INDEX";

src/common/catalog/src/error.rs

@@ -16,29 +16,26 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::prelude::{Snafu, StatusCode};
-use snafu::{Backtrace, ErrorCompat};
+use snafu::Location;
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
     #[snafu(display("Invalid catalog info: {}", key))]
-    InvalidCatalog { key: String, backtrace: Backtrace },
+    InvalidCatalog { key: String, location: Location },
 
     #[snafu(display("Failed to deserialize catalog entry value: {}", raw))]
     DeserializeCatalogEntryValue {
         raw: String,
-        backtrace: Backtrace,
+        location: Location,
         source: serde_json::error::Error,
     },
 
     #[snafu(display("Failed to serialize catalog entry value"))]
     SerializeCatalogEntryValue {
-        backtrace: Backtrace,
+        location: Location,
         source: serde_json::error::Error,
     },
-
-    #[snafu(display("Failed to parse node id: {}", key))]
-    ParseNodeId { key: String, backtrace: Backtrace },
 }
 
 impl ErrorExt for Error {
@@ -47,14 +44,9 @@ impl ErrorExt for Error {
             Error::InvalidCatalog { .. }
             | Error::DeserializeCatalogEntryValue { .. }
             | Error::SerializeCatalogEntryValue { .. } => StatusCode::Unexpected,
-            Error::ParseNodeId { .. } => StatusCode::InvalidArguments,
         }
     }
 
-    fn backtrace_opt(&self) -> Option<&Backtrace> {
-        ErrorCompat::backtrace(self)
-    }
-
     fn as_any(&self) -> &dyn Any {
         self
     }

src/common/catalog/src/lib.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use consts::DEFAULT_CATALOG_NAME;
+
 pub mod consts;
 pub mod error;
 
@@ -20,3 +22,23 @@ pub mod error;
 pub fn format_full_table_name(catalog: &str, schema: &str, table: &str) -> String {
     format!("{catalog}.{schema}.{table}")
 }
+
+/// Build db name from catalog and schema string
+pub fn build_db_string(catalog: &str, schema: &str) -> String {
+    if catalog == DEFAULT_CATALOG_NAME {
+        schema.to_string()
+    } else {
+        format!("{catalog}-{schema}")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_db_string() {
+        assert_eq!("test", build_db_string(DEFAULT_CATALOG_NAME, "test"));
+        assert_eq!("a0b1c2d3-test", build_db_string("a0b1c2d3", "test"));
+    }
+}

src/common/datasource/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "common-datasource"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+arrow.workspace = true
+arrow-schema.workspace = true
+async-compression = { version = "0.3", features = [
+    "bzip2",
+    "gzip",
+    "xz",
+    "zstd",
+    "futures-io",
+    "tokio",
+] }
+async-trait.workspace = true
+bytes = "1.1"
+common-base = { path = "../base" }
+common-error = { path = "../error" }
+common-runtime = { path = "../runtime" }
+datafusion.workspace = true
+derive_builder = "0.12"
+futures.workspace = true
+object-store = { path = "../../object-store" }
+regex = "1.7"
+snafu.workspace = true
+tokio.workspace = true
+tokio-util.workspace = true
+url = "2.3"
+paste = "1.0"
+
+[dev-dependencies]
+common-test-util = { path = "../test-util" }

src/common/datasource/src/buffered_writer.rs

@@ -0,0 +1,167 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::future::Future;
+
+use arrow::record_batch::RecordBatch;
+use async_trait::async_trait;
+use datafusion::parquet::format::FileMetaData;
+use snafu::{OptionExt, ResultExt};
+use tokio::io::{AsyncWrite, AsyncWriteExt};
+
+use crate::error::{self, Result};
+use crate::share_buffer::SharedBuffer;
+
+pub struct LazyBufferedWriter<T, U, F> {
+    path: String,
+    writer_factory: F,
+    writer: Option<T>,
+    /// None stands for [`LazyBufferedWriter`] closed.
+    encoder: Option<U>,
+    buffer: SharedBuffer,
+    rows_written: usize,
+    bytes_written: u64,
+    threshold: usize,
+}
+
+pub trait DfRecordBatchEncoder {
+    fn write(&mut self, batch: &RecordBatch) -> Result<()>;
+}
+
+#[async_trait]
+pub trait ArrowWriterCloser {
+    async fn close(mut self) -> Result<FileMetaData>;
+}
+
+impl<
+        T: AsyncWrite + Send + Unpin,
+        U: DfRecordBatchEncoder + ArrowWriterCloser,
+        F: FnMut(String) -> Fut,
+        Fut: Future<Output = Result<T>>,
+    > LazyBufferedWriter<T, U, F>
+{
+    /// Closes `LazyBufferedWriter` and optionally flushes all data to underlying storage
+    /// if any row's been written.
+    pub async fn close_with_arrow_writer(mut self) -> Result<(FileMetaData, u64)> {
+        let encoder = self
+            .encoder
+            .take()
+            .context(error::BufferedWriterClosedSnafu)?;
+        let metadata = encoder.close().await?;
+
+        // Use `rows_written` to keep a track of if any rows have been written.
+        // If no row's been written, then we can simply close the underlying
+        // writer without flush so that no file will be actually created.
+        if self.rows_written != 0 {
+            self.bytes_written += self.try_flush(true).await?;
+        }
+        // It's important to shut down! flushes all pending writes
+        self.close_inner_writer().await?;
+        Ok((metadata, self.bytes_written))
+    }
+}
+
+impl<
+        T: AsyncWrite + Send + Unpin,
+        U: DfRecordBatchEncoder,
+        F: FnMut(String) -> Fut,
+        Fut: Future<Output = Result<T>>,
+    > LazyBufferedWriter<T, U, F>
+{
+    /// Closes the writer without flushing the buffer data.
+    pub async fn close_inner_writer(&mut self) -> Result<()> {
+        if let Some(writer) = &mut self.writer {
+            writer.shutdown().await.context(error::AsyncWriteSnafu)?;
+        }
+        Ok(())
+    }
+
+    pub fn new(
+        threshold: usize,
+        buffer: SharedBuffer,
+        encoder: U,
+        path: impl AsRef<str>,
+        writer_factory: F,
+    ) -> Self {
+        Self {
+            path: path.as_ref().to_string(),
+            threshold,
+            encoder: Some(encoder),
+            buffer,
+            rows_written: 0,
+            bytes_written: 0,
+            writer_factory,
+            writer: None,
+        }
+    }
+
+    pub async fn write(&mut self, batch: &RecordBatch) -> Result<()> {
+        let encoder = self
+            .encoder
+            .as_mut()
+            .context(error::BufferedWriterClosedSnafu)?;
+        encoder.write(batch)?;
+        self.rows_written += batch.num_rows();
+        self.bytes_written += self.try_flush(false).await?;
+        Ok(())
+    }
+
+    pub async fn try_flush(&mut self, all: bool) -> Result<u64> {
+        let mut bytes_written: u64 = 0;
+
+        // Once buffered data size reaches threshold, split the data in chunks (typically 4MB)
+        // and write to underlying storage.
+        while self.buffer.buffer.lock().unwrap().len() >= self.threshold {
+            let chunk = {
+                let mut buffer = self.buffer.buffer.lock().unwrap();
+                buffer.split_to(self.threshold)
+            };
+            let size = chunk.len();
+
+            self.maybe_init_writer()
+                .await?
+                .write_all(&chunk)
+                .await
+                .context(error::AsyncWriteSnafu)?;
+
+            bytes_written += size as u64;
+        }
+
+        if all {
+            bytes_written += self.try_flush_all().await?;
+        }
+        Ok(bytes_written)
+    }
+
+    /// Only initiates underlying file writer when rows have been written.
+    async fn maybe_init_writer(&mut self) -> Result<&mut T> {
+        if let Some(ref mut writer) = self.writer {
+            Ok(writer)
+        } else {
+            let writer = (self.writer_factory)(self.path.clone()).await?;
+            Ok(self.writer.insert(writer))
+        }
+    }
+
+    async fn try_flush_all(&mut self) -> Result<u64> {
+        let remain = self.buffer.buffer.lock().unwrap().split();
+        let size = remain.len();
+        self.maybe_init_writer()
+            .await?
+            .write_all(&remain)
+            .await
+            .context(error::AsyncWriteSnafu)?;
+        Ok(size as u64)
+    }
+}

src/common/datasource/src/compression.rs

@@ -0,0 +1,180 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Display;
+use std::io;
+use std::str::FromStr;
+
+use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZstdDecoder};
+use async_compression::tokio::write;
+use bytes::Bytes;
+use futures::Stream;
+use tokio::io::{AsyncRead, AsyncWriteExt, BufReader};
+use tokio_util::io::{ReaderStream, StreamReader};
+
+use crate::error::{self, Error, Result};
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum CompressionType {
+    /// Gzip-ed file
+    Gzip,
+    /// Bzip2-ed file
+    Bzip2,
+    /// Xz-ed file (liblzma)
+    Xz,
+    /// Zstd-ed file,
+    Zstd,
+    /// Uncompressed file
+    Uncompressed,
+}
+
+impl FromStr for CompressionType {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        let s = s.to_uppercase();
+        match s.as_str() {
+            "GZIP" | "GZ" => Ok(Self::Gzip),
+            "BZIP2" | "BZ2" => Ok(Self::Bzip2),
+            "XZ" => Ok(Self::Xz),
+            "ZST" | "ZSTD" => Ok(Self::Zstd),
+            "" => Ok(Self::Uncompressed),
+            _ => error::UnsupportedCompressionTypeSnafu {
+                compression_type: s,
+            }
+            .fail(),
+        }
+    }
+}
+
+impl Display for CompressionType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(match self {
+            Self::Gzip => "GZIP",
+            Self::Bzip2 => "BZIP2",
+            Self::Xz => "XZ",
+            Self::Zstd => "ZSTD",
+            Self::Uncompressed => "",
+        })
+    }
+}
+
+impl CompressionType {
+    pub const fn is_compressed(&self) -> bool {
+        !matches!(self, &Self::Uncompressed)
+    }
+
+    pub const fn file_extension(&self) -> &'static str {
+        match self {
+            Self::Gzip => "gz",
+            Self::Bzip2 => "bz2",
+            Self::Xz => "xz",
+            Self::Zstd => "zst",
+            Self::Uncompressed => "",
+        }
+    }
+}
+
+macro_rules! impl_compression_type {
+    ($(($enum_item:ident, $prefix:ident)),*) => {
+        paste::item! {
+            impl CompressionType {
+                pub async fn encode(&self, content: impl AsRef<[u8]>) -> io::Result<Vec<u8>> {
+                    match self {
+                        $(
+                            CompressionType::$enum_item => {
+                                let mut buffer = Vec::with_capacity(content.as_ref().len());
+                                let mut encoder = write::[<$prefix Encoder>]::new(&mut buffer);
+                                encoder.write_all(content.as_ref()).await?;
+                                encoder.shutdown().await?;
+                                Ok(buffer)
+                            }
+                        )*
+                        CompressionType::Uncompressed => Ok(content.as_ref().to_vec()),
+                    }
+                }
+
+                pub async fn decode(&self, content: impl AsRef<[u8]>) -> io::Result<Vec<u8>> {
+                    match self {
+                        $(
+                            CompressionType::$enum_item => {
+                                let mut buffer = Vec::with_capacity(content.as_ref().len() * 2);
+                                let mut encoder = write::[<$prefix Decoder>]::new(&mut buffer);
+                                encoder.write_all(content.as_ref()).await?;
+                                encoder.shutdown().await?;
+                                Ok(buffer)
+                            }
+                        )*
+                        CompressionType::Uncompressed => Ok(content.as_ref().to_vec()),
+                    }
+                }
+
+                pub fn convert_async_read<T: AsyncRead + Unpin + Send + 'static>(
+                    &self,
+                    s: T,
+                ) -> Box<dyn AsyncRead + Unpin + Send> {
+                    match self {
+                        $(CompressionType::$enum_item => Box::new([<$prefix Decoder>]::new(BufReader::new(s))),)*
+                        CompressionType::Uncompressed => Box::new(s),
+                    }
+                }
+
+                pub fn convert_stream<T: Stream<Item = io::Result<Bytes>> + Unpin + Send + 'static>(
+                    &self,
+                    s: T,
+                ) -> Box<dyn Stream<Item = io::Result<Bytes>> + Send + Unpin> {
+                    match self {
+                        $(CompressionType::$enum_item => Box::new(ReaderStream::new([<$prefix Decoder>]::new(StreamReader::new(s)))),)*
+                        CompressionType::Uncompressed => Box::new(s),
+                    }
+                }
+            }
+
+            #[cfg(test)]
+            mod tests {
+                use super::CompressionType;
+
+                $(
+                #[tokio::test]
+                async fn [<test_ $enum_item:lower _compression>]() {
+                    let string = "foo_bar".as_bytes().to_vec();
+                    let compress = CompressionType::$enum_item
+                        .encode(&string)
+                        .await
+                        .unwrap();
+                    let decompress = CompressionType::$enum_item
+                        .decode(&compress)
+                        .await
+                        .unwrap();
+                    assert_eq!(decompress, string);
+                })*
+
+                #[tokio::test]
+                async fn test_uncompression() {
+                    let string = "foo_bar".as_bytes().to_vec();
+                    let compress = CompressionType::Uncompressed
+                        .encode(&string)
+                        .await
+                        .unwrap();
+                    let decompress = CompressionType::Uncompressed
+                        .decode(&compress)
+                        .await
+                        .unwrap();
+                    assert_eq!(decompress, string);
+                }
+            }
+        }
+    };
+}
+
+impl_compression_type!((Gzip, Gzip), (Bzip2, Bz), (Xz, Xz), (Zstd, Zstd));

src/common/datasource/src/error.rs

@@ -0,0 +1,209 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+
+use arrow_schema::ArrowError;
+use common_error::prelude::*;
+use datafusion::parquet::errors::ParquetError;
+use snafu::Location;
+use url::ParseError;
+
+#[derive(Debug, Snafu)]
+#[snafu(visibility(pub))]
+pub enum Error {
+    #[snafu(display("Unsupported compression type: {}", compression_type))]
+    UnsupportedCompressionType {
+        compression_type: String,
+        location: Location,
+    },
+
+    #[snafu(display("Unsupported backend protocol: {}", protocol))]
+    UnsupportedBackendProtocol {
+        protocol: String,
+        location: Location,
+    },
+
+    #[snafu(display("Unsupported format protocol: {}", format))]
+    UnsupportedFormat { format: String, location: Location },
+
+    #[snafu(display("empty host: {}", url))]
+    EmptyHostPath { url: String, location: Location },
+
+    #[snafu(display("Invalid url: {}, error :{}", url, source))]
+    InvalidUrl {
+        url: String,
+        source: ParseError,
+        location: Location,
+    },
+
+    #[snafu(display("Failed to build backend, source: {}", source))]
+    BuildBackend {
+        source: object_store::Error,
+        location: Location,
+    },
+
+    #[snafu(display("Failed to read object from path: {}, source: {}", path, source))]
+    ReadObject {
+        path: String,
+        location: Location,
+        source: object_store::Error,
+    },
+
+    #[snafu(display("Failed to write object to path: {}, source: {}", path, source))]
+    WriteObject {
+        path: String,
+        location: Location,
+        source: object_store::Error,
+    },
+
+    #[snafu(display("Failed to write: {}", source))]
+    AsyncWrite {
+        source: std::io::Error,
+        location: Location,
+    },
+
+    #[snafu(display("Failed to write record batch: {}", source))]
+    WriteRecordBatch {
+        location: Location,
+        source: ArrowError,
+    },
+
+    #[snafu(display("Failed to encode record batch: {}", source))]
+    EncodeRecordBatch {
+        location: Location,
+        source: ParquetError,
+    },
+
+    #[snafu(display("Failed to read record batch: {}", source))]
+    ReadRecordBatch {
+        location: Location,
+        source: datafusion::error::DataFusionError,
+    },
+
+    #[snafu(display("Failed to read parquet source: {}", source))]
+    ReadParquetSnafu {
+        location: Location,
+        source: datafusion::parquet::errors::ParquetError,
+    },
+
+    #[snafu(display("Failed to convert parquet to schema: {}", source))]
+    ParquetToSchema {
+        location: Location,
+        source: datafusion::parquet::errors::ParquetError,
+    },
+
+    #[snafu(display("Failed to infer schema from file, source: {}", source))]
+    InferSchema {
+        location: Location,
+        source: arrow_schema::ArrowError,
+    },
+
+    #[snafu(display("Failed to list object in path: {}, source: {}", path, source))]
+    ListObjects {
+        path: String,
+        location: Location,
+        source: object_store::Error,
+    },
+
+    #[snafu(display("Invalid connection: {}", msg))]
+    InvalidConnection { msg: String, location: Location },
+
+    #[snafu(display("Failed to join handle: {}", source))]
+    JoinHandle {
+        location: Location,
+        source: tokio::task::JoinError,
+    },
+
+    #[snafu(display("Failed to parse format {} with value: {}", key, value))]
+    ParseFormat {
+        key: &'static str,
+        value: String,
+        location: Location,
+    },
+
+    #[snafu(display("Failed to merge schema: {}", source))]
+    MergeSchema {
+        source: arrow_schema::ArrowError,
+        location: Location,
+    },
+
+    #[snafu(display("Buffered writer closed"))]
+    BufferedWriterClosed { location: Location },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl ErrorExt for Error {
+    fn status_code(&self) -> StatusCode {
+        use Error::*;
+        match self {
+            BuildBackend { .. }
+            | ListObjects { .. }
+            | ReadObject { .. }
+            | WriteObject { .. }
+            | AsyncWrite { .. } => StatusCode::StorageUnavailable,
+
+            UnsupportedBackendProtocol { .. }
+            | UnsupportedCompressionType { .. }
+            | UnsupportedFormat { .. }
+            | InvalidConnection { .. }
+            | InvalidUrl { .. }
+            | EmptyHostPath { .. }
+            | InferSchema { .. }
+            | ReadParquetSnafu { .. }
+            | ParquetToSchema { .. }
+            | ParseFormat { .. }
+            | MergeSchema { .. } => StatusCode::InvalidArguments,
+
+            JoinHandle { .. }
+            | ReadRecordBatch { .. }
+            | WriteRecordBatch { .. }
+            | EncodeRecordBatch { .. }
+            | BufferedWriterClosed { .. } => StatusCode::Unexpected,
+        }
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn location_opt(&self) -> Option<common_error::snafu::Location> {
+        use Error::*;
+        match self {
+            BuildBackend { location, .. } => Some(*location),
+            ReadObject { location, .. } => Some(*location),
+            ListObjects { location, .. } => Some(*location),
+            InferSchema { location, .. } => Some(*location),
+            ReadParquetSnafu { location, .. } => Some(*location),
+            ParquetToSchema { location, .. } => Some(*location),
+            JoinHandle { location, .. } => Some(*location),
+            ParseFormat { location, .. } => Some(*location),
+            MergeSchema { location, .. } => Some(*location),
+            WriteObject { location, .. } => Some(*location),
+            ReadRecordBatch { location, .. } => Some(*location),
+            WriteRecordBatch { location, .. } => Some(*location),
+            AsyncWrite { location, .. } => Some(*location),
+            EncodeRecordBatch { location, .. } => Some(*location),
+            BufferedWriterClosed { location, .. } => Some(*location),
+
+            UnsupportedBackendProtocol { location, .. } => Some(*location),
+            EmptyHostPath { location, .. } => Some(*location),
+            InvalidUrl { location, .. } => Some(*location),
+            InvalidConnection { location, .. } => Some(*location),
+            UnsupportedCompressionType { location, .. } => Some(*location),
+            UnsupportedFormat { location, .. } => Some(*location),
+        }
+    }
+}

src/common/datasource/src/file_format.rs

@@ -0,0 +1,215 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod csv;
+pub mod json;
+pub mod parquet;
+#[cfg(test)]
+pub mod tests;
+
+pub const DEFAULT_SCHEMA_INFER_MAX_RECORD: usize = 1000;
+
+use std::collections::HashMap;
+use std::result;
+use std::sync::Arc;
+use std::task::Poll;
+
+use arrow::record_batch::RecordBatch;
+use arrow_schema::{ArrowError, Schema as ArrowSchema};
+use async_trait::async_trait;
+use bytes::{Buf, Bytes};
+use datafusion::error::{DataFusionError, Result as DataFusionResult};
+use datafusion::physical_plan::file_format::FileOpenFuture;
+use datafusion::physical_plan::SendableRecordBatchStream;
+use futures::StreamExt;
+use object_store::ObjectStore;
+use snafu::ResultExt;
+
+use self::csv::CsvFormat;
+use self::json::JsonFormat;
+use self::parquet::ParquetFormat;
+use crate::buffered_writer::{DfRecordBatchEncoder, LazyBufferedWriter};
+use crate::compression::CompressionType;
+use crate::error::{self, Result};
+use crate::share_buffer::SharedBuffer;
+
+pub const FORMAT_COMPRESSION_TYPE: &str = "compression_type";
+pub const FORMAT_DELIMITER: &str = "delimiter";
+pub const FORMAT_SCHEMA_INFER_MAX_RECORD: &str = "schema_infer_max_record";
+pub const FORMAT_HAS_HEADER: &str = "has_header";
+pub const FORMAT_TYPE: &str = "format";
+pub const FILE_PATTERN: &str = "pattern";
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Format {
+    Csv(CsvFormat),
+    Json(JsonFormat),
+    Parquet(ParquetFormat),
+}
+
+impl Format {
+    pub fn suffix(&self) -> &'static str {
+        match self {
+            Format::Csv(_) => ".csv",
+            Format::Json(_) => ".json",
+            Format::Parquet(_) => ".parquet",
+        }
+    }
+}
+
+impl TryFrom<&HashMap<String, String>> for Format {
+    type Error = error::Error;
+
+    fn try_from(options: &HashMap<String, String>) -> Result<Self> {
+        let format = options
+            .get(FORMAT_TYPE)
+            .map(|format| format.to_ascii_uppercase())
+            .unwrap_or_else(|| "PARQUET".to_string());
+
+        match format.as_str() {
+            "CSV" => Ok(Self::Csv(CsvFormat::try_from(options)?)),
+            "JSON" => Ok(Self::Json(JsonFormat::try_from(options)?)),
+            "PARQUET" => Ok(Self::Parquet(ParquetFormat::default())),
+            _ => error::UnsupportedFormatSnafu { format: &format }.fail(),
+        }
+    }
+}
+
+#[async_trait]
+pub trait FileFormat: Send + Sync + std::fmt::Debug {
+    async fn infer_schema(&self, store: &ObjectStore, path: &str) -> Result<ArrowSchema>;
+}
+
+pub trait ArrowDecoder: Send + 'static {
+    /// Decode records from `buf` returning the number of bytes read.
+    ///
+    /// This method returns `Ok(0)` once `batch_size` objects have been parsed since the
+    /// last call to [`Self::flush`], or `buf` is exhausted.
+    ///
+    /// Any remaining bytes should be included in the next call to [`Self::decode`].
+    fn decode(&mut self, buf: &[u8]) -> result::Result<usize, ArrowError>;
+
+    /// Flushes the currently buffered data to a [`RecordBatch`].
+    ///
+    /// This should only be called after [`Self::decode`] has returned `Ok(0)`,
+    /// otherwise may return an error if part way through decoding a record
+    ///
+    /// Returns `Ok(None)` if no buffered data.
+    fn flush(&mut self) -> result::Result<Option<RecordBatch>, ArrowError>;
+}
+
+impl ArrowDecoder for arrow::csv::reader::Decoder {
+    fn decode(&mut self, buf: &[u8]) -> result::Result<usize, ArrowError> {
+        self.decode(buf)
+    }
+
+    fn flush(&mut self) -> result::Result<Option<RecordBatch>, ArrowError> {
+        self.flush()
+    }
+}
+
+#[allow(deprecated)]
+impl ArrowDecoder for arrow::json::RawDecoder {
+    fn decode(&mut self, buf: &[u8]) -> result::Result<usize, ArrowError> {
+        self.decode(buf)
+    }
+
+    fn flush(&mut self) -> result::Result<Option<RecordBatch>, ArrowError> {
+        self.flush()
+    }
+}
+
+pub fn open_with_decoder<T: ArrowDecoder, F: Fn() -> DataFusionResult<T>>(
+    object_store: Arc<ObjectStore>,
+    path: String,
+    compression_type: CompressionType,
+    decoder_factory: F,
+) -> DataFusionResult<FileOpenFuture> {
+    let mut decoder = decoder_factory()?;
+    Ok(Box::pin(async move {
+        let reader = object_store
+            .reader(&path)
+            .await
+            .map_err(|e| DataFusionError::External(Box::new(e)))?;
+
+        let mut upstream = compression_type.convert_stream(reader).fuse();
+
+        let mut buffered = Bytes::new();
+
+        let stream = futures::stream::poll_fn(move |cx| {
+            loop {
+                if buffered.is_empty() {
+                    if let Some(result) = futures::ready!(upstream.poll_next_unpin(cx)) {
+                        buffered = result?;
+                    };
+                }
+
+                let decoded = decoder.decode(buffered.as_ref())?;
+
+                if decoded == 0 {
+                    break;
+                } else {
+                    buffered.advance(decoded);
+                }
+            }
+
+            Poll::Ready(decoder.flush().transpose())
+        });
+
+        Ok(stream.boxed())
+    }))
+}
+
+pub async fn infer_schemas(
+    store: &ObjectStore,
+    files: &[String],
+    file_format: &dyn FileFormat,
+) -> Result<ArrowSchema> {
+    let mut schemas = Vec::with_capacity(files.len());
+    for file in files {
+        schemas.push(file_format.infer_schema(store, file).await?)
+    }
+    ArrowSchema::try_merge(schemas).context(error::MergeSchemaSnafu)
+}
+
+pub async fn stream_to_file<T: DfRecordBatchEncoder, U: Fn(SharedBuffer) -> T>(
+    mut stream: SendableRecordBatchStream,
+    store: ObjectStore,
+    path: &str,
+    threshold: usize,
+    encoder_factory: U,
+) -> Result<usize> {
+    let buffer = SharedBuffer::with_capacity(threshold);
+    let encoder = encoder_factory(buffer.clone());
+    let mut writer = LazyBufferedWriter::new(threshold, buffer, encoder, path, |path| async {
+        store
+            .writer(&path)
+            .await
+            .context(error::WriteObjectSnafu { path })
+    });
+
+    let mut rows = 0;
+
+    while let Some(batch) = stream.next().await {
+        let batch = batch.context(error::ReadRecordBatchSnafu)?;
+        writer.write(&batch).await?;
+        rows += batch.num_rows();
+    }
+
+    // Flushes all pending writes
+    writer.try_flush(true).await?;
+    writer.close_inner_writer().await?;
+
+    Ok(rows)
+}

src/common/datasource/src/file_format/csv.rs

@@ -0,0 +1,320 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use arrow::csv;
+#[allow(deprecated)]
+use arrow::csv::reader::infer_reader_schema as infer_csv_schema;
+use arrow::record_batch::RecordBatch;
+use arrow_schema::{Schema, SchemaRef};
+use async_trait::async_trait;
+use common_runtime;
+use datafusion::error::Result as DataFusionResult;
+use datafusion::physical_plan::file_format::{FileMeta, FileOpenFuture, FileOpener};
+use datafusion::physical_plan::SendableRecordBatchStream;
+use derive_builder::Builder;
+use object_store::ObjectStore;
+use snafu::ResultExt;
+use tokio_util::io::SyncIoBridge;
+
+use super::stream_to_file;
+use crate::buffered_writer::DfRecordBatchEncoder;
+use crate::compression::CompressionType;
+use crate::error::{self, Result};
+use crate::file_format::{self, open_with_decoder, FileFormat};
+use crate::share_buffer::SharedBuffer;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct CsvFormat {
+    pub has_header: bool,
+    pub delimiter: u8,
+    pub schema_infer_max_record: Option<usize>,
+    pub compression_type: CompressionType,
+}
+
+impl TryFrom<&HashMap<String, String>> for CsvFormat {
+    type Error = error::Error;
+
+    fn try_from(value: &HashMap<String, String>) -> Result<Self> {
+        let mut format = CsvFormat::default();
+        if let Some(delimiter) = value.get(file_format::FORMAT_DELIMITER) {
+            // TODO(weny): considers to support parse like "\t" (not only b'\t')
+            format.delimiter = u8::from_str(delimiter).map_err(|_| {
+                error::ParseFormatSnafu {
+                    key: file_format::FORMAT_DELIMITER,
+                    value: delimiter,
+                }
+                .build()
+            })?;
+        };
+        if let Some(compression_type) = value.get(file_format::FORMAT_COMPRESSION_TYPE) {
+            format.compression_type = CompressionType::from_str(compression_type)?;
+        };
+        if let Some(schema_infer_max_record) =
+            value.get(file_format::FORMAT_SCHEMA_INFER_MAX_RECORD)
+        {
+            format.schema_infer_max_record =
+                Some(schema_infer_max_record.parse::<usize>().map_err(|_| {
+                    error::ParseFormatSnafu {
+                        key: file_format::FORMAT_SCHEMA_INFER_MAX_RECORD,
+                        value: schema_infer_max_record,
+                    }
+                    .build()
+                })?);
+        };
+        if let Some(has_header) = value.get(file_format::FORMAT_HAS_HEADER) {
+            format.has_header = has_header.parse().map_err(|_| {
+                error::ParseFormatSnafu {
+                    key: file_format::FORMAT_HAS_HEADER,
+                    value: has_header,
+                }
+                .build()
+            })?;
+        }
+        Ok(format)
+    }
+}
+
+impl Default for CsvFormat {
+    fn default() -> Self {
+        Self {
+            has_header: true,
+            delimiter: b',',
+            schema_infer_max_record: Some(file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD),
+            compression_type: CompressionType::Uncompressed,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Builder)]
+pub struct CsvConfig {
+    batch_size: usize,
+    file_schema: SchemaRef,
+    #[builder(default = "None")]
+    file_projection: Option<Vec<usize>>,
+    #[builder(default = "true")]
+    has_header: bool,
+    #[builder(default = "b','")]
+    delimiter: u8,
+}
+
+impl CsvConfig {
+    fn builder(&self) -> csv::ReaderBuilder {
+        let mut builder = csv::ReaderBuilder::new(self.file_schema.clone())
+            .with_delimiter(self.delimiter)
+            .with_batch_size(self.batch_size)
+            .has_header(self.has_header);
+
+        if let Some(proj) = &self.file_projection {
+            builder = builder.with_projection(proj.clone());
+        }
+
+        builder
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct CsvOpener {
+    config: Arc<CsvConfig>,
+    object_store: Arc<ObjectStore>,
+    compression_type: CompressionType,
+}
+
+impl CsvOpener {
+    /// Return a new [`CsvOpener`]. The caller must ensure [`CsvConfig`].file_schema must correspond to the opening file.
+    pub fn new(
+        config: CsvConfig,
+        object_store: ObjectStore,
+        compression_type: CompressionType,
+    ) -> Self {
+        CsvOpener {
+            config: Arc::new(config),
+            object_store: Arc::new(object_store),
+            compression_type,
+        }
+    }
+}
+
+impl FileOpener for CsvOpener {
+    fn open(&self, meta: FileMeta) -> DataFusionResult<FileOpenFuture> {
+        open_with_decoder(
+            self.object_store.clone(),
+            meta.location().to_string(),
+            self.compression_type,
+            || Ok(self.config.builder().build_decoder()),
+        )
+    }
+}
+
+#[allow(deprecated)]
+#[async_trait]
+impl FileFormat for CsvFormat {
+    async fn infer_schema(&self, store: &ObjectStore, path: &str) -> Result<Schema> {
+        let reader = store
+            .reader(path)
+            .await
+            .context(error::ReadObjectSnafu { path })?;
+
+        let decoded = self.compression_type.convert_async_read(reader);
+
+        let delimiter = self.delimiter;
+        let schema_infer_max_record = self.schema_infer_max_record;
+        let has_header = self.has_header;
+
+        common_runtime::spawn_blocking_read(move || {
+            let reader = SyncIoBridge::new(decoded);
+
+            let (schema, _records_read) =
+                infer_csv_schema(reader, delimiter, schema_infer_max_record, has_header)
+                    .context(error::InferSchemaSnafu)?;
+            Ok(schema)
+        })
+        .await
+        .context(error::JoinHandleSnafu)?
+    }
+}
+
+pub async fn stream_to_csv(
+    stream: SendableRecordBatchStream,
+    store: ObjectStore,
+    path: &str,
+    threshold: usize,
+) -> Result<usize> {
+    stream_to_file(stream, store, path, threshold, |buffer| {
+        csv::Writer::new(buffer)
+    })
+    .await
+}
+
+impl DfRecordBatchEncoder for csv::Writer<SharedBuffer> {
+    fn write(&mut self, batch: &RecordBatch) -> Result<()> {
+        self.write(batch).context(error::WriteRecordBatchSnafu)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use crate::file_format::{
+        FileFormat, FORMAT_COMPRESSION_TYPE, FORMAT_DELIMITER, FORMAT_HAS_HEADER,
+        FORMAT_SCHEMA_INFER_MAX_RECORD,
+    };
+    use crate::test_util::{self, format_schema, test_store};
+
+    fn test_data_root() -> String {
+        test_util::get_data_dir("tests/csv").display().to_string()
+    }
+
+    #[tokio::test]
+    async fn infer_schema_basic() {
+        let csv = CsvFormat::default();
+        let store = test_store(&test_data_root());
+        let schema = csv.infer_schema(&store, "simple.csv").await.unwrap();
+        let formatted: Vec<_> = format_schema(schema);
+
+        assert_eq!(
+            vec![
+                "c1: Utf8: NULL",
+                "c2: Int64: NULL",
+                "c3: Int64: NULL",
+                "c4: Int64: NULL",
+                "c5: Int64: NULL",
+                "c6: Int64: NULL",
+                "c7: Int64: NULL",
+                "c8: Int64: NULL",
+                "c9: Int64: NULL",
+                "c10: Int64: NULL",
+                "c11: Float64: NULL",
+                "c12: Float64: NULL",
+                "c13: Utf8: NULL"
+            ],
+            formatted,
+        );
+    }
+
+    #[tokio::test]
+    async fn infer_schema_with_limit() {
+        let json = CsvFormat {
+            schema_infer_max_record: Some(3),
+            ..CsvFormat::default()
+        };
+        let store = test_store(&test_data_root());
+        let schema = json
+            .infer_schema(&store, "schema_infer_limit.csv")
+            .await
+            .unwrap();
+        let formatted: Vec<_> = format_schema(schema);
+
+        assert_eq!(
+            vec![
+                "a: Int64: NULL",
+                "b: Float64: NULL",
+                "c: Int64: NULL",
+                "d: Int64: NULL"
+            ],
+            formatted
+        );
+
+        let json = CsvFormat::default();
+        let store = test_store(&test_data_root());
+        let schema = json
+            .infer_schema(&store, "schema_infer_limit.csv")
+            .await
+            .unwrap();
+        let formatted: Vec<_> = format_schema(schema);
+
+        assert_eq!(
+            vec![
+                "a: Int64: NULL",
+                "b: Float64: NULL",
+                "c: Int64: NULL",
+                "d: Utf8: NULL"
+            ],
+            formatted
+        );
+    }
+
+    #[test]
+    fn test_try_from() {
+        let mut map = HashMap::new();
+        let format: CsvFormat = CsvFormat::try_from(&map).unwrap();
+
+        assert_eq!(format, CsvFormat::default());
+
+        map.insert(
+            FORMAT_SCHEMA_INFER_MAX_RECORD.to_string(),
+            "2000".to_string(),
+        );
+
+        map.insert(FORMAT_COMPRESSION_TYPE.to_string(), "zstd".to_string());
+        map.insert(FORMAT_DELIMITER.to_string(), b'\t'.to_string());
+        map.insert(FORMAT_HAS_HEADER.to_string(), "false".to_string());
+
+        let format = CsvFormat::try_from(&map).unwrap();
+
+        assert_eq!(
+            format,
+            CsvFormat {
+                compression_type: CompressionType::Zstd,
+                schema_infer_max_record: Some(2000),
+                delimiter: b'\t',
+                has_header: false,
+            }
+        );
+    }
+}

src/common/datasource/src/file_format/json.rs

@@ -0,0 +1,239 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::io::BufReader;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use arrow::datatypes::SchemaRef;
+use arrow::json::reader::{infer_json_schema_from_iterator, ValueIter};
+use arrow::json::writer::LineDelimited;
+#[allow(deprecated)]
+use arrow::json::{self, RawReaderBuilder};
+use arrow::record_batch::RecordBatch;
+use arrow_schema::Schema;
+use async_trait::async_trait;
+use common_runtime;
+use datafusion::error::{DataFusionError, Result as DataFusionResult};
+use datafusion::physical_plan::file_format::{FileMeta, FileOpenFuture, FileOpener};
+use datafusion::physical_plan::SendableRecordBatchStream;
+use object_store::ObjectStore;
+use snafu::ResultExt;
+use tokio_util::io::SyncIoBridge;
+
+use super::stream_to_file;
+use crate::buffered_writer::DfRecordBatchEncoder;
+use crate::compression::CompressionType;
+use crate::error::{self, Result};
+use crate::file_format::{self, open_with_decoder, FileFormat};
+use crate::share_buffer::SharedBuffer;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct JsonFormat {
+    pub schema_infer_max_record: Option<usize>,
+    pub compression_type: CompressionType,
+}
+
+impl TryFrom<&HashMap<String, String>> for JsonFormat {
+    type Error = error::Error;
+
+    fn try_from(value: &HashMap<String, String>) -> Result<Self> {
+        let mut format = JsonFormat::default();
+        if let Some(compression_type) = value.get(file_format::FORMAT_COMPRESSION_TYPE) {
+            format.compression_type = CompressionType::from_str(compression_type)?
+        };
+        if let Some(schema_infer_max_record) =
+            value.get(file_format::FORMAT_SCHEMA_INFER_MAX_RECORD)
+        {
+            format.schema_infer_max_record =
+                Some(schema_infer_max_record.parse::<usize>().map_err(|_| {
+                    error::ParseFormatSnafu {
+                        key: file_format::FORMAT_SCHEMA_INFER_MAX_RECORD,
+                        value: schema_infer_max_record,
+                    }
+                    .build()
+                })?);
+        };
+        Ok(format)
+    }
+}
+
+impl Default for JsonFormat {
+    fn default() -> Self {
+        Self {
+            schema_infer_max_record: Some(file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD),
+            compression_type: CompressionType::Uncompressed,
+        }
+    }
+}
+
+#[async_trait]
+impl FileFormat for JsonFormat {
+    async fn infer_schema(&self, store: &ObjectStore, path: &str) -> Result<Schema> {
+        let reader = store
+            .reader(path)
+            .await
+            .context(error::ReadObjectSnafu { path })?;
+
+        let decoded = self.compression_type.convert_async_read(reader);
+
+        let schema_infer_max_record = self.schema_infer_max_record;
+
+        common_runtime::spawn_blocking_read(move || {
+            let mut reader = BufReader::new(SyncIoBridge::new(decoded));
+
+            let iter = ValueIter::new(&mut reader, schema_infer_max_record);
+
+            let schema = infer_json_schema_from_iterator(iter).context(error::InferSchemaSnafu)?;
+
+            Ok(schema)
+        })
+        .await
+        .context(error::JoinHandleSnafu)?
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct JsonOpener {
+    batch_size: usize,
+    projected_schema: SchemaRef,
+    object_store: Arc<ObjectStore>,
+    compression_type: CompressionType,
+}
+
+impl JsonOpener {
+    /// Return a new [`JsonOpener`]. Any fields not present in `projected_schema` will be ignored.
+    pub fn new(
+        batch_size: usize,
+        projected_schema: SchemaRef,
+        object_store: ObjectStore,
+        compression_type: CompressionType,
+    ) -> Self {
+        Self {
+            batch_size,
+            projected_schema,
+            object_store: Arc::new(object_store),
+            compression_type,
+        }
+    }
+}
+
+#[allow(deprecated)]
+impl FileOpener for JsonOpener {
+    fn open(&self, meta: FileMeta) -> DataFusionResult<FileOpenFuture> {
+        open_with_decoder(
+            self.object_store.clone(),
+            meta.location().to_string(),
+            self.compression_type,
+            || {
+                RawReaderBuilder::new(self.projected_schema.clone())
+                    .with_batch_size(self.batch_size)
+                    .build_decoder()
+                    .map_err(DataFusionError::from)
+            },
+        )
+    }
+}
+
+pub async fn stream_to_json(
+    stream: SendableRecordBatchStream,
+    store: ObjectStore,
+    path: &str,
+    threshold: usize,
+) -> Result<usize> {
+    stream_to_file(stream, store, path, threshold, |buffer| {
+        json::LineDelimitedWriter::new(buffer)
+    })
+    .await
+}
+
+impl DfRecordBatchEncoder for json::Writer<SharedBuffer, LineDelimited> {
+    fn write(&mut self, batch: &RecordBatch) -> Result<()> {
+        self.write(batch).context(error::WriteRecordBatchSnafu)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::file_format::{FileFormat, FORMAT_COMPRESSION_TYPE, FORMAT_SCHEMA_INFER_MAX_RECORD};
+    use crate::test_util::{self, format_schema, test_store};
+
+    fn test_data_root() -> String {
+        test_util::get_data_dir("tests/json").display().to_string()
+    }
+
+    #[tokio::test]
+    async fn infer_schema_basic() {
+        let json = JsonFormat::default();
+        let store = test_store(&test_data_root());
+        let schema = json.infer_schema(&store, "simple.json").await.unwrap();
+        let formatted: Vec<_> = format_schema(schema);
+
+        assert_eq!(
+            vec![
+                "a: Int64: NULL",
+                "b: Float64: NULL",
+                "c: Boolean: NULL",
+                "d: Utf8: NULL",
+            ],
+            formatted
+        );
+    }
+
+    #[tokio::test]
+    async fn infer_schema_with_limit() {
+        let json = JsonFormat {
+            schema_infer_max_record: Some(3),
+            ..JsonFormat::default()
+        };
+        let store = test_store(&test_data_root());
+        let schema = json
+            .infer_schema(&store, "schema_infer_limit.json")
+            .await
+            .unwrap();
+        let formatted: Vec<_> = format_schema(schema);
+
+        assert_eq!(
+            vec!["a: Int64: NULL", "b: Float64: NULL", "c: Boolean: NULL"],
+            formatted
+        );
+    }
+
+    #[test]
+    fn test_try_from() {
+        let mut map = HashMap::new();
+        let format = JsonFormat::try_from(&map).unwrap();
+
+        assert_eq!(format, JsonFormat::default());
+
+        map.insert(
+            FORMAT_SCHEMA_INFER_MAX_RECORD.to_string(),
+            "2000".to_string(),
+        );
+
+        map.insert(FORMAT_COMPRESSION_TYPE.to_string(), "zstd".to_string());
+
+        let format = JsonFormat::try_from(&map).unwrap();
+
+        assert_eq!(
+            format,
+            JsonFormat {
+                compression_type: CompressionType::Zstd,
+                schema_infer_max_record: Some(2000),
+            }
+        );
+    }
+}

src/common/datasource/src/file_format/parquet.rs

@@ -0,0 +1,179 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::result;
+use std::sync::Arc;
+
+use arrow::record_batch::RecordBatch;
+use arrow_schema::Schema;
+use async_trait::async_trait;
+use datafusion::error::Result as DatafusionResult;
+use datafusion::parquet::arrow::async_reader::AsyncFileReader;
+use datafusion::parquet::arrow::{parquet_to_arrow_schema, ArrowWriter};
+use datafusion::parquet::errors::{ParquetError, Result as ParquetResult};
+use datafusion::parquet::file::metadata::ParquetMetaData;
+use datafusion::parquet::format::FileMetaData;
+use datafusion::physical_plan::file_format::{FileMeta, ParquetFileReaderFactory};
+use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
+use futures::future::BoxFuture;
+use object_store::{ObjectStore, Reader};
+use snafu::ResultExt;
+
+use crate::buffered_writer::{ArrowWriterCloser, DfRecordBatchEncoder};
+use crate::error::{self, Result};
+use crate::file_format::FileFormat;
+use crate::share_buffer::SharedBuffer;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct ParquetFormat {}
+
+#[async_trait]
+impl FileFormat for ParquetFormat {
+    async fn infer_schema(&self, store: &ObjectStore, path: &str) -> Result<Schema> {
+        let mut reader = store
+            .reader(path)
+            .await
+            .context(error::ReadObjectSnafu { path })?;
+
+        let metadata = reader
+            .get_metadata()
+            .await
+            .context(error::ReadParquetSnafuSnafu)?;
+
+        let file_metadata = metadata.file_metadata();
+        let schema = parquet_to_arrow_schema(
+            file_metadata.schema_descr(),
+            file_metadata.key_value_metadata(),
+        )
+        .context(error::ParquetToSchemaSnafu)?;
+
+        Ok(schema)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct DefaultParquetFileReaderFactory {
+    object_store: ObjectStore,
+}
+
+/// Returns a AsyncFileReader factory
+impl DefaultParquetFileReaderFactory {
+    pub fn new(object_store: ObjectStore) -> Self {
+        Self { object_store }
+    }
+}
+
+impl ParquetFileReaderFactory for DefaultParquetFileReaderFactory {
+    // TODO(weny): Supports [`metadata_size_hint`].
+    // The upstream has a implementation supports [`metadata_size_hint`],
+    // however it coupled with Box<dyn ObjectStore>.
+    fn create_reader(
+        &self,
+        _partition_index: usize,
+        file_meta: FileMeta,
+        _metadata_size_hint: Option<usize>,
+        _metrics: &ExecutionPlanMetricsSet,
+    ) -> DatafusionResult<Box<dyn AsyncFileReader + Send>> {
+        let path = file_meta.location().to_string();
+        let object_store = self.object_store.clone();
+
+        Ok(Box::new(LazyParquetFileReader::new(object_store, path)))
+    }
+}
+
+pub struct LazyParquetFileReader {
+    object_store: ObjectStore,
+    reader: Option<Reader>,
+    path: String,
+}
+
+impl LazyParquetFileReader {
+    pub fn new(object_store: ObjectStore, path: String) -> Self {
+        LazyParquetFileReader {
+            object_store,
+            path,
+            reader: None,
+        }
+    }
+
+    /// Must initialize the reader, or throw an error from the future.
+    async fn maybe_initialize(&mut self) -> result::Result<(), object_store::Error> {
+        if self.reader.is_none() {
+            let reader = self.object_store.reader(&self.path).await?;
+            self.reader = Some(reader);
+        }
+
+        Ok(())
+    }
+}
+
+impl AsyncFileReader for LazyParquetFileReader {
+    fn get_bytes(
+        &mut self,
+        range: std::ops::Range<usize>,
+    ) -> BoxFuture<'_, ParquetResult<bytes::Bytes>> {
+        Box::pin(async move {
+            self.maybe_initialize()
+                .await
+                .map_err(|e| ParquetError::External(Box::new(e)))?;
+            // Safety: Must initialized
+            self.reader.as_mut().unwrap().get_bytes(range).await
+        })
+    }
+
+    fn get_metadata(&mut self) -> BoxFuture<'_, ParquetResult<Arc<ParquetMetaData>>> {
+        Box::pin(async move {
+            self.maybe_initialize()
+                .await
+                .map_err(|e| ParquetError::External(Box::new(e)))?;
+            // Safety: Must initialized
+            self.reader.as_mut().unwrap().get_metadata().await
+        })
+    }
+}
+
+impl DfRecordBatchEncoder for ArrowWriter<SharedBuffer> {
+    fn write(&mut self, batch: &RecordBatch) -> Result<()> {
+        self.write(batch).context(error::EncodeRecordBatchSnafu)
+    }
+}
+
+#[async_trait]
+impl ArrowWriterCloser for ArrowWriter<SharedBuffer> {
+    async fn close(self) -> Result<FileMetaData> {
+        self.close().context(error::EncodeRecordBatchSnafu)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_util::{self, format_schema, test_store};
+
+    fn test_data_root() -> String {
+        test_util::get_data_dir("tests/parquet")
+            .display()
+            .to_string()
+    }
+
+    #[tokio::test]
+    async fn infer_schema_basic() {
+        let json = ParquetFormat::default();
+        let store = test_store(&test_data_root());
+        let schema = json.infer_schema(&store, "basic.parquet").await.unwrap();
+        let formatted: Vec<_> = format_schema(schema);
+
+        assert_eq!(vec!["num: Int64: NULL", "str: Utf8: NULL"], formatted);
+    }
+}

src/common/datasource/src/file_format/tests.rs

@@ -0,0 +1,228 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::assert_matches::assert_matches;
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::vec;
+
+use datafusion::assert_batches_eq;
+use datafusion::execution::context::TaskContext;
+use datafusion::physical_plan::file_format::{FileOpener, FileScanConfig, FileStream, ParquetExec};
+use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::prelude::SessionContext;
+use futures::StreamExt;
+
+use super::FORMAT_TYPE;
+use crate::compression::CompressionType;
+use crate::error;
+use crate::file_format::csv::{CsvConfigBuilder, CsvOpener};
+use crate::file_format::json::JsonOpener;
+use crate::file_format::parquet::DefaultParquetFileReaderFactory;
+use crate::file_format::Format;
+use crate::test_util::{self, scan_config, test_basic_schema, test_store};
+
+struct Test<'a, T: FileOpener> {
+    config: FileScanConfig,
+    opener: T,
+    expected: Vec<&'a str>,
+}
+
+impl<'a, T: FileOpener> Test<'a, T> {
+    pub async fn run(self) {
+        let result = FileStream::new(
+            &self.config,
+            0,
+            self.opener,
+            &ExecutionPlanMetricsSet::new(),
+        )
+        .unwrap()
+        .map(|b| b.unwrap())
+        .collect::<Vec<_>>()
+        .await;
+
+        assert_batches_eq!(self.expected, &result);
+    }
+}
+
+#[tokio::test]
+async fn test_json_opener() {
+    let store = test_store("/");
+
+    let schema = test_basic_schema();
+
+    let json_opener = JsonOpener::new(
+        100,
+        schema.clone(),
+        store.clone(),
+        CompressionType::Uncompressed,
+    );
+
+    let path = &test_util::get_data_dir("tests/json/basic.json")
+        .display()
+        .to_string();
+    let tests = [
+        Test {
+            config: scan_config(schema.clone(), None, path),
+            opener: json_opener.clone(),
+            expected: vec![
+                "+-----+-------+",
+                "| num | str   |",
+                "+-----+-------+",
+                "| 5   | test  |",
+                "| 2   | hello |",
+                "| 4   | foo   |",
+                "+-----+-------+",
+            ],
+        },
+        Test {
+            config: scan_config(schema.clone(), Some(1), path),
+            opener: json_opener.clone(),
+            expected: vec![
+                "+-----+------+",
+                "| num | str  |",
+                "+-----+------+",
+                "| 5   | test |",
+                "+-----+------+",
+            ],
+        },
+    ];
+
+    for test in tests {
+        test.run().await;
+    }
+}
+
+#[tokio::test]
+async fn test_csv_opener() {
+    let store = test_store("/");
+
+    let schema = test_basic_schema();
+    let path = &test_util::get_data_dir("tests/csv/basic.csv")
+        .display()
+        .to_string();
+    let csv_conf = CsvConfigBuilder::default()
+        .batch_size(test_util::TEST_BATCH_SIZE)
+        .file_schema(schema.clone())
+        .build()
+        .unwrap();
+
+    let csv_opener = CsvOpener::new(csv_conf, store, CompressionType::Uncompressed);
+
+    let tests = [
+        Test {
+            config: scan_config(schema.clone(), None, path),
+            opener: csv_opener.clone(),
+            expected: vec![
+                "+-----+-------+",
+                "| num | str   |",
+                "+-----+-------+",
+                "| 5   | test  |",
+                "| 2   | hello |",
+                "| 4   | foo   |",
+                "+-----+-------+",
+            ],
+        },
+        Test {
+            config: scan_config(schema.clone(), Some(1), path),
+            opener: csv_opener.clone(),
+            expected: vec![
+                "+-----+------+",
+                "| num | str  |",
+                "+-----+------+",
+                "| 5   | test |",
+                "+-----+------+",
+            ],
+        },
+    ];
+
+    for test in tests {
+        test.run().await;
+    }
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn test_parquet_exec() {
+    let store = test_store("/");
+
+    let schema = test_basic_schema();
+
+    let path = &test_util::get_data_dir("tests/parquet/basic.parquet")
+        .display()
+        .to_string();
+    let base_config = scan_config(schema.clone(), None, path);
+
+    let exec = ParquetExec::new(base_config, None, None)
+        .with_parquet_file_reader_factory(Arc::new(DefaultParquetFileReaderFactory::new(store)));
+
+    let ctx = SessionContext::new();
+
+    let context = Arc::new(TaskContext::from(&ctx));
+
+    // The stream batch size can be set by ctx.session_config.batch_size
+    let result = exec
+        .execute(0, context)
+        .unwrap()
+        .map(|b| b.unwrap())
+        .collect::<Vec<_>>()
+        .await;
+
+    assert_batches_eq!(
+        vec![
+            "+-----+-------+",
+            "| num | str   |",
+            "+-----+-------+",
+            "| 5   | test  |",
+            "| 2   | hello |",
+            "| 4   | foo   |",
+            "+-----+-------+",
+        ],
+        &result
+    );
+}
+
+#[test]
+fn test_format() {
+    let value = [(FORMAT_TYPE.to_string(), "csv".to_string())]
+        .into_iter()
+        .collect::<HashMap<_, _>>();
+
+    assert_matches!(Format::try_from(&value).unwrap(), Format::Csv(_));
+
+    let value = [(FORMAT_TYPE.to_string(), "Parquet".to_string())]
+        .into_iter()
+        .collect::<HashMap<_, _>>();
+
+    assert_matches!(Format::try_from(&value).unwrap(), Format::Parquet(_));
+
+    let value = [(FORMAT_TYPE.to_string(), "JSON".to_string())]
+        .into_iter()
+        .collect::<HashMap<_, _>>();
+
+    assert_matches!(Format::try_from(&value).unwrap(), Format::Json(_));
+
+    let value = [(FORMAT_TYPE.to_string(), "Foobar".to_string())]
+        .into_iter()
+        .collect::<HashMap<_, _>>();
+
+    assert_matches!(
+        Format::try_from(&value).unwrap_err(),
+        error::Error::UnsupportedFormat { .. }
+    );
+
+    let value = HashMap::new();
+
+    assert_matches!(Format::try_from(&value).unwrap(), Format::Parquet(_));
+}

src/common/datasource/src/lib.rs

@@ -0,0 +1,28 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(assert_matches)]
+
+pub mod buffered_writer;
+pub mod compression;
+pub mod error;
+pub mod file_format;
+pub mod lister;
+pub mod object_store;
+pub mod share_buffer;
+#[cfg(test)]
+pub mod test_util;
+#[cfg(test)]
+pub mod tests;
+pub mod util;

src/common/datasource/src/lister.rs

@@ -0,0 +1,83 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use futures::{future, TryStreamExt};
+use object_store::{Entry, ObjectStore};
+use regex::Regex;
+use snafu::ResultExt;
+
+use crate::error::{self, Result};
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Source {
+    Filename(String),
+    Dir,
+}
+
+pub struct Lister {
+    object_store: ObjectStore,
+    source: Source,
+    path: String,
+    regex: Option<Regex>,
+}
+
+impl Lister {
+    pub fn new(
+        object_store: ObjectStore,
+        source: Source,
+        path: String,
+        regex: Option<Regex>,
+    ) -> Self {
+        Lister {
+            object_store,
+            source,
+            path,
+            regex,
+        }
+    }
+
+    pub async fn list(&self) -> Result<Vec<Entry>> {
+        match &self.source {
+            Source::Dir => {
+                let streamer = self
+                    .object_store
+                    .list(&self.path)
+                    .await
+                    .context(error::ListObjectsSnafu { path: &self.path })?;
+
+                streamer
+                    .try_filter(|f| {
+                        let res = self
+                            .regex
+                            .as_ref()
+                            .map(|x| x.is_match(f.name()))
+                            .unwrap_or(true);
+                        future::ready(res)
+                    })
+                    .try_collect::<Vec<_>>()
+                    .await
+                    .context(error::ListObjectsSnafu { path: &self.path })
+            }
+            Source::Filename(filename) => {
+                // make sure this file exists
+                let file_full_path = format!("{}{}", self.path, filename);
+                let _ = self.object_store.stat(&file_full_path).await.context(
+                    error::ListObjectsSnafu {
+                        path: &file_full_path,
+                    },
+                )?;
+                Ok(vec![Entry::new(&file_full_path)])
+            }
+        }
+    }
+}

src/common/datasource/src/object_store.rs

@@ -0,0 +1,60 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod fs;
+pub mod s3;
+use std::collections::HashMap;
+
+use object_store::ObjectStore;
+use snafu::{OptionExt, ResultExt};
+use url::{ParseError, Url};
+
+use self::fs::build_fs_backend;
+use self::s3::build_s3_backend;
+use crate::error::{self, Result};
+
+pub const FS_SCHEMA: &str = "FS";
+pub const S3_SCHEMA: &str = "S3";
+
+/// Returns (schema, Option<host>, path)
+pub fn parse_url(url: &str) -> Result<(String, Option<String>, String)> {
+    let parsed_url = Url::parse(url);
+    match parsed_url {
+        Ok(url) => Ok((
+            url.scheme().to_string(),
+            url.host_str().map(|s| s.to_string()),
+            url.path().to_string(),
+        )),
+        Err(ParseError::RelativeUrlWithoutBase) => {
+            Ok((FS_SCHEMA.to_string(), None, url.to_string()))
+        }
+        Err(err) => Err(err).context(error::InvalidUrlSnafu { url }),
+    }
+}
+
+pub fn build_backend(url: &str, connection: &HashMap<String, String>) -> Result<ObjectStore> {
+    let (schema, host, _path) = parse_url(url)?;
+
+    match schema.to_uppercase().as_str() {
+        S3_SCHEMA => {
+            let host = host.context(error::EmptyHostPathSnafu {
+                url: url.to_string(),
+            })?;
+            Ok(build_s3_backend(&host, "/", connection)?)
+        }
+        FS_SCHEMA => Ok(build_fs_backend("/")?),
+
+        _ => error::UnsupportedBackendProtocolSnafu { protocol: schema }.fail(),
+    }
+}