test: more reduce tests

.github/actions/build-dev-builder-image/action.yml

@@ -62,16 +62,6 @@ runs:
         IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
         IMAGE_TAG=${{ inputs.version }}
 
-    - name: Build and push android dev builder image to dockerhub
-      shell: bash
-      run:
-        make dev-builder \
-        BASE_IMAGE=android \
-        BUILDX_MULTI_PLATFORM_BUILD=true \
-        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
-        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-        IMAGE_TAG=${{ inputs.version }}
-
     - name: Login to ACR
       uses: docker/login-action@v2
       continue-on-error: true

.github/actions/build-macos-artifacts/action.yml

@@ -34,10 +34,6 @@ inputs:
   aws-region:
     description: AWS region
     required: true
-  upload-to-s3:
-    description: Upload to S3
-    required: false
-    default: 'true'
 runs:
   using: composite
   steps:
@@ -107,4 +103,3 @@ runs:
         aws-access-key-id: ${{ inputs.aws-access-key-id }}
         aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
         aws-region: ${{ inputs.aws-region }}
-        upload-to-s3: ${{ inputs.upload-to-s3 }}

.github/actions/upload-artifacts/action.yml

@@ -37,7 +37,7 @@ inputs:
   upload-retry-timeout:
     description: Timeout for uploading artifacts to S3
     required: false
-    default: "30" # minutes
+    default: "10" # minutes
   working-dir:
     description: Working directory to upload the artifacts
     required: false

.github/workflows/develop.yml

@@ -217,7 +217,7 @@ jobs:
       - name: Install WSL distribution
         uses: Vampire/setup-wsl@v2
         with:
-         distribution: Ubuntu-22.04
+          distribution: Ubuntu-22.04
       - name: Running tests
         run: cargo nextest run -F pyo3_backend,dashboard
         env:

.github/workflows/release.yml

@@ -91,7 +91,7 @@ env:
   # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
   NIGHTLY_RELEASE_PREFIX: nightly
   # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
-  NEXT_RELEASE_VERSION: v0.5.0
+  NEXT_RELEASE_VERSION: v0.4.0
 
 jobs:
   allocate-runners:
@@ -181,7 +181,6 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
@@ -205,7 +204,6 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}
 
   build-macos-artifacts:
     name: Build macOS artifacts
@@ -252,7 +250,6 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub

Cargo.toml

@@ -8,11 +8,10 @@ members = [
     "src/cmd",
     "src/common/base",
     "src/common/catalog",
-    "src/common/config",
     "src/common/datasource",
     "src/common/error",
     "src/common/function",
-    "src/common/macro",
+    "src/common/function-macro",
     "src/common/greptimedb-telemetry",
     "src/common/grpc",
     "src/common/grpc-expr",
@@ -30,16 +29,15 @@ members = [
     "src/common/version",
     "src/datanode",
     "src/datatypes",
-    "src/file-engine",
+    "src/file-table-engine",
     "src/frontend",
     "src/log-store",
     "src/meta-client",
     "src/meta-srv",
+    "src/mito",
     "src/mito2",
     "src/object-store",
-    "src/operator",
     "src/partition",
-    "src/plugins",
     "src/promql",
     "src/query",
     "src/script",
@@ -48,60 +46,51 @@ members = [
     "src/sql",
     "src/storage",
     "src/store-api",
+    "src/flow",
     "src/table",
+    "src/table-procedure",
     "tests-integration",
     "tests/runner",
 ]
 resolver = "2"
 
 [workspace.package]
-version = "0.4.0"
+version = "0.4.0-nightly"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.dependencies]
-aquamarine = "0.3"
 arrow = { version = "43.0" }
+etcd-client = "0.11"
 arrow-array = "43.0"
 arrow-flight = "43.0"
 arrow-schema = { version = "43.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
-datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
-datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
-datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
-datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
-datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
-datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
-datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
 derive_builder = "0.12"
-etcd-client = "0.11"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "1f1dd532a111e3834cc3019c5605e2993ffb9dc3" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "4a277f27caa035a801d5b9c020a0449777736614" }
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
-meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
-metrics = "0.20"
-moka = "0.12"
 once_cell = "1.18"
 opentelemetry-proto = { version = "0.2", features = ["gen-tonic", "metrics"] }
 parquet = "43.0"
 paste = "1.0"
 prost = "0.11"
-raft-engine = { git = "https://github.com/tikv/raft-engine.git", rev = "22dfb426cd994602b57725ef080287d3e53db479" }
 rand = "0.8"
 regex = "1.8"
-reqwest = { version = "0.11", default-features = false, features = [
-    "json",
-    "rustls-tls-native-roots",
-    "stream",
-] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-smallvec = "1"
 snafu = { version = "0.7", features = ["backtraces"] }
 sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4", features = [
     "visitor",
@@ -113,6 +102,8 @@ tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.7"
 tonic = { version = "0.9", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
+metrics = "0.20"
+meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
 ## workspaces members
 api = { path = "src/api" }
 auth = { path = "src/auth" }
@@ -121,29 +112,29 @@ client = { path = "src/client" }
 cmd = { path = "src/cmd" }
 common-base = { path = "src/common/base" }
 common-catalog = { path = "src/common/catalog" }
-common-config = { path = "src/common/config" }
 common-datasource = { path = "src/common/datasource" }
 common-error = { path = "src/common/error" }
 common-function = { path = "src/common/function" }
+common-function-macro = { path = "src/common/function-macro" }
 common-greptimedb-telemetry = { path = "src/common/greptimedb-telemetry" }
 common-grpc = { path = "src/common/grpc" }
 common-grpc-expr = { path = "src/common/grpc-expr" }
-common-macro = { path = "src/common/macro" }
 common-mem-prof = { path = "src/common/mem-prof" }
 common-meta = { path = "src/common/meta" }
-common-pprof = { path = "src/common/pprof" }
 common-procedure = { path = "src/common/procedure" }
 common-procedure-test = { path = "src/common/procedure-test" }
+common-pprof = { path = "src/common/pprof" }
 common-query = { path = "src/common/query" }
 common-recordbatch = { path = "src/common/recordbatch" }
 common-runtime = { path = "src/common/runtime" }
+substrait = { path = "src/common/substrait" }
 common-telemetry = { path = "src/common/telemetry" }
 common-test-util = { path = "src/common/test-util" }
 common-time = { path = "src/common/time" }
 common-version = { path = "src/common/version" }
 datanode = { path = "src/datanode" }
 datatypes = { path = "src/datatypes" }
-file-engine = { path = "src/file-engine" }
+file-table-engine = { path = "src/file-table-engine" }
 frontend = { path = "src/frontend" }
 log-store = { path = "src/log-store" }
 meta-client = { path = "src/meta-client" }
@@ -151,9 +142,7 @@ meta-srv = { path = "src/meta-srv" }
 mito = { path = "src/mito" }
 mito2 = { path = "src/mito2" }
 object-store = { path = "src/object-store" }
-operator = { path = "src/operator" }
 partition = { path = "src/partition" }
-plugins = { path = "src/plugins" }
 promql = { path = "src/promql" }
 query = { path = "src/query" }
 script = { path = "src/script" }
@@ -162,8 +151,8 @@ session = { path = "src/session" }
 sql = { path = "src/sql" }
 storage = { path = "src/storage" }
 store-api = { path = "src/store-api" }
-substrait = { path = "src/common/substrait" }
 table = { path = "src/table" }
+table-procedure = { path = "src/table-procedure" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"

Makefile

@@ -55,15 +55,11 @@ else
 	BUILDX_MULTI_PLATFORM_BUILD_OPTS := -o type=docker
 endif
 
-ifneq ($(strip $(CARGO_BUILD_EXTRA_OPTS)),)
-	CARGO_BUILD_OPTS += ${CARGO_BUILD_EXTRA_OPTS}
-endif
-
 ##@ Build
 
 .PHONY: build
 build: ## Build debug version greptime.
-	cargo ${CARGO_EXTENSION} build ${CARGO_BUILD_OPTS}
+	cargo build ${CARGO_BUILD_OPTS}
 
 .POHNY: build-by-dev-builder
 build-by-dev-builder: ## Build greptime by dev-builder.
@@ -71,34 +67,11 @@ build-by-dev-builder: ## Build greptime by dev-builder.
 	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
 	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
 	make build \
-	CARGO_EXTENSION="${CARGO_EXTENSION}" \
 	CARGO_PROFILE=${CARGO_PROFILE} \
 	FEATURES=${FEATURES} \
 	TARGET_DIR=${TARGET_DIR} \
 	TARGET=${TARGET} \
-	RELEASE=${RELEASE} \
+	RELEASE=${RELEASE}
-	CARGO_BUILD_EXTRA_OPTS="${CARGO_BUILD_EXTRA_OPTS}"
-
-.PHONY: build-android-bin
-build-android-bin: ## Build greptime binary for android.
-	docker run --network=host \
-	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:latest \
-	make build \
-	CARGO_EXTENSION="ndk --platform 23 -t aarch64-linux-android" \
-	CARGO_PROFILE=release \
-	FEATURES="${FEATURES}" \
-	TARGET_DIR="${TARGET_DIR}" \
-	TARGET="${TARGET}" \
-	RELEASE="${RELEASE}" \
-	CARGO_BUILD_EXTRA_OPTS="--bin greptime --no-default-features"
-
-.PHONY: strip-android-bin
-strip-android-bin: ## Strip greptime binary for android.
-	docker run --network=host \
-	-v ${PWD}:/greptimedb \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:latest \
-	bash -c '$${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip /greptimedb/target/aarch64-linux-android/release/greptime'
 
 .PHONY: clean
 clean: ## Clean the project.

README.md

@@ -27,14 +27,6 @@
     <a href="https://greptime.com/slack"><img src="https://img.shields.io/badge/slack-GreptimeDB-0abd59?logo=slack" alt="slack" /></a>
 </p>
 
-## Upcoming Event
-Come and meet us in **KubeCon + CloudNativeCon North America 2023!**
-<p align="center">
-  <picture>
-    <img alt="KubeCon + CloudNativeCon North Logo" src="./docs/banner/KCCNC_NA_2023_1000x200_Email Banner.png" width="800px">
-  </picture>
-</p>
-
 ## What is GreptimeDB
 
 GreptimeDB is an open-source time-series database with a special focus on

benchmarks/src/bin/nyc-taxi.rs

@@ -27,7 +27,7 @@ use arrow::record_batch::RecordBatch;
 use clap::Parser;
 use client::api::v1::column::Values;
 use client::api::v1::{
-    Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests, SemanticType,
+    Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests,
 };
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
@@ -105,6 +105,7 @@ async fn write_data(
         let (columns, row_count) = convert_record_batch(record_batch);
         let request = InsertRequest {
             table_name: TABLE_NAME.to_string(),
+            region_number: 0,
             columns,
             row_count,
         };
@@ -188,7 +189,7 @@ fn build_values(column: &ArrayRef) -> (Values, ColumnDataType) {
             let values = array.values();
             (
                 Values {
-                    timestamp_microsecond_values: values.to_vec(),
+                    ts_microsecond_values: values.to_vec(),
                     ..Default::default()
                 },
                 ColumnDataType::TimestampMicrosecond,
@@ -252,161 +253,124 @@ fn create_table_expr() -> CreateTableExpr {
         column_defs: vec![
             ColumnDef {
                 name: "VendorID".to_string(),
-                data_type: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Tag as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "tpep_pickup_datetime".to_string(),
-                data_type: ColumnDataType::TimestampMicrosecond as i32,
+                datatype: ColumnDataType::TimestampMicrosecond as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Timestamp as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "tpep_dropoff_datetime".to_string(),
-                data_type: ColumnDataType::TimestampMicrosecond as i32,
+                datatype: ColumnDataType::TimestampMicrosecond as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "passenger_count".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "trip_distance".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "RatecodeID".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "store_and_fwd_flag".to_string(),
-                data_type: ColumnDataType::String as i32,
+                datatype: ColumnDataType::String as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "PULocationID".to_string(),
-                data_type: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "DOLocationID".to_string(),
-                data_type: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "payment_type".to_string(),
-                data_type: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "fare_amount".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "extra".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "mta_tax".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "tip_amount".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "tolls_amount".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "improvement_surcharge".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "total_amount".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "congestion_surcharge".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "airport_fee".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
         ],
         time_index: "tpep_pickup_datetime".to_string(),
         primary_keys: vec!["VendorID".to_string()],
         create_if_not_exists: false,
         table_options: Default::default(),
+        region_numbers: vec![0],
         table_id: None,
         engine: "mito".to_string(),
     }

config/datanode.example.toml

@@ -1,5 +1,7 @@
 # Node running mode, see `standalone.example.toml`.
 mode = "distributed"
+# Whether to use in-memory catalog, see `standalone.example.toml`.
+enable_memory_catalog = false
 # The datanode identifier, should be unique.
 node_id = 42
 # gRPC server address, "127.0.0.1:3001" by default.
@@ -8,24 +10,19 @@ rpc_addr = "127.0.0.1:3001"
 rpc_hostname = "127.0.0.1"
 # The number of gRPC server worker threads, 8 by default.
 rpc_runtime_size = 8
-# Start services after regions have obtained leases.
-# It will block the datanode start if it can't receive leases in the heartbeat from metasrv.
-require_lease_before_startup = false
 
 [heartbeat]
-# Interval for sending heartbeat messages to the Metasrv in milliseconds, 3000 by default.
+# Interval for sending heartbeat messages to the Metasrv in milliseconds, 5000 by default.
-interval_millis = 3000
+interval_millis = 5000
 
 # Metasrv client options.
-[meta_client]
+[meta_client_options]
 # Metasrv address list.
 metasrv_addrs = ["127.0.0.1:3002"]
-# Heartbeat timeout in milliseconds, 500 by default.
-heartbeat_timeout_millis = 500
 # Operation timeout in milliseconds, 3000 by default.
 timeout_millis = 3000
 # Connect server timeout in milliseconds, 5000 by default.
-connect_timeout_millis = 1000
+connect_timeout_millis = 5000
 # `TCP_NODELAY` option for accepted connections, true by default.
 tcp_nodelay = true
 
@@ -47,12 +44,6 @@ type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
 
-# Cache configuration for object storage such as 'S3' etc.
-# The local file cache directory
-# cache_path = "/path/local_cache"
-# The local file cache capacity in bytes.
-# cache_capacity = "256Mib"
-
 # Compaction options, see `standalone.example.toml`.
 [storage.compaction]
 max_inflight_tasks = 4
@@ -80,27 +71,10 @@ auto_flush_interval = "1h"
 # Global write buffer size for all regions.
 global_write_buffer_size = "1GB"
 
-# Mito engine options
+# Procedure storage options, see `standalone.example.toml`.
-[[region_engine]]
+[procedure]
-[region_engine.mito]
+max_retry_times = 3
-# Number of region workers
+retry_delay = "500ms"
-num_workers = 8
-# Request channel size of each worker
-worker_channel_size = 128
-# Max batch size for a worker to handle requests
-worker_request_batch_size = 64
-# Number of meta action updated to trigger a new checkpoint for the manifest
-manifest_checkpoint_distance = 10
-# Manifest compression type
-manifest_compress_type = "Uncompressed"
-# Max number of running background jobs
-max_background_jobs = 4
-# Interval to auto flush a region if it has not flushed yet.
-auto_flush_interval = "1h"
-# Global write buffer size for all regions.
-global_write_buffer_size = "1GB"
-# Global write buffer size threshold to reject write requests (default 2G).
-global_write_buffer_reject_size = "2GB"
 
 # Log options
 # [logging]

config/frontend.example.toml

@@ -8,61 +8,58 @@ interval_millis = 5000
 retry_interval_millis = 5000
 
 # HTTP server options, see `standalone.example.toml`.
-[http]
+[http_options]
 addr = "127.0.0.1:4000"
 timeout = "30s"
 body_limit = "64MB"
 
 # gRPC server options, see `standalone.example.toml`.
-[grpc]
+[grpc_options]
 addr = "127.0.0.1:4001"
 runtime_size = 8
 
 # MySQL server options, see `standalone.example.toml`.
-[mysql]
+[mysql_options]
-enable = true
 addr = "127.0.0.1:4002"
 runtime_size = 2
 
 # MySQL server TLS options, see `standalone.example.toml`.
-[mysql.tls]
+[mysql_options.tls]
 mode = "disable"
 cert_path = ""
 key_path = ""
 
 # PostgresSQL server options, see `standalone.example.toml`.
-[postgres]
+[postgres_options]
-enable = true
 addr = "127.0.0.1:4003"
 runtime_size = 2
 
 # PostgresSQL server TLS options, see `standalone.example.toml`.
-[postgres.tls]
+[postgres_options.tls]
 mode = "disable"
 cert_path = ""
 key_path = ""
 
 # OpenTSDB protocol options, see `standalone.example.toml`.
-[opentsdb]
+[opentsdb_options]
-enable = true
 addr = "127.0.0.1:4242"
 runtime_size = 2
 
 # InfluxDB protocol options, see `standalone.example.toml`.
-[influxdb]
+[influxdb_options]
 enable = true
 
 # Prometheus remote storage options, see `standalone.example.toml`.
-[prom_store]
+[prom_store_options]
 enable = true
 
 # Metasrv client options, see `datanode.example.toml`.
-[meta_client]
+[meta_client_options]
 metasrv_addrs = ["127.0.0.1:3002"]
 timeout_millis = 3000
 # DDL timeouts options.
 ddl_timeout_millis = 10000
-connect_timeout_millis = 1000
+connect_timeout_millis = 5000
 tcp_nodelay = true
 
 # Log options, see `standalone.example.toml`

config/metasrv.example.toml

@@ -6,6 +6,8 @@ bind_addr = "127.0.0.1:3002"
 server_addr = "127.0.0.1:3002"
 # Etcd server address, "127.0.0.1:2379" by default.
 store_addr = "127.0.0.1:2379"
+# Datanode lease in seconds, 15 seconds by default.
+datanode_lease_secs = 15
 # Datanode selector type.
 # - "LeaseBased" (default value).
 # - "LoadBased"

config/standalone.example.toml

@@ -1,10 +1,12 @@
 # Node running mode, "standalone" or "distributed".
 mode = "standalone"
+# Whether to use in-memory catalog, `false` by default.
+enable_memory_catalog = false
 # Whether to enable greptimedb telemetry, true by default.
 enable_telemetry = true
 
 # HTTP server options.
-[http]
+[http_options]
 # Server address, "127.0.0.1:4000" by default.
 addr = "127.0.0.1:4000"
 # HTTP request timeout, 30s by default.
@@ -14,23 +16,21 @@ timeout = "30s"
 body_limit = "64MB"
 
 # gRPC server options.
-[grpc]
+[grpc_options]
 # Server address, "127.0.0.1:4001" by default.
 addr = "127.0.0.1:4001"
 # The number of server worker threads, 8 by default.
 runtime_size = 8
 
 # MySQL server options.
-[mysql]
+[mysql_options]
-# Whether to enable
-enable = true
 # Server address, "127.0.0.1:4002" by default.
 addr = "127.0.0.1:4002"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # MySQL server TLS options.
-[mysql.tls]
+[mysql_options.tls]
 # TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html
 # - "disable" (default value)
 # - "prefer"
@@ -44,16 +44,14 @@ cert_path = ""
 key_path = ""
 
 # PostgresSQL server options.
-[postgres]
+[postgres_options]
-# Whether to enable
-enable = true
 # Server address, "127.0.0.1:4003" by default.
 addr = "127.0.0.1:4003"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # PostgresSQL server TLS options, see `[mysql_options.tls]` section.
-[postgres.tls]
+[postgres_options.tls]
 # TLS mode.
 mode = "disable"
 # certificate file path.
@@ -62,26 +60,26 @@ cert_path = ""
 key_path = ""
 
 # OpenTSDB protocol options.
-[opentsdb]
+[opentsdb_options]
-# Whether to enable
-enable = true
 # OpenTSDB telnet API server address, "127.0.0.1:4242" by default.
 addr = "127.0.0.1:4242"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # InfluxDB protocol options.
-[influxdb]
+[influxdb_options]
 # Whether to enable InfluxDB protocol in HTTP API, true by default.
 enable = true
 
 # Prometheus remote storage options
-[prom_store]
+[prom_store_options]
 # Whether to enable Prometheus remote write and read in HTTP API, true by default.
 enable = true
 
 # WAL options.
 [wal]
+# WAL data directory
+# dir = "/tmp/greptimedb/wal"
 # WAL file size in bytes.
 file_size = "256MB"
 # WAL purge threshold.
@@ -93,20 +91,6 @@ read_batch_size = 128
 # Whether to sync log file after every write.
 sync_write = false
 
-# Kv options.
-[kv_store]
-# Kv file size in bytes.
-file_size = "256MB"
-# Kv purge threshold.
-purge_threshold = "4GB"
-
-# Procedure storage options.
-[procedure]
-# Procedure max retry time.
-max_retry_times = 3
-# Initial retry delay of procedures, increases exponentially
-retry_delay = "500ms"
-
 # Storage options.
 [storage]
 # The working home directory.
@@ -115,10 +99,6 @@ data_home = "/tmp/greptimedb/"
 type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
-# Cache configuration for object storage such as 'S3' etc.
-# cache_path = "/path/local_cache"
-# The local file cache capacity in bytes.
-# cache_capacity = "256Mib"
 
 # Compaction options.
 [storage.compaction]
@@ -150,6 +130,13 @@ auto_flush_interval = "1h"
 # Global write buffer size for all regions.
 global_write_buffer_size = "1GB"
 
+# Procedure storage options.
+[procedure]
+# Procedure max retry time.
+max_retry_times = 3
+# Initial retry delay of procedures, increases exponentially
+retry_delay = "500ms"
+
 # Log options
 # [logging]
 # Specify logs directory.

docker/dev-builder/android/Dockerfile

@@ -1,41 +0,0 @@
-FROM --platform=linux/amd64 saschpe/android-ndk:34-jdk17.0.8_7-ndk25.2.9519653-cmake3.22.1
-
-ENV LANG en_US.utf8
-WORKDIR /greptimedb
-
-# Rename libunwind to libgcc
-RUN cp ${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/14.0.7/lib/linux/aarch64/libunwind.a ${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/14.0.7/lib/linux/aarch64/libgcc.a
-
-# Install dependencies.
-RUN apt-get update && apt-get install -y \
-    libssl-dev \
-    protobuf-compiler \
-    curl \
-    git \
-    build-essential \
-    pkg-config \
-    python3 \
-    python3-dev \
-    python3-pip \
-    && pip3 install --upgrade pip \
-    && pip3 install pyarrow
-
-# Trust workdir
-RUN git config --global --add safe.directory /greptimedb
-
-# Install Rust.
-SHELL ["/bin/bash", "-c"]
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
-ENV PATH /root/.cargo/bin/:$PATH
-
-# Add android toolchains
-ARG RUST_TOOLCHAIN
-RUN rustup toolchain install ${RUST_TOOLCHAIN}
-RUN rustup target add aarch64-linux-android
-
-# Install cargo-ndk
-RUN cargo install cargo-ndk
-ENV ANDROID_NDK_HOME $NDK_ROOT
-
-# Builder entrypoint.
-CMD ["cargo", "ndk", "--platform", "23", "-t", "aarch64-linux-android", "build", "--bin", "greptime", "--profile", "release", "--no-default-features"]

src/api/Cargo.toml

@@ -7,7 +7,6 @@ license.workspace = true
 [dependencies]
 common-base = { workspace = true }
 common-error = { workspace = true }
-common-macro = { workspace = true }
 common-time = { workspace = true }
 datatypes = { workspace = true }
 greptime-proto.workspace = true

src/api/src/error.rs

@@ -16,16 +16,14 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use datatypes::prelude::ConcreteDataType;
 use snafu::prelude::*;
 use snafu::Location;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unknown proto column datatype: {}", datatype))]
     UnknownColumnDataType { datatype: i32, location: Location },
@@ -36,14 +34,22 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to convert column default constraint, column: {}", column))]
+    #[snafu(display(
+        "Failed to convert column default constraint, column: {}, source: {}",
+        column,
+        source
+    ))]
     ConvertColumnDefaultConstraint {
         column: String,
         location: Location,
         source: datatypes::error::Error,
     },
 
-    #[snafu(display("Invalid column default constraint, column: {}", column))]
+    #[snafu(display(
+        "Invalid column default constraint, column: {}, source: {}",
+        column,
+        source
+    ))]
     InvalidColumnDefaultConstraint {
         column: String,
         location: Location,

src/api/src/helper.rs

@@ -18,32 +18,32 @@ use common_base::BitVec;
 use common_time::interval::IntervalUnit;
 use common_time::time::Time;
 use common_time::timestamp::TimeUnit;
-use common_time::{Date, DateTime, Duration, Interval, Timestamp};
+use common_time::{Date, DateTime, Interval, Timestamp};
 use datatypes::prelude::{ConcreteDataType, ValueRef};
 use datatypes::scalars::ScalarVector;
 use datatypes::types::{
-    DurationType, Int16Type, Int8Type, IntervalType, TimeType, TimestampType, UInt16Type, UInt8Type,
+    Int16Type, Int8Type, IntervalType, TimeType, TimestampType, UInt16Type, UInt8Type,
 };
 use datatypes::value::{OrderedF32, OrderedF64, Value};
 use datatypes::vectors::{
-    BinaryVector, BooleanVector, DateTimeVector, DateVector, DurationMicrosecondVector,
+    BinaryVector, BooleanVector, DateTimeVector, DateVector, Float32Vector, Float64Vector,
-    DurationMillisecondVector, DurationNanosecondVector, DurationSecondVector, Float32Vector,
+    Int32Vector, Int64Vector, IntervalDayTimeVector, IntervalMonthDayNanoVector,
-    Float64Vector, Int32Vector, Int64Vector, IntervalDayTimeVector, IntervalMonthDayNanoVector,
     IntervalYearMonthVector, PrimitiveVector, StringVector, TimeMicrosecondVector,
     TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector, TimestampMicrosecondVector,
     TimestampMillisecondVector, TimestampNanosecondVector, TimestampSecondVector, UInt32Vector,
     UInt64Vector, VectorRef,
 };
+use greptime_proto::v1;
 use greptime_proto::v1::ddl_request::Expr;
 use greptime_proto::v1::greptime_request::Request;
 use greptime_proto::v1::query_request::Query;
 use greptime_proto::v1::value::ValueData;
-use greptime_proto::v1::{self, DdlRequest, IntervalMonthDayNano, QueryRequest, Row, SemanticType};
+use greptime_proto::v1::{DdlRequest, IntervalMonthDayNano, QueryRequest, SemanticType};
 use snafu::prelude::*;
 
 use crate::error::{self, Result};
 use crate::v1::column::Values;
-use crate::v1::{Column, ColumnDataType, Value as GrpcValue};
+use crate::v1::{Column, ColumnDataType};
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct ColumnDataTypeWrapper(ColumnDataType);
@@ -101,14 +101,6 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
             ColumnDataType::IntervalMonthDayNano => {
                 ConcreteDataType::interval_month_day_nano_datatype()
             }
-            ColumnDataType::DurationSecond => ConcreteDataType::duration_second_datatype(),
-            ColumnDataType::DurationMillisecond => {
-                ConcreteDataType::duration_millisecond_datatype()
-            }
-            ColumnDataType::DurationMicrosecond => {
-                ConcreteDataType::duration_microsecond_datatype()
-            }
-            ColumnDataType::DurationNanosecond => ConcreteDataType::duration_nanosecond_datatype(),
         }
     }
 }
@@ -150,12 +142,6 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
                 IntervalType::DayTime(_) => ColumnDataType::IntervalDayTime,
                 IntervalType::MonthDayNano(_) => ColumnDataType::IntervalMonthDayNano,
             },
-            ConcreteDataType::Duration(d) => match d {
-                DurationType::Second(_) => ColumnDataType::DurationSecond,
-                DurationType::Millisecond(_) => ColumnDataType::DurationMillisecond,
-                DurationType::Microsecond(_) => ColumnDataType::DurationMicrosecond,
-                DurationType::Nanosecond(_) => ColumnDataType::DurationNanosecond,
-            },
             ConcreteDataType::Null(_)
             | ConcreteDataType::List(_)
             | ConcreteDataType::Dictionary(_) => {
@@ -229,19 +215,19 @@ pub fn values_with_capacity(datatype: ColumnDataType, capacity: usize) -> Values
             ..Default::default()
         },
         ColumnDataType::TimestampSecond => Values {
-            timestamp_second_values: Vec::with_capacity(capacity),
+            ts_second_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
         ColumnDataType::TimestampMillisecond => Values {
-            timestamp_millisecond_values: Vec::with_capacity(capacity),
+            ts_millisecond_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
         ColumnDataType::TimestampMicrosecond => Values {
-            timestamp_microsecond_values: Vec::with_capacity(capacity),
+            ts_microsecond_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
         ColumnDataType::TimestampNanosecond => Values {
-            timestamp_nanosecond_values: Vec::with_capacity(capacity),
+            ts_nanosecond_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
         ColumnDataType::TimeSecond => Values {
@@ -272,22 +258,6 @@ pub fn values_with_capacity(datatype: ColumnDataType, capacity: usize) -> Values
             interval_month_day_nano_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
-        ColumnDataType::DurationSecond => Values {
-            duration_second_values: Vec::with_capacity(capacity),
-            ..Default::default()
-        },
-        ColumnDataType::DurationMillisecond => Values {
-            duration_millisecond_values: Vec::with_capacity(capacity),
-            ..Default::default()
-        },
-        ColumnDataType::DurationMicrosecond => Values {
-            duration_microsecond_values: Vec::with_capacity(capacity),
-            ..Default::default()
-        },
-        ColumnDataType::DurationNanosecond => Values {
-            duration_nanosecond_values: Vec::with_capacity(capacity),
-            ..Default::default()
-        },
     }
 }
 
@@ -317,10 +287,10 @@ pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
         Value::Date(val) => values.date_values.push(val.val()),
         Value::DateTime(val) => values.datetime_values.push(val.val()),
         Value::Timestamp(val) => match val.unit() {
-            TimeUnit::Second => values.timestamp_second_values.push(val.value()),
+            TimeUnit::Second => values.ts_second_values.push(val.value()),
-            TimeUnit::Millisecond => values.timestamp_millisecond_values.push(val.value()),
+            TimeUnit::Millisecond => values.ts_millisecond_values.push(val.value()),
-            TimeUnit::Microsecond => values.timestamp_microsecond_values.push(val.value()),
+            TimeUnit::Microsecond => values.ts_microsecond_values.push(val.value()),
-            TimeUnit::Nanosecond => values.timestamp_nanosecond_values.push(val.value()),
+            TimeUnit::Nanosecond => values.ts_nanosecond_values.push(val.value()),
         },
         Value::Time(val) => match val.unit() {
             TimeUnit::Second => values.time_second_values.push(val.value()),
@@ -335,12 +305,6 @@ pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
                 .interval_month_day_nano_values
                 .push(convert_i128_to_interval(val.to_i128())),
         },
-        Value::Duration(val) => match val.unit() {
-            TimeUnit::Second => values.duration_second_values.push(val.value()),
-            TimeUnit::Millisecond => values.duration_millisecond_values.push(val.value()),
-            TimeUnit::Microsecond => values.duration_microsecond_values.push(val.value()),
-            TimeUnit::Nanosecond => values.duration_nanosecond_values.push(val.value()),
-        },
         Value::List(_) => unreachable!(),
     });
     column.null_mask = null_mask.into_vec();
@@ -375,6 +339,8 @@ fn ddl_request_type(request: &DdlRequest) -> &'static str {
         Some(Expr::CreateTable(_)) => "ddl.create_table",
         Some(Expr::Alter(_)) => "ddl.alter",
         Some(Expr::DropTable(_)) => "ddl.drop_table",
+        Some(Expr::FlushTable(_)) => "ddl.flush_table",
+        Some(Expr::CompactTable(_)) => "ddl.compact_table",
         Some(Expr::TruncateTable(_)) => "ddl.truncate_table",
         None => "ddl.empty",
     }
@@ -412,16 +378,10 @@ pub fn pb_value_to_value_ref(value: &v1::Value) -> ValueRef {
         ValueData::StringValue(string) => ValueRef::String(string.as_str()),
         ValueData::DateValue(d) => ValueRef::Date(Date::from(*d)),
         ValueData::DatetimeValue(d) => ValueRef::DateTime(DateTime::new(*d)),
-        ValueData::TimestampSecondValue(t) => ValueRef::Timestamp(Timestamp::new_second(*t)),
+        ValueData::TsSecondValue(t) => ValueRef::Timestamp(Timestamp::new_second(*t)),
-        ValueData::TimestampMillisecondValue(t) => {
+        ValueData::TsMillisecondValue(t) => ValueRef::Timestamp(Timestamp::new_millisecond(*t)),
-            ValueRef::Timestamp(Timestamp::new_millisecond(*t))
+        ValueData::TsMicrosecondValue(t) => ValueRef::Timestamp(Timestamp::new_microsecond(*t)),
-        }
+        ValueData::TsNanosecondValue(t) => ValueRef::Timestamp(Timestamp::new_nanosecond(*t)),
-        ValueData::TimestampMicrosecondValue(t) => {
-            ValueRef::Timestamp(Timestamp::new_microsecond(*t))
-        }
-        ValueData::TimestampNanosecondValue(t) => {
-            ValueRef::Timestamp(Timestamp::new_nanosecond(*t))
-        }
         ValueData::TimeSecondValue(t) => ValueRef::Time(Time::new_second(*t)),
         ValueData::TimeMillisecondValue(t) => ValueRef::Time(Time::new_millisecond(*t)),
         ValueData::TimeMicrosecondValue(t) => ValueRef::Time(Time::new_microsecond(*t)),
@@ -432,10 +392,6 @@ pub fn pb_value_to_value_ref(value: &v1::Value) -> ValueRef {
             let interval = Interval::from_month_day_nano(v.months, v.days, v.nanoseconds);
             ValueRef::Interval(interval)
         }
-        ValueData::DurationSecondValue(v) => ValueRef::Duration(Duration::new_second(*v)),
-        ValueData::DurationMillisecondValue(v) => ValueRef::Duration(Duration::new_millisecond(*v)),
-        ValueData::DurationMicrosecondValue(v) => ValueRef::Duration(Duration::new_microsecond(*v)),
-        ValueData::DurationNanosecondValue(v) => ValueRef::Duration(Duration::new_nanosecond(*v)),
     }
 }
 
@@ -465,17 +421,17 @@ pub fn pb_values_to_vector_ref(data_type: &ConcreteDataType, values: Values) ->
         ConcreteDataType::Date(_) => Arc::new(DateVector::from_vec(values.date_values)),
         ConcreteDataType::DateTime(_) => Arc::new(DateTimeVector::from_vec(values.datetime_values)),
         ConcreteDataType::Timestamp(unit) => match unit {
-            TimestampType::Second(_) => Arc::new(TimestampSecondVector::from_vec(
+            TimestampType::Second(_) => {
-                values.timestamp_second_values,
+                Arc::new(TimestampSecondVector::from_vec(values.ts_second_values))
-            )),
+            }
             TimestampType::Millisecond(_) => Arc::new(TimestampMillisecondVector::from_vec(
-                values.timestamp_millisecond_values,
+                values.ts_millisecond_values,
             )),
             TimestampType::Microsecond(_) => Arc::new(TimestampMicrosecondVector::from_vec(
-                values.timestamp_microsecond_values,
+                values.ts_microsecond_values,
             )),
             TimestampType::Nanosecond(_) => Arc::new(TimestampNanosecondVector::from_vec(
-                values.timestamp_nanosecond_values,
+                values.ts_nanosecond_values,
             )),
         },
         ConcreteDataType::Time(unit) => match unit {
@@ -508,20 +464,6 @@ pub fn pb_values_to_vector_ref(data_type: &ConcreteDataType, values: Values) ->
                 ))
             }
         },
-        ConcreteDataType::Duration(unit) => match unit {
-            DurationType::Second(_) => Arc::new(DurationSecondVector::from_vec(
-                values.duration_second_values,
-            )),
-            DurationType::Millisecond(_) => Arc::new(DurationMillisecondVector::from_vec(
-                values.duration_millisecond_values,
-            )),
-            DurationType::Microsecond(_) => Arc::new(DurationMicrosecondVector::from_vec(
-                values.duration_microsecond_values,
-            )),
-            DurationType::Nanosecond(_) => Arc::new(DurationNanosecondVector::from_vec(
-                values.duration_nanosecond_values,
-            )),
-        },
         ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
             unreachable!()
         }
@@ -611,22 +553,22 @@ pub fn pb_values_to_values(data_type: &ConcreteDataType, values: Values) -> Vec<
             .map(|v| Value::Date(v.into()))
             .collect(),
         ConcreteDataType::Timestamp(TimestampType::Second(_)) => values
-            .timestamp_second_values
+            .ts_second_values
             .into_iter()
             .map(|v| Value::Timestamp(Timestamp::new_second(v)))
             .collect(),
         ConcreteDataType::Timestamp(TimestampType::Millisecond(_)) => values
-            .timestamp_millisecond_values
+            .ts_millisecond_values
             .into_iter()
             .map(|v| Value::Timestamp(Timestamp::new_millisecond(v)))
             .collect(),
         ConcreteDataType::Timestamp(TimestampType::Microsecond(_)) => values
-            .timestamp_microsecond_values
+            .ts_microsecond_values
             .into_iter()
             .map(|v| Value::Timestamp(Timestamp::new_microsecond(v)))
             .collect(),
         ConcreteDataType::Timestamp(TimestampType::Nanosecond(_)) => values
-            .timestamp_nanosecond_values
+            .ts_nanosecond_values
             .into_iter()
             .map(|v| Value::Timestamp(Timestamp::new_nanosecond(v)))
             .collect(),
@@ -672,26 +614,6 @@ pub fn pb_values_to_values(data_type: &ConcreteDataType, values: Values) -> Vec<
                 ))
             })
             .collect(),
-        ConcreteDataType::Duration(DurationType::Second(_)) => values
-            .duration_second_values
-            .into_iter()
-            .map(|v| Value::Duration(Duration::new_second(v)))
-            .collect(),
-        ConcreteDataType::Duration(DurationType::Millisecond(_)) => values
-            .duration_millisecond_values
-            .into_iter()
-            .map(|v| Value::Duration(Duration::new_millisecond(v)))
-            .collect(),
-        ConcreteDataType::Duration(DurationType::Microsecond(_)) => values
-            .duration_microsecond_values
-            .into_iter()
-            .map(|v| Value::Duration(Duration::new_microsecond(v)))
-            .collect(),
-        ConcreteDataType::Duration(DurationType::Nanosecond(_)) => values
-            .duration_nanosecond_values
-            .into_iter()
-            .map(|v| Value::Duration(Duration::new_nanosecond(v)))
-            .collect(),
         ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
             unreachable!()
         }
@@ -763,16 +685,16 @@ pub fn to_proto_value(value: Value) -> Option<v1::Value> {
         },
         Value::Timestamp(v) => match v.unit() {
             TimeUnit::Second => v1::Value {
-                value_data: Some(ValueData::TimestampSecondValue(v.value())),
+                value_data: Some(ValueData::TsSecondValue(v.value())),
             },
             TimeUnit::Millisecond => v1::Value {
-                value_data: Some(ValueData::TimestampMillisecondValue(v.value())),
+                value_data: Some(ValueData::TsMillisecondValue(v.value())),
             },
             TimeUnit::Microsecond => v1::Value {
-                value_data: Some(ValueData::TimestampMicrosecondValue(v.value())),
+                value_data: Some(ValueData::TsMicrosecondValue(v.value())),
             },
             TimeUnit::Nanosecond => v1::Value {
-                value_data: Some(ValueData::TimestampNanosecondValue(v.value())),
+                value_data: Some(ValueData::TsNanosecondValue(v.value())),
             },
         },
         Value::Time(v) => match v.unit() {
@@ -802,20 +724,6 @@ pub fn to_proto_value(value: Value) -> Option<v1::Value> {
                 )),
             },
         },
-        Value::Duration(v) => match v.unit() {
-            TimeUnit::Second => v1::Value {
-                value_data: Some(ValueData::DurationSecondValue(v.value())),
-            },
-            TimeUnit::Millisecond => v1::Value {
-                value_data: Some(ValueData::DurationMillisecondValue(v.value())),
-            },
-            TimeUnit::Microsecond => v1::Value {
-                value_data: Some(ValueData::DurationMicrosecondValue(v.value())),
-            },
-            TimeUnit::Nanosecond => v1::Value {
-                value_data: Some(ValueData::DurationNanosecondValue(v.value())),
-            },
-        },
         Value::List(_) => return None,
     };
 
@@ -842,10 +750,10 @@ pub fn proto_value_type(value: &v1::Value) -> Option<ColumnDataType> {
         ValueData::StringValue(_) => ColumnDataType::String,
         ValueData::DateValue(_) => ColumnDataType::Date,
         ValueData::DatetimeValue(_) => ColumnDataType::Datetime,
-        ValueData::TimestampSecondValue(_) => ColumnDataType::TimestampSecond,
+        ValueData::TsSecondValue(_) => ColumnDataType::TimestampSecond,
-        ValueData::TimestampMillisecondValue(_) => ColumnDataType::TimestampMillisecond,
+        ValueData::TsMillisecondValue(_) => ColumnDataType::TimestampMillisecond,
-        ValueData::TimestampMicrosecondValue(_) => ColumnDataType::TimestampMicrosecond,
+        ValueData::TsMicrosecondValue(_) => ColumnDataType::TimestampMicrosecond,
-        ValueData::TimestampNanosecondValue(_) => ColumnDataType::TimestampNanosecond,
+        ValueData::TsNanosecondValue(_) => ColumnDataType::TimestampNanosecond,
         ValueData::TimeSecondValue(_) => ColumnDataType::TimeSecond,
         ValueData::TimeMillisecondValue(_) => ColumnDataType::TimeMillisecond,
         ValueData::TimeMicrosecondValue(_) => ColumnDataType::TimeMicrosecond,
@@ -853,10 +761,6 @@ pub fn proto_value_type(value: &v1::Value) -> Option<ColumnDataType> {
         ValueData::IntervalYearMonthValues(_) => ColumnDataType::IntervalYearMonth,
         ValueData::IntervalDayTimeValues(_) => ColumnDataType::IntervalDayTime,
         ValueData::IntervalMonthDayNanoValues(_) => ColumnDataType::IntervalMonthDayNano,
-        ValueData::DurationSecondValue(_) => ColumnDataType::DurationSecond,
-        ValueData::DurationMillisecondValue(_) => ColumnDataType::DurationMillisecond,
-        ValueData::DurationMicrosecondValue(_) => ColumnDataType::DurationMicrosecond,
-        ValueData::DurationNanosecondValue(_) => ColumnDataType::DurationNanosecond,
     };
     Some(value_type)
 }
@@ -893,92 +797,15 @@ pub fn to_column_data_type(data_type: &ConcreteDataType) -> Option<ColumnDataTyp
         ConcreteDataType::Time(TimeType::Millisecond(_)) => ColumnDataType::TimeMillisecond,
         ConcreteDataType::Time(TimeType::Microsecond(_)) => ColumnDataType::TimeMicrosecond,
         ConcreteDataType::Time(TimeType::Nanosecond(_)) => ColumnDataType::TimeNanosecond,
-        ConcreteDataType::Duration(DurationType::Second(_)) => ColumnDataType::DurationSecond,
+        ConcreteDataType::Null(_)
-        ConcreteDataType::Duration(DurationType::Millisecond(_)) => {
+        | ConcreteDataType::Interval(_)
-            ColumnDataType::DurationMillisecond
+        | ConcreteDataType::List(_)
-        }
+        | ConcreteDataType::Dictionary(_) => return None,
-        ConcreteDataType::Duration(DurationType::Microsecond(_)) => {
-            ColumnDataType::DurationMicrosecond
-        }
-        ConcreteDataType::Duration(DurationType::Nanosecond(_)) => {
-            ColumnDataType::DurationNanosecond
-        }
-        ConcreteDataType::Interval(IntervalType::YearMonth(_)) => ColumnDataType::IntervalYearMonth,
-        ConcreteDataType::Interval(IntervalType::MonthDayNano(_)) => {
-            ColumnDataType::IntervalMonthDayNano
-        }
-        ConcreteDataType::Interval(IntervalType::DayTime(_)) => ColumnDataType::IntervalDayTime,
-        ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
-            return None
-        }
     };
 
     Some(column_data_type)
 }
 
-pub fn vectors_to_rows<'a>(
-    columns: impl Iterator<Item = &'a VectorRef>,
-    row_count: usize,
-) -> Vec<Row> {
-    let mut rows = vec![Row { values: vec![] }; row_count];
-    for column in columns {
-        for (row_index, row) in rows.iter_mut().enumerate() {
-            row.values.push(value_to_grpc_value(column.get(row_index)))
-        }
-    }
-
-    rows
-}
-
-pub fn value_to_grpc_value(value: Value) -> GrpcValue {
-    GrpcValue {
-        value_data: match value {
-            Value::Null => None,
-            Value::Boolean(v) => Some(ValueData::BoolValue(v)),
-            Value::UInt8(v) => Some(ValueData::U8Value(v as _)),
-            Value::UInt16(v) => Some(ValueData::U16Value(v as _)),
-            Value::UInt32(v) => Some(ValueData::U32Value(v)),
-            Value::UInt64(v) => Some(ValueData::U64Value(v)),
-            Value::Int8(v) => Some(ValueData::I8Value(v as _)),
-            Value::Int16(v) => Some(ValueData::I16Value(v as _)),
-            Value::Int32(v) => Some(ValueData::I32Value(v)),
-            Value::Int64(v) => Some(ValueData::I64Value(v)),
-            Value::Float32(v) => Some(ValueData::F32Value(*v)),
-            Value::Float64(v) => Some(ValueData::F64Value(*v)),
-            Value::String(v) => Some(ValueData::StringValue(v.as_utf8().to_string())),
-            Value::Binary(v) => Some(ValueData::BinaryValue(v.to_vec())),
-            Value::Date(v) => Some(ValueData::DateValue(v.val())),
-            Value::DateTime(v) => Some(ValueData::DatetimeValue(v.val())),
-            Value::Timestamp(v) => Some(match v.unit() {
-                TimeUnit::Second => ValueData::TimestampSecondValue(v.value()),
-                TimeUnit::Millisecond => ValueData::TimestampMillisecondValue(v.value()),
-                TimeUnit::Microsecond => ValueData::TimestampMicrosecondValue(v.value()),
-                TimeUnit::Nanosecond => ValueData::TimestampNanosecondValue(v.value()),
-            }),
-            Value::Time(v) => Some(match v.unit() {
-                TimeUnit::Second => ValueData::TimeSecondValue(v.value()),
-                TimeUnit::Millisecond => ValueData::TimeMillisecondValue(v.value()),
-                TimeUnit::Microsecond => ValueData::TimeMicrosecondValue(v.value()),
-                TimeUnit::Nanosecond => ValueData::TimeNanosecondValue(v.value()),
-            }),
-            Value::Interval(v) => Some(match v.unit() {
-                IntervalUnit::YearMonth => ValueData::IntervalYearMonthValues(v.to_i32()),
-                IntervalUnit::DayTime => ValueData::IntervalDayTimeValues(v.to_i64()),
-                IntervalUnit::MonthDayNano => {
-                    ValueData::IntervalMonthDayNanoValues(convert_i128_to_interval(v.to_i128()))
-                }
-            }),
-            Value::Duration(v) => Some(match v.unit() {
-                TimeUnit::Second => ValueData::DurationSecondValue(v.value()),
-                TimeUnit::Millisecond => ValueData::DurationMillisecondValue(v.value()),
-                TimeUnit::Microsecond => ValueData::DurationMicrosecondValue(v.value()),
-                TimeUnit::Nanosecond => ValueData::DurationNanosecondValue(v.value()),
-            }),
-            Value::List(_) => unreachable!(),
-        },
-    }
-}
-
 /// Returns true if the column type is equal to expected type.
 fn is_column_type_eq(column_type: ColumnDataType, expect_type: &ConcreteDataType) -> bool {
     if let Some(expect) = to_column_data_type(expect_type) {
@@ -993,16 +820,14 @@ mod tests {
     use std::sync::Arc;
 
     use datatypes::types::{
-        DurationMillisecondType, DurationSecondType, Int32Type, IntervalDayTimeType,
+        IntervalDayTimeType, IntervalMonthDayNanoType, IntervalYearMonthType, TimeMillisecondType,
-        IntervalMonthDayNanoType, IntervalYearMonthType, TimeMillisecondType, TimeSecondType,
+        TimeSecondType, TimestampMillisecondType, TimestampSecondType,
-        TimestampMillisecondType, TimestampSecondType, UInt32Type,
     };
     use datatypes::vectors::{
-        BooleanVector, DurationMicrosecondVector, DurationMillisecondVector,
+        BooleanVector, IntervalDayTimeVector, IntervalMonthDayNanoVector, IntervalYearMonthVector,
-        DurationNanosecondVector, DurationSecondVector, IntervalDayTimeVector,
+        TimeMicrosecondVector, TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector,
-        IntervalMonthDayNanoVector, IntervalYearMonthVector, TimeMicrosecondVector,
+        TimestampMicrosecondVector, TimestampMillisecondVector, TimestampNanosecondVector,
-        TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector, TimestampMicrosecondVector,
+        TimestampSecondVector, Vector,
-        TimestampMillisecondVector, TimestampNanosecondVector, TimestampSecondVector, Vector,
     };
     use paste::paste;
 
@@ -1063,7 +888,7 @@ mod tests {
         assert_eq!(2, values.capacity());
 
         let values = values_with_capacity(ColumnDataType::TimestampMillisecond, 2);
-        let values = values.timestamp_millisecond_values;
+        let values = values.ts_millisecond_values;
         assert_eq!(2, values.capacity());
 
         let values = values_with_capacity(ColumnDataType::TimeMillisecond, 2);
@@ -1077,10 +902,6 @@ mod tests {
         let values = values_with_capacity(ColumnDataType::IntervalMonthDayNano, 2);
         let values = values.interval_month_day_nano_values;
         assert_eq!(2, values.capacity());
-
-        let values = values_with_capacity(ColumnDataType::DurationMillisecond, 2);
-        let values = values.duration_millisecond_values;
-        assert_eq!(2, values.capacity());
     }
 
     #[test]
@@ -1165,10 +986,6 @@ mod tests {
             ConcreteDataType::interval_datatype(IntervalUnit::MonthDayNano),
             ColumnDataTypeWrapper(ColumnDataType::IntervalMonthDayNano).into()
         );
-        assert_eq!(
-            ConcreteDataType::duration_millisecond_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::DurationMillisecond).into()
-        )
     }
 
     #[test]
@@ -1257,12 +1074,6 @@ mod tests {
                 .try_into()
                 .unwrap()
         );
-        assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::DurationMillisecond),
-            ConcreteDataType::duration_millisecond_datatype()
-                .try_into()
-                .unwrap()
-        );
 
         let result: Result<ColumnDataTypeWrapper> = ConcreteDataType::null_datatype().try_into();
         assert!(result.is_err());
@@ -1296,28 +1107,28 @@ mod tests {
         push_vals(&mut column, 3, vector);
         assert_eq!(
             vec![1, 2, 3],
-            column.values.as_ref().unwrap().timestamp_nanosecond_values
+            column.values.as_ref().unwrap().ts_nanosecond_values
         );
 
         let vector = Arc::new(TimestampMillisecondVector::from_vec(vec![4, 5, 6]));
         push_vals(&mut column, 3, vector);
         assert_eq!(
             vec![4, 5, 6],
-            column.values.as_ref().unwrap().timestamp_millisecond_values
+            column.values.as_ref().unwrap().ts_millisecond_values
         );
 
         let vector = Arc::new(TimestampMicrosecondVector::from_vec(vec![7, 8, 9]));
         push_vals(&mut column, 3, vector);
         assert_eq!(
             vec![7, 8, 9],
-            column.values.as_ref().unwrap().timestamp_microsecond_values
+            column.values.as_ref().unwrap().ts_microsecond_values
         );
 
         let vector = Arc::new(TimestampSecondVector::from_vec(vec![10, 11, 12]));
         push_vals(&mut column, 3, vector);
         assert_eq!(
             vec![10, 11, 12],
-            column.values.as_ref().unwrap().timestamp_second_values
+            column.values.as_ref().unwrap().ts_second_values
         );
     }
 
@@ -1406,47 +1217,6 @@ mod tests {
         });
     }
 
-    #[test]
-    fn test_column_put_duration_values() {
-        let mut column = Column {
-            column_name: "test".to_string(),
-            semantic_type: 0,
-            values: Some(Values {
-                ..Default::default()
-            }),
-            null_mask: vec![],
-            datatype: 0,
-        };
-
-        let vector = Arc::new(DurationNanosecondVector::from_vec(vec![1, 2, 3]));
-        push_vals(&mut column, 3, vector);
-        assert_eq!(
-            vec![1, 2, 3],
-            column.values.as_ref().unwrap().duration_nanosecond_values
-        );
-
-        let vector = Arc::new(DurationMicrosecondVector::from_vec(vec![7, 8, 9]));
-        push_vals(&mut column, 3, vector);
-        assert_eq!(
-            vec![7, 8, 9],
-            column.values.as_ref().unwrap().duration_microsecond_values
-        );
-
-        let vector = Arc::new(DurationMillisecondVector::from_vec(vec![4, 5, 6]));
-        push_vals(&mut column, 3, vector);
-        assert_eq!(
-            vec![4, 5, 6],
-            column.values.as_ref().unwrap().duration_millisecond_values
-        );
-
-        let vector = Arc::new(DurationSecondVector::from_vec(vec![10, 11, 12]));
-        push_vals(&mut column, 3, vector);
-        assert_eq!(
-            vec![10, 11, 12],
-            column.values.as_ref().unwrap().duration_second_values
-        );
-    }
-
     #[test]
     fn test_column_put_vector() {
         use crate::v1::SemanticType;
@@ -1487,7 +1257,7 @@ mod tests {
         let actual = pb_values_to_values(
             &ConcreteDataType::Timestamp(TimestampType::Second(TimestampSecondType)),
             Values {
-                timestamp_second_values: vec![1_i64, 2_i64, 3_i64],
+                ts_second_values: vec![1_i64, 2_i64, 3_i64],
                 ..Default::default()
             },
         );
@@ -1502,7 +1272,7 @@ mod tests {
         let actual = pb_values_to_values(
             &ConcreteDataType::Timestamp(TimestampType::Millisecond(TimestampMillisecondType)),
             Values {
-                timestamp_millisecond_values: vec![1_i64, 2_i64, 3_i64],
+                ts_millisecond_values: vec![1_i64, 2_i64, 3_i64],
                 ..Default::default()
             },
         );
@@ -1547,39 +1317,6 @@ mod tests {
         assert_eq!(expect, actual);
     }
 
-    #[test]
-    fn test_convert_duration_values() {
-        // second
-        let actual = pb_values_to_values(
-            &ConcreteDataType::Duration(DurationType::Second(DurationSecondType)),
-            Values {
-                duration_second_values: vec![1_i64, 2_i64, 3_i64],
-                ..Default::default()
-            },
-        );
-        let expect = vec![
-            Value::Duration(Duration::new_second(1_i64)),
-            Value::Duration(Duration::new_second(2_i64)),
-            Value::Duration(Duration::new_second(3_i64)),
-        ];
-        assert_eq!(expect, actual);
-
-        // millisecond
-        let actual = pb_values_to_values(
-            &ConcreteDataType::Duration(DurationType::Millisecond(DurationMillisecondType)),
-            Values {
-                duration_millisecond_values: vec![1_i64, 2_i64, 3_i64],
-                ..Default::default()
-            },
-        );
-        let expect = vec![
-            Value::Duration(Duration::new_millisecond(1_i64)),
-            Value::Duration(Duration::new_millisecond(2_i64)),
-            Value::Duration(Duration::new_millisecond(3_i64)),
-        ];
-        assert_eq!(expect, actual);
-    }
-
     #[test]
     fn test_convert_interval_values() {
         // year_month
@@ -1787,76 +1524,4 @@ mod tests {
             Value::DateTime(3.into())
         ]
     );
-
-    #[test]
-    fn test_vectors_to_rows_for_different_types() {
-        let boolean_vec = BooleanVector::from_vec(vec![true, false, true]);
-        let int8_vec = PrimitiveVector::<Int8Type>::from_iter_values(vec![1, 2, 3]);
-        let int32_vec = PrimitiveVector::<Int32Type>::from_iter_values(vec![100, 200, 300]);
-        let uint8_vec = PrimitiveVector::<UInt8Type>::from_iter_values(vec![10, 20, 30]);
-        let uint32_vec = PrimitiveVector::<UInt32Type>::from_iter_values(vec![1000, 2000, 3000]);
-        let float32_vec = Float32Vector::from_vec(vec![1.1, 2.2, 3.3]);
-        let date_vec = DateVector::from_vec(vec![10, 20, 30]);
-        let string_vec = StringVector::from_vec(vec!["a", "b", "c"]);
-
-        let vector_refs: Vec<VectorRef> = vec![
-            Arc::new(boolean_vec),
-            Arc::new(int8_vec),
-            Arc::new(int32_vec),
-            Arc::new(uint8_vec),
-            Arc::new(uint32_vec),
-            Arc::new(float32_vec),
-            Arc::new(date_vec),
-            Arc::new(string_vec),
-        ];
-
-        let result = vectors_to_rows(vector_refs.iter(), 3);
-
-        assert_eq!(result.len(), 3);
-
-        assert_eq!(result[0].values.len(), 8);
-        let values = result[0]
-            .values
-            .iter()
-            .map(|v| v.value_data.clone().unwrap())
-            .collect::<Vec<_>>();
-        assert_eq!(values[0], ValueData::BoolValue(true));
-        assert_eq!(values[1], ValueData::I8Value(1));
-        assert_eq!(values[2], ValueData::I32Value(100));
-        assert_eq!(values[3], ValueData::U8Value(10));
-        assert_eq!(values[4], ValueData::U32Value(1000));
-        assert_eq!(values[5], ValueData::F32Value(1.1));
-        assert_eq!(values[6], ValueData::DateValue(10));
-        assert_eq!(values[7], ValueData::StringValue("a".to_string()));
-
-        assert_eq!(result[1].values.len(), 8);
-        let values = result[1]
-            .values
-            .iter()
-            .map(|v| v.value_data.clone().unwrap())
-            .collect::<Vec<_>>();
-        assert_eq!(values[0], ValueData::BoolValue(false));
-        assert_eq!(values[1], ValueData::I8Value(2));
-        assert_eq!(values[2], ValueData::I32Value(200));
-        assert_eq!(values[3], ValueData::U8Value(20));
-        assert_eq!(values[4], ValueData::U32Value(2000));
-        assert_eq!(values[5], ValueData::F32Value(2.2));
-        assert_eq!(values[6], ValueData::DateValue(20));
-        assert_eq!(values[7], ValueData::StringValue("b".to_string()));
-
-        assert_eq!(result[2].values.len(), 8);
-        let values = result[2]
-            .values
-            .iter()
-            .map(|v| v.value_data.clone().unwrap())
-            .collect::<Vec<_>>();
-        assert_eq!(values[0], ValueData::BoolValue(true));
-        assert_eq!(values[1], ValueData::I8Value(3));
-        assert_eq!(values[2], ValueData::I32Value(300));
-        assert_eq!(values[3], ValueData::U8Value(30));
-        assert_eq!(values[4], ValueData::U32Value(3000));
-        assert_eq!(values[5], ValueData::F32Value(3.3));
-        assert_eq!(values[6], ValueData::DateValue(30));
-        assert_eq!(values[7], ValueData::StringValue("c".to_string()));
-    }
 }

src/api/src/v1/column_def.rs

@@ -12,9 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema};
-
-use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, COMMENT_KEY};
 use snafu::ResultExt;
 
 use crate::error::{self, Result};
@@ -22,7 +20,7 @@ use crate::helper::ColumnDataTypeWrapper;
 use crate::v1::ColumnDef;
 
 pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
-    let data_type = ColumnDataTypeWrapper::try_new(column_def.data_type)?;
+    let data_type = ColumnDataTypeWrapper::try_new(column_def.datatype)?;
 
     let constraint = if column_def.default_constraint.is_empty() {
         None
@@ -36,17 +34,9 @@ pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
         )
     };
 
-    let mut metadata = HashMap::new();
+    ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
-    if !column_def.comment.is_empty() {
+        .with_default_constraint(constraint)
-        metadata.insert(COMMENT_KEY.to_string(), column_def.comment.clone());
+        .context(error::InvalidColumnDefaultConstraintSnafu {
-    }
+            column: &column_def.name,
-
+        })
-    Ok(
-        ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
-            .with_default_constraint(constraint)
-            .context(error::InvalidColumnDefaultConstraintSnafu {
-                column: &column_def.name,
-            })?
-            .with_metadata(metadata),
-    )
 }

src/auth/Cargo.toml

@@ -4,6 +4,8 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
 [features]
 default = []
 testing = []
@@ -12,7 +14,6 @@ testing = []
 api.workspace = true
 async-trait.workspace = true
 common-error.workspace = true
-common-macro.workspace = true
 digest = "0.10"
 hex = { version = "0.4" }
 secrecy = { version = "0.8", features = ["serde", "alloc"] }

src/auth/src/common.rs

@@ -26,7 +26,7 @@ use crate::{UserInfoRef, UserProviderRef};
 
 pub(crate) const DEFAULT_USERNAME: &str = "greptime";
 
-/// construct a [`UserInfo`](crate::user_info::UserInfo) impl with name
+/// construct a [`UserInfo`] impl with name
 /// use default username `greptime` if None is provided
 pub fn userinfo_by_name(username: Option<String>) -> UserInfoRef {
     DefaultUserInfo::with_name(username.unwrap_or_else(|| DEFAULT_USERNAME.to_string()))

src/auth/src/error.rs

@@ -14,12 +14,10 @@
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Invalid config value: {}, {}", value, msg))]
     InvalidConfig { value: String, msg: String },
@@ -30,14 +28,13 @@ pub enum Error {
     #[snafu(display("Internal state error: {}", msg))]
     InternalState { msg: String },
 
-    #[snafu(display("IO error"))]
+    #[snafu(display("IO error, source: {}", source))]
     Io {
-        #[snafu(source)]
+        source: std::io::Error,
-        error: std::io::Error,
         location: Location,
     },
 
-    #[snafu(display("Auth failed"))]
+    #[snafu(display("Auth failed, source: {}", source))]
     AuthBackend {
         location: Location,
         source: BoxedError,

src/auth/src/user_provider.rs

@@ -22,15 +22,15 @@ use crate::UserInfoRef;
 pub trait UserProvider: Send + Sync {
     fn name(&self) -> &str;
 
-    /// Checks whether a user is valid and allowed to access the database.
+    /// [`authenticate`] checks whether a user is valid and allowed to access the database.
     async fn authenticate(&self, id: Identity<'_>, password: Password<'_>) -> Result<UserInfoRef>;
 
-    /// Checks whether a connection request
+    /// [`authorize`] checks whether a connection request
     /// from a certain user to a certain catalog/schema is legal.
-    /// This method should be called after [authenticate()](UserProvider::authenticate()).
+    /// This method should be called after [`authenticate`].
     async fn authorize(&self, catalog: &str, schema: &str, user_info: &UserInfoRef) -> Result<()>;
 
-    /// Combination of [authenticate()](UserProvider::authenticate()) and [authorize()](UserProvider::authorize()).
+    /// [`auth`] is a combination of [`authenticate`] and [`authorize`].
     /// In most cases it's preferred for both convenience and performance.
     async fn auth(
         &self,

src/catalog/Cargo.toml

@@ -16,7 +16,6 @@ async-trait = "0.1"
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
-common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
@@ -31,9 +30,8 @@ futures-util.workspace = true
 lazy_static.workspace = true
 meta-client = { workspace = true }
 metrics.workspace = true
-moka = { workspace = true, features = ["future"] }
+moka = { version = "0.11", features = ["future"] }
 parking_lot = "0.12"
-partition.workspace = true
 regex.workspace = true
 serde.workspace = true
 serde_json = "1.0"
@@ -48,6 +46,7 @@ catalog = { workspace = true, features = ["testing"] }
 chrono.workspace = true
 common-test-util = { workspace = true }
 log-store = { workspace = true }
+mito = { workspace = true, features = ["test"] }
 object-store = { workspace = true }
 storage = { workspace = true }
 tokio.workspace = true

src/catalog/src/error.rs

@@ -17,48 +17,54 @@ use std::fmt::Debug;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
-use table::metadata::TableId;
 use tokio::task::JoinError;
 
-#[derive(Snafu)]
+use crate::DeregisterTableRequest;
+
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to list catalogs"))]
+    #[snafu(display("Failed to list catalogs, source: {}", source))]
     ListCatalogs {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to list {}'s schemas", catalog))]
+    #[snafu(display("Failed to list {}'s schemas, source: {}", catalog, source))]
     ListSchemas {
         location: Location,
         catalog: String,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to re-compile script due to internal error"))]
+    #[snafu(display(
+        "Failed to re-compile script due to internal error, source: {}",
+        source
+    ))]
     CompileScriptInternal {
         location: Location,
         source: BoxedError,
     },
-    #[snafu(display("Failed to open system catalog table"))]
+    #[snafu(display("Failed to open system catalog table, source: {}", source))]
     OpenSystemCatalog {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to create system catalog table"))]
+    #[snafu(display("Failed to create system catalog table, source: {}", source))]
     CreateSystemCatalog {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to create table, table info: {}", table_info))]
+    #[snafu(display(
+        "Failed to create table, table info: {}, source: {}",
+        table_info,
+        source
+    ))]
     CreateTable {
         table_info: String,
         location: Location,
@@ -92,14 +98,13 @@ pub enum Error {
     #[snafu(display("Catalog value is not present"))]
     EmptyValue { location: Location },
 
-    #[snafu(display("Failed to deserialize value"))]
+    #[snafu(display("Failed to deserialize value, source: {}", source))]
     ValueDeserialize {
-        #[snafu(source)]
+        source: serde_json::error::Error,
-        error: serde_json::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Table engine not found: {}", engine_name))]
+    #[snafu(display("Table engine not found: {}, source: {}", engine_name, source))]
     TableEngineNotFound {
         engine_name: String,
         location: Location,
@@ -137,18 +142,15 @@ pub enum Error {
     #[snafu(display("Operation {} not supported", op))]
     NotSupported { op: String, location: Location },
 
-    #[snafu(display("Failed to open table {table_id}"))]
+    #[snafu(display("Failed to open table, table info: {}, source: {}", table_info, source))]
     OpenTable {
-        table_id: TableId,
+        table_info: String,
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to open table in parallel"))]
+    #[snafu(display("Failed to open table in parallel, source: {}", source))]
-    ParallelOpenTable {
+    ParallelOpenTable { source: JoinError },
-        #[snafu(source)]
-        error: JoinError,
-    },
 
     #[snafu(display("Table not found while opening table, table info: {}", table_info))]
     TableNotFound {
@@ -162,52 +164,72 @@ pub enum Error {
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to create recordbatch"))]
+    #[snafu(display("Failed to create recordbatch, source: {}", source))]
     CreateRecordBatch {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to insert table creation record to system catalog"))]
+    #[snafu(display(
+        "Failed to insert table creation record to system catalog, source: {}",
+        source
+    ))]
     InsertCatalogRecord {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to scan system catalog table"))]
+    #[snafu(display(
+        "Failed to deregister table, request: {:?}, source: {}",
+        request,
+        source
+    ))]
+    DeregisterTable {
+        request: DeregisterTableRequest,
+        location: Location,
+        source: table::error::Error,
+    },
+
+    #[snafu(display("Illegal catalog manager state: {}", msg))]
+    IllegalManagerState { location: Location, msg: String },
+
+    #[snafu(display("Failed to scan system catalog table, source: {}", source))]
     SystemCatalogTableScan {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display(""))]
+    #[snafu(display("{source}"))]
     Internal {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to upgrade weak catalog manager reference"))]
+    #[snafu(display(
+        "Failed to upgrade weak catalog manager reference. location: {}",
+        location
+    ))]
     UpgradeWeakCatalogManagerRef { location: Location },
 
-    #[snafu(display("Failed to execute system catalog table scan"))]
+    #[snafu(display("Failed to execute system catalog table scan, source: {}", source))]
     SystemCatalogTableScanExec {
         location: Location,
         source: common_query::error::Error,
     },
 
-    #[snafu(display("Cannot parse catalog value"))]
+    #[snafu(display("Cannot parse catalog value, source: {}", source))]
     InvalidCatalogValue {
         location: Location,
         source: common_catalog::error::Error,
     },
 
-    #[snafu(display("Failed to perform metasrv operation"))]
+    #[snafu(display("Failed to perform metasrv operation, source: {}", source))]
     MetaSrv {
         location: Location,
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Invalid table info in catalog"))]
+    #[snafu(display("Invalid table info in catalog, source: {}", source))]
     InvalidTableInfoInCatalog {
         location: Location,
         source: datatypes::error::Error,
@@ -216,14 +238,17 @@ pub enum Error {
     #[snafu(display("Illegal access to catalog: {} and schema: {}", catalog, schema))]
     QueryAccessDenied { catalog: String, schema: String },
 
-    #[snafu(display(""))]
+    #[snafu(display("Invalid system table definition: {err_msg}"))]
+    InvalidSystemTableDef { err_msg: String, location: Location },
+
+    #[snafu(display("{}: {}", msg, source))]
     Datafusion {
-        #[snafu(source)]
+        msg: String,
-        error: DataFusionError,
+        source: DataFusionError,
         location: Location,
     },
 
-    #[snafu(display("Table schema mismatch"))]
+    #[snafu(display("Table schema mismatch, source: {}", source))]
     TableSchemaMismatch {
         location: Location,
         source: table::error::Error,
@@ -232,7 +257,7 @@ pub enum Error {
     #[snafu(display("A generic error has occurred, msg: {}", msg))]
     Generic { msg: String, location: Location },
 
-    #[snafu(display("Table metadata manager error"))]
+    #[snafu(display("Table metadata manager error: {}", source))]
     TableMetadataManager {
         source: common_meta::error::Error,
         location: Location,
@@ -247,8 +272,10 @@ impl ErrorExt for Error {
             Error::InvalidKey { .. }
             | Error::SchemaNotFound { .. }
             | Error::TableNotFound { .. }
+            | Error::IllegalManagerState { .. }
             | Error::CatalogNotFound { .. }
             | Error::InvalidEntryType { .. }
+            | Error::InvalidSystemTableDef { .. }
             | Error::ParallelOpenTable { .. } => StatusCode::Unexpected,
 
             Error::SystemCatalog { .. }
@@ -279,6 +306,7 @@ impl ErrorExt for Error {
             | Error::InsertCatalogRecord { source, .. }
             | Error::OpenTable { source, .. }
             | Error::CreateTable { source, .. }
+            | Error::DeregisterTable { source, .. }
             | Error::TableSchemaMismatch { source, .. } => source.status_code(),
 
             Error::MetaSrv { source, .. } => source.status_code(),

src/catalog/src/information_schema/columns.rs

@@ -158,7 +158,7 @@ impl InformationSchemaColumnsBuilder {
 
         for schema_name in catalog_manager.schema_names(&catalog_name).await? {
             if !catalog_manager
-                .schema_exists(&catalog_name, &schema_name)
+                .schema_exist(&catalog_name, &schema_name)
                 .await?
             {
                 continue;

src/catalog/src/information_schema/tables.rs

@@ -154,7 +154,7 @@ impl InformationSchemaTablesBuilder {
 
         for schema_name in catalog_manager.schema_names(&catalog_name).await? {
             if !catalog_manager
-                .schema_exists(&catalog_name, &schema_name)
+                .schema_exist(&catalog_name, &schema_name)
                 .await?
             {
                 continue;

src/catalog/src/kvbackend/manager.rs

@@ -1,292 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::collections::BTreeSet;
-use std::sync::{Arc, Weak};
-
-use common_catalog::consts::{DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, NUMBERS_TABLE_ID};
-use common_error::ext::BoxedError;
-use common_meta::cache_invalidator::{CacheInvalidator, CacheInvalidatorRef, Context};
-use common_meta::datanode_manager::DatanodeManagerRef;
-use common_meta::error::Result as MetaResult;
-use common_meta::key::catalog_name::CatalogNameKey;
-use common_meta::key::schema_name::SchemaNameKey;
-use common_meta::key::table_name::TableNameKey;
-use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
-use common_meta::kv_backend::KvBackendRef;
-use common_meta::table_name::TableName;
-use futures_util::TryStreamExt;
-use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
-use snafu::prelude::*;
-use table::dist_table::DistTable;
-use table::metadata::TableId;
-use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
-use table::TableRef;
-
-use crate::error::{
-    self as catalog_err, ListCatalogsSnafu, ListSchemasSnafu, Result as CatalogResult,
-    TableMetadataManagerSnafu,
-};
-use crate::information_schema::{InformationSchemaProvider, COLUMNS, TABLES};
-use crate::CatalogManager;
-
-/// Access all existing catalog, schema and tables.
-///
-/// The result comes from two source, all the user tables are presented in
-/// a kv-backend which persists the metadata of a table. And system tables
-/// comes from `SystemCatalog`, which is static and read-only.
-#[derive(Clone)]
-pub struct KvBackendCatalogManager {
-    // TODO(LFC): Maybe use a real implementation for Standalone mode.
-    // Now we use `NoopKvCacheInvalidator` for Standalone mode. In Standalone mode, the KV backend
-    // is implemented by RaftEngine. Maybe we need a cache for it?
-    cache_invalidator: CacheInvalidatorRef,
-    partition_manager: PartitionRuleManagerRef,
-    table_metadata_manager: TableMetadataManagerRef,
-    datanode_manager: DatanodeManagerRef,
-    /// A sub-CatalogManager that handles system tables
-    system_catalog: SystemCatalog,
-}
-
-#[async_trait::async_trait]
-impl CacheInvalidator for KvBackendCatalogManager {
-    async fn invalidate_table_name(&self, ctx: &Context, table_name: TableName) -> MetaResult<()> {
-        self.cache_invalidator
-            .invalidate_table_name(ctx, table_name)
-            .await
-    }
-
-    async fn invalidate_table_id(&self, ctx: &Context, table_id: TableId) -> MetaResult<()> {
-        self.cache_invalidator
-            .invalidate_table_id(ctx, table_id)
-            .await
-    }
-}
-
-impl KvBackendCatalogManager {
-    pub fn new(
-        backend: KvBackendRef,
-        cache_invalidator: CacheInvalidatorRef,
-        datanode_manager: DatanodeManagerRef,
-    ) -> Arc<Self> {
-        Arc::new_cyclic(|me| Self {
-            partition_manager: Arc::new(PartitionRuleManager::new(backend.clone())),
-            table_metadata_manager: Arc::new(TableMetadataManager::new(backend)),
-            cache_invalidator,
-            datanode_manager,
-            system_catalog: SystemCatalog {
-                catalog_manager: me.clone(),
-            },
-        })
-    }
-
-    pub fn partition_manager(&self) -> PartitionRuleManagerRef {
-        self.partition_manager.clone()
-    }
-
-    pub fn table_metadata_manager_ref(&self) -> &TableMetadataManagerRef {
-        &self.table_metadata_manager
-    }
-
-    pub fn datanode_manager(&self) -> DatanodeManagerRef {
-        self.datanode_manager.clone()
-    }
-}
-
-#[async_trait::async_trait]
-impl CatalogManager for KvBackendCatalogManager {
-    async fn catalog_names(&self) -> CatalogResult<Vec<String>> {
-        let stream = self
-            .table_metadata_manager
-            .catalog_manager()
-            .catalog_names()
-            .await;
-
-        let keys = stream
-            .try_collect::<Vec<_>>()
-            .await
-            .map_err(BoxedError::new)
-            .context(ListCatalogsSnafu)?;
-
-        Ok(keys)
-    }
-
-    async fn schema_names(&self, catalog: &str) -> CatalogResult<Vec<String>> {
-        let stream = self
-            .table_metadata_manager
-            .schema_manager()
-            .schema_names(catalog)
-            .await;
-        let mut keys = stream
-            .try_collect::<BTreeSet<_>>()
-            .await
-            .map_err(BoxedError::new)
-            .context(ListSchemasSnafu { catalog })?
-            .into_iter()
-            .collect::<Vec<_>>();
-
-        keys.extend_from_slice(&self.system_catalog.schema_names());
-
-        Ok(keys)
-    }
-
-    async fn table_names(&self, catalog: &str, schema: &str) -> CatalogResult<Vec<String>> {
-        let mut tables = self
-            .table_metadata_manager
-            .table_name_manager()
-            .tables(catalog, schema)
-            .await
-            .context(TableMetadataManagerSnafu)?
-            .into_iter()
-            .map(|(k, _)| k)
-            .collect::<Vec<String>>();
-        tables.extend_from_slice(&self.system_catalog.table_names(schema));
-
-        Ok(tables)
-    }
-
-    async fn catalog_exists(&self, catalog: &str) -> CatalogResult<bool> {
-        self.table_metadata_manager
-            .catalog_manager()
-            .exists(CatalogNameKey::new(catalog))
-            .await
-            .context(TableMetadataManagerSnafu)
-    }
-
-    async fn schema_exists(&self, catalog: &str, schema: &str) -> CatalogResult<bool> {
-        if self.system_catalog.schema_exist(schema) {
-            return Ok(true);
-        }
-
-        self.table_metadata_manager
-            .schema_manager()
-            .exists(SchemaNameKey::new(catalog, schema))
-            .await
-            .context(TableMetadataManagerSnafu)
-    }
-
-    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> CatalogResult<bool> {
-        if self.system_catalog.table_exist(schema, table) {
-            return Ok(true);
-        }
-
-        let key = TableNameKey::new(catalog, schema, table);
-        self.table_metadata_manager
-            .table_name_manager()
-            .get(key)
-            .await
-            .context(TableMetadataManagerSnafu)
-            .map(|x| x.is_some())
-    }
-
-    async fn table(
-        &self,
-        catalog: &str,
-        schema: &str,
-        table_name: &str,
-    ) -> CatalogResult<Option<TableRef>> {
-        if let Some(table) = self.system_catalog.table(catalog, schema, table_name) {
-            return Ok(Some(table));
-        }
-
-        let key = TableNameKey::new(catalog, schema, table_name);
-        let Some(table_name_value) = self
-            .table_metadata_manager
-            .table_name_manager()
-            .get(key)
-            .await
-            .context(TableMetadataManagerSnafu)?
-        else {
-            return Ok(None);
-        };
-        let table_id = table_name_value.table_id();
-
-        let Some(table_info_value) = self
-            .table_metadata_manager
-            .table_info_manager()
-            .get(table_id)
-            .await
-            .context(TableMetadataManagerSnafu)?
-            .map(|v| v.into_inner())
-        else {
-            return Ok(None);
-        };
-        let table_info = Arc::new(
-            table_info_value
-                .table_info
-                .try_into()
-                .context(catalog_err::InvalidTableInfoInCatalogSnafu)?,
-        );
-        Ok(Some(DistTable::table(table_info)))
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-// TODO: This struct can hold a static map of all system tables when
-// the upper layer (e.g., procedure) can inform the catalog manager
-// a new catalog is created.
-/// Existing system tables:
-/// - public.numbers
-/// - information_schema.tables
-/// - information_schema.columns
-#[derive(Clone)]
-struct SystemCatalog {
-    catalog_manager: Weak<KvBackendCatalogManager>,
-}
-
-impl SystemCatalog {
-    fn schema_names(&self) -> Vec<String> {
-        vec![INFORMATION_SCHEMA_NAME.to_string()]
-    }
-
-    fn table_names(&self, schema: &str) -> Vec<String> {
-        if schema == INFORMATION_SCHEMA_NAME {
-            vec![TABLES.to_string(), COLUMNS.to_string()]
-        } else if schema == DEFAULT_SCHEMA_NAME {
-            vec![NUMBERS_TABLE_NAME.to_string()]
-        } else {
-            vec![]
-        }
-    }
-
-    fn schema_exist(&self, schema: &str) -> bool {
-        schema == INFORMATION_SCHEMA_NAME
-    }
-
-    fn table_exist(&self, schema: &str, table: &str) -> bool {
-        if schema == INFORMATION_SCHEMA_NAME {
-            table == TABLES || table == COLUMNS
-        } else if schema == DEFAULT_SCHEMA_NAME {
-            table == NUMBERS_TABLE_NAME
-        } else {
-            false
-        }
-    }
-
-    fn table(&self, catalog: &str, schema: &str, table_name: &str) -> Option<TableRef> {
-        if schema == INFORMATION_SCHEMA_NAME {
-            let information_schema_provider =
-                InformationSchemaProvider::new(catalog.to_string(), self.catalog_manager.clone());
-            information_schema_provider.table(table_name)
-        } else if schema == DEFAULT_SCHEMA_NAME && table_name == NUMBERS_TABLE_NAME {
-            Some(NumbersTable::table(NUMBERS_TABLE_ID))
-        } else {
-            None
-        }
-    }
-}

src/catalog/src/lib.rs

@@ -17,38 +17,79 @@
 #![feature(try_blocks)]
 
 use std::any::Any;
+use std::collections::HashMap;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
-use futures::future::BoxFuture;
+use api::v1::meta::{RegionStat, TableIdent, TableName};
-use table::metadata::TableId;
+use common_telemetry::{info, warn};
+use snafu::ResultExt;
+use table::engine::{EngineContext, TableEngineRef};
+use table::metadata::{TableId, TableType};
 use table::requests::CreateTableRequest;
 use table::TableRef;
 
-use crate::error::Result;
+use crate::error::{CreateTableSnafu, Result};
 
 pub mod error;
 pub mod information_schema;
-pub mod kvbackend;
+pub mod local;
-pub mod memory;
 mod metrics;
+pub mod remote;
+pub mod system;
 pub mod table_source;
+pub mod tables;
 
 #[async_trait::async_trait]
 pub trait CatalogManager: Send + Sync {
     fn as_any(&self) -> &dyn Any;
 
+    /// Starts a catalog manager.
+    async fn start(&self) -> Result<()>;
+
+    /// Registers a catalog to catalog manager, returns whether the catalog exist before.
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool>;
+
+    /// Register a schema with catalog name and schema name. Retuens whether the
+    /// schema registered.
+    ///
+    /// # Errors
+    ///
+    /// This method will/should fail if catalog not exist
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool>;
+
+    /// Deregisters a database within given catalog/schema to catalog manager
+    async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool>;
+
+    /// Registers a table within given catalog/schema to catalog manager,
+    /// returns whether the table registered.
+    ///
+    /// # Errors
+    ///
+    /// This method will/should fail if catalog or schema not exist
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool>;
+
+    /// Deregisters a table within given catalog/schema to catalog manager
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()>;
+
+    /// Rename a table to [RenameTableRequest::new_table_name], returns whether the table is renamed.
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool>;
+
+    /// Register a system table, should be called before starting the manager.
+    async fn register_system_table(&self, request: RegisterSystemTableRequest)
+        -> error::Result<()>;
+
     async fn catalog_names(&self) -> Result<Vec<String>>;
 
     async fn schema_names(&self, catalog: &str) -> Result<Vec<String>>;
 
     async fn table_names(&self, catalog: &str, schema: &str) -> Result<Vec<String>>;
 
-    async fn catalog_exists(&self, catalog: &str) -> Result<bool>;
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool>;
 
-    async fn schema_exists(&self, catalog: &str, schema: &str) -> Result<bool>;
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool>;
 
-    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> Result<bool>;
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool>;
 
     /// Returns the table by catalog, schema and table name.
     async fn table(
@@ -62,8 +103,7 @@ pub trait CatalogManager: Send + Sync {
 pub type CatalogManagerRef = Arc<dyn CatalogManager>;
 
 /// Hook called after system table opening.
-pub type OpenSystemTableHook =
+pub type OpenSystemTableHook = Arc<dyn Fn(TableRef) -> Result<()> + Send + Sync>;
-    Box<dyn Fn(TableRef) -> BoxFuture<'static, Result<()>> + Send + Sync>;
 
 /// Register system table request:
 /// - When system table is already created and registered, the hook will be called
@@ -122,3 +162,120 @@ pub struct RegisterSchemaRequest {
     pub catalog: String,
     pub schema: String,
 }
+
+pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
+    manager: &'a M,
+    engine: TableEngineRef,
+    sys_table_requests: &'a mut Vec<RegisterSystemTableRequest>,
+) -> Result<()> {
+    for req in sys_table_requests.drain(..) {
+        let catalog_name = &req.create_table_request.catalog_name;
+        let schema_name = &req.create_table_request.schema_name;
+        let table_name = &req.create_table_request.table_name;
+        let table_id = req.create_table_request.id;
+
+        let table = manager.table(catalog_name, schema_name, table_name).await?;
+        let table = if let Some(table) = table {
+            table
+        } else {
+            let table = engine
+                .create_table(&EngineContext::default(), req.create_table_request.clone())
+                .await
+                .with_context(|_| CreateTableSnafu {
+                    table_info: common_catalog::format_full_table_name(
+                        catalog_name,
+                        schema_name,
+                        table_name,
+                    ),
+                })?;
+            let _ = manager
+                .register_table(RegisterTableRequest {
+                    catalog: catalog_name.clone(),
+                    schema: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    table_id,
+                    table: table.clone(),
+                })
+                .await?;
+            info!("Created and registered system table: {table_name}");
+            table
+        };
+        if let Some(hook) = req.open_hook {
+            (hook)(table)?;
+        }
+    }
+    Ok(())
+}
+
+/// The stat of regions in the datanode node.
+/// The number of regions can be got from len of vec.
+///
+/// Ignores any errors occurred during iterating regions. The intention of this method is to
+/// collect region stats that will be carried in Datanode's heartbeat to Metasrv, so it's a
+/// "try our best" job.
+pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<RegionStat>) {
+    let mut region_number: u64 = 0;
+    let mut region_stats = Vec::new();
+
+    let Ok(catalog_names) = catalog_manager.catalog_names().await else {
+        return (region_number, region_stats);
+    };
+    for catalog_name in catalog_names {
+        let Ok(schema_names) = catalog_manager.schema_names(&catalog_name).await else {
+            continue;
+        };
+        for schema_name in schema_names {
+            let Ok(table_names) = catalog_manager
+                .table_names(&catalog_name, &schema_name)
+                .await
+            else {
+                continue;
+            };
+            for table_name in table_names {
+                let Ok(Some(table)) = catalog_manager
+                    .table(&catalog_name, &schema_name, &table_name)
+                    .await
+                else {
+                    continue;
+                };
+
+                if table.table_type() != TableType::Base {
+                    continue;
+                }
+
+                let table_info = table.table_info();
+                let region_numbers = &table_info.meta.region_numbers;
+                region_number += region_numbers.len() as u64;
+
+                let engine = &table_info.meta.engine;
+                let table_id = table_info.ident.table_id;
+
+                match table.region_stats() {
+                    Ok(stats) => {
+                        let stats = stats.into_iter().map(|stat| RegionStat {
+                            region_id: stat.region_id,
+                            table_ident: Some(TableIdent {
+                                table_id,
+                                table_name: Some(TableName {
+                                    catalog_name: catalog_name.clone(),
+                                    schema_name: schema_name.clone(),
+                                    table_name: table_name.clone(),
+                                }),
+                                engine: engine.clone(),
+                            }),
+                            approximate_bytes: stat.disk_usage_bytes as i64,
+                            attrs: HashMap::from([("engine_name".to_owned(), engine.clone())]),
+                            ..Default::default()
+                        });
+
+                        region_stats.extend(stats);
+                    }
+                    Err(e) => {
+                        warn!("Failed to get region status, err: {:?}", e);
+                    }
+                };
+            }
+        }
+    }
+    (region_number, region_stats)
+}

src/catalog/src/local.rs

@@ -0,0 +1,19 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod manager;
+pub mod memory;
+
+pub use manager::LocalCatalogManager;
+pub use memory::{new_memory_catalog_manager, MemoryCatalogManager};

src/catalog/src/local/manager.rs

@@ -0,0 +1,633 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
+    MITO_ENGINE, NUMBERS_TABLE_ID, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID,
+    SYSTEM_CATALOG_TABLE_NAME,
+};
+use common_catalog::format_full_table_name;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use common_telemetry::{error, info};
+use datatypes::prelude::ScalarVector;
+use datatypes::vectors::{BinaryVector, UInt8Vector};
+use futures_util::lock::Mutex;
+use metrics::increment_gauge;
+use snafu::{ensure, OptionExt, ResultExt};
+use table::engine::manager::TableEngineManagerRef;
+use table::engine::EngineContext;
+use table::metadata::TableId;
+use table::requests::OpenTableRequest;
+use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
+use table::table::TableIdProvider;
+use table::TableRef;
+
+use crate::error::{
+    self, CatalogNotFoundSnafu, IllegalManagerStateSnafu, OpenTableSnafu, ReadSystemCatalogSnafu,
+    Result, SchemaExistsSnafu, SchemaNotFoundSnafu, SystemCatalogSnafu,
+    SystemCatalogTypeMismatchSnafu, TableEngineNotFoundSnafu, TableExistsSnafu, TableNotExistSnafu,
+    TableNotFoundSnafu, UnimplementedSnafu,
+};
+use crate::local::memory::MemoryCatalogManager;
+use crate::system::{
+    decode_system_catalog, Entry, SystemCatalogTable, TableEntry, ENTRY_TYPE_INDEX, KEY_INDEX,
+    VALUE_INDEX,
+};
+use crate::tables::SystemCatalog;
+use crate::{
+    handle_system_table_request, CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest,
+    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
+};
+
+/// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
+pub struct LocalCatalogManager {
+    system: Arc<SystemCatalog>,
+    catalogs: Arc<MemoryCatalogManager>,
+    engine_manager: TableEngineManagerRef,
+    next_table_id: AtomicU32,
+    init_lock: Mutex<bool>,
+    register_lock: Mutex<()>,
+    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
+}
+
+impl LocalCatalogManager {
+    /// Create a new [CatalogManager] with given user catalogs and mito engine
+    pub async fn try_new(engine_manager: TableEngineManagerRef) -> Result<Self> {
+        let engine = engine_manager
+            .engine(MITO_ENGINE)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: MITO_ENGINE,
+            })?;
+        let table = SystemCatalogTable::new(engine.clone()).await?;
+        let memory_catalog_manager = crate::local::memory::new_memory_catalog_manager()?;
+        let system_catalog = Arc::new(SystemCatalog::new(table));
+        Ok(Self {
+            system: system_catalog,
+            catalogs: memory_catalog_manager,
+            engine_manager,
+            next_table_id: AtomicU32::new(MIN_USER_TABLE_ID),
+            init_lock: Mutex::new(false),
+            register_lock: Mutex::new(()),
+            system_table_requests: Mutex::new(Vec::default()),
+        })
+    }
+
+    /// Scan all entries from system catalog table
+    pub async fn init(&self) -> Result<()> {
+        self.init_system_catalog().await?;
+        let system_records = self.system.information_schema.system.records().await?;
+        let entries = self.collect_system_catalog_entries(system_records).await?;
+        let max_table_id = self.handle_system_catalog_entries(entries).await?;
+
+        info!(
+            "All system catalog entries processed, max table id: {}",
+            max_table_id
+        );
+        self.next_table_id
+            .store((max_table_id + 1).max(MIN_USER_TABLE_ID), Ordering::Relaxed);
+        *self.init_lock.lock().await = true;
+
+        // Processing system table hooks
+        let mut sys_table_requests = self.system_table_requests.lock().await;
+        let engine = self
+            .engine_manager
+            .engine(MITO_ENGINE)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: MITO_ENGINE,
+            })?;
+
+        handle_system_table_request(self, engine, &mut sys_table_requests).await?;
+        Ok(())
+    }
+
+    async fn init_system_catalog(&self) -> Result<()> {
+        // register default catalog and default schema
+        self.catalogs
+            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string())?;
+        self.catalogs.register_schema_sync(RegisterSchemaRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+        })?;
+
+        // register SystemCatalogTable
+        self.catalogs
+            .register_catalog_sync(SYSTEM_CATALOG_NAME.to_string())?;
+        self.catalogs.register_schema_sync(RegisterSchemaRequest {
+            catalog: SYSTEM_CATALOG_NAME.to_string(),
+            schema: INFORMATION_SCHEMA_NAME.to_string(),
+        })?;
+        let register_table_req = RegisterTableRequest {
+            catalog: SYSTEM_CATALOG_NAME.to_string(),
+            schema: INFORMATION_SCHEMA_NAME.to_string(),
+            table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
+            table_id: SYSTEM_CATALOG_TABLE_ID,
+            table: self.system.information_schema.system.as_table_ref(),
+        };
+        self.catalogs.register_table(register_table_req).await?;
+
+        // Add numbers table for test
+        let register_number_table_req = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: NUMBERS_TABLE_NAME.to_string(),
+            table_id: NUMBERS_TABLE_ID,
+            table: NumbersTable::table(NUMBERS_TABLE_ID),
+        };
+
+        self.catalogs
+            .register_table(register_number_table_req)
+            .await?;
+
+        Ok(())
+    }
+
+    /// Collect stream of system catalog entries to `Vec<Entry>`
+    async fn collect_system_catalog_entries(
+        &self,
+        stream: SendableRecordBatchStream,
+    ) -> Result<Vec<Entry>> {
+        let record_batch = common_recordbatch::util::collect(stream)
+            .await
+            .context(ReadSystemCatalogSnafu)?;
+        let rbs = record_batch
+            .into_iter()
+            .map(Self::record_batch_to_entry)
+            .collect::<Result<Vec<_>>>()?;
+        Ok(rbs.into_iter().flat_map(Vec::into_iter).collect::<_>())
+    }
+
+    /// Convert `RecordBatch` to a vector of `Entry`.
+    fn record_batch_to_entry(rb: RecordBatch) -> Result<Vec<Entry>> {
+        ensure!(
+            rb.num_columns() >= 6,
+            SystemCatalogSnafu {
+                msg: format!("Length mismatch: {}", rb.num_columns())
+            }
+        );
+
+        let entry_type = rb
+            .column(ENTRY_TYPE_INDEX)
+            .as_any()
+            .downcast_ref::<UInt8Vector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(ENTRY_TYPE_INDEX).data_type(),
+            })?;
+
+        let key = rb
+            .column(KEY_INDEX)
+            .as_any()
+            .downcast_ref::<BinaryVector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(KEY_INDEX).data_type(),
+            })?;
+
+        let value = rb
+            .column(VALUE_INDEX)
+            .as_any()
+            .downcast_ref::<BinaryVector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(VALUE_INDEX).data_type(),
+            })?;
+
+        let mut res = Vec::with_capacity(rb.num_rows());
+        for ((t, k), v) in entry_type
+            .iter_data()
+            .zip(key.iter_data())
+            .zip(value.iter_data())
+        {
+            let entry = decode_system_catalog(t, k, v)?;
+            res.push(entry);
+        }
+        Ok(res)
+    }
+
+    /// Processes records from system catalog table and returns the max table id persisted
+    /// in system catalog table.
+    async fn handle_system_catalog_entries(&self, entries: Vec<Entry>) -> Result<TableId> {
+        let entries = Self::sort_entries(entries);
+        let mut max_table_id = 0;
+        for entry in entries {
+            match entry {
+                Entry::Catalog(c) => {
+                    self.catalogs
+                        .register_catalog_sync(c.catalog_name.clone())?;
+                    info!("Register catalog: {}", c.catalog_name);
+                }
+                Entry::Schema(s) => {
+                    let req = RegisterSchemaRequest {
+                        catalog: s.catalog_name.clone(),
+                        schema: s.schema_name.clone(),
+                    };
+                    let _ = self.catalogs.register_schema_sync(req)?;
+                    info!("Registered schema: {:?}", s);
+                }
+                Entry::Table(t) => {
+                    max_table_id = max_table_id.max(t.table_id);
+                    if t.is_deleted {
+                        continue;
+                    }
+                    self.open_and_register_table(&t).await?;
+                    info!("Registered table: {:?}", t);
+                }
+            }
+        }
+        Ok(max_table_id)
+    }
+
+    /// Sort catalog entries to ensure catalog entries comes first, then schema entries,
+    /// and table entries is the last.
+    fn sort_entries(mut entries: Vec<Entry>) -> Vec<Entry> {
+        entries.sort();
+        entries
+    }
+
+    async fn open_and_register_table(&self, t: &TableEntry) -> Result<()> {
+        self.check_catalog_schema_exist(&t.catalog_name, &t.schema_name)
+            .await?;
+
+        let context = EngineContext {};
+        let open_request = OpenTableRequest {
+            catalog_name: t.catalog_name.clone(),
+            schema_name: t.schema_name.clone(),
+            table_name: t.table_name.clone(),
+            table_id: t.table_id,
+            region_numbers: vec![0],
+        };
+        let engine = self
+            .engine_manager
+            .engine(&t.engine)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: &t.engine,
+            })?;
+
+        let table_ref = engine
+            .open_table(&context, open_request)
+            .await
+            .with_context(|_| OpenTableSnafu {
+                table_info: format!(
+                    "{}.{}.{}, id: {}",
+                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
+                ),
+            })?
+            .with_context(|| TableNotFoundSnafu {
+                table_info: format!(
+                    "{}.{}.{}, id: {}",
+                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
+                ),
+            })?;
+
+        let register_request = RegisterTableRequest {
+            catalog: t.catalog_name.clone(),
+            schema: t.schema_name.clone(),
+            table_name: t.table_name.clone(),
+            table_id: t.table_id,
+            table: table_ref,
+        };
+        let _ = self.catalogs.register_table(register_request).await?;
+
+        Ok(())
+    }
+
+    async fn check_state(&self) -> Result<()> {
+        let started = self.init_lock.lock().await;
+        ensure!(
+            *started,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager not started",
+            }
+        );
+        Ok(())
+    }
+
+    async fn check_catalog_schema_exist(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+    ) -> Result<()> {
+        if !self.catalogs.catalog_exist(catalog_name).await? {
+            return CatalogNotFoundSnafu { catalog_name }.fail()?;
+        }
+        if !self
+            .catalogs
+            .schema_exist(catalog_name, schema_name)
+            .await?
+        {
+            return SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            }
+            .fail()?;
+        }
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl TableIdProvider for LocalCatalogManager {
+    async fn next_table_id(&self) -> table::Result<TableId> {
+        Ok(self.next_table_id.fetch_add(1, Ordering::Relaxed))
+    }
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for LocalCatalogManager {
+    /// Start [LocalCatalogManager] to load all information from system catalog table.
+    /// Make sure table engine is initialized before starting [MemoryCatalogManager].
+    async fn start(&self) -> Result<()> {
+        self.init().await
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        self.check_state().await?;
+
+        let catalog_name = request.catalog.clone();
+        let schema_name = request.schema.clone();
+
+        self.check_catalog_schema_exist(&catalog_name, &schema_name)
+            .await?;
+
+        {
+            let _lock = self.register_lock.lock().await;
+            if let Some(existing) = self
+                .catalogs
+                .table(&request.catalog, &request.schema, &request.table_name)
+                .await?
+            {
+                if existing.table_info().ident.table_id != request.table_id {
+                    error!(
+                        "Unexpected table register request: {:?}, existing: {:?}",
+                        request,
+                        existing.table_info()
+                    );
+                    return TableExistsSnafu {
+                        table: format_full_table_name(
+                            &catalog_name,
+                            &schema_name,
+                            &request.table_name,
+                        ),
+                    }
+                    .fail();
+                }
+                // Try to register table with same table id, just ignore.
+                Ok(false)
+            } else {
+                // table does not exist
+                let engine = request.table.table_info().meta.engine.to_string();
+                let table_name = request.table_name.clone();
+                let table_id = request.table_id;
+                let _ = self.catalogs.register_table(request).await?;
+                let _ = self
+                    .system
+                    .register_table(
+                        catalog_name.clone(),
+                        schema_name.clone(),
+                        table_name,
+                        table_id,
+                        engine,
+                    )
+                    .await?;
+                increment_gauge!(
+                    crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+                    1.0,
+                    &[crate::metrics::db_label(&catalog_name, &schema_name)],
+                );
+                Ok(true)
+            }
+        }
+    }
+
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
+        self.check_state().await?;
+
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+
+        self.check_catalog_schema_exist(catalog_name, schema_name)
+            .await?;
+        ensure!(
+            self.catalogs
+                .table(catalog_name, schema_name, &request.new_table_name)
+                .await?
+                .is_none(),
+            TableExistsSnafu {
+                table: &request.new_table_name
+            }
+        );
+
+        let _lock = self.register_lock.lock().await;
+        let old_table = self
+            .catalogs
+            .table(catalog_name, schema_name, &request.table_name)
+            .await?
+            .context(TableNotExistSnafu {
+                table: &request.table_name,
+            })?;
+
+        let engine = old_table.table_info().meta.engine.to_string();
+        // rename table in system catalog
+        let _ = self
+            .system
+            .register_table(
+                catalog_name.clone(),
+                schema_name.clone(),
+                request.new_table_name.clone(),
+                request.table_id,
+                engine,
+            )
+            .await?;
+
+        self.catalogs.rename_table(request).await
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
+        self.check_state().await?;
+
+        {
+            let _ = self.register_lock.lock().await;
+
+            let DeregisterTableRequest {
+                catalog,
+                schema,
+                table_name,
+            } = &request;
+            let table_id = self
+                .catalogs
+                .table(catalog, schema, table_name)
+                .await?
+                .with_context(|| error::TableNotExistSnafu {
+                    table: format_full_table_name(catalog, schema, table_name),
+                })?
+                .table_info()
+                .ident
+                .table_id;
+
+            self.system.deregister_table(&request, table_id).await?;
+            self.catalogs.deregister_table(request).await
+        }
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        self.check_state().await?;
+
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+
+        if !self.catalogs.catalog_exist(catalog_name).await? {
+            return CatalogNotFoundSnafu { catalog_name }.fail()?;
+        }
+
+        {
+            let _lock = self.register_lock.lock().await;
+            ensure!(
+                !self
+                    .catalogs
+                    .schema_exist(catalog_name, schema_name)
+                    .await?,
+                SchemaExistsSnafu {
+                    schema: schema_name,
+                }
+            );
+            let _ = self
+                .system
+                .register_schema(request.catalog.clone(), schema_name.clone())
+                .await?;
+            self.catalogs.register_schema_sync(request)
+        }
+    }
+
+    async fn deregister_schema(&self, _request: DeregisterSchemaRequest) -> Result<bool> {
+        UnimplementedSnafu {
+            operation: "deregister schema",
+        }
+        .fail()
+    }
+
+    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
+        let catalog_name = request.create_table_request.catalog_name.clone();
+        let schema_name = request.create_table_request.schema_name.clone();
+
+        let mut sys_table_requests = self.system_table_requests.lock().await;
+        sys_table_requests.push(request);
+        increment_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&catalog_name, &schema_name)],
+        );
+        Ok(())
+    }
+
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
+        self.catalogs.schema_exist(catalog, schema).await
+    }
+
+    async fn table(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        self.catalogs
+            .table(catalog_name, schema_name, table_name)
+            .await
+    }
+
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
+        if catalog.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
+            Ok(true)
+        } else {
+            self.catalogs.catalog_exist(catalog).await
+        }
+    }
+
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
+        self.catalogs.table_exist(catalog, schema, table).await
+    }
+
+    async fn catalog_names(&self) -> Result<Vec<String>> {
+        self.catalogs.catalog_names().await
+    }
+
+    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
+        self.catalogs.schema_names(catalog_name).await
+    }
+
+    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
+        self.catalogs.table_names(catalog_name, schema_name).await
+    }
+
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
+        self.catalogs.clone().register_catalog(name).await
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+
+    use mito::engine::MITO_ENGINE;
+
+    use super::*;
+    use crate::system::{CatalogEntry, SchemaEntry};
+
+    #[test]
+    fn test_sort_entry() {
+        let vec = vec![
+            Entry::Table(TableEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+                table_name: "T1".to_string(),
+                table_id: 1,
+                engine: MITO_ENGINE.to_string(),
+                is_deleted: false,
+            }),
+            Entry::Catalog(CatalogEntry {
+                catalog_name: "C2".to_string(),
+            }),
+            Entry::Schema(SchemaEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+            }),
+            Entry::Schema(SchemaEntry {
+                catalog_name: "C2".to_string(),
+                schema_name: "S2".to_string(),
+            }),
+            Entry::Catalog(CatalogEntry {
+                catalog_name: "".to_string(),
+            }),
+            Entry::Table(TableEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+                table_name: "T2".to_string(),
+                table_id: 2,
+                engine: MITO_ENGINE.to_string(),
+                is_deleted: false,
+            }),
+        ];
+        let res = LocalCatalogManager::sort_entries(vec);
+        assert_matches!(res[0], Entry::Catalog(..));
+        assert_matches!(res[1], Entry::Catalog(..));
+        assert_matches!(res[2], Entry::Schema(..));
+        assert_matches!(res[3], Entry::Schema(..));
+        assert_matches!(res[4], Entry::Table(..));
+        assert_matches!(res[5], Entry::Table(..));
+    }
+}

src/catalog/src/local/memory.rs

@@ -15,29 +15,129 @@
 use std::any::Any;
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
+use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::{Arc, RwLock, Weak};
 
-use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME};
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
+};
 use metrics::{decrement_gauge, increment_gauge};
 use snafu::OptionExt;
+use table::metadata::TableId;
+use table::table::TableIdProvider;
 use table::TableRef;
 
-use crate::error::{CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu};
+use crate::error::{
+    CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu, TableNotFoundSnafu,
+};
 use crate::information_schema::InformationSchemaProvider;
-use crate::{CatalogManager, DeregisterTableRequest, RegisterSchemaRequest, RegisterTableRequest};
+use crate::{
+    CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest, RegisterSchemaRequest,
+    RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
+};
 
 type SchemaEntries = HashMap<String, HashMap<String, TableRef>>;
 
 /// Simple in-memory list of catalogs
-#[derive(Clone)]
 pub struct MemoryCatalogManager {
     /// Collection of catalogs containing schemas and ultimately Tables
-    catalogs: Arc<RwLock<HashMap<String, SchemaEntries>>>,
+    pub catalogs: RwLock<HashMap<String, SchemaEntries>>,
+    pub table_id: AtomicU32,
+}
+
+#[async_trait::async_trait]
+impl TableIdProvider for MemoryCatalogManager {
+    async fn next_table_id(&self) -> table::error::Result<TableId> {
+        Ok(self.table_id.fetch_add(1, Ordering::Relaxed))
+    }
 }
 
 #[async_trait::async_trait]
 impl CatalogManager for MemoryCatalogManager {
-    async fn schema_exists(&self, catalog: &str, schema: &str) -> Result<bool> {
+    async fn start(&self) -> Result<()> {
+        self.table_id.store(MIN_USER_TABLE_ID, Ordering::Relaxed);
+        Ok(())
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        self.register_table_sync(request)
+    }
+
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schema = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .get_mut(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
+
+        // check old and new table names
+        if !schema.contains_key(&request.table_name) {
+            return TableNotFoundSnafu {
+                table_info: request.table_name.to_string(),
+            }
+            .fail()?;
+        }
+        if schema.contains_key(&request.new_table_name) {
+            return TableExistsSnafu {
+                table: &request.new_table_name,
+            }
+            .fail();
+        }
+
+        let table = schema.remove(&request.table_name).unwrap();
+        let _ = schema.insert(request.new_table_name, table);
+
+        Ok(true)
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
+        self.deregister_table_sync(request)
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        self.register_schema_sync(request)
+    }
+
+    async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schemas = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?;
+        let table_count = schemas
+            .remove(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?
+            .len();
+        decrement_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            table_count as f64,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
+
+        decrement_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
+        Ok(true)
+    }
+
+    async fn register_system_table(&self, _request: RegisterSystemTableRequest) -> Result<()> {
+        // TODO(ruihang): support register system table request
+        Ok(())
+    }
+
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
         self.schema_exist_sync(catalog, schema)
     }
 
@@ -59,11 +159,11 @@ impl CatalogManager for MemoryCatalogManager {
         Ok(result)
     }
 
-    async fn catalog_exists(&self, catalog: &str) -> Result<bool> {
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
         self.catalog_exist_sync(catalog)
     }
 
-    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
         let catalogs = self.catalogs.read().unwrap();
         Ok(catalogs
             .get(catalog)
@@ -108,27 +208,28 @@ impl CatalogManager for MemoryCatalogManager {
             .collect())
     }
 
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
+        self.register_catalog_sync(name)
+    }
+
     fn as_any(&self) -> &dyn Any {
         self
     }
 }
 
 impl MemoryCatalogManager {
-    pub fn new() -> Arc<Self> {
-        Arc::new(Self {
-            catalogs: Default::default(),
-        })
-    }
-
     /// Creates a manager with some default setups
     /// (e.g. default catalog/schema and information schema)
     pub fn with_default_setup() -> Arc<Self> {
         let manager = Arc::new(Self {
+            table_id: AtomicU32::new(MIN_USER_TABLE_ID),
             catalogs: Default::default(),
         });
 
         // Safety: default catalog/schema is registered in order so no CatalogNotFound error will occur
-        manager.register_catalog_sync(DEFAULT_CATALOG_NAME).unwrap();
+        manager
+            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string())
+            .unwrap();
         manager
             .register_schema_sync(RegisterSchemaRequest {
                 catalog: DEFAULT_CATALOG_NAME.to_string(),
@@ -156,15 +257,12 @@ impl MemoryCatalogManager {
     }
 
     /// Registers a catalog if it does not exist and returns false if the schema exists.
-    pub fn register_catalog_sync(&self, name: &str) -> Result<bool> {
+    pub fn register_catalog_sync(self: &Arc<Self>, name: String) -> Result<bool> {
-        let name = name.to_string();
-
         let mut catalogs = self.catalogs.write().unwrap();
 
         match catalogs.entry(name.clone()) {
             Entry::Vacant(e) => {
-                let arc_self = Arc::new(self.clone());
+                let catalog = self.create_catalog_entry(name);
-                let catalog = arc_self.create_catalog_entry(name);
                 e.insert(catalog);
                 increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
                 Ok(true)
@@ -263,7 +361,7 @@ impl MemoryCatalogManager {
         let schema = &table.table_info().schema_name;
 
         if !manager.catalog_exist_sync(catalog).unwrap() {
-            manager.register_catalog_sync(catalog).unwrap();
+            manager.register_catalog_sync(catalog.to_string()).unwrap();
         }
 
         if !manager.schema_exist_sync(catalog, schema).unwrap() {
@@ -295,6 +393,8 @@ pub fn new_memory_catalog_manager() -> Result<Arc<MemoryCatalogManager>> {
 #[cfg(test)]
 mod tests {
     use common_catalog::consts::*;
+    use common_error::ext::ErrorExt;
+    use common_error::status_code::StatusCode;
     use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
 
     use super::*;
@@ -311,7 +411,7 @@ mod tests {
             table: NumbersTable::table(NUMBERS_TABLE_ID),
         };
 
-        catalog_list.register_table_sync(register_request).unwrap();
+        let _ = catalog_list.register_table(register_request).await.unwrap();
         let table = catalog_list
             .table(
                 DEFAULT_CATALOG_NAME,
@@ -328,11 +428,130 @@ mod tests {
             .is_none());
     }
 
+    #[tokio::test]
+    async fn test_mem_manager_rename_table() {
+        let catalog = MemoryCatalogManager::with_default_setup();
+        let table_name = "test_table";
+        assert!(!catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap());
+        // register test table
+        let table_id = 2333;
+        let register_request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id,
+            table: NumbersTable::table(table_id),
+        };
+        assert!(catalog.register_table(register_request).await.unwrap());
+        assert!(catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap());
+
+        // rename test table
+        let new_table_name = "test_table_renamed";
+        let rename_request = RenameTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            new_table_name: new_table_name.to_string(),
+            table_id,
+        };
+        let _ = catalog.rename_table(rename_request).await.unwrap();
+
+        // test old table name not exist
+        assert!(!catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap());
+
+        // test new table name exists
+        assert!(catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap());
+        let registered_table = catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(registered_table.table_info().ident.table_id, table_id);
+
+        let dup_register_request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: new_table_name.to_string(),
+            table_id: table_id + 1,
+            table: NumbersTable::table(table_id + 1),
+        };
+        let result = catalog.register_table(dup_register_request).await;
+        let err = result.err().unwrap();
+        assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
+    }
+
+    #[tokio::test]
+    async fn test_catalog_rename_table() {
+        let catalog = MemoryCatalogManager::with_default_setup();
+        let table_name = "num";
+        let table_id = 2333;
+        let table = NumbersTable::table(table_id);
+
+        // register table
+        let register_table_req = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id,
+            table,
+        };
+        assert!(catalog.register_table(register_table_req).await.unwrap());
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_some());
+
+        // rename table
+        let new_table_name = "numbers_new";
+        let rename_table_req = RenameTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            new_table_name: new_table_name.to_string(),
+            table_id,
+        };
+        assert!(catalog.rename_table(rename_table_req).await.unwrap());
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_none());
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .is_some());
+
+        let registered_table = catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(registered_table.table_info().ident.table_id, table_id);
+    }
+
     #[test]
     pub fn test_register_catalog_sync() {
         let list = MemoryCatalogManager::with_default_setup();
-        assert!(list.register_catalog_sync("test_catalog").unwrap());
+        assert!(list
-        assert!(!list.register_catalog_sync("test_catalog").unwrap());
+            .register_catalog_sync("test_catalog".to_string())
+            .unwrap());
+        assert!(!list
+            .register_catalog_sync("test_catalog".to_string())
+            .unwrap());
     }
 
     #[tokio::test]
@@ -347,7 +566,7 @@ mod tests {
             table_id: 2333,
             table: NumbersTable::table(2333),
         };
-        catalog.register_table_sync(register_table_req).unwrap();
+        let _ = catalog.register_table(register_table_req).await.unwrap();
         assert!(catalog
             .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
             .await
@@ -359,11 +578,53 @@ mod tests {
             schema: DEFAULT_SCHEMA_NAME.to_string(),
             table_name: table_name.to_string(),
         };
-        catalog.deregister_table_sync(deregister_table_req).unwrap();
+        catalog
+            .deregister_table(deregister_table_req)
+            .await
+            .unwrap();
         assert!(catalog
             .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
             .await
             .unwrap()
             .is_none());
     }
+
+    #[tokio::test]
+    async fn test_catalog_deregister_schema() {
+        let catalog = MemoryCatalogManager::with_default_setup();
+
+        // Registers a catalog, a schema, and a table.
+        let catalog_name = "foo_catalog".to_string();
+        let schema_name = "foo_schema".to_string();
+        let table_name = "foo_table".to_string();
+        let schema = RegisterSchemaRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+        };
+        let table = RegisterTableRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+            table_name,
+            table_id: 0,
+            table: NumbersTable::table(0),
+        };
+        catalog
+            .clone()
+            .register_catalog(catalog_name.clone())
+            .await
+            .unwrap();
+        catalog.register_schema(schema).await.unwrap();
+        catalog.register_table(table).await.unwrap();
+
+        let request = DeregisterSchemaRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+        };
+
+        assert!(catalog.deregister_schema(request).await.unwrap());
+        assert!(!catalog
+            .schema_exist(&catalog_name, &schema_name)
+            .await
+            .unwrap());
+    }
 }

src/catalog/src/remote.rs

@@ -12,11 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+
 pub use client::{CachedMetaKvBackend, MetaKvBackend};
+pub use manager::RemoteCatalogManager;
 
 mod client;
 mod manager;
 
 #[cfg(feature = "testing")]
 pub mod mock;
-pub use manager::KvBackendCatalogManager;
+pub mod region_alive_keeper;
+
+#[async_trait::async_trait]
+pub trait KvCacheInvalidator: Send + Sync {
+    async fn invalidate_key(&self, key: &[u8]);
+}
+
+pub type KvCacheInvalidatorRef = Arc<dyn KvCacheInvalidator>;

src/catalog/src/remote/client.rs

@@ -18,9 +18,8 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use common_error::ext::BoxedError;
-use common_meta::cache_invalidator::KvCacheInvalidator;
 use common_meta::error::Error::{CacheNotGet, GetKvCache};
-use common_meta::error::{CacheNotGetSnafu, Error, ExternalSnafu, Result};
+use common_meta::error::{CacheNotGetSnafu, Error, MetaSrvSnafu, Result};
 use common_meta::kv_backend::{KvBackend, KvBackendRef, TxnService};
 use common_meta::rpc::store::{
     BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
@@ -29,11 +28,12 @@ use common_meta::rpc::store::{
     RangeRequest, RangeResponse,
 };
 use common_meta::rpc::KeyValue;
-use common_telemetry::{debug, timer};
+use common_telemetry::timer;
 use meta_client::client::MetaClient;
 use moka::future::{Cache, CacheBuilder};
 use snafu::{OptionExt, ResultExt};
 
+use super::KvCacheInvalidator;
 use crate::metrics::{METRIC_CATALOG_KV_GET, METRIC_CATALOG_KV_REMOTE_GET};
 
 const CACHE_MAX_CAPACITY: u64 = 10000;
@@ -197,8 +197,7 @@ impl KvBackend for CachedMetaKvBackend {
 #[async_trait::async_trait]
 impl KvCacheInvalidator for CachedMetaKvBackend {
     async fn invalidate_key(&self, key: &[u8]) {
-        self.cache.invalidate(key).await;
+        self.cache.invalidate(key).await
-        debug!("invalidated cache key: {}", String::from_utf8_lossy(key));
     }
 }
 
@@ -252,7 +251,7 @@ impl KvBackend for MetaKvBackend {
             .range(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
@@ -261,7 +260,7 @@ impl KvBackend for MetaKvBackend {
             .range(RangeRequest::new().with_key(key))
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)?;
+            .context(MetaSrvSnafu)?;
         Ok(response.take_kvs().get_mut(0).map(|kv| KeyValue {
             key: kv.take_key(),
             value: kv.take_value(),
@@ -273,7 +272,7 @@ impl KvBackend for MetaKvBackend {
             .batch_put(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn put(&self, req: PutRequest) -> Result<PutResponse> {
@@ -281,7 +280,7 @@ impl KvBackend for MetaKvBackend {
             .put(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn delete_range(&self, req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
@@ -289,7 +288,7 @@ impl KvBackend for MetaKvBackend {
             .delete_range(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
@@ -297,7 +296,7 @@ impl KvBackend for MetaKvBackend {
             .batch_delete(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
@@ -305,7 +304,7 @@ impl KvBackend for MetaKvBackend {
             .batch_get(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn compare_and_put(
@@ -316,7 +315,7 @@ impl KvBackend for MetaKvBackend {
             .compare_and_put(request)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
@@ -324,7 +323,7 @@ impl KvBackend for MetaKvBackend {
             .move_value(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     fn as_any(&self) -> &dyn Any {

src/catalog/src/remote/manager.rs

@@ -0,0 +1,436 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_catalog::consts::MITO_ENGINE;
+use common_meta::ident::TableIdent;
+use common_meta::key::catalog_name::CatalogNameKey;
+use common_meta::key::datanode_table::DatanodeTableValue;
+use common_meta::key::schema_name::SchemaNameKey;
+use common_meta::key::TableMetadataManagerRef;
+use common_telemetry::{error, info, warn};
+use futures_util::TryStreamExt;
+use metrics::increment_gauge;
+use snafu::{ensure, OptionExt, ResultExt};
+use table::engine::manager::TableEngineManagerRef;
+use table::engine::EngineContext;
+use table::requests::OpenTableRequest;
+use table::TableRef;
+use tokio::sync::Mutex;
+
+use crate::error::{
+    OpenTableSnafu, ParallelOpenTableSnafu, Result, TableEngineNotFoundSnafu, TableExistsSnafu,
+    TableMetadataManagerSnafu, TableNotFoundSnafu, UnimplementedSnafu,
+};
+use crate::local::MemoryCatalogManager;
+use crate::remote::region_alive_keeper::RegionAliveKeepers;
+use crate::{
+    handle_system_table_request, CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest,
+    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
+};
+
+/// Catalog manager based on metasrv.
+pub struct RemoteCatalogManager {
+    node_id: u64,
+    engine_manager: TableEngineManagerRef,
+    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
+    memory_catalog_manager: Arc<MemoryCatalogManager>,
+    table_metadata_manager: TableMetadataManagerRef,
+}
+
+impl RemoteCatalogManager {
+    pub fn new(
+        engine_manager: TableEngineManagerRef,
+        node_id: u64,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
+        table_metadata_manager: TableMetadataManagerRef,
+    ) -> Self {
+        Self {
+            engine_manager,
+            node_id,
+            system_table_requests: Default::default(),
+            region_alive_keepers,
+            memory_catalog_manager: MemoryCatalogManager::with_default_setup(),
+            table_metadata_manager,
+        }
+    }
+
+    async fn initiate_catalogs(&self) -> Result<()> {
+        let tables = self
+            .table_metadata_manager
+            .datanode_table_manager()
+            .tables(self.node_id)
+            .try_collect::<Vec<_>>()
+            .await
+            .context(TableMetadataManagerSnafu)?;
+
+        let joins = tables
+            .into_iter()
+            .map(|datanode_table_value| {
+                let engine_manager = self.engine_manager.clone();
+                let memory_catalog_manager = self.memory_catalog_manager.clone();
+                let table_metadata_manager = self.table_metadata_manager.clone();
+                let region_alive_keepers = self.region_alive_keepers.clone();
+                common_runtime::spawn_bg(async move {
+                    let table_id = datanode_table_value.table_id;
+                    if let Err(e) = open_and_register_table(
+                        engine_manager,
+                        datanode_table_value,
+                        memory_catalog_manager,
+                        table_metadata_manager,
+                        region_alive_keepers,
+                    )
+                    .await
+                    {
+                        // Note that we don't return error here if table opened failed. This is because
+                        // we don't want those broken tables to impede the startup of Datanode.
+                        // However, this could be changed in the future.
+                        error!(e; "Failed to open or register table, id = {table_id}")
+                    }
+                })
+            })
+            .collect::<Vec<_>>();
+
+        let _ = futures::future::try_join_all(joins)
+            .await
+            .context(ParallelOpenTableSnafu)?;
+        Ok(())
+    }
+}
+
+async fn open_and_register_table(
+    engine_manager: TableEngineManagerRef,
+    datanode_table_value: DatanodeTableValue,
+    memory_catalog_manager: Arc<MemoryCatalogManager>,
+    table_metadata_manager: TableMetadataManagerRef,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
+) -> Result<()> {
+    let context = EngineContext {};
+
+    let table_id = datanode_table_value.table_id;
+    let region_numbers = datanode_table_value.regions;
+
+    let table_info_value = table_metadata_manager
+        .table_info_manager()
+        .get(table_id)
+        .await
+        .context(TableMetadataManagerSnafu)?
+        .context(TableNotFoundSnafu {
+            table_info: format!("table id: {table_id}"),
+        })?;
+    let table_info = &table_info_value.table_info;
+    let catalog_name = table_info.catalog_name.clone();
+    let schema_name = table_info.schema_name.clone();
+    let table_name = table_info.name.clone();
+
+    let request = OpenTableRequest {
+        catalog_name: catalog_name.clone(),
+        schema_name: schema_name.clone(),
+        table_name: table_name.clone(),
+        table_id,
+        region_numbers: region_numbers.clone(),
+    };
+    let engine =
+        engine_manager
+            .engine(&table_info.meta.engine)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: &table_info.meta.engine,
+            })?;
+
+    let table_ident = TableIdent {
+        catalog: catalog_name,
+        schema: schema_name,
+        table: table_name,
+        table_id,
+        engine: table_info.meta.engine.clone(),
+    };
+
+    let table = engine
+        .open_table(&context, request)
+        .await
+        .with_context(|_| OpenTableSnafu {
+            table_info: table_ident.to_string(),
+        })?
+        .with_context(|| TableNotFoundSnafu {
+            table_info: table_ident.to_string(),
+        })?;
+    info!("Successfully opened table, {table_ident}");
+
+    if !memory_catalog_manager
+        .catalog_exist(&table_ident.catalog)
+        .await?
+    {
+        memory_catalog_manager.register_catalog_sync(table_ident.catalog.clone())?;
+    }
+
+    if !memory_catalog_manager
+        .schema_exist(&table_ident.catalog, &table_ident.schema)
+        .await?
+    {
+        memory_catalog_manager.register_schema_sync(RegisterSchemaRequest {
+            catalog: table_ident.catalog.clone(),
+            schema: table_ident.schema.clone(),
+        })?;
+    }
+
+    let request = RegisterTableRequest {
+        catalog: table_ident.catalog.clone(),
+        schema: table_ident.schema.clone(),
+        table_name: table_ident.table.clone(),
+        table_id,
+        table,
+    };
+    let registered =
+        register_table(&memory_catalog_manager, &region_alive_keepers, request).await?;
+    ensure!(
+        registered,
+        TableExistsSnafu {
+            table: table_ident.to_string(),
+        }
+    );
+    info!("Successfully registered table, {table_ident}");
+    Ok(())
+}
+
+async fn register_table(
+    memory_catalog_manager: &Arc<MemoryCatalogManager>,
+    region_alive_keepers: &Arc<RegionAliveKeepers>,
+    request: RegisterTableRequest,
+) -> Result<bool> {
+    let table = request.table.clone();
+
+    let registered = memory_catalog_manager.register_table_sync(request)?;
+
+    if registered {
+        let table_info = table.table_info();
+        let table_ident = TableIdent {
+            catalog: table_info.catalog_name.clone(),
+            schema: table_info.schema_name.clone(),
+            table: table_info.name.clone(),
+            table_id: table_info.table_id(),
+            engine: table_info.meta.engine.clone(),
+        };
+        region_alive_keepers
+            .register_table(table_ident, table, memory_catalog_manager.clone())
+            .await?;
+    }
+
+    Ok(registered)
+}
+
+#[async_trait]
+impl CatalogManager for RemoteCatalogManager {
+    async fn start(&self) -> Result<()> {
+        self.initiate_catalogs().await?;
+
+        let mut system_table_requests = self.system_table_requests.lock().await;
+        let engine = self
+            .engine_manager
+            .engine(MITO_ENGINE)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: MITO_ENGINE,
+            })?;
+        handle_system_table_request(self, engine, &mut system_table_requests).await?;
+        info!("All system table opened");
+        Ok(())
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        register_table(
+            &self.memory_catalog_manager,
+            &self.region_alive_keepers,
+            request,
+        )
+        .await
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
+        let Some(table) = self
+            .memory_catalog_manager
+            .table(&request.catalog, &request.schema, &request.table_name)
+            .await?
+        else {
+            return Ok(());
+        };
+
+        let table_info = table.table_info();
+        let table_ident = TableIdent {
+            catalog: request.catalog.clone(),
+            schema: request.schema.clone(),
+            table: request.table_name.clone(),
+            table_id: table_info.ident.table_id,
+            engine: table_info.meta.engine.clone(),
+        };
+        if let Some(keeper) = self
+            .region_alive_keepers
+            .deregister_table(&table_ident)
+            .await
+        {
+            warn!(
+                "Table {} is deregistered from region alive keepers",
+                keeper.table_ident(),
+            );
+        }
+
+        self.memory_catalog_manager.deregister_table(request).await
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        self.memory_catalog_manager.register_schema_sync(request)
+    }
+
+    async fn deregister_schema(&self, _request: DeregisterSchemaRequest) -> Result<bool> {
+        UnimplementedSnafu {
+            operation: "deregister schema",
+        }
+        .fail()
+    }
+
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
+        self.memory_catalog_manager.rename_table(request).await
+    }
+
+    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
+        let catalog_name = request.create_table_request.catalog_name.clone();
+        let schema_name = request.create_table_request.schema_name.clone();
+
+        let mut requests = self.system_table_requests.lock().await;
+        requests.push(request);
+        increment_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&catalog_name, &schema_name)],
+        );
+        Ok(())
+    }
+
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
+        if !self.catalog_exist(catalog).await? {
+            return Ok(false);
+        }
+
+        if self
+            .memory_catalog_manager
+            .schema_exist(catalog, schema)
+            .await?
+        {
+            return Ok(true);
+        }
+
+        let remote_schema_exists = self
+            .table_metadata_manager
+            .schema_manager()
+            .exist(SchemaNameKey::new(catalog, schema))
+            .await
+            .context(TableMetadataManagerSnafu)?;
+        // Create schema locally if remote schema exists. Since local schema is managed by memory
+        // catalog manager, creating a local schema is relatively cheap (just a HashMap).
+        // Besides, if this method ("schema_exist) is called, it's very likely that someone wants to
+        // create a table in this schema. We should create the schema now.
+        if remote_schema_exists
+            && self
+                .memory_catalog_manager
+                .register_schema(RegisterSchemaRequest {
+                    catalog: catalog.to_string(),
+                    schema: schema.to_string(),
+                })
+                .await?
+        {
+            info!("register schema '{catalog}/{schema}' on demand");
+        }
+
+        Ok(remote_schema_exists)
+    }
+
+    async fn table(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        self.memory_catalog_manager
+            .table(catalog_name, schema_name, table_name)
+            .await
+    }
+
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
+        if self.memory_catalog_manager.catalog_exist(catalog).await? {
+            return Ok(true);
+        }
+
+        let key = CatalogNameKey::new(catalog);
+        let remote_catalog_exists = self
+            .table_metadata_manager
+            .catalog_manager()
+            .exist(key)
+            .await
+            .context(TableMetadataManagerSnafu)?;
+
+        // Create catalog locally if remote catalog exists. Since local catalog is managed by memory
+        // catalog manager, creating a local catalog is relatively cheap (just a HashMap).
+        // Besides, if this method ("catalog_exist) is called, it's very likely that someone wants to
+        // create a table in this catalog. We should create the catalog now.
+        if remote_catalog_exists
+            && self
+                .memory_catalog_manager
+                .clone()
+                .register_catalog(catalog.to_string())
+                .await?
+        {
+            info!("register catalog '{catalog}' on demand");
+        }
+
+        Ok(remote_catalog_exists)
+    }
+
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
+        if !self.catalog_exist(catalog).await? {
+            return Ok(false);
+        }
+
+        if !self.schema_exist(catalog, schema).await? {
+            return Ok(false);
+        }
+
+        self.memory_catalog_manager
+            .table_exist(catalog, schema, table)
+            .await
+    }
+
+    async fn catalog_names(&self) -> Result<Vec<String>> {
+        self.memory_catalog_manager.catalog_names().await
+    }
+
+    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
+        self.memory_catalog_manager.schema_names(catalog_name).await
+    }
+
+    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
+        self.memory_catalog_manager
+            .table_names(catalog_name, schema_name)
+            .await
+    }
+
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
+        self.memory_catalog_manager.register_catalog_sync(name)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/catalog/src/remote/mock.rs

@@ -55,14 +55,14 @@ impl TableEngine for MockTableEngine {
 
         let data = vec![Arc::new(StringVector::from(vec!["a", "b", "c"])) as _];
         let record_batch = RecordBatch::new(schema, data).unwrap();
-        let table = MemTable::new_with_catalog(
+        let table: TableRef = Arc::new(MemTable::new_with_catalog(
             &request.table_name,
             record_batch,
             table_id,
             request.catalog_name,
             request.schema_name,
             vec![0],
-        );
+        )) as Arc<_>;
 
         let mut tables = self.tables.write().unwrap();
         let _ = tables.insert(table_id, table.clone() as TableRef);

src/catalog/src/remote/region_alive_keeper.rs

@@ -0,0 +1,867 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_meta::error::InvalidProtoMsgSnafu;
+use common_meta::heartbeat::handler::{
+    HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
+};
+use common_meta::ident::TableIdent;
+use common_meta::RegionIdent;
+use common_telemetry::{debug, error, info, warn};
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::RegionNumber;
+use table::engine::manager::TableEngineManagerRef;
+use table::engine::{CloseTableResult, EngineContext, TableEngineRef};
+use table::metadata::TableId;
+use table::requests::CloseTableRequest;
+use table::TableRef;
+use tokio::sync::{mpsc, oneshot, Mutex};
+use tokio::task::JoinHandle;
+use tokio::time::{Duration, Instant};
+
+use crate::error::{Result, TableEngineNotFoundSnafu};
+use crate::local::MemoryCatalogManager;
+use crate::DeregisterTableRequest;
+
+/// [RegionAliveKeepers] manages all [RegionAliveKeeper] in a scope of tables.
+pub struct RegionAliveKeepers {
+    table_engine_manager: TableEngineManagerRef,
+    keepers: Arc<Mutex<HashMap<TableId, Arc<RegionAliveKeeper>>>>,
+    heartbeat_interval_millis: u64,
+    started: AtomicBool,
+
+    /// The epoch when [RegionAliveKeepers] is created. It's used to get a monotonically non-decreasing
+    /// elapsed time when submitting heartbeats to Metasrv (because [Instant] is monotonically
+    /// non-decreasing). The heartbeat request will carry the duration since this epoch, and the
+    /// duration acts like an "invariant point" for region's keep alive lease.
+    epoch: Instant,
+}
+
+impl RegionAliveKeepers {
+    pub fn new(
+        table_engine_manager: TableEngineManagerRef,
+        heartbeat_interval_millis: u64,
+    ) -> Self {
+        Self {
+            table_engine_manager,
+            keepers: Arc::new(Mutex::new(HashMap::new())),
+            heartbeat_interval_millis,
+            started: AtomicBool::new(false),
+            epoch: Instant::now(),
+        }
+    }
+
+    pub async fn find_keeper(&self, table_id: TableId) -> Option<Arc<RegionAliveKeeper>> {
+        self.keepers.lock().await.get(&table_id).cloned()
+    }
+
+    pub async fn register_table(
+        &self,
+        table_ident: TableIdent,
+        table: TableRef,
+        catalog_manager: Arc<MemoryCatalogManager>,
+    ) -> Result<()> {
+        let table_id = table_ident.table_id;
+        let keeper = self.find_keeper(table_id).await;
+        if keeper.is_some() {
+            return Ok(());
+        }
+
+        let table_engine = self
+            .table_engine_manager
+            .engine(&table_ident.engine)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: &table_ident.engine,
+            })?;
+
+        let keeper = Arc::new(RegionAliveKeeper::new(
+            table_engine,
+            catalog_manager,
+            table_ident.clone(),
+            self.heartbeat_interval_millis,
+        ));
+        for r in table.table_info().meta.region_numbers.iter() {
+            keeper.register_region(*r).await;
+        }
+
+        let mut keepers = self.keepers.lock().await;
+        let _ = keepers.insert(table_id, keeper.clone());
+
+        if self.started.load(Ordering::Relaxed) {
+            keeper.start().await;
+
+            info!("RegionAliveKeeper for table {table_ident} is started!");
+        } else {
+            info!("RegionAliveKeeper for table {table_ident} is registered but not started yet!");
+        }
+        Ok(())
+    }
+
+    pub async fn deregister_table(
+        &self,
+        table_ident: &TableIdent,
+    ) -> Option<Arc<RegionAliveKeeper>> {
+        let table_id = table_ident.table_id;
+        self.keepers.lock().await.remove(&table_id).map(|x| {
+            info!("Deregister RegionAliveKeeper for table {table_ident}");
+            x
+        })
+    }
+
+    pub async fn register_region(&self, region_ident: &RegionIdent) {
+        let table_id = region_ident.table_ident.table_id;
+        let Some(keeper) = self.find_keeper(table_id).await else {
+            // Alive keeper could be affected by lagging msg, just warn and ignore.
+            warn!("Alive keeper for region {region_ident} is not found!");
+            return;
+        };
+        keeper.register_region(region_ident.region_number).await
+    }
+
+    pub async fn deregister_region(&self, region_ident: &RegionIdent) {
+        let table_id = region_ident.table_ident.table_id;
+        let Some(keeper) = self.find_keeper(table_id).await else {
+            // Alive keeper could be affected by lagging msg, just warn and ignore.
+            warn!("Alive keeper for region {region_ident} is not found!");
+            return;
+        };
+        let _ = keeper.deregister_region(region_ident.region_number).await;
+    }
+
+    pub async fn start(&self) {
+        let keepers = self.keepers.lock().await;
+        for keeper in keepers.values() {
+            keeper.start().await;
+        }
+        self.started.store(true, Ordering::Relaxed);
+
+        info!(
+            "RegionAliveKeepers for tables {:?} are started!",
+            keepers.keys().map(|x| x.to_string()).collect::<Vec<_>>(),
+        );
+    }
+
+    pub fn epoch(&self) -> Instant {
+        self.epoch
+    }
+}
+
+#[async_trait]
+impl HeartbeatResponseHandler for RegionAliveKeepers {
+    fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
+        !ctx.response.region_leases.is_empty()
+    }
+
+    async fn handle(
+        &self,
+        ctx: &mut HeartbeatResponseHandlerContext,
+    ) -> common_meta::error::Result<HandleControl> {
+        let leases = ctx.response.region_leases.drain(..).collect::<Vec<_>>();
+        for lease in leases {
+            let table_ident: TableIdent = match lease
+                .table_ident
+                .context(InvalidProtoMsgSnafu {
+                    err_msg: "'table_ident' is missing in RegionLease",
+                })
+                .and_then(|x| x.try_into())
+            {
+                Ok(x) => x,
+                Err(e) => {
+                    error!(e; "");
+                    continue;
+                }
+            };
+
+            let table_id = table_ident.table_id;
+            let Some(keeper) = self.keepers.lock().await.get(&table_id).cloned() else {
+                // Alive keeper could be affected by lagging msg, just warn and ignore.
+                warn!("Alive keeper for table {table_ident} is not found!");
+                continue;
+            };
+
+            let start_instant = self.epoch + Duration::from_millis(lease.duration_since_epoch);
+            let deadline = start_instant + Duration::from_secs(lease.lease_seconds);
+            keeper.keep_lived(lease.regions, deadline).await;
+        }
+        Ok(HandleControl::Continue)
+    }
+}
+
+/// [RegionAliveKeeper] starts a countdown for each region in a table. When deadline is reached,
+/// the region will be closed.
+/// The deadline is controlled by Metasrv. It works like "lease" for regions: a Datanode submits its
+/// opened regions to Metasrv, in heartbeats. If Metasrv decides some region could be resided in this
+/// Datanode, it will "extend" the region's "lease", with a deadline for [RegionAliveKeeper] to
+/// countdown.
+pub struct RegionAliveKeeper {
+    catalog_manager: Arc<MemoryCatalogManager>,
+    table_engine: TableEngineRef,
+    table_ident: TableIdent,
+    countdown_task_handles: Arc<Mutex<HashMap<RegionNumber, Arc<CountdownTaskHandle>>>>,
+    heartbeat_interval_millis: u64,
+    started: AtomicBool,
+}
+
+impl RegionAliveKeeper {
+    fn new(
+        table_engine: TableEngineRef,
+        catalog_manager: Arc<MemoryCatalogManager>,
+        table_ident: TableIdent,
+        heartbeat_interval_millis: u64,
+    ) -> Self {
+        Self {
+            catalog_manager,
+            table_engine,
+            table_ident,
+            countdown_task_handles: Arc::new(Mutex::new(HashMap::new())),
+            heartbeat_interval_millis,
+            started: AtomicBool::new(false),
+        }
+    }
+
+    async fn find_handle(&self, region: &RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
+        self.countdown_task_handles
+            .lock()
+            .await
+            .get(region)
+            .cloned()
+    }
+
+    async fn register_region(&self, region: RegionNumber) {
+        if self.find_handle(&region).await.is_some() {
+            return;
+        }
+
+        let countdown_task_handles = Arc::downgrade(&self.countdown_task_handles);
+        let on_task_finished = async move {
+            if let Some(x) = countdown_task_handles.upgrade() {
+                let _ = x.lock().await.remove(&region);
+            } // Else the countdown task handles map could be dropped because the keeper is dropped.
+        };
+        let catalog_manager = self.catalog_manager.clone();
+        let ident = self.table_ident.clone();
+        let handle = Arc::new(CountdownTaskHandle::new(
+            self.table_engine.clone(),
+            self.table_ident.clone(),
+            region,
+            move |result: Option<CloseTableResult>| {
+                if matches!(result, Some(CloseTableResult::Released(_))) {
+                    let result = catalog_manager.deregister_table_sync(DeregisterTableRequest {
+                        catalog: ident.catalog.to_string(),
+                        schema: ident.schema.to_string(),
+                        table_name: ident.table.to_string(),
+                    });
+
+                    info!(
+                        "Deregister table: {} after countdown task finished, result: {result:?}",
+                        ident.table_id
+                    );
+                } else {
+                    debug!("Countdown task returns: {result:?}");
+                }
+                on_task_finished
+            },
+        ));
+
+        let mut handles = self.countdown_task_handles.lock().await;
+        let _ = handles.insert(region, handle.clone());
+
+        if self.started.load(Ordering::Relaxed) {
+            handle.start(self.heartbeat_interval_millis).await;
+
+            info!(
+                "Region alive countdown for region {region} in table {} is started!",
+                self.table_ident
+            );
+        } else {
+            info!(
+                "Region alive countdown for region {region} in table {} is registered but not started yet!",
+                self.table_ident
+            );
+        }
+    }
+
+    async fn deregister_region(&self, region: RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
+        self.countdown_task_handles
+            .lock()
+            .await
+            .remove(&region)
+            .map(|x| {
+                info!(
+                    "Deregister alive countdown for region {region} in table {}",
+                    self.table_ident
+                );
+                x
+            })
+    }
+
+    async fn start(&self) {
+        let handles = self.countdown_task_handles.lock().await;
+        for handle in handles.values() {
+            handle.start(self.heartbeat_interval_millis).await;
+        }
+
+        self.started.store(true, Ordering::Relaxed);
+        info!(
+            "Region alive countdowns for regions {:?} in table {} are started!",
+            handles.keys().copied().collect::<Vec<_>>(),
+            self.table_ident
+        );
+    }
+
+    async fn keep_lived(&self, designated_regions: Vec<RegionNumber>, deadline: Instant) {
+        for region in designated_regions {
+            if let Some(handle) = self.find_handle(&region).await {
+                handle.reset_deadline(deadline).await;
+            }
+            // Else the region alive keeper might be triggered by lagging messages, we can safely ignore it.
+        }
+    }
+
+    pub async fn deadline(&self, region: RegionNumber) -> Option<Instant> {
+        let mut deadline = None;
+        if let Some(handle) = self.find_handle(&region).await {
+            let (s, r) = oneshot::channel();
+            if handle.tx.send(CountdownCommand::Deadline(s)).await.is_ok() {
+                deadline = r.await.ok()
+            }
+        }
+        deadline
+    }
+
+    pub fn table_ident(&self) -> &TableIdent {
+        &self.table_ident
+    }
+}
+
+#[derive(Debug)]
+enum CountdownCommand {
+    Start(u64),
+    Reset(Instant),
+    Deadline(oneshot::Sender<Instant>),
+}
+
+struct CountdownTaskHandle {
+    tx: mpsc::Sender<CountdownCommand>,
+    handler: JoinHandle<()>,
+    table_ident: TableIdent,
+    region: RegionNumber,
+}
+
+impl CountdownTaskHandle {
+    /// Creates a new [CountdownTaskHandle] and starts the countdown task.
+    /// # Params
+    /// - `on_task_finished`: a callback to be invoked when the task is finished. Note that it will not
+    ///   be invoked if the task is cancelled (by dropping the handle). This is because we want something
+    ///   meaningful to be done when the task is finished, e.g. deregister the handle from the map.
+    ///   While dropping the handle does not necessarily mean the task is finished.
+    fn new<Fut>(
+        table_engine: TableEngineRef,
+        table_ident: TableIdent,
+        region: RegionNumber,
+        on_task_finished: impl FnOnce(Option<CloseTableResult>) -> Fut + Send + 'static,
+    ) -> Self
+    where
+        Fut: Future<Output = ()> + Send,
+    {
+        let (tx, rx) = mpsc::channel(1024);
+
+        let mut countdown_task = CountdownTask {
+            table_engine,
+            table_ident: table_ident.clone(),
+            region,
+            rx,
+        };
+        let handler = common_runtime::spawn_bg(async move {
+            let result = countdown_task.run().await;
+            on_task_finished(result).await;
+        });
+
+        Self {
+            tx,
+            handler,
+            table_ident,
+            region,
+        }
+    }
+
+    async fn start(&self, heartbeat_interval_millis: u64) {
+        if let Err(e) = self
+            .tx
+            .send(CountdownCommand::Start(heartbeat_interval_millis))
+            .await
+        {
+            warn!(
+                "Failed to start region alive keeper countdown: {e}. \
+                Maybe the task is stopped due to region been closed."
+            );
+        }
+    }
+
+    async fn reset_deadline(&self, deadline: Instant) {
+        if let Err(e) = self.tx.send(CountdownCommand::Reset(deadline)).await {
+            warn!(
+                "Failed to reset region alive keeper deadline: {e}. \
+                Maybe the task is stopped due to region been closed."
+            );
+        }
+    }
+}
+
+impl Drop for CountdownTaskHandle {
+    fn drop(&mut self) {
+        debug!(
+            "Aborting region alive countdown task for region {} in table {}",
+            self.region, self.table_ident,
+        );
+        self.handler.abort();
+    }
+}
+
+struct CountdownTask {
+    table_engine: TableEngineRef,
+    table_ident: TableIdent,
+    region: RegionNumber,
+    rx: mpsc::Receiver<CountdownCommand>,
+}
+
+impl CountdownTask {
+    // returns true if
+    async fn run(&mut self) -> Option<CloseTableResult> {
+        // 30 years. See `Instant::far_future`.
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 30);
+
+        // Make sure the alive countdown is not gonna happen before heartbeat task is started (the
+        // "start countdown" command will be sent from heartbeat task).
+        let countdown = tokio::time::sleep_until(far_future);
+        tokio::pin!(countdown);
+
+        let region = &self.region;
+        let table_ident = &self.table_ident;
+        loop {
+            tokio::select! {
+                command = self.rx.recv() => {
+                    match command {
+                        Some(CountdownCommand::Start(heartbeat_interval_millis)) => {
+                            // Set first deadline in 4 heartbeats (roughly after 20 seconds from now if heartbeat
+                            // interval is set to default 5 seconds), to make Datanode and Metasrv more tolerable to
+                            // network or other jitters during startup.
+                            let first_deadline = Instant::now() + Duration::from_millis(heartbeat_interval_millis) * 4;
+                            countdown.set(tokio::time::sleep_until(first_deadline));
+                        },
+                        Some(CountdownCommand::Reset(deadline)) => {
+                            if countdown.deadline() < deadline {
+                                debug!(
+                                    "Reset deadline of region {region} of table {table_ident} to approximately {} seconds later",
+                                    (deadline - Instant::now()).as_secs_f32(),
+                                );
+                                countdown.set(tokio::time::sleep_until(deadline));
+                            }
+                            // Else the countdown could be either:
+                            // - not started yet;
+                            // - during startup protection;
+                            // - received a lagging heartbeat message.
+                            // All can be safely ignored.
+                        },
+                        None => {
+                            info!(
+                                "The handle of countdown task for region {region} of table {table_ident} \
+                                is dropped, RegionAliveKeeper out."
+                            );
+                            break;
+                        },
+                        Some(CountdownCommand::Deadline(tx)) => {
+                            let _ = tx.send(countdown.deadline());
+                        }
+                    }
+                }
+                () = &mut countdown => {
+                    let result = self.close_region().await;
+                    warn!(
+                        "Region {region} of table {table_ident} is closed, result: {result:?}. \
+                        RegionAliveKeeper out.",
+                    );
+                    return Some(result);
+                }
+            }
+        }
+        None
+    }
+
+    async fn close_region(&self) -> CloseTableResult {
+        let ctx = EngineContext::default();
+        let region = self.region;
+        let table_ident = &self.table_ident;
+        loop {
+            let request = CloseTableRequest {
+                catalog_name: table_ident.catalog.clone(),
+                schema_name: table_ident.schema.clone(),
+                table_name: table_ident.table.clone(),
+                table_id: table_ident.table_id,
+                region_numbers: vec![region],
+                flush: true,
+            };
+            match self.table_engine.close_table(&ctx, request).await {
+                Ok(result) => return result,
+                // If region is failed to close, immediately retry. Maybe we should panic instead?
+                Err(e) => error!(e;
+                    "Failed to close region {region} of table {table_ident}. \
+                    For the integrity of data, retry closing and retry without wait.",
+                ),
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::Arc;
+
+    use api::v1::meta::{HeartbeatResponse, RegionLease};
+    use common_meta::heartbeat::mailbox::HeartbeatMailbox;
+    use datatypes::schema::RawSchema;
+    use table::engine::manager::MemoryTableEngineManager;
+    use table::engine::TableEngine;
+    use table::requests::{CreateTableRequest, TableOptions};
+    use table::test_util::EmptyTable;
+
+    use super::*;
+    use crate::remote::mock::MockTableEngine;
+
+    async fn prepare_keepers() -> (TableIdent, RegionAliveKeepers) {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_engine_manager = Arc::new(MemoryTableEngineManager::new(table_engine));
+        let keepers = RegionAliveKeepers::new(table_engine_manager, 5000);
+
+        let catalog = "my_catalog";
+        let schema = "my_schema";
+        let table = "my_table";
+        let table_ident = TableIdent {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table: table.to_string(),
+            table_id: 1,
+            engine: "MockTableEngine".to_string(),
+        };
+        let table = Arc::new(EmptyTable::new(CreateTableRequest {
+            id: 1,
+            catalog_name: catalog.to_string(),
+            schema_name: schema.to_string(),
+            table_name: table.to_string(),
+            desc: None,
+            schema: RawSchema {
+                column_schemas: vec![],
+                timestamp_index: None,
+                version: 0,
+            },
+            region_numbers: vec![1, 2, 3],
+            primary_key_indices: vec![],
+            create_if_not_exists: false,
+            table_options: TableOptions::default(),
+            engine: "MockTableEngine".to_string(),
+        }));
+        let catalog_manager = MemoryCatalogManager::new_with_table(table.clone());
+        keepers
+            .register_table(table_ident.clone(), table, catalog_manager)
+            .await
+            .unwrap();
+        assert!(keepers
+            .keepers
+            .lock()
+            .await
+            .contains_key(&table_ident.table_id));
+
+        (table_ident, keepers)
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_handle_heartbeat_response() {
+        let (table_ident, keepers) = prepare_keepers().await;
+
+        keepers.start().await;
+        let startup_protection_until = Instant::now() + Duration::from_secs(21);
+
+        let duration_since_epoch = (Instant::now() - keepers.epoch).as_millis() as _;
+        let lease_seconds = 100;
+        let response = HeartbeatResponse {
+            region_leases: vec![RegionLease {
+                table_ident: Some(table_ident.clone().into()),
+                regions: vec![1, 3], // Not extending region 2's lease time.
+                duration_since_epoch,
+                lease_seconds,
+            }],
+            ..Default::default()
+        };
+        let keep_alive_until = keepers.epoch
+            + Duration::from_millis(duration_since_epoch)
+            + Duration::from_secs(lease_seconds);
+
+        let (tx, _) = mpsc::channel(8);
+        let mailbox = Arc::new(HeartbeatMailbox::new(tx));
+        let mut ctx = HeartbeatResponseHandlerContext::new(mailbox, response);
+
+        assert!(keepers.handle(&mut ctx).await.unwrap() == HandleControl::Continue);
+
+        // sleep to wait for background task spawned in `handle`
+        tokio::time::sleep(Duration::from_secs(1)).await;
+
+        async fn test(
+            keeper: &Arc<RegionAliveKeeper>,
+            region_number: RegionNumber,
+            startup_protection_until: Instant,
+            keep_alive_until: Instant,
+            is_kept_live: bool,
+        ) {
+            let deadline = keeper.deadline(region_number).await.unwrap();
+            if is_kept_live {
+                assert!(deadline > startup_protection_until && deadline == keep_alive_until);
+            } else {
+                assert!(deadline <= startup_protection_until);
+            }
+        }
+
+        let keeper = &keepers
+            .keepers
+            .lock()
+            .await
+            .get(&table_ident.table_id)
+            .cloned()
+            .unwrap();
+
+        // Test region 1 and 3 is kept lived. Their deadlines are updated to desired instant.
+        test(keeper, 1, startup_protection_until, keep_alive_until, true).await;
+        test(keeper, 3, startup_protection_until, keep_alive_until, true).await;
+
+        // Test region 2 is not kept lived. It's deadline is not updated: still during startup protection period.
+        test(keeper, 2, startup_protection_until, keep_alive_until, false).await;
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_region_alive_keepers() {
+        let (table_ident, keepers) = prepare_keepers().await;
+
+        keepers
+            .register_region(&RegionIdent {
+                cluster_id: 1,
+                datanode_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 4,
+            })
+            .await;
+
+        keepers.start().await;
+        for keeper in keepers.keepers.lock().await.values() {
+            let regions = {
+                let handles = keeper.countdown_task_handles.lock().await;
+                handles.keys().copied().collect::<Vec<_>>()
+            };
+            for region in regions {
+                // assert countdown tasks are started
+                let deadline = keeper.deadline(region).await.unwrap();
+                assert!(deadline <= Instant::now() + Duration::from_secs(20));
+            }
+        }
+
+        keepers
+            .deregister_region(&RegionIdent {
+                cluster_id: 1,
+                datanode_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 1,
+            })
+            .await;
+        let mut regions = keepers
+            .find_keeper(table_ident.table_id)
+            .await
+            .unwrap()
+            .countdown_task_handles
+            .lock()
+            .await
+            .keys()
+            .copied()
+            .collect::<Vec<_>>();
+        regions.sort();
+        assert_eq!(regions, vec![2, 3, 4]);
+
+        let keeper = keepers.deregister_table(&table_ident).await.unwrap();
+        assert!(Arc::try_unwrap(keeper).is_ok(), "keeper is not dropped");
+        assert!(keepers.keepers.lock().await.is_empty());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_region_alive_keeper() {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_ident = TableIdent {
+            catalog: "my_catalog".to_string(),
+            schema: "my_schema".to_string(),
+            table: "my_table".to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let catalog_manager = MemoryCatalogManager::with_default_setup();
+        let keeper = RegionAliveKeeper::new(table_engine, catalog_manager, table_ident, 1000);
+
+        let region = 1;
+        assert!(keeper.find_handle(&region).await.is_none());
+        keeper.register_region(region).await;
+        let _ = keeper.find_handle(&region).await.unwrap();
+
+        let ten_seconds_later = || Instant::now() + Duration::from_secs(10);
+
+        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
+        assert!(keeper.find_handle(&2).await.is_none());
+        assert!(keeper.find_handle(&3).await.is_none());
+
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
+        // assert if keeper is not started, keep_lived is of no use
+        assert!(keeper.deadline(region).await.unwrap() > far_future);
+
+        keeper.start().await;
+        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
+        // assert keep_lived works if keeper is started
+        assert!(keeper.deadline(region).await.unwrap() <= ten_seconds_later());
+
+        let handle = keeper.deregister_region(region).await.unwrap();
+        assert!(Arc::try_unwrap(handle).is_ok(), "handle is not dropped");
+        assert!(keeper.find_handle(&region).await.is_none());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_countdown_task_handle() {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_ident = TableIdent {
+            catalog: "my_catalog".to_string(),
+            schema: "my_schema".to_string(),
+            table: "my_table".to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let finished = Arc::new(AtomicBool::new(false));
+        let finished_clone = finished.clone();
+        let handle = CountdownTaskHandle::new(
+            table_engine.clone(),
+            table_ident.clone(),
+            1,
+            |_| async move { finished_clone.store(true, Ordering::Relaxed) },
+        );
+        let tx = handle.tx.clone();
+
+        // assert countdown task is running
+        tx.send(CountdownCommand::Start(5000)).await.unwrap();
+        assert!(!finished.load(Ordering::Relaxed));
+
+        drop(handle);
+        tokio::time::sleep(Duration::from_secs(1)).await;
+
+        // assert countdown task is stopped
+        assert!(tx
+            .try_send(CountdownCommand::Reset(
+                Instant::now() + Duration::from_secs(10)
+            ))
+            .is_err());
+        // assert `on_task_finished` is not called (because the task is aborted by the handle's drop)
+        assert!(!finished.load(Ordering::Relaxed));
+
+        let finished = Arc::new(AtomicBool::new(false));
+        let finished_clone = finished.clone();
+        let handle = CountdownTaskHandle::new(table_engine, table_ident, 1, |_| async move {
+            finished_clone.store(true, Ordering::Relaxed)
+        });
+        handle.tx.send(CountdownCommand::Start(100)).await.unwrap();
+        tokio::time::sleep(Duration::from_secs(1)).await;
+        // assert `on_task_finished` is called when task is finished normally
+        assert!(finished.load(Ordering::Relaxed));
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_countdown_task_run() {
+        let ctx = &EngineContext::default();
+        let catalog = "my_catalog";
+        let schema = "my_schema";
+        let table = "my_table";
+        let table_id = 1;
+        let request = CreateTableRequest {
+            id: table_id,
+            catalog_name: catalog.to_string(),
+            schema_name: schema.to_string(),
+            table_name: table.to_string(),
+            desc: None,
+            schema: RawSchema {
+                column_schemas: vec![],
+                timestamp_index: None,
+                version: 0,
+            },
+            region_numbers: vec![],
+            primary_key_indices: vec![],
+            create_if_not_exists: false,
+            table_options: TableOptions::default(),
+            engine: "mito".to_string(),
+        };
+
+        let table_engine = Arc::new(MockTableEngine::default());
+        let _ = table_engine.create_table(ctx, request).await.unwrap();
+
+        let table_ident = TableIdent {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table: table.to_string(),
+            table_id,
+            engine: "mito".to_string(),
+        };
+        let (tx, rx) = mpsc::channel(10);
+        let mut task = CountdownTask {
+            table_engine: table_engine.clone(),
+            table_ident,
+            region: 1,
+            rx,
+        };
+        let _handle = common_runtime::spawn_bg(async move {
+            task.run().await;
+        });
+
+        async fn deadline(tx: &mpsc::Sender<CountdownCommand>) -> Instant {
+            let (s, r) = oneshot::channel();
+            tx.send(CountdownCommand::Deadline(s)).await.unwrap();
+            r.await.unwrap()
+        }
+
+        // if countdown task is not started, its deadline is set to far future
+        assert!(deadline(&tx).await > Instant::now() + Duration::from_secs(86400 * 365 * 29));
+
+        // start countdown in 250ms * 4 = 1s
+        tx.send(CountdownCommand::Start(250)).await.unwrap();
+        // assert deadline is correctly set
+        assert!(deadline(&tx).await <= Instant::now() + Duration::from_secs(1));
+
+        // reset countdown in 1.5s
+        tx.send(CountdownCommand::Reset(
+            Instant::now() + Duration::from_millis(1500),
+        ))
+        .await
+        .unwrap();
+
+        // assert the table is closed after deadline is reached
+        assert!(table_engine.table_exists(ctx, table_id));
+        // spare 500ms for the task to close the table
+        tokio::time::sleep(Duration::from_millis(2000)).await;
+        assert!(!table_engine.table_exists(ctx, table_id));
+    }
+}

src/catalog/src/system.rs

@@ -0,0 +1,634 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MITO_ENGINE,
+    SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID, SYSTEM_CATALOG_TABLE_NAME,
+};
+use common_recordbatch::SendableRecordBatchStream;
+use common_telemetry::{debug, warn};
+use common_time::util;
+use datatypes::prelude::{ConcreteDataType, ScalarVector, VectorRef};
+use datatypes::schema::{ColumnSchema, RawSchema};
+use datatypes::vectors::{BinaryVector, TimestampMillisecondVector, UInt8Vector};
+use serde::{Deserialize, Serialize};
+use snafu::{ensure, OptionExt, ResultExt};
+use store_api::storage::ScanRequest;
+use table::engine::{EngineContext, TableEngineRef};
+use table::metadata::TableId;
+use table::requests::{CreateTableRequest, InsertRequest, OpenTableRequest, TableOptions};
+use table::TableRef;
+
+use crate::error::{
+    self, CreateSystemCatalogSnafu, DeregisterTableSnafu, EmptyValueSnafu, Error,
+    InsertCatalogRecordSnafu, InvalidEntryTypeSnafu, InvalidKeySnafu, OpenSystemCatalogSnafu,
+    Result, ValueDeserializeSnafu,
+};
+use crate::DeregisterTableRequest;
+
+pub const ENTRY_TYPE_INDEX: usize = 0;
+pub const KEY_INDEX: usize = 1;
+pub const VALUE_INDEX: usize = 3;
+
+pub struct SystemCatalogTable(TableRef);
+
+impl SystemCatalogTable {
+    pub async fn new(engine: TableEngineRef) -> Result<Self> {
+        let request = OpenTableRequest {
+            catalog_name: SYSTEM_CATALOG_NAME.to_string(),
+            schema_name: INFORMATION_SCHEMA_NAME.to_string(),
+            table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
+            table_id: SYSTEM_CATALOG_TABLE_ID,
+            region_numbers: vec![0],
+        };
+        let schema = build_system_catalog_schema();
+        let ctx = EngineContext::default();
+
+        if let Some(table) = engine
+            .open_table(&ctx, request)
+            .await
+            .context(OpenSystemCatalogSnafu)?
+        {
+            Ok(Self(table))
+        } else {
+            // system catalog table is not yet created, try to create
+            let request = CreateTableRequest {
+                id: SYSTEM_CATALOG_TABLE_ID,
+                catalog_name: SYSTEM_CATALOG_NAME.to_string(),
+                schema_name: INFORMATION_SCHEMA_NAME.to_string(),
+                table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
+                desc: Some("System catalog table".to_string()),
+                schema,
+                region_numbers: vec![0],
+                primary_key_indices: vec![ENTRY_TYPE_INDEX, KEY_INDEX],
+                create_if_not_exists: true,
+                table_options: TableOptions::default(),
+                engine: engine.name().to_string(),
+            };
+
+            let table = engine
+                .create_table(&ctx, request)
+                .await
+                .context(CreateSystemCatalogSnafu)?;
+            Ok(Self(table))
+        }
+    }
+
+    pub async fn register_table(
+        &self,
+        catalog: String,
+        schema: String,
+        table_name: String,
+        table_id: TableId,
+        engine: String,
+    ) -> Result<usize> {
+        let insert_request =
+            build_table_insert_request(catalog, schema, table_name, table_id, engine);
+        self.0
+            .insert(insert_request)
+            .await
+            .context(InsertCatalogRecordSnafu)
+    }
+
+    pub(crate) async fn deregister_table(
+        &self,
+        request: &DeregisterTableRequest,
+        table_id: TableId,
+    ) -> Result<()> {
+        let deletion_request = build_table_deletion_request(request, table_id);
+        self.0
+            .insert(deletion_request)
+            .await
+            .map(|x| {
+                if x != 1 {
+                    let table = common_catalog::format_full_table_name(
+                        &request.catalog,
+                        &request.schema,
+                        &request.table_name
+                    );
+                    warn!("Failed to delete table record from information_schema, unexpected returned result: {x}, table: {table}");
+                }
+            })
+            .with_context(|_| DeregisterTableSnafu {
+                request: request.clone(),
+            })
+    }
+
+    pub async fn register_schema(&self, catalog: String, schema: String) -> Result<usize> {
+        let insert_request = build_schema_insert_request(catalog, schema);
+        self.0
+            .insert(insert_request)
+            .await
+            .context(InsertCatalogRecordSnafu)
+    }
+
+    /// Create a stream of all entries inside system catalog table
+    pub async fn records(&self) -> Result<SendableRecordBatchStream> {
+        let full_projection = None;
+        let scan_req = ScanRequest {
+            sequence: None,
+            projection: full_projection,
+            filters: vec![],
+            output_ordering: None,
+            limit: None,
+        };
+        let stream = self
+            .0
+            .scan_to_stream(scan_req)
+            .await
+            .context(error::SystemCatalogTableScanSnafu)?;
+        Ok(stream)
+    }
+
+    pub fn as_table_ref(&self) -> TableRef {
+        self.0.clone()
+    }
+}
+
+/// Build system catalog table schema.
+/// A system catalog table consists of 6 columns, namely
+/// - entry_type: type of entry in current row, can be any variant of [EntryType].
+/// - key: a binary encoded key of entry, differs according to different entry type.
+/// - timestamp: currently not used.
+/// - value: JSON-encoded value of entry's metadata.
+/// - gmt_created: create time of this metadata.
+/// - gmt_modified: last updated time of this metadata.
+fn build_system_catalog_schema() -> RawSchema {
+    let cols = vec![
+        ColumnSchema::new(
+            "entry_type".to_string(),
+            ConcreteDataType::uint8_datatype(),
+            false,
+        ),
+        ColumnSchema::new(
+            "key".to_string(),
+            ConcreteDataType::binary_datatype(),
+            false,
+        ),
+        ColumnSchema::new(
+            "timestamp".to_string(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new(
+            "value".to_string(),
+            ConcreteDataType::binary_datatype(),
+            false,
+        ),
+        ColumnSchema::new(
+            "gmt_created".to_string(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        ),
+        ColumnSchema::new(
+            "gmt_modified".to_string(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        ),
+    ];
+
+    RawSchema::new(cols)
+}
+
+/// Formats key string for table entry in system catalog
+#[inline]
+pub fn format_table_entry_key(catalog: &str, schema: &str, table_id: TableId) -> String {
+    format!("{catalog}.{schema}.{table_id}")
+}
+
+pub fn build_table_insert_request(
+    catalog: String,
+    schema: String,
+    table_name: String,
+    table_id: TableId,
+    engine: String,
+) -> InsertRequest {
+    let entry_key = format_table_entry_key(&catalog, &schema, table_id);
+    build_insert_request(
+        EntryType::Table,
+        entry_key.as_bytes(),
+        serde_json::to_string(&TableEntryValue {
+            table_name,
+            engine,
+            is_deleted: false,
+        })
+        .unwrap()
+        .as_bytes(),
+    )
+}
+
+pub(crate) fn build_table_deletion_request(
+    request: &DeregisterTableRequest,
+    table_id: TableId,
+) -> InsertRequest {
+    let entry_key = format_table_entry_key(&request.catalog, &request.schema, table_id);
+    build_insert_request(
+        EntryType::Table,
+        entry_key.as_bytes(),
+        serde_json::to_string(&TableEntryValue {
+            table_name: "".to_string(),
+            engine: "".to_string(),
+            is_deleted: true,
+        })
+        .unwrap()
+        .as_bytes(),
+    )
+}
+
+fn build_primary_key_columns(entry_type: EntryType, key: &[u8]) -> HashMap<String, VectorRef> {
+    HashMap::from([
+        (
+            "entry_type".to_string(),
+            Arc::new(UInt8Vector::from_slice([entry_type as u8])) as VectorRef,
+        ),
+        (
+            "key".to_string(),
+            Arc::new(BinaryVector::from_slice(&[key])) as VectorRef,
+        ),
+        (
+            "timestamp".to_string(),
+            // Timestamp in key part is intentionally left to 0
+            Arc::new(TimestampMillisecondVector::from_slice([0])) as VectorRef,
+        ),
+    ])
+}
+
+pub fn build_schema_insert_request(catalog_name: String, schema_name: String) -> InsertRequest {
+    let full_schema_name = format!("{catalog_name}.{schema_name}");
+    build_insert_request(
+        EntryType::Schema,
+        full_schema_name.as_bytes(),
+        serde_json::to_string(&SchemaEntryValue {})
+            .unwrap()
+            .as_bytes(),
+    )
+}
+
+pub fn build_insert_request(entry_type: EntryType, key: &[u8], value: &[u8]) -> InsertRequest {
+    let primary_key_columns = build_primary_key_columns(entry_type, key);
+
+    let mut columns_values = HashMap::with_capacity(6);
+    columns_values.extend(primary_key_columns);
+
+    let _ = columns_values.insert(
+        "value".to_string(),
+        Arc::new(BinaryVector::from_slice(&[value])) as _,
+    );
+
+    let now = util::current_time_millis();
+    let _ = columns_values.insert(
+        "gmt_created".to_string(),
+        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
+    );
+
+    let _ = columns_values.insert(
+        "gmt_modified".to_string(),
+        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
+    );
+
+    InsertRequest {
+        catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+        table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
+        columns_values,
+        region_number: 0, // system catalog table has only one region
+    }
+}
+
+pub fn decode_system_catalog(
+    entry_type: Option<u8>,
+    key: Option<&[u8]>,
+    value: Option<&[u8]>,
+) -> Result<Entry> {
+    debug!(
+        "Decode system catalog entry: {:?}, {:?}, {:?}",
+        entry_type, key, value
+    );
+    let entry_type = entry_type.context(InvalidKeySnafu { key: None })?;
+    let key = String::from_utf8_lossy(key.context(InvalidKeySnafu { key: None })?);
+
+    match EntryType::try_from(entry_type)? {
+        EntryType::Catalog => {
+            // As for catalog entry, the key is a string with format: `<catalog_name>`
+            // and the value is current not used.
+            let catalog_name = key.to_string();
+            Ok(Entry::Catalog(CatalogEntry { catalog_name }))
+        }
+        EntryType::Schema => {
+            // As for schema entry, the key is a string with format: `<catalog_name>.<schema_name>`
+            // and the value is current not used.
+            let schema_parts = key.split('.').collect::<Vec<_>>();
+            ensure!(
+                schema_parts.len() == 2,
+                InvalidKeySnafu {
+                    key: Some(key.to_string())
+                }
+            );
+            Ok(Entry::Schema(SchemaEntry {
+                catalog_name: schema_parts[0].to_string(),
+                schema_name: schema_parts[1].to_string(),
+            }))
+        }
+
+        EntryType::Table => {
+            // As for table entry, the key is a string with format: `<catalog_name>.<schema_name>.<table_id>`
+            // and the value is a JSON string with format: `{"table_name": <table_name>}`
+            let table_parts = key.split('.').collect::<Vec<_>>();
+            ensure!(
+                table_parts.len() >= 3,
+                InvalidKeySnafu {
+                    key: Some(key.to_string())
+                }
+            );
+            let value = value.context(EmptyValueSnafu)?;
+            debug!("Table meta value: {}", String::from_utf8_lossy(value));
+            let table_meta: TableEntryValue =
+                serde_json::from_slice(value).context(ValueDeserializeSnafu)?;
+            let table_id = table_parts[2].parse::<TableId>().unwrap();
+            Ok(Entry::Table(TableEntry {
+                catalog_name: table_parts[0].to_string(),
+                schema_name: table_parts[1].to_string(),
+                table_name: table_meta.table_name,
+                table_id,
+                engine: table_meta.engine,
+                is_deleted: table_meta.is_deleted,
+            }))
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum EntryType {
+    Catalog = 1,
+    Schema = 2,
+    Table = 3,
+}
+
+impl TryFrom<u8> for EntryType {
+    type Error = Error;
+
+    fn try_from(value: u8) -> std::result::Result<Self, Self::Error> {
+        match value {
+            b if b == Self::Catalog as u8 => Ok(Self::Catalog),
+            b if b == Self::Schema as u8 => Ok(Self::Schema),
+            b if b == Self::Table as u8 => Ok(Self::Table),
+            b => InvalidEntryTypeSnafu {
+                entry_type: Some(b),
+            }
+            .fail(),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
+pub enum Entry {
+    Catalog(CatalogEntry),
+    Schema(SchemaEntry),
+    Table(TableEntry),
+}
+
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
+pub struct CatalogEntry {
+    pub catalog_name: String,
+}
+
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
+pub struct SchemaEntry {
+    pub catalog_name: String,
+    pub schema_name: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SchemaEntryValue;
+
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
+pub struct TableEntry {
+    pub catalog_name: String,
+    pub schema_name: String,
+    pub table_name: String,
+    pub table_id: TableId,
+    pub engine: String,
+    pub is_deleted: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct TableEntryValue {
+    pub table_name: String,
+
+    #[serde(default = "mito_engine")]
+    pub engine: String,
+
+    #[serde(default = "not_deleted")]
+    pub is_deleted: bool,
+}
+
+fn mito_engine() -> String {
+    MITO_ENGINE.to_string()
+}
+
+fn not_deleted() -> bool {
+    false
+}
+
+#[cfg(test)]
+mod tests {
+    use common_recordbatch::RecordBatches;
+    use common_test_util::temp_dir::{create_temp_dir, TempDir};
+    use datatypes::value::Value;
+    use log_store::NoopLogStore;
+    use mito::config::EngineConfig;
+    use mito::engine::{MitoEngine, MITO_ENGINE};
+    use object_store::ObjectStore;
+    use storage::compaction::noop::NoopCompactionScheduler;
+    use storage::config::EngineConfig as StorageEngineConfig;
+    use storage::EngineImpl;
+    use table::metadata::TableType;
+    use table::metadata::TableType::Base;
+
+    use super::*;
+
+    #[test]
+    pub fn test_decode_catalog_entry() {
+        let entry = decode_system_catalog(
+            Some(EntryType::Catalog as u8),
+            Some("some_catalog".as_bytes()),
+            None,
+        )
+        .unwrap();
+        if let Entry::Catalog(e) = entry {
+            assert_eq!("some_catalog", e.catalog_name);
+        } else {
+            panic!("Unexpected type: {entry:?}");
+        }
+    }
+
+    #[test]
+    pub fn test_decode_schema_entry() {
+        let entry = decode_system_catalog(
+            Some(EntryType::Schema as u8),
+            Some("some_catalog.some_schema".as_bytes()),
+            None,
+        )
+        .unwrap();
+
+        if let Entry::Schema(e) = entry {
+            assert_eq!("some_catalog", e.catalog_name);
+            assert_eq!("some_schema", e.schema_name);
+        } else {
+            panic!("Unexpected type: {entry:?}");
+        }
+    }
+
+    #[test]
+    pub fn test_decode_table() {
+        let entry = decode_system_catalog(
+            Some(EntryType::Table as u8),
+            Some("some_catalog.some_schema.42".as_bytes()),
+            Some("{\"table_name\":\"some_table\"}".as_bytes()),
+        )
+        .unwrap();
+
+        if let Entry::Table(e) = entry {
+            assert_eq!("some_catalog", e.catalog_name);
+            assert_eq!("some_schema", e.schema_name);
+            assert_eq!("some_table", e.table_name);
+            assert_eq!(42, e.table_id);
+        } else {
+            panic!("Unexpected type: {entry:?}");
+        }
+    }
+
+    #[test]
+    pub fn test_decode_mismatch() {
+        assert!(decode_system_catalog(
+            Some(EntryType::Table as u8),
+            Some("some_catalog.some_schema.42".as_bytes()),
+            None,
+        )
+        .is_err());
+    }
+
+    #[test]
+    pub fn test_entry_type() {
+        assert_eq!(EntryType::Catalog, EntryType::try_from(1).unwrap());
+        assert_eq!(EntryType::Schema, EntryType::try_from(2).unwrap());
+        assert_eq!(EntryType::Table, EntryType::try_from(3).unwrap());
+        assert!(EntryType::try_from(4).is_err());
+    }
+
+    pub async fn prepare_table_engine() -> (TempDir, TableEngineRef) {
+        let dir = create_temp_dir("system-table-test");
+        let store_dir = dir.path().to_string_lossy();
+        let mut builder = object_store::services::Fs::default();
+        let _ = builder.root(&store_dir);
+        let object_store = ObjectStore::new(builder).unwrap().finish();
+        let noop_compaction_scheduler = Arc::new(NoopCompactionScheduler::default());
+        let table_engine = Arc::new(MitoEngine::new(
+            EngineConfig::default(),
+            EngineImpl::new(
+                StorageEngineConfig::default(),
+                Arc::new(NoopLogStore),
+                object_store.clone(),
+                noop_compaction_scheduler,
+            )
+            .unwrap(),
+            object_store,
+        ));
+        (dir, table_engine)
+    }
+
+    #[tokio::test]
+    async fn test_system_table_type() {
+        let (_dir, table_engine) = prepare_table_engine().await;
+        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
+        assert_eq!(Base, system_table.as_table_ref().table_type());
+    }
+
+    #[tokio::test]
+    async fn test_system_table_info() {
+        let (_dir, table_engine) = prepare_table_engine().await;
+        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
+        let info = system_table.as_table_ref().table_info();
+        assert_eq!(TableType::Base, info.table_type);
+        assert_eq!(SYSTEM_CATALOG_TABLE_NAME, info.name);
+        assert_eq!(SYSTEM_CATALOG_TABLE_ID, info.ident.table_id);
+        assert_eq!(SYSTEM_CATALOG_NAME, info.catalog_name);
+        assert_eq!(INFORMATION_SCHEMA_NAME, info.schema_name);
+    }
+
+    #[tokio::test]
+    async fn test_system_catalog_table_records() {
+        let (_, table_engine) = prepare_table_engine().await;
+        let catalog_table = SystemCatalogTable::new(table_engine).await.unwrap();
+
+        let result = catalog_table
+            .register_table(
+                DEFAULT_CATALOG_NAME.to_string(),
+                DEFAULT_SCHEMA_NAME.to_string(),
+                "my_table".to_string(),
+                1,
+                MITO_ENGINE.to_string(),
+            )
+            .await
+            .unwrap();
+        assert_eq!(result, 1);
+
+        let records = catalog_table.records().await.unwrap();
+        let mut batches = RecordBatches::try_collect(records).await.unwrap().take();
+        assert_eq!(batches.len(), 1);
+        let batch = batches.remove(0);
+        assert_eq!(batch.num_rows(), 1);
+
+        let row = batch.rows().next().unwrap();
+        let Value::UInt8(entry_type) = row[0] else {
+            unreachable!()
+        };
+        let Value::Binary(key) = row[1].clone() else {
+            unreachable!()
+        };
+        let Value::Binary(value) = row[3].clone() else {
+            unreachable!()
+        };
+        let entry = decode_system_catalog(Some(entry_type), Some(&*key), Some(&*value)).unwrap();
+        let expected = Entry::Table(TableEntry {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "my_table".to_string(),
+            table_id: 1,
+            engine: MITO_ENGINE.to_string(),
+            is_deleted: false,
+        });
+        assert_eq!(entry, expected);
+
+        catalog_table
+            .deregister_table(
+                &DeregisterTableRequest {
+                    catalog: DEFAULT_CATALOG_NAME.to_string(),
+                    schema: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "my_table".to_string(),
+                },
+                1,
+            )
+            .await
+            .unwrap();
+
+        let records = catalog_table.records().await.unwrap();
+        let batches = RecordBatches::try_collect(records).await.unwrap().take();
+        assert_eq!(batches.len(), 1);
+    }
+}

src/catalog/src/table_source.rs

@@ -123,7 +123,7 @@ mod tests {
     use session::context::QueryContext;
 
     use super::*;
-    use crate::memory::MemoryCatalogManager;
+    use crate::local::MemoryCatalogManager;
 
     #[test]
     fn test_validate_table_ref() {

src/catalog/src/tables.rs

@@ -0,0 +1,77 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The `tables` table in system catalog keeps a record of all tables created by user.
+
+use std::sync::Arc;
+
+use table::metadata::TableId;
+
+use crate::system::SystemCatalogTable;
+use crate::DeregisterTableRequest;
+
+pub struct InformationSchema {
+    pub system: Arc<SystemCatalogTable>,
+}
+
+pub struct SystemCatalog {
+    pub information_schema: Arc<InformationSchema>,
+}
+
+impl SystemCatalog {
+    pub(crate) fn new(system: SystemCatalogTable) -> Self {
+        let schema = InformationSchema {
+            system: Arc::new(system),
+        };
+        Self {
+            information_schema: Arc::new(schema),
+        }
+    }
+
+    pub async fn register_table(
+        &self,
+        catalog: String,
+        schema: String,
+        table_name: String,
+        table_id: TableId,
+        engine: String,
+    ) -> crate::error::Result<usize> {
+        self.information_schema
+            .system
+            .register_table(catalog, schema, table_name, table_id, engine)
+            .await
+    }
+
+    pub(crate) async fn deregister_table(
+        &self,
+        request: &DeregisterTableRequest,
+        table_id: TableId,
+    ) -> crate::error::Result<()> {
+        self.information_schema
+            .system
+            .deregister_table(request, table_id)
+            .await
+    }
+
+    pub async fn register_schema(
+        &self,
+        catalog: String,
+        schema: String,
+    ) -> crate::error::Result<usize> {
+        self.information_schema
+            .system
+            .register_schema(catalog, schema)
+            .await
+    }
+}

src/catalog/tests/local_catalog_tests.rs

@@ -0,0 +1,175 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use catalog::local::LocalCatalogManager;
+    use catalog::{CatalogManager, RegisterTableRequest, RenameTableRequest};
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use common_telemetry::{error, info};
+    use common_test_util::temp_dir::TempDir;
+    use mito::config::EngineConfig;
+    use table::engine::manager::MemoryTableEngineManager;
+    use table::table::numbers::NumbersTable;
+    use tokio::sync::Mutex;
+
+    async fn create_local_catalog_manager(
+    ) -> Result<(TempDir, LocalCatalogManager), catalog::error::Error> {
+        let (dir, object_store) =
+            mito::table::test_util::new_test_object_store("setup_mock_engine_and_table").await;
+        let mock_engine = Arc::new(mito::table::test_util::MockMitoEngine::new(
+            EngineConfig::default(),
+            mito::table::test_util::MockEngine::default(),
+            object_store,
+        ));
+        let engine_manager = Arc::new(MemoryTableEngineManager::new(mock_engine.clone()));
+        let catalog_manager = LocalCatalogManager::try_new(engine_manager).await.unwrap();
+        catalog_manager.start().await?;
+        Ok((dir, catalog_manager))
+    }
+
+    #[tokio::test]
+    async fn test_rename_table() {
+        common_telemetry::init_default_ut_logging();
+        let (_dir, catalog_manager) = create_local_catalog_manager().await.unwrap();
+        // register table
+        let table_name = "test_table";
+        let table_id = 42;
+        let request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id,
+            table: NumbersTable::table(table_id),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        // rename table
+        let new_table_name = "table_t";
+        let rename_table_req = RenameTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            new_table_name: new_table_name.to_string(),
+            table_id,
+        };
+        assert!(catalog_manager
+            .rename_table(rename_table_req)
+            .await
+            .unwrap());
+
+        let registered_table = catalog_manager
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(registered_table.table_info().ident.table_id, table_id);
+    }
+
+    #[tokio::test]
+    async fn test_duplicate_register() {
+        let (_dir, catalog_manager) = create_local_catalog_manager().await.unwrap();
+        let request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "test_table".to_string(),
+            table_id: 42,
+            table: NumbersTable::table(42),
+        };
+        assert!(catalog_manager
+            .register_table(request.clone())
+            .await
+            .unwrap());
+
+        // register table with same table id will succeed with 0 as return val.
+        assert!(!catalog_manager.register_table(request).await.unwrap());
+
+        let err = catalog_manager
+            .register_table(RegisterTableRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_SCHEMA_NAME.to_string(),
+                table_name: "test_table".to_string(),
+                table_id: 43,
+                table: NumbersTable::table(43),
+            })
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("Table `greptime.public.test_table` already exists"),
+            "Actual error message: {err}",
+        );
+    }
+
+    #[test]
+    fn test_concurrent_register() {
+        common_telemetry::init_default_ut_logging();
+        let rt = Arc::new(tokio::runtime::Builder::new_multi_thread().build().unwrap());
+        let (_dir, catalog_manager) =
+            rt.block_on(async { create_local_catalog_manager().await.unwrap() });
+        let catalog_manager = Arc::new(catalog_manager);
+
+        let succeed = Arc::new(Mutex::new(None));
+
+        let mut handles = Vec::with_capacity(8);
+        for i in 0..8 {
+            let catalog = catalog_manager.clone();
+            let succeed = succeed.clone();
+            let handle = rt.spawn(async move {
+                let table_id = 42 + i;
+                let table = NumbersTable::table(table_id);
+                let table_info = table.table_info();
+                let req = RegisterTableRequest {
+                    catalog: DEFAULT_CATALOG_NAME.to_string(),
+                    schema: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "test_table".to_string(),
+                    table_id,
+                    table,
+                };
+                match catalog.register_table(req).await {
+                    Ok(res) => {
+                        if res {
+                            let mut succeed = succeed.lock().await;
+                            info!("Successfully registered table: {}", table_id);
+                            *succeed = Some(table_info);
+                        }
+                    }
+                    Err(_) => {
+                        error!("Failed to register table {}", table_id);
+                    }
+                }
+            });
+            handles.push(handle);
+        }
+
+        rt.block_on(async move {
+            for handle in handles {
+                handle.await.unwrap();
+            }
+            let guard = succeed.lock().await;
+            let table_info = guard.as_ref().unwrap();
+            let table_registered = catalog_manager
+                .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "test_table")
+                .await
+                .unwrap()
+                .unwrap();
+            assert_eq!(
+                table_registered.table_info().ident.table_id,
+                table_info.ident.table_id
+            );
+        });
+    }
+}

src/catalog/tests/remote_catalog_tests.rs

@@ -0,0 +1,453 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(assert_matches)]
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+    use std::collections::HashSet;
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    use catalog::remote::mock::MockTableEngine;
+    use catalog::remote::region_alive_keeper::RegionAliveKeepers;
+    use catalog::remote::{CachedMetaKvBackend, RemoteCatalogManager};
+    use catalog::{CatalogManager, RegisterSchemaRequest, RegisterTableRequest};
+    use common_catalog::consts::{
+        DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MITO_ENGINE,
+    };
+    use common_meta::helper::CatalogValue;
+    use common_meta::ident::TableIdent;
+    use common_meta::key::catalog_name::CatalogNameKey;
+    use common_meta::key::TableMetadataManager;
+    use common_meta::kv_backend::memory::MemoryKvBackend;
+    use common_meta::kv_backend::KvBackend;
+    use common_meta::rpc::store::{CompareAndPutRequest, PutRequest};
+    use datatypes::schema::RawSchema;
+    use table::engine::manager::{MemoryTableEngineManager, TableEngineManagerRef};
+    use table::engine::{EngineContext, TableEngineRef};
+    use table::requests::CreateTableRequest;
+    use table::test_util::EmptyTable;
+    use tokio::time::Instant;
+
+    struct TestingComponents {
+        catalog_manager: Arc<RemoteCatalogManager>,
+        table_engine_manager: TableEngineManagerRef,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
+    }
+
+    impl TestingComponents {
+        fn table_engine(&self) -> TableEngineRef {
+            self.table_engine_manager.engine(MITO_ENGINE).unwrap()
+        }
+    }
+
+    #[tokio::test]
+    async fn test_cached_backend() {
+        let backend = CachedMetaKvBackend::wrap(Arc::new(MemoryKvBackend::default()));
+
+        let default_catalog_key = CatalogNameKey::new(DEFAULT_CATALOG_NAME).to_string();
+
+        let req = PutRequest::new()
+            .with_key(default_catalog_key.as_bytes())
+            .with_value(CatalogValue.as_bytes().unwrap());
+        backend.put(req).await.unwrap();
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        let _ = ret.unwrap();
+
+        let req = CompareAndPutRequest::new()
+            .with_key(b"__catalog_name/greptime".to_vec())
+            .with_expect(CatalogValue.as_bytes().unwrap())
+            .with_value(b"123".to_vec());
+        let _ = backend.compare_and_put(req).await.unwrap();
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        assert_eq!(b"123", ret.as_ref().unwrap().value.as_slice());
+
+        let req = PutRequest::new()
+            .with_key(b"__catalog_name/greptime".to_vec())
+            .with_value(b"1234".to_vec());
+        let _ = backend.put(req).await;
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        assert_eq!(b"1234", ret.unwrap().value.as_slice());
+
+        backend
+            .delete(b"__catalog_name/greptime", false)
+            .await
+            .unwrap();
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        assert!(ret.is_none());
+    }
+
+    async fn prepare_components(node_id: u64) -> TestingComponents {
+        let backend = Arc::new(MemoryKvBackend::default());
+
+        let req = PutRequest::new()
+            .with_key(b"__catalog_name/greptime".to_vec())
+            .with_value(b"".to_vec());
+        backend.put(req).await.unwrap();
+
+        let req = PutRequest::new()
+            .with_key(b"__schema_name/greptime-public".to_vec())
+            .with_value(b"".to_vec());
+        backend.put(req).await.unwrap();
+
+        let cached_backend = Arc::new(CachedMetaKvBackend::wrap(backend));
+
+        let table_engine = Arc::new(MockTableEngine::default());
+        let engine_manager = Arc::new(MemoryTableEngineManager::alias(
+            MITO_ENGINE.to_string(),
+            table_engine,
+        ));
+
+        let region_alive_keepers = Arc::new(RegionAliveKeepers::new(engine_manager.clone(), 5000));
+
+        let catalog_manager = RemoteCatalogManager::new(
+            engine_manager.clone(),
+            node_id,
+            region_alive_keepers.clone(),
+            Arc::new(TableMetadataManager::new(cached_backend)),
+        );
+        catalog_manager.start().await.unwrap();
+
+        TestingComponents {
+            catalog_manager: Arc::new(catalog_manager),
+            table_engine_manager: engine_manager,
+            region_alive_keepers,
+        }
+    }
+
+    #[tokio::test]
+    async fn test_remote_catalog_default() {
+        common_telemetry::init_default_ut_logging();
+        let node_id = 42;
+        let TestingComponents {
+            catalog_manager, ..
+        } = prepare_components(node_id).await;
+        assert_eq!(
+            vec![DEFAULT_CATALOG_NAME.to_string()],
+            catalog_manager.catalog_names().await.unwrap()
+        );
+
+        let mut schema_names = catalog_manager
+            .schema_names(DEFAULT_CATALOG_NAME)
+            .await
+            .unwrap();
+        schema_names.sort_unstable();
+        assert_eq!(
+            vec![
+                INFORMATION_SCHEMA_NAME.to_string(),
+                DEFAULT_SCHEMA_NAME.to_string()
+            ],
+            schema_names
+        );
+    }
+
+    #[tokio::test]
+    async fn test_remote_catalog_register_nonexistent() {
+        common_telemetry::init_default_ut_logging();
+        let node_id = 42;
+        let components = prepare_components(node_id).await;
+
+        // register a new table with an nonexistent catalog
+        let catalog_name = "nonexistent_catalog".to_string();
+        let schema_name = "nonexistent_schema".to_string();
+        let table_name = "fail_table".to_string();
+        // this schema has no effect
+        let table_schema = RawSchema::new(vec![]);
+        let table = components
+            .table_engine()
+            .create_table(
+                &EngineContext {},
+                CreateTableRequest {
+                    id: 1,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    desc: None,
+                    schema: table_schema,
+                    region_numbers: vec![0],
+                    primary_key_indices: vec![],
+                    create_if_not_exists: false,
+                    table_options: Default::default(),
+                    engine: MITO_ENGINE.to_string(),
+                },
+            )
+            .await
+            .unwrap();
+        let reg_req = RegisterTableRequest {
+            catalog: catalog_name,
+            schema: schema_name,
+            table_name,
+            table_id: 1,
+            table,
+        };
+        let res = components.catalog_manager.register_table(reg_req).await;
+
+        // because nonexistent_catalog does not exist yet.
+        assert_matches!(
+            res.err().unwrap(),
+            catalog::error::Error::CatalogNotFound { .. }
+        );
+    }
+
+    #[tokio::test]
+    async fn test_register_table() {
+        let node_id = 42;
+        let components = prepare_components(node_id).await;
+        let mut schema_names = components
+            .catalog_manager
+            .schema_names(DEFAULT_CATALOG_NAME)
+            .await
+            .unwrap();
+        schema_names.sort_unstable();
+        assert_eq!(
+            vec![
+                INFORMATION_SCHEMA_NAME.to_string(),
+                DEFAULT_SCHEMA_NAME.to_string(),
+            ],
+            schema_names
+        );
+
+        // register a new table with an nonexistent catalog
+        let catalog_name = DEFAULT_CATALOG_NAME.to_string();
+        let schema_name = DEFAULT_SCHEMA_NAME.to_string();
+        let table_name = "test_table".to_string();
+        let table_id = 1;
+        // this schema has no effect
+        let table_schema = RawSchema::new(vec![]);
+        let table = components
+            .table_engine()
+            .create_table(
+                &EngineContext {},
+                CreateTableRequest {
+                    id: table_id,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    desc: None,
+                    schema: table_schema,
+                    region_numbers: vec![0],
+                    primary_key_indices: vec![],
+                    create_if_not_exists: false,
+                    table_options: Default::default(),
+                    engine: MITO_ENGINE.to_string(),
+                },
+            )
+            .await
+            .unwrap();
+        let reg_req = RegisterTableRequest {
+            catalog: catalog_name,
+            schema: schema_name,
+            table_name: table_name.clone(),
+            table_id,
+            table,
+        };
+        assert!(components
+            .catalog_manager
+            .register_table(reg_req)
+            .await
+            .unwrap());
+        assert_eq!(
+            vec![table_name],
+            components
+                .catalog_manager
+                .table_names(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
+                .await
+                .unwrap()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_register_catalog_schema_table() {
+        let node_id = 42;
+        let components = prepare_components(node_id).await;
+
+        let catalog_name = "test_catalog".to_string();
+        let schema_name = "nonexistent_schema".to_string();
+
+        // register catalog to catalog manager
+        assert!(components
+            .catalog_manager
+            .clone()
+            .register_catalog(catalog_name.clone())
+            .await
+            .is_ok());
+        assert_eq!(
+            HashSet::<String>::from_iter(vec![
+                DEFAULT_CATALOG_NAME.to_string(),
+                catalog_name.clone()
+            ]),
+            HashSet::from_iter(components.catalog_manager.catalog_names().await.unwrap())
+        );
+
+        let table_to_register = components
+            .table_engine()
+            .create_table(
+                &EngineContext {},
+                CreateTableRequest {
+                    id: 2,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: "".to_string(),
+                    desc: None,
+                    schema: RawSchema::new(vec![]),
+                    region_numbers: vec![0],
+                    primary_key_indices: vec![],
+                    create_if_not_exists: false,
+                    table_options: Default::default(),
+                    engine: MITO_ENGINE.to_string(),
+                },
+            )
+            .await
+            .unwrap();
+
+        let reg_req = RegisterTableRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+            table_name: " fail_table".to_string(),
+            table_id: 2,
+            table: table_to_register,
+        };
+        // this register will fail since schema does not exist yet
+        assert_matches!(
+            components
+                .catalog_manager
+                .register_table(reg_req.clone())
+                .await
+                .unwrap_err(),
+            catalog::error::Error::SchemaNotFound { .. }
+        );
+
+        let register_schema_request = RegisterSchemaRequest {
+            catalog: catalog_name.to_string(),
+            schema: schema_name.to_string(),
+        };
+        assert!(components
+            .catalog_manager
+            .register_schema(register_schema_request)
+            .await
+            .expect("Register schema should not fail"));
+        assert!(components
+            .catalog_manager
+            .register_table(reg_req)
+            .await
+            .unwrap());
+
+        assert_eq!(
+            HashSet::from([schema_name.clone(), INFORMATION_SCHEMA_NAME.to_string()]),
+            components
+                .catalog_manager
+                .schema_names(&catalog_name)
+                .await
+                .unwrap()
+                .into_iter()
+                .collect()
+        )
+    }
+
+    #[tokio::test]
+    async fn test_register_table_before_and_after_region_alive_keeper_started() {
+        let components = prepare_components(42).await;
+        let catalog_manager = &components.catalog_manager;
+        let region_alive_keepers = &components.region_alive_keepers;
+
+        let table_before = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "table_before".to_string(),
+            table_id: 1,
+            engine: MITO_ENGINE.to_string(),
+        };
+        let request = RegisterTableRequest {
+            catalog: table_before.catalog.clone(),
+            schema: table_before.schema.clone(),
+            table_name: table_before.table.clone(),
+            table_id: table_before.table_id,
+            table: Arc::new(EmptyTable::new(CreateTableRequest {
+                id: table_before.table_id,
+                catalog_name: table_before.catalog.clone(),
+                schema_name: table_before.schema.clone(),
+                table_name: table_before.table.clone(),
+                desc: None,
+                schema: RawSchema::new(vec![]),
+                region_numbers: vec![0],
+                primary_key_indices: vec![],
+                create_if_not_exists: false,
+                table_options: Default::default(),
+                engine: MITO_ENGINE.to_string(),
+            })),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        let keeper = region_alive_keepers
+            .find_keeper(table_before.table_id)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
+        // assert region alive countdown is not started
+        assert!(deadline > far_future);
+
+        region_alive_keepers.start().await;
+
+        let table_after = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "table_after".to_string(),
+            table_id: 2,
+            engine: MITO_ENGINE.to_string(),
+        };
+        let request = RegisterTableRequest {
+            catalog: table_after.catalog.clone(),
+            schema: table_after.schema.clone(),
+            table_name: table_after.table.clone(),
+            table_id: table_after.table_id,
+            table: Arc::new(EmptyTable::new(CreateTableRequest {
+                id: table_after.table_id,
+                catalog_name: table_after.catalog.clone(),
+                schema_name: table_after.schema.clone(),
+                table_name: table_after.table.clone(),
+                desc: None,
+                schema: RawSchema::new(vec![]),
+                region_numbers: vec![0],
+                primary_key_indices: vec![],
+                create_if_not_exists: false,
+                table_options: Default::default(),
+                engine: MITO_ENGINE.to_string(),
+            })),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        let keeper = region_alive_keepers
+            .find_keeper(table_after.table_id)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        // assert countdown is started for the table registered after [RegionAliveKeepers] started
+        assert!(deadline <= Instant::now() + Duration::from_secs(20));
+
+        let keeper = region_alive_keepers
+            .find_keeper(table_before.table_id)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        // assert countdown is started for the table registered before [RegionAliveKeepers] started, too
+        assert!(deadline <= Instant::now() + Duration::from_secs(20));
+    }
+}

src/client/Cargo.toml

@@ -11,12 +11,10 @@ testing = []
 api = { workspace = true }
 arrow-flight.workspace = true
 async-stream.workspace = true
-async-trait.workspace = true
 common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
-common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
@@ -27,11 +25,10 @@ datatypes = { workspace = true }
 derive_builder.workspace = true
 enum_dispatch = "0.3"
 futures-util.workspace = true
-moka = { workspace = true, features = ["future"] }
+moka = { version = "0.9", features = ["future"] }
 parking_lot = "0.12"
 prost.workspace = true
 rand.workspace = true
-session = { workspace = true }
 snafu.workspace = true
 tokio-stream = { version = "0.1", features = ["net"] }
 tokio.workspace = true

src/client/examples/logical.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, SemanticType, TableId};
+use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, TableId};
 use client::{Client, Database};
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
 use prost::Message;
@@ -41,27 +41,21 @@ async fn run() {
         column_defs: vec![
             ColumnDef {
                 name: "timestamp".to_string(),
-                data_type: ColumnDataType::TimestampMillisecond as i32,
+                datatype: ColumnDataType::TimestampMillisecond as i32,
                 is_nullable: false,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Timestamp as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "key".to_string(),
-                data_type: ColumnDataType::Uint64 as i32,
+                datatype: ColumnDataType::Uint64 as i32,
                 is_nullable: false,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Tag as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "value".to_string(),
-                data_type: ColumnDataType::Uint64 as i32,
+                datatype: ColumnDataType::Uint64 as i32,
                 is_nullable: false,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
         ],
         time_index: "timestamp".to_string(),
@@ -69,6 +63,7 @@ async fn run() {
         create_if_not_exists: false,
         table_options: Default::default(),
         table_id: Some(TableId { id: 1024 }),
+        region_numbers: vec![0],
         engine: MITO_ENGINE.to_string(),
     };
 
@@ -78,7 +73,7 @@ async fn run() {
 
     let logical = mock_logical_plan();
     event!(Level::INFO, "plan size: {:#?}", logical.len());
-    let result = db.logical_plan(logical, 0).await.unwrap();
+    let result = db.logical_plan(logical, None).await.unwrap();
 
     event!(Level::INFO, "result: {:#?}", result);
 }

src/client/examples/stream_ingest.rs

@@ -42,14 +42,14 @@ async fn run() {
         .insert(vec![to_insert_request(weather_records_1())])
         .await
     {
-        error!("Error: {e:?}");
+        error!("Error: {e}");
     }
 
     if let Err(e) = stream_inserter
         .insert(vec![to_insert_request(weather_records_2())])
         .await
     {
-        error!("Error: {e:?}");
+        error!("Error: {e}");
     }
 
     let result = stream_inserter.finish().await;
@@ -59,7 +59,7 @@ async fn run() {
             info!("Rows written: {rows}");
         }
         Err(e) => {
-            error!("Error: {e:?}");
+            error!("Error: {e}");
         }
     };
 }
@@ -131,7 +131,7 @@ fn to_insert_request(records: Vec<WeatherRecord>) -> InsertRequest {
         Column {
             column_name: "ts".to_owned(),
             values: Some(column::Values {
-                timestamp_millisecond_values: timestamp_millis,
+                ts_millisecond_values: timestamp_millis,
                 ..Default::default()
             }),
             semantic_type: SemanticType::Timestamp as i32,
@@ -177,5 +177,6 @@ fn to_insert_request(records: Vec<WeatherRecord>) -> InsertRequest {
         table_name: "weather_demo".to_owned(),
         columns,
         row_count: rows as u32,
+        ..Default::default()
     }
 }

src/client/src/client.rs

@@ -138,38 +138,24 @@ impl Client {
         Ok((addr, channel))
     }
 
-    fn max_grpc_recv_message_size(&self) -> usize {
-        self.inner.channel_manager.config().max_recv_message_size
-    }
-
-    fn max_grpc_send_message_size(&self) -> usize {
-        self.inner.channel_manager.config().max_send_message_size
-    }
-
     pub(crate) fn make_flight_client(&self) -> Result<FlightClient> {
         let (addr, channel) = self.find_channel()?;
         Ok(FlightClient {
             addr,
-            client: FlightServiceClient::new(channel)
+            client: FlightServiceClient::new(channel),
-                .max_decoding_message_size(self.max_grpc_recv_message_size())
-                .max_encoding_message_size(self.max_grpc_send_message_size()),
         })
     }
 
     pub(crate) fn make_database_client(&self) -> Result<DatabaseClient> {
         let (_, channel) = self.find_channel()?;
         Ok(DatabaseClient {
-            inner: GreptimeDatabaseClient::new(channel)
+            inner: GreptimeDatabaseClient::new(channel),
-                .max_decoding_message_size(self.max_grpc_recv_message_size())
-                .max_encoding_message_size(self.max_grpc_send_message_size()),
         })
     }
 
     pub(crate) fn raw_region_client(&self) -> Result<PbRegionClient<Channel>> {
         let (_, channel) = self.find_channel()?;
-        Ok(PbRegionClient::new(channel)
+        Ok(PbRegionClient::new(channel))
-            .max_decoding_message_size(self.max_grpc_recv_message_size())
-            .max_encoding_message_size(self.max_grpc_send_message_size()))
     }
 
     pub fn make_prometheus_gateway_client(&self) -> Result<PrometheusGatewayClient<Channel>> {

src/client/src/client_manager.rs

@@ -13,15 +13,12 @@
 // limitations under the License.
 
 use std::fmt::{Debug, Formatter};
-use std::sync::Arc;
 use std::time::Duration;
 
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
-use common_meta::datanode_manager::{Datanode, DatanodeManager};
 use common_meta::peer::Peer;
 use moka::future::{Cache, CacheBuilder};
 
-use crate::region::RegionRequester;
 use crate::Client;
 
 pub struct DatanodeClients {
@@ -43,15 +40,6 @@ impl Debug for DatanodeClients {
     }
 }
 
-#[async_trait::async_trait]
-impl DatanodeManager for DatanodeClients {
-    async fn datanode(&self, datanode: &Peer) -> Arc<dyn Datanode> {
-        let client = self.get_client(datanode).await;
-
-        Arc::new(RegionRequester::new(client))
-    }
-}
-
 impl DatanodeClients {
     pub fn new(config: ChannelConfig) -> Self {
         Self {

src/client/src/database.rs

@@ -17,9 +17,9 @@ use api::v1::ddl_request::Expr as DdlExpr;
 use api::v1::greptime_request::Request;
 use api::v1::query_request::Query;
 use api::v1::{
-    AlterExpr, AuthHeader, CreateTableExpr, DdlRequest, DeleteRequests, DropTableExpr,
+    AlterExpr, AuthHeader, CompactTableExpr, CreateTableExpr, DdlRequest, DeleteRequests,
-    GreptimeRequest, InsertRequests, PromRangeQuery, QueryRequest, RequestHeader,
+    DropTableExpr, FlushTableExpr, GreptimeRequest, InsertRequests, PromRangeQuery, QueryRequest,
-    RowInsertRequests, TruncateTableExpr,
+    RequestHeader, RowInsertRequests, TruncateTableExpr,
 };
 use arrow_flight::Ticket;
 use async_stream::stream;
@@ -147,13 +147,13 @@ impl Database {
 
     async fn handle(&self, request: Request) -> Result<u32> {
         let mut client = self.client.make_database_client()?.inner;
-        let request = self.to_rpc_request(request, 0);
+        let request = self.to_rpc_request(request, None);
         let response = client.handle(request).await?.into_inner();
         from_grpc_response(response)
     }
 
     #[inline]
-    fn to_rpc_request(&self, request: Request, trace_id: u64) -> GreptimeRequest {
+    fn to_rpc_request(&self, request: Request, trace_id: Option<u64>) -> GreptimeRequest {
         GreptimeRequest {
             header: Some(RequestHeader {
                 catalog: self.catalog.clone(),
@@ -161,7 +161,7 @@ impl Database {
                 authorization: self.ctx.auth_header.clone(),
                 dbname: self.dbname.clone(),
                 trace_id,
-                span_id: 0,
+                span_id: None,
             }),
             request: Some(request),
         }
@@ -173,12 +173,16 @@ impl Database {
             Request::Query(QueryRequest {
                 query: Some(Query::Sql(sql.to_string())),
             }),
-            0,
+            None,
         )
         .await
     }
 
-    pub async fn logical_plan(&self, logical_plan: Vec<u8>, trace_id: u64) -> Result<Output> {
+    pub async fn logical_plan(
+        &self,
+        logical_plan: Vec<u8>,
+        trace_id: Option<u64>,
+    ) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_LOGICAL_PLAN);
         self.do_get(
             Request::Query(QueryRequest {
@@ -206,7 +210,7 @@ impl Database {
                     step: step.to_string(),
                 })),
             }),
-            0,
+            None,
         )
         .await
     }
@@ -217,7 +221,7 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::CreateTable(expr)),
             }),
-            0,
+            None,
         )
         .await
     }
@@ -228,7 +232,7 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::Alter(expr)),
             }),
-            0,
+            None,
         )
         .await
     }
@@ -239,7 +243,29 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::DropTable(expr)),
             }),
-            0,
+            None,
+        )
+        .await
+    }
+
+    pub async fn flush_table(&self, expr: FlushTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_FLUSH_TABLE);
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::FlushTable(expr)),
+            }),
+            None,
+        )
+        .await
+    }
+
+    pub async fn compact_table(&self, expr: CompactTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_COMPACT_TABLE);
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::CompactTable(expr)),
+            }),
+            None,
         )
         .await
     }
@@ -250,12 +276,12 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::TruncateTable(expr)),
             }),
-            0,
+            None,
         )
         .await
     }
 
-    async fn do_get(&self, request: Request, trace_id: u64) -> Result<Output> {
+    async fn do_get(&self, request: Request, trace_id: Option<u64>) -> Result<Output> {
         // FIXME(paomian): should be added some labels for metrics
         let _timer = timer!(metrics::METRIC_GRPC_DO_GET);
         let request = self.to_rpc_request(request, trace_id);
@@ -276,7 +302,7 @@ impl Database {
                 source: BoxedError::new(ServerSnafu { code, msg }.build()),
             };
             logging::error!(
-                "Failed to do Flight get, addr: {}, code: {}, source: {:?}",
+                "Failed to do Flight get, addr: {}, code: {}, source: {}",
                 client.addr(),
                 tonic_code,
                 error

src/client/src/error.rs

@@ -17,37 +17,29 @@ use std::any::Any;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_error::{GREPTIME_ERROR_CODE, GREPTIME_ERROR_MSG};
-use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 use tonic::{Code, Status};
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Illegal Flight messages, reason: {}", reason))]
     IllegalFlightMessages { reason: String, location: Location },
 
-    #[snafu(display("Failed to do Flight get, code: {}", tonic_code))]
+    #[snafu(display("Failed to do Flight get, code: {}, source: {}", tonic_code, source))]
     FlightGet {
         addr: String,
         tonic_code: Code,
         source: BoxedError,
     },
 
-    #[snafu(display("Failure occurs during handling request"))]
+    #[snafu(display("Failed to convert FlightData, source: {}", source))]
-    HandleRequest {
-        location: Location,
-        source: BoxedError,
-    },
-
-    #[snafu(display("Failed to convert FlightData"))]
     ConvertFlightData {
         location: Location,
         source: common_grpc::Error,
     },
 
-    #[snafu(display("Column datatype error"))]
+    #[snafu(display("Column datatype error, source: {}", source))]
     ColumnDataType {
         location: Location,
         source: api::error::Error,
@@ -59,16 +51,17 @@ pub enum Error {
     #[snafu(display("Missing required field in protobuf, field: {}", field))]
     MissingField { field: String, location: Location },
 
-    #[snafu(display("Failed to create gRPC channel, peer address: {}", addr))]
+    #[snafu(display(
+        "Failed to create gRPC channel, peer address: {}, source: {}",
+        addr,
+        source
+    ))]
     CreateChannel {
         addr: String,
         location: Location,
         source: common_grpc::error::Error,
     },
 
-    #[snafu(display("Failed to request RegionServer, code: {}", code))]
-    RegionServer { code: Code, source: BoxedError },
-
     // Server error carried in Tonic Status's metadata.
     #[snafu(display("{}", msg))]
     Server { code: StatusCode, msg: String },
@@ -92,9 +85,7 @@ impl ErrorExt for Error {
             | Error::ClientStreaming { .. } => StatusCode::Internal,
 
             Error::Server { code, .. } => *code,
-            Error::FlightGet { source, .. }
+            Error::FlightGet { source, .. } => source.status_code(),
-            | Error::HandleRequest { source, .. }
-            | Error::RegionServer { source, .. } => source.status_code(),
             Error::CreateChannel { source, .. } | Error::ConvertFlightData { source, .. } => {
                 source.status_code()
             }

src/client/src/metrics.rs

@@ -21,6 +21,8 @@ pub const METRIC_GRPC_SQL: &str = "grpc.sql";
 pub const METRIC_GRPC_LOGICAL_PLAN: &str = "grpc.logical_plan";
 pub const METRIC_GRPC_ALTER: &str = "grpc.alter";
 pub const METRIC_GRPC_DROP_TABLE: &str = "grpc.drop_table";
+pub const METRIC_GRPC_FLUSH_TABLE: &str = "grpc.flush_table";
+pub const METRIC_GRPC_COMPACT_TABLE: &str = "grpc.compact_table";
 pub const METRIC_GRPC_TRUNCATE_TABLE: &str = "grpc.truncate_table";
 pub const METRIC_GRPC_DO_GET: &str = "grpc.do_get";
 pub(crate) const METRIC_REGION_REQUEST_GRPC: &str = "grpc.region_request";

src/client/src/region.rs

@@ -12,146 +12,44 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::region::{QueryRequest, RegionRequest, RegionResponse};
+use api::v1::region::{region_request, RegionRequest, RegionRequestHeader, RegionResponse};
 use api::v1::ResponseHeader;
-use arrow_flight::Ticket;
-use async_stream::stream;
-use async_trait::async_trait;
-use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use common_grpc::flight::{FlightDecoder, FlightMessage};
+use common_telemetry::timer;
-use common_meta::datanode_manager::{AffectedRows, Datanode};
+use snafu::OptionExt;
-use common_meta::error::{self as meta_error, Result as MetaResult};
-use common_recordbatch::error::ExternalSnafu;
-use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
-use common_telemetry::{error, timer};
-use prost::Message;
-use snafu::{location, Location, OptionExt, ResultExt};
-use tokio_stream::StreamExt;
 
-use crate::error::Error::RegionServer;
+use crate::error::{IllegalDatabaseResponseSnafu, Result, ServerSnafu};
-use crate::error::{
+use crate::{metrics, Client};
-    self, ConvertFlightDataSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
+
-    MissingFieldSnafu, Result, ServerSnafu,
+type AffectedRows = u64;
-};
-use crate::{metrics, Client, Error};
 
 #[derive(Debug)]
 pub struct RegionRequester {
+    trace_id: Option<u64>,
+    span_id: Option<u64>,
     client: Client,
 }
 
-#[async_trait]
-impl Datanode for RegionRequester {
-    async fn handle(&self, request: RegionRequest) -> MetaResult<AffectedRows> {
-        self.handle_inner(request).await.map_err(|err| {
-            if matches!(err, RegionServer { .. }) {
-                meta_error::Error::RetryLater {
-                    source: BoxedError::new(err),
-                }
-            } else {
-                meta_error::Error::External {
-                    source: BoxedError::new(err),
-                    location: location!(),
-                }
-            }
-        })
-    }
-
-    async fn handle_query(&self, request: QueryRequest) -> MetaResult<SendableRecordBatchStream> {
-        let ticket = Ticket {
-            ticket: request.encode_to_vec().into(),
-        };
-        self.do_get_inner(ticket)
-            .await
-            .map_err(BoxedError::new)
-            .context(meta_error::ExternalSnafu)
-    }
-}
-
 impl RegionRequester {
     pub fn new(client: Client) -> Self {
-        Self { client }
+        // TODO(LFC): Pass in trace_id and span_id from some context when we have it.
+        Self {
+            trace_id: None,
+            span_id: None,
+            client,
+        }
     }
 
-    pub async fn do_get_inner(&self, ticket: Ticket) -> Result<SendableRecordBatchStream> {
+    pub async fn handle(self, request: region_request::Body) -> Result<AffectedRows> {
-        let mut flight_client = self.client.make_flight_client()?;
+        let request_type = request.as_ref().to_string();
-        let response = flight_client
-            .mut_inner()
-            .do_get(ticket)
-            .await
-            .map_err(|e| {
-                let tonic_code = e.code();
-                let e: error::Error = e.into();
-                let code = e.status_code();
-                let msg = e.to_string();
-                let error = Error::FlightGet {
-                    tonic_code,
-                    addr: flight_client.addr().to_string(),
-                    source: BoxedError::new(ServerSnafu { code, msg }.build()),
-                };
-                error!(
-                    e; "Failed to do Flight get, addr: {}, code: {}",
-                    flight_client.addr(),
-                    tonic_code
-                );
-                error
-            })?;
 
-        let flight_data_stream = response.into_inner();
+        let request = RegionRequest {
-        let mut decoder = FlightDecoder::default();
+            header: Some(RegionRequestHeader {
-
+                trace_id: self.trace_id,
-        let mut flight_message_stream = flight_data_stream.map(move |flight_data| {
+                span_id: self.span_id,
-            flight_data
+            }),
-                .map_err(Error::from)
+            body: Some(request),
-                .and_then(|data| decoder.try_decode(data).context(ConvertFlightDataSnafu))
-        });
-
-        let Some(first_flight_message) = flight_message_stream.next().await else {
-            return IllegalFlightMessagesSnafu {
-                reason: "Expect the response not to be empty",
-            }
-            .fail();
         };
-        let FlightMessage::Schema(schema) = first_flight_message? else {
-            return IllegalFlightMessagesSnafu {
-                reason: "Expect schema to be the first flight message",
-            }
-            .fail();
-        };
-
-        let stream = Box::pin(stream!({
-            while let Some(flight_message) = flight_message_stream.next().await {
-                let flight_message = flight_message
-                    .map_err(BoxedError::new)
-                    .context(ExternalSnafu)?;
-                let FlightMessage::Recordbatch(record_batch) = flight_message else {
-                    yield IllegalFlightMessagesSnafu {
-                            reason: "A Schema message must be succeeded exclusively by a set of RecordBatch messages"
-                        }
-                        .fail()
-                        .map_err(BoxedError::new)
-                        .context(ExternalSnafu);
-                    break;
-                };
-                yield Ok(record_batch);
-            }
-        }));
-        let record_batch_stream = RecordBatchStreamAdaptor {
-            schema,
-            stream,
-            output_ordering: None,
-        };
-        Ok(Box::pin(record_batch_stream))
-    }
-
-    async fn handle_inner(&self, request: RegionRequest) -> Result<AffectedRows> {
-        let request_type = request
-            .body
-            .as_ref()
-            .with_context(|| MissingFieldSnafu { field: "body" })?
-            .as_ref()
-            .to_string();
 
         let _timer = timer!(
             metrics::METRIC_REGION_REQUEST_GRPC,
@@ -163,31 +61,15 @@ impl RegionRequester {
         let RegionResponse {
             header,
             affected_rows,
-        } = client
+        } = client.handle(request).await?.into_inner();
-            .handle(request)
-            .await
-            .map_err(|e| {
-                let code = e.code();
-                let err: error::Error = e.into();
-                // Uses `Error::RegionServer` instead of `Error::Server`
-                error::Error::RegionServer {
-                    code,
-                    source: BoxedError::new(err),
-                }
-            })?
-            .into_inner();
 
         check_response_header(header)?;
 
         Ok(affected_rows)
     }
-
-    pub async fn handle(&self, request: RegionRequest) -> Result<AffectedRows> {
-        self.handle_inner(request).await
-    }
 }
 
-pub fn check_response_header(header: Option<ResponseHeader>) -> Result<()> {
+fn check_response_header(header: Option<ResponseHeader>) -> Result<()> {
     let status = header
         .and_then(|header| header.status)
         .context(IllegalDatabaseResponseSnafu {

src/cmd/Cargo.toml

@@ -23,12 +23,8 @@ chrono.workspace = true
 clap = { version = "3.1", features = ["derive"] }
 client = { workspace = true }
 common-base = { workspace = true }
-common-catalog = { workspace = true }
-common-config = { workspace = true }
 common-error = { workspace = true }
-common-macro = { workspace = true }
 common-meta = { workspace = true }
-common-procedure = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
 common-telemetry = { workspace = true, features = [
@@ -39,24 +35,18 @@ datanode = { workspace = true }
 datatypes = { workspace = true }
 either = "1.8"
 etcd-client.workspace = true
-file-engine = { workspace = true }
 frontend = { workspace = true }
 futures.workspace = true
-lazy_static.workspace = true
 meta-client = { workspace = true }
 meta-srv = { workspace = true }
 metrics.workspace = true
-mito2 = { workspace = true }
 nu-ansi-term = "0.46"
 partition = { workspace = true }
-plugins.workspace = true
 prost.workspace = true
 query = { workspace = true }
 rand.workspace = true
-regex.workspace = true
 rustyline = "10.1"
 serde.workspace = true
-serde_json.workspace = true
 servers = { workspace = true }
 session = { workspace = true }
 snafu.workspace = true

src/cmd/src/bin/greptime.rs

@@ -116,7 +116,7 @@ impl SubCommand {
                 Ok(Application::Metasrv(app))
             }
             (SubCommand::Standalone(cmd), Options::Standalone(opts)) => {
-                let app = cmd.build(*opts).await?;
+                let app = cmd.build(opts.fe_opts, opts.dn_opts).await?;
                 Ok(Application::Standalone(app))
             }
             (SubCommand::Cli(cmd), Options::Cli(_)) => {

src/cmd/src/cli/bench.rs

@@ -120,6 +120,7 @@ fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
         created_on: chrono::DateTime::default(),
         primary_key_indices: vec![],
         next_column_id: columns as u32 + 1,
+        engine_options: Default::default(),
         value_indices: vec![],
         options: Default::default(),
         region_numbers: (1..=100).collect(),

src/cmd/src/cli/repl.rs

@@ -16,16 +16,20 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;
 
-use catalog::kvbackend::{CachedMetaKvBackend, KvBackendCatalogManager};
+use catalog::remote::CachedMetaKvBackend;
 use client::client_manager::DatanodeClients;
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_base::Plugins;
 use common_error::ext::ErrorExt;
+use common_meta::key::TableMetadataManager;
 use common_query::Output;
 use common_recordbatch::RecordBatches;
 use common_telemetry::logging;
 use either::Either;
+use frontend::catalog::FrontendCatalogManager;
 use meta_client::client::MetaClientBuilder;
+use partition::manager::PartitionRuleManager;
+use partition::route::TableRoutes;
 use query::datafusion::DatafusionQueryEngine;
 use query::logical_optimizer::LogicalOptimizer;
 use query::parser::QueryLanguageParser;
@@ -35,7 +39,7 @@ use query::QueryEngine;
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 use session::context::QueryContext;
-use snafu::ResultExt;
+use snafu::{ErrorCompat, ResultExt};
 use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
 
 use crate::cli::cmd::ReplCommand;
@@ -148,7 +152,7 @@ impl Repl {
             .await
             .map_err(|e| {
                 let status_code = e.status_code();
-                let root_cause = e.output_msg();
+                let root_cause = e.iter_chain().last().unwrap();
                 println!("Error: {}({status_code}), {root_cause}", status_code as u32)
             })
             .is_ok()
@@ -176,7 +180,7 @@ impl Repl {
                 .encode(&plan)
                 .context(SubstraitEncodeLogicalPlanSnafu)?;
 
-            self.database.logical_plan(plan.to_vec(), 0).await
+            self.database.logical_plan(plan.to_vec(), None).await
         } else {
             self.database.sql(&sql).await
         }
@@ -250,19 +254,24 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
 
     let cached_meta_backend = Arc::new(CachedMetaKvBackend::new(meta_client.clone()));
 
+    let table_routes = Arc::new(TableRoutes::new(meta_client));
+    let partition_manager = Arc::new(PartitionRuleManager::new(table_routes));
+
     let datanode_clients = Arc::new(DatanodeClients::default());
 
-    let catalog_list = KvBackendCatalogManager::new(
+    let catalog_list = Arc::new(FrontendCatalogManager::new(
         cached_meta_backend.clone(),
         cached_meta_backend.clone(),
+        partition_manager,
         datanode_clients,
-    );
+        Arc::new(TableMetadataManager::new(cached_meta_backend)),
-    let plugins: Plugins = Default::default();
+    ));
+    let plugins: Arc<Plugins> = Default::default();
     let state = Arc::new(QueryEngineState::new(
         catalog_list,
-        None,
-        None,
         false,
+        None,
+        None,
         plugins.clone(),
     ));
 

src/cmd/src/cli/upgrade.rs

@@ -17,15 +17,15 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use clap::Parser;
 use client::api::v1::meta::TableRouteValue;
-use common_meta::ddl::utils::region_storage_path;
 use common_meta::error as MetaError;
+use common_meta::helper::{CatalogKey as v1CatalogKey, SchemaKey as v1SchemaKey, TableGlobalValue};
 use common_meta::key::catalog_name::{CatalogNameKey, CatalogNameValue};
-use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue, RegionInfo};
+use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue};
 use common_meta::key::schema_name::{SchemaNameKey, SchemaNameValue};
 use common_meta::key::table_info::{TableInfoKey, TableInfoValue};
 use common_meta::key::table_name::{TableNameKey, TableNameValue};
 use common_meta::key::table_region::{TableRegionKey, TableRegionValue};
-use common_meta::key::table_route::{TableRouteKey, TableRouteValue as NextTableRouteValue};
+use common_meta::key::table_route::{NextTableRouteKey, TableRouteValue as NextTableRouteValue};
 use common_meta::key::{RegionDistribution, TableMetaKey};
 use common_meta::range_stream::PaginationStream;
 use common_meta::rpc::router::TableRoute;
@@ -39,7 +39,6 @@ use meta_srv::service::store::etcd::EtcdStore;
 use meta_srv::service::store::kv::{KvBackendAdapter, KvStoreRef};
 use prost::Message;
 use snafu::ResultExt;
-use v1_helper::{CatalogKey as v1CatalogKey, SchemaKey as v1SchemaKey, TableGlobalValue};
 
 use crate::cli::{Instance, Tool};
 use crate::error::{self, ConnectEtcdSnafu, Result};
@@ -155,7 +154,7 @@ impl MigrateTableMetadata {
         let new_table_value = NextTableRouteValue::new(table_route.region_routes);
 
         let table_id = table_route.table.id as u32;
-        let new_key = TableRouteKey::new(table_id);
+        let new_key = NextTableRouteKey::new(table_id);
         info!("Creating '{new_key}'");
 
         if self.dryrun {
@@ -387,11 +386,6 @@ impl MigrateTableMetadata {
 
     async fn create_datanode_table_keys(&self, value: &TableGlobalValue) {
         let table_id = value.table_id();
-        let engine = value.table_info.meta.engine.as_str();
-        let region_storage_path = region_storage_path(
-            &value.table_info.catalog_name,
-            &value.table_info.schema_name,
-        );
         let region_distribution: RegionDistribution =
             value.regions_id_map.clone().into_iter().collect();
 
@@ -400,18 +394,7 @@ impl MigrateTableMetadata {
             .map(|(datanode_id, regions)| {
                 let k = DatanodeTableKey::new(datanode_id, table_id);
                 info!("Creating DatanodeTableKey '{k}' => {regions:?}");
-                (
+                (k, DatanodeTableValue::new(table_id, regions))
-                    k,
-                    DatanodeTableValue::new(
-                        table_id,
-                        regions,
-                        RegionInfo {
-                            engine: engine.to_string(),
-                            region_storage_path: region_storage_path.clone(),
-                            region_options: (&value.table_info.meta.options).into(),
-                        },
-                    ),
-                )
             })
             .collect::<Vec<_>>();
 
@@ -426,148 +409,3 @@ impl MigrateTableMetadata {
         }
     }
 }
-
-#[deprecated(since = "0.4.0", note = "Used for migrate old version(v0.3) metadata")]
-mod v1_helper {
-    use std::collections::HashMap;
-    use std::fmt::{Display, Formatter};
-
-    use err::{DeserializeCatalogEntryValueSnafu, Error, InvalidCatalogSnafu};
-    use lazy_static::lazy_static;
-    use regex::Regex;
-    use serde::{Deserialize, Serialize};
-    use snafu::{ensure, OptionExt, ResultExt};
-    use table::metadata::{RawTableInfo, TableId};
-
-    pub const CATALOG_KEY_PREFIX: &str = "__c";
-    pub const SCHEMA_KEY_PREFIX: &str = "__s";
-
-    /// The pattern of a valid catalog, schema or table name.
-    const NAME_PATTERN: &str = "[a-zA-Z_:][a-zA-Z0-9_:]*";
-
-    lazy_static! {
-        static ref CATALOG_KEY_PATTERN: Regex =
-            Regex::new(&format!("^{CATALOG_KEY_PREFIX}-({NAME_PATTERN})$")).unwrap();
-    }
-
-    lazy_static! {
-        static ref SCHEMA_KEY_PATTERN: Regex = Regex::new(&format!(
-            "^{SCHEMA_KEY_PREFIX}-({NAME_PATTERN})-({NAME_PATTERN})$"
-        ))
-        .unwrap();
-    }
-
-    /// Table global info contains necessary info for a datanode to create table regions, including
-    /// table id, table meta(schema...), region id allocation across datanodes.
-    #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-    pub struct TableGlobalValue {
-        /// Id of datanode that created the global table info kv. only for debugging.
-        pub node_id: u64,
-        /// Allocation of region ids across all datanodes.
-        pub regions_id_map: HashMap<u64, Vec<u32>>,
-        pub table_info: RawTableInfo,
-    }
-
-    impl TableGlobalValue {
-        pub fn table_id(&self) -> TableId {
-            self.table_info.ident.table_id
-        }
-    }
-
-    pub struct CatalogKey {
-        pub catalog_name: String,
-    }
-
-    impl Display for CatalogKey {
-        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-            f.write_str(CATALOG_KEY_PREFIX)?;
-            f.write_str("-")?;
-            f.write_str(&self.catalog_name)
-        }
-    }
-
-    impl CatalogKey {
-        pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-            let key = s.as_ref();
-            let captures = CATALOG_KEY_PATTERN
-                .captures(key)
-                .context(InvalidCatalogSnafu { key })?;
-            ensure!(captures.len() == 2, InvalidCatalogSnafu { key });
-            Ok(Self {
-                catalog_name: captures[1].to_string(),
-            })
-        }
-    }
-
-    #[derive(Debug, Serialize, Deserialize)]
-    pub struct CatalogValue;
-
-    pub struct SchemaKey {
-        pub catalog_name: String,
-        pub schema_name: String,
-    }
-
-    impl Display for SchemaKey {
-        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-            f.write_str(SCHEMA_KEY_PREFIX)?;
-            f.write_str("-")?;
-            f.write_str(&self.catalog_name)?;
-            f.write_str("-")?;
-            f.write_str(&self.schema_name)
-        }
-    }
-
-    impl SchemaKey {
-        pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-            let key = s.as_ref();
-            let captures = SCHEMA_KEY_PATTERN
-                .captures(key)
-                .context(InvalidCatalogSnafu { key })?;
-            ensure!(captures.len() == 3, InvalidCatalogSnafu { key });
-            Ok(Self {
-                catalog_name: captures[1].to_string(),
-                schema_name: captures[2].to_string(),
-            })
-        }
-    }
-
-    #[derive(Debug, Serialize, Deserialize)]
-    pub struct SchemaValue;
-
-    macro_rules! define_catalog_value {
-    ( $($val_ty: ty), *) => {
-            $(
-                impl $val_ty {
-                    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-                        serde_json::from_str(s.as_ref())
-                            .context(DeserializeCatalogEntryValueSnafu { raw: s.as_ref() })
-                    }
-
-                    pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result<Self, Error> {
-                         Self::parse(&String::from_utf8_lossy(bytes.as_ref()))
-                    }
-                }
-            )*
-        }
-    }
-
-    define_catalog_value!(TableGlobalValue);
-
-    mod err {
-        use snafu::{Location, Snafu};
-
-        #[derive(Debug, Snafu)]
-        #[snafu(visibility(pub))]
-        pub enum Error {
-            #[snafu(display("Invalid catalog info: {}", key))]
-            InvalidCatalog { key: String, location: Location },
-
-            #[snafu(display("Failed to deserialize catalog entry value: {}", raw))]
-            DeserializeCatalogEntryValue {
-                raw: String,
-                location: Location,
-                source: serde_json::error::Error,
-            },
-        }
-    }
-}

src/cmd/src/datanode.rs

@@ -16,8 +16,7 @@ use std::time::Duration;
 
 use clap::Parser;
 use common_telemetry::logging;
-use datanode::config::DatanodeOptions;
+use datanode::datanode::{Datanode, DatanodeOptions};
-use datanode::datanode::{Datanode, DatanodeBuilder};
 use meta_client::MetaClientOptions;
 use servers::Mode;
 use snafu::ResultExt;
@@ -31,10 +30,6 @@ pub struct Instance {
 
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
-        plugins::start_datanode_plugins(self.datanode.plugins())
-            .await
-            .context(StartDatanodeSnafu)?;
-
         self.datanode.start().await.context(StartDatanodeSnafu)
     }
 
@@ -96,6 +91,8 @@ struct StartCommand {
     #[clap(long)]
     data_home: Option<String>,
     #[clap(long)]
+    wal_dir: Option<String>,
+    #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
     http_timeout: Option<u64>,
@@ -132,7 +129,7 @@ impl StartCommand {
         }
 
         if let Some(metasrv_addrs) = &self.metasrv_addr {
-            opts.meta_client
+            opts.meta_client_options
                 .get_or_insert_with(MetaClientOptions::default)
                 .metasrv_addrs = metasrv_addrs.clone();
             opts.mode = Mode::Distributed;
@@ -149,30 +146,29 @@ impl StartCommand {
             opts.storage.data_home = data_home.clone();
         }
 
+        if let Some(wal_dir) = &self.wal_dir {
+            opts.wal.dir = Some(wal_dir.clone());
+        }
+
         if let Some(http_addr) = &self.http_addr {
-            opts.http.addr = http_addr.clone();
+            opts.http_opts.addr = http_addr.clone();
         }
 
         if let Some(http_timeout) = self.http_timeout {
-            opts.http.timeout = Duration::from_secs(http_timeout)
+            opts.http_opts.timeout = Duration::from_secs(http_timeout)
         }
 
         // Disable dashboard in datanode.
-        opts.http.disable_dashboard = true;
+        opts.http_opts.disable_dashboard = true;
 
         Ok(Options::Datanode(Box::new(opts)))
     }
 
-    async fn build(self, mut opts: DatanodeOptions) -> Result<Instance> {
+    async fn build(self, opts: DatanodeOptions) -> Result<Instance> {
-        let plugins = plugins::setup_datanode_plugins(&mut opts)
-            .await
-            .context(StartDatanodeSnafu)?;
-
         logging::info!("Datanode start command: {:#?}", self);
         logging::info!("Datanode options: {:#?}", opts);
 
-        let datanode = DatanodeBuilder::new(opts, None, plugins)
+        let datanode = Datanode::new(opts, Default::default())
-            .build()
             .await
             .context(StartDatanodeSnafu)?;
 
@@ -187,7 +183,9 @@ mod tests {
 
     use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::config::{CompactionConfig, FileConfig, ObjectStoreConfig, RegionManifestConfig};
+    use datanode::datanode::{
+        CompactionConfig, FileConfig, ObjectStoreConfig, RegionManifestConfig,
+    };
     use servers::Mode;
 
     use super::*;
@@ -204,7 +202,7 @@ mod tests {
             rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
 
-            [meta_client]
+            [meta_client_options]
             metasrv_addrs = ["127.0.0.1:3002"]
             timeout_millis = 3000
             connect_timeout_millis = 5000
@@ -252,6 +250,7 @@ mod tests {
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
         assert_eq!(Some(42), options.node_id);
 
+        assert_eq!("/other/wal", options.wal.dir.unwrap());
         assert_eq!(Duration::from_secs(600), options.wal.purge_interval);
         assert_eq!(1024 * 1024 * 1024, options.wal.file_size.0);
         assert_eq!(1024 * 1024 * 1024 * 50, options.wal.purge_threshold.0);
@@ -263,8 +262,7 @@ mod tests {
             connect_timeout_millis,
             tcp_nodelay,
             ddl_timeout_millis,
-            ..
+        } = options.meta_client_options.unwrap();
-        } = options.meta_client.unwrap();
 
         assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);
         assert_eq!(5000, connect_timeout_millis);
@@ -362,7 +360,7 @@ mod tests {
             rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
 
-            [meta_client]
+            [meta_client_options]
             timeout_millis = 3000
             connect_timeout_millis = 5000
             tcp_nodelay = true
@@ -415,10 +413,10 @@ mod tests {
                     Some("99"),
                 ),
                 (
-                    // meta_client.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
                     [
                         env_prefix.to_string(),
-                        "meta_client".to_uppercase(),
+                        "meta_client_options".to_uppercase(),
                         "metasrv_addrs".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -428,6 +426,7 @@ mod tests {
             || {
                 let command = StartCommand {
                     config_file: Some(file.path().to_str().unwrap().to_string()),
+                    wal_dir: Some("/other/wal/dir".to_string()),
                     env_prefix: env_prefix.to_string(),
                     ..Default::default()
                 };
@@ -444,7 +443,7 @@ mod tests {
                     Some(Duration::from_secs(9))
                 );
                 assert_eq!(
-                    opts.meta_client.unwrap().metasrv_addrs,
+                    opts.meta_client_options.unwrap().metasrv_addrs,
                     vec![
                         "127.0.0.1:3001".to_string(),
                         "127.0.0.1:3002".to_string(),
@@ -455,6 +454,9 @@ mod tests {
                 // Should be read from config file, config file > env > default values.
                 assert_eq!(opts.storage.compaction.max_purge_tasks, 32);
 
+                // Should be read from cli, cli > config file > env > default values.
+                assert_eq!(opts.wal.dir.unwrap(), "/other/wal/dir");
+
                 // Should be default value.
                 assert_eq!(
                     opts.storage.manifest.checkpoint_margin,

src/cmd/src/error.rs

@@ -16,64 +16,56 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use config::ConfigError;
 use rustyline::error::ReadlineError;
 use snafu::{Location, Snafu};
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to create default catalog and schema"))]
+    #[snafu(display("Failed to iter stream, source: {}", source))]
-    InitMetadata {
-        location: Location,
-        source: common_meta::error::Error,
-    },
-
-    #[snafu(display("Failed to iter stream"))]
     IterStream {
         location: Location,
         source: common_meta::error::Error,
     },
 
-    #[snafu(display("Failed to start datanode"))]
+    #[snafu(display("Failed to start datanode, source: {}", source))]
     StartDatanode {
         location: Location,
         source: datanode::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown datanode"))]
+    #[snafu(display("Failed to shutdown datanode, source: {}", source))]
     ShutdownDatanode {
         location: Location,
         source: datanode::error::Error,
     },
 
-    #[snafu(display("Failed to start frontend"))]
+    #[snafu(display("Failed to start frontend, source: {}", source))]
     StartFrontend {
         location: Location,
         source: frontend::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown frontend"))]
+    #[snafu(display("Failed to shutdown frontend, source: {}", source))]
     ShutdownFrontend {
         location: Location,
         source: frontend::error::Error,
     },
 
-    #[snafu(display("Failed to build meta server"))]
+    #[snafu(display("Failed to build meta server, source: {}", source))]
     BuildMetaServer {
         location: Location,
         source: meta_srv::error::Error,
     },
 
-    #[snafu(display("Failed to start meta server"))]
+    #[snafu(display("Failed to start meta server, source: {}", source))]
     StartMetaServer {
         location: Location,
         source: meta_srv::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown meta server"))]
+    #[snafu(display("Failed to shutdown meta server, source: {}", source))]
     ShutdownMetaServer {
         location: Location,
         source: meta_srv::error::Error,
@@ -85,7 +77,13 @@ pub enum Error {
     #[snafu(display("Illegal config: {}", msg))]
     IllegalConfig { msg: String, location: Location },
 
-    #[snafu(display("Unsupported selector type: {}", selector_type))]
+    #[snafu(display("Illegal auth config: {}", source))]
+    IllegalAuthConfig {
+        location: Location,
+        source: auth::error::Error,
+    },
+
+    #[snafu(display("Unsupported selector type, {} source: {}", selector_type, source))]
     UnsupportedSelectorType {
         selector_type: String,
         location: Location,
@@ -95,89 +93,78 @@ pub enum Error {
     #[snafu(display("Invalid REPL command: {reason}"))]
     InvalidReplCommand { reason: String },
 
-    #[snafu(display("Cannot create REPL"))]
+    #[snafu(display("Cannot create REPL: {}", source))]
     ReplCreation {
-        #[snafu(source)]
+        source: ReadlineError,
-        error: ReadlineError,
         location: Location,
     },
 
-    #[snafu(display("Error reading command"))]
+    #[snafu(display("Error reading command: {}", source))]
     Readline {
-        #[snafu(source)]
+        source: ReadlineError,
-        error: ReadlineError,
         location: Location,
     },
 
-    #[snafu(display("Failed to request database, sql: {sql}"))]
+    #[snafu(display("Failed to request database, sql: {sql}, source: {source}"))]
     RequestDatabase {
         sql: String,
         location: Location,
         source: client::Error,
     },
 
-    #[snafu(display("Failed to collect RecordBatches"))]
+    #[snafu(display("Failed to collect RecordBatches, source: {source}"))]
     CollectRecordBatches {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to pretty print Recordbatches"))]
+    #[snafu(display("Failed to pretty print Recordbatches, source: {source}"))]
     PrettyPrintRecordBatches {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to start Meta client"))]
+    #[snafu(display("Failed to start Meta client, source: {}", source))]
     StartMetaClient {
         location: Location,
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Failed to parse SQL: {}", sql))]
+    #[snafu(display("Failed to parse SQL: {}, source: {}", sql, source))]
     ParseSql {
         sql: String,
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to plan statement"))]
+    #[snafu(display("Failed to plan statement, source: {}", source))]
     PlanStatement {
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to encode logical plan in substrait"))]
+    #[snafu(display("Failed to encode logical plan in substrait, source: {}", source))]
     SubstraitEncodeLogicalPlan {
         location: Location,
         source: substrait::error::Error,
     },
 
-    #[snafu(display("Failed to load layered config"))]
+    #[snafu(display("Failed to load layered config, source: {}", source))]
     LoadLayeredConfig {
-        #[snafu(source)]
+        source: ConfigError,
-        error: ConfigError,
         location: Location,
     },
 
-    #[snafu(display("Failed to start catalog manager"))]
+    #[snafu(display("Failed to start catalog manager, source: {}", source))]
     StartCatalogManager {
         location: Location,
         source: catalog::error::Error,
     },
 
-    #[snafu(display("Failed to connect to Etcd at {etcd_addr}"))]
+    #[snafu(display("Failed to connect to Etcd at {etcd_addr}, source: {}", source))]
     ConnectEtcd {
         etcd_addr: String,
-        #[snafu(source)]
+        source: etcd_client::Error,
-        error: etcd_client::Error,
-        location: Location,
-    },
-
-    #[snafu(display("Failed to serde json"))]
-    SerdeJson {
-        #[snafu(source)]
-        error: serde_json::error::Error,
         location: Location,
     },
 }
@@ -195,13 +182,12 @@ impl ErrorExt for Error {
             Error::ShutdownMetaServer { source, .. } => source.status_code(),
             Error::BuildMetaServer { source, .. } => source.status_code(),
             Error::UnsupportedSelectorType { source, .. } => source.status_code(),
-            Error::IterStream { source, .. } | Error::InitMetadata { source, .. } => {
+            Error::IterStream { source, .. } => source.status_code(),
-                source.status_code()
-            }
             Error::MissingConfig { .. }
             | Error::LoadLayeredConfig { .. }
             | Error::IllegalConfig { .. }
             | Error::InvalidReplCommand { .. }
+            | Error::IllegalAuthConfig { .. }
             | Error::ConnectEtcd { .. } => StatusCode::InvalidArguments,
 
             Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,
@@ -214,8 +200,6 @@ impl ErrorExt for Error {
             }
             Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(),
             Error::StartCatalogManager { source, .. } => source.status_code(),
-
-            Error::SerdeJson { .. } => StatusCode::Unexpected,
         }
     }
 

src/cmd/src/frontend.rs

@@ -12,18 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::time::Duration;
+use std::sync::Arc;
 
+use auth::UserProviderRef;
 use clap::Parser;
+use common_base::Plugins;
 use common_telemetry::logging;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
+use frontend::service_config::InfluxdbOptions;
 use meta_client::MetaClientOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
 use snafu::ResultExt;
 
-use crate::error::{self, Result, StartFrontendSnafu};
+use crate::error::{self, IllegalAuthConfigSnafu, Result, StartCatalogManagerSnafu};
 use crate::options::{Options, TopLevelOptions};
 
 pub struct Instance {
@@ -32,11 +35,16 @@ pub struct Instance {
 
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
-        plugins::start_frontend_plugins(self.frontend.plugins().clone())
+        self.frontend
+            .catalog_manager()
+            .start()
             .await
-            .context(StartFrontendSnafu)?;
+            .context(StartCatalogManagerSnafu)?;
 
-        self.frontend.start().await.context(StartFrontendSnafu)
+        self.frontend
+            .start()
+            .await
+            .context(error::StartFrontendSnafu)
     }
 
     pub async fn stop(&self) -> Result<()> {
@@ -87,8 +95,6 @@ pub struct StartCommand {
     #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
-    http_timeout: Option<u64>,
-    #[clap(long)]
     grpc_addr: Option<String>,
     #[clap(long)]
     mysql_addr: Option<String>,
@@ -139,75 +145,86 @@ impl StartCommand {
         );
 
         if let Some(addr) = &self.http_addr {
-            opts.http.addr = addr.clone()
+            if let Some(http_opts) = &mut opts.http_options {
-        }
+                http_opts.addr = addr.clone()
-
+            }
-        if let Some(http_timeout) = self.http_timeout {
-            opts.http.timeout = Duration::from_secs(http_timeout)
         }
 
         if let Some(disable_dashboard) = self.disable_dashboard {
-            opts.http.disable_dashboard = disable_dashboard;
+            opts.http_options
+                .get_or_insert_with(Default::default)
+                .disable_dashboard = disable_dashboard;
         }
 
         if let Some(addr) = &self.grpc_addr {
-            opts.grpc.addr = addr.clone()
+            if let Some(grpc_opts) = &mut opts.grpc_options {
+                grpc_opts.addr = addr.clone()
+            }
         }
 
         if let Some(addr) = &self.mysql_addr {
-            opts.mysql.enable = true;
+            if let Some(mysql_opts) = &mut opts.mysql_options {
-            opts.mysql.addr = addr.clone();
+                mysql_opts.addr = addr.clone();
-            opts.mysql.tls = tls_opts.clone();
+                mysql_opts.tls = tls_opts.clone();
+            }
         }
 
         if let Some(addr) = &self.postgres_addr {
-            opts.postgres.enable = true;
+            if let Some(postgres_opts) = &mut opts.postgres_options {
-            opts.postgres.addr = addr.clone();
+                postgres_opts.addr = addr.clone();
-            opts.postgres.tls = tls_opts;
+                postgres_opts.tls = tls_opts;
+            }
         }
 
         if let Some(addr) = &self.opentsdb_addr {
-            opts.opentsdb.enable = true;
+            if let Some(opentsdb_addr) = &mut opts.opentsdb_options {
-            opts.opentsdb.addr = addr.clone();
+                opentsdb_addr.addr = addr.clone();
+            }
         }
 
         if let Some(enable) = self.influxdb_enable {
-            opts.influxdb.enable = enable;
+            opts.influxdb_options = Some(InfluxdbOptions { enable });
         }
 
         if let Some(metasrv_addrs) = &self.metasrv_addr {
-            opts.meta_client
+            opts.meta_client_options
                 .get_or_insert_with(MetaClientOptions::default)
                 .metasrv_addrs = metasrv_addrs.clone();
             opts.mode = Mode::Distributed;
         }
 
-        opts.user_provider = self.user_provider.clone();
-
         Ok(Options::Frontend(Box::new(opts)))
     }
 
-    async fn build(self, mut opts: FrontendOptions) -> Result<Instance> {
+    async fn build(self, opts: FrontendOptions) -> Result<Instance> {
-        let plugins = plugins::setup_frontend_plugins(&mut opts)
-            .await
-            .context(StartFrontendSnafu)?;
-
         logging::info!("Frontend start command: {:#?}", self);
         logging::info!("Frontend options: {:#?}", opts);
 
+        let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
+
         let mut instance = FeInstance::try_new_distributed(&opts, plugins.clone())
             .await
-            .context(StartFrontendSnafu)?;
+            .context(error::StartFrontendSnafu)?;
 
         instance
             .build_servers(&opts)
             .await
-            .context(StartFrontendSnafu)?;
+            .context(error::StartFrontendSnafu)?;
 
         Ok(Instance { frontend: instance })
     }
 }
 
+pub fn load_frontend_plugins(user_provider: &Option<String>) -> Result<Plugins> {
+    let plugins = Plugins::new();
+
+    if let Some(provider) = user_provider {
+        let provider = auth::user_provider_from_option(provider).context(IllegalAuthConfigSnafu)?;
+        plugins.insert::<UserProviderRef>(provider);
+    }
+    Ok(plugins)
+}
+
 #[cfg(test)]
 mod tests {
     use std::io::Write;
@@ -217,7 +234,6 @@ mod tests {
     use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
     use frontend::service_config::GrpcOptions;
-    use servers::http::HttpOptions;
 
     use super::*;
     use crate::options::ENV_VAR_SEP;
@@ -239,29 +255,40 @@ mod tests {
             unreachable!()
         };
 
-        assert_eq!(opts.http.addr, "127.0.0.1:1234");
+        assert_eq!(opts.http_options.as_ref().unwrap().addr, "127.0.0.1:1234");
-        assert_eq!(ReadableSize::mb(64), opts.http.body_limit);
+        assert_eq!(
-        assert_eq!(opts.mysql.addr, "127.0.0.1:5678");
+            ReadableSize::mb(64),
-        assert_eq!(opts.postgres.addr, "127.0.0.1:5432");
+            opts.http_options.as_ref().unwrap().body_limit
-        assert_eq!(opts.opentsdb.addr, "127.0.0.1:4321");
+        );
+        assert_eq!(opts.mysql_options.as_ref().unwrap().addr, "127.0.0.1:5678");
+        assert_eq!(
+            opts.postgres_options.as_ref().unwrap().addr,
+            "127.0.0.1:5432"
+        );
+        assert_eq!(
+            opts.opentsdb_options.as_ref().unwrap().addr,
+            "127.0.0.1:4321"
+        );
 
         let default_opts = FrontendOptions::default();
-
-        assert_eq!(opts.grpc.addr, default_opts.grpc.addr);
-        assert!(opts.mysql.enable);
-        assert_eq!(opts.mysql.runtime_size, default_opts.mysql.runtime_size);
-        assert!(opts.postgres.enable);
         assert_eq!(
-            opts.postgres.runtime_size,
+            opts.grpc_options.unwrap().addr,
-            default_opts.postgres.runtime_size
+            default_opts.grpc_options.unwrap().addr
         );
-        assert!(opts.opentsdb.enable);
         assert_eq!(
-            opts.opentsdb.runtime_size,
+            opts.mysql_options.as_ref().unwrap().runtime_size,
-            default_opts.opentsdb.runtime_size
+            default_opts.mysql_options.as_ref().unwrap().runtime_size
+        );
+        assert_eq!(
+            opts.postgres_options.as_ref().unwrap().runtime_size,
+            default_opts.postgres_options.as_ref().unwrap().runtime_size
+        );
+        assert_eq!(
+            opts.opentsdb_options.as_ref().unwrap().runtime_size,
+            default_opts.opentsdb_options.as_ref().unwrap().runtime_size
         );
 
-        assert!(!opts.influxdb.enable);
+        assert!(!opts.influxdb_options.unwrap().enable);
     }
 
     #[test]
@@ -270,7 +297,7 @@ mod tests {
         let toml_str = r#"
             mode = "distributed"
 
-            [http]
+            [http_options]
             addr = "127.0.0.1:4000"
             timeout = "30s"
             body_limit = "2GB"
@@ -292,10 +319,19 @@ mod tests {
             unreachable!()
         };
         assert_eq!(Mode::Distributed, fe_opts.mode);
-        assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
+        assert_eq!(
-        assert_eq!(Duration::from_secs(30), fe_opts.http.timeout);
+            "127.0.0.1:4000".to_string(),
+            fe_opts.http_options.as_ref().unwrap().addr
+        );
+        assert_eq!(
+            Duration::from_secs(30),
+            fe_opts.http_options.as_ref().unwrap().timeout
+        );
 
-        assert_eq!(ReadableSize::gb(2), fe_opts.http.body_limit);
+        assert_eq!(
+            ReadableSize::gb(2),
+            fe_opts.http_options.as_ref().unwrap().body_limit
+        );
 
         assert_eq!("debug", fe_opts.logging.level.as_ref().unwrap());
         assert_eq!("/tmp/greptimedb/test/logs".to_string(), fe_opts.logging.dir);
@@ -303,17 +339,14 @@ mod tests {
 
     #[tokio::test]
     async fn test_try_from_start_command_to_anymap() {
-        let mut fe_opts = FrontendOptions {
+        let command = StartCommand {
-            http: HttpOptions {
-                disable_dashboard: false,
-                ..Default::default()
-            },
             user_provider: Some("static_user_provider:cmd:test=test".to_string()),
+            disable_dashboard: Some(false),
             ..Default::default()
         };
 
-        let plugins = plugins::setup_frontend_plugins(&mut fe_opts).await.unwrap();
+        let plugins = load_frontend_plugins(&command.user_provider);
-
+        let plugins = plugins.unwrap();
         let provider = plugins.get::<UserProviderRef>().unwrap();
         let result = provider
             .authenticate(
@@ -349,15 +382,15 @@ mod tests {
         let toml_str = r#"
             mode = "distributed"
 
-            [http]
+            [http_options]
             addr = "127.0.0.1:4000"
 
-            [meta_client]
+            [meta_client_options]
             timeout_millis = 3000
             connect_timeout_millis = 5000
             tcp_nodelay = true
 
-            [mysql]
+            [mysql_options]
             addr = "127.0.0.1:4002"
         "#;
         write!(file, "{}", toml_str).unwrap();
@@ -366,40 +399,40 @@ mod tests {
         temp_env::with_vars(
             [
                 (
-                    // mysql.addr = 127.0.0.1:14002
+                    // mysql_options.addr = 127.0.0.1:14002
                     [
                         env_prefix.to_string(),
-                        "mysql".to_uppercase(),
+                        "mysql_options".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
                     Some("127.0.0.1:14002"),
                 ),
                 (
-                    // mysql.runtime_size = 11
+                    // mysql_options.runtime_size = 11
                     [
                         env_prefix.to_string(),
-                        "mysql".to_uppercase(),
+                        "mysql_options".to_uppercase(),
                         "runtime_size".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
                     Some("11"),
                 ),
                 (
-                    // http.addr = 127.0.0.1:24000
+                    // http_options.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
-                        "http".to_uppercase(),
+                        "http_options".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
                     Some("127.0.0.1:24000"),
                 ),
                 (
-                    // meta_client.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
                     [
                         env_prefix.to_string(),
-                        "meta_client".to_uppercase(),
+                        "meta_client_options".to_uppercase(),
                         "metasrv_addrs".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -424,9 +457,9 @@ mod tests {
                 };
 
                 // Should be read from env, env > default values.
-                assert_eq!(fe_opts.mysql.runtime_size, 11);
+                assert_eq!(fe_opts.mysql_options.as_ref().unwrap().runtime_size, 11);
                 assert_eq!(
-                    fe_opts.meta_client.unwrap().metasrv_addrs,
+                    fe_opts.meta_client_options.unwrap().metasrv_addrs,
                     vec![
                         "127.0.0.1:3001".to_string(),
                         "127.0.0.1:3002".to_string(),
@@ -435,13 +468,22 @@ mod tests {
                 );
 
                 // Should be read from config file, config file > env > default values.
-                assert_eq!(fe_opts.mysql.addr, "127.0.0.1:4002");
+                assert_eq!(
+                    fe_opts.mysql_options.as_ref().unwrap().addr,
+                    "127.0.0.1:4002"
+                );
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(fe_opts.http.addr, "127.0.0.1:14000");
+                assert_eq!(
+                    fe_opts.http_options.as_ref().unwrap().addr,
+                    "127.0.0.1:14000"
+                );
 
                 // Should be default value.
-                assert_eq!(fe_opts.grpc.addr, GrpcOptions::default().addr);
+                assert_eq!(
+                    fe_opts.grpc_options.as_ref().unwrap().addr,
+                    GrpcOptions::default().addr
+                );
             },
         );
     }

src/cmd/src/metasrv.rs

@@ -20,7 +20,7 @@ use meta_srv::bootstrap::MetaSrvInstance;
 use meta_srv::metasrv::MetaSrvOptions;
 use snafu::ResultExt;
 
-use crate::error::{self, Result, StartMetaServerSnafu};
+use crate::error::{self, Result};
 use crate::options::{Options, TopLevelOptions};
 
 pub struct Instance {
@@ -29,10 +29,10 @@ pub struct Instance {
 
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
-        plugins::start_meta_srv_plugins(self.instance.plugins())
+        self.instance
+            .start()
             .await
-            .context(StartMetaServerSnafu)?;
+            .context(error::StartMetaServerSnafu)
-        self.instance.start().await.context(StartMetaServerSnafu)
     }
 
     pub async fn stop(&self) -> Result<()> {
@@ -93,7 +93,7 @@ struct StartCommand {
     #[clap(long)]
     use_memory_store: Option<bool>,
     #[clap(long)]
-    enable_region_failover: Option<bool>,
+    disable_region_failover: Option<bool>,
     #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
@@ -140,33 +140,30 @@ impl StartCommand {
             opts.use_memory_store = use_memory_store;
         }
 
-        if let Some(enable_region_failover) = self.enable_region_failover {
+        if let Some(disable_region_failover) = self.disable_region_failover {
-            opts.enable_region_failover = enable_region_failover;
+            opts.disable_region_failover = disable_region_failover;
         }
 
         if let Some(http_addr) = &self.http_addr {
-            opts.http.addr = http_addr.clone();
+            opts.http_opts.addr = http_addr.clone();
         }
 
         if let Some(http_timeout) = self.http_timeout {
-            opts.http.timeout = Duration::from_secs(http_timeout);
+            opts.http_opts.timeout = Duration::from_secs(http_timeout);
         }
 
         // Disable dashboard in metasrv.
-        opts.http.disable_dashboard = true;
+        opts.http_opts.disable_dashboard = true;
 
         Ok(Options::Metasrv(Box::new(opts)))
     }
 
-    async fn build(self, mut opts: MetaSrvOptions) -> Result<Instance> {
+    async fn build(self, opts: MetaSrvOptions) -> Result<Instance> {
-        let plugins = plugins::setup_meta_srv_plugins(&mut opts)
-            .await
-            .context(StartMetaServerSnafu)?;
-
         logging::info!("MetaSrv start command: {:#?}", self);
+
         logging::info!("MetaSrv options: {:#?}", opts);
 
-        let instance = MetaSrvInstance::new(opts, plugins)
+        let instance = MetaSrvInstance::new(opts)
             .await
             .context(error::BuildMetaServerSnafu)?;
 
@@ -210,6 +207,7 @@ mod tests {
             bind_addr = "127.0.0.1:3002"
             server_addr = "127.0.0.1:3002"
             store_addr = "127.0.0.1:2379"
+            datanode_lease_secs = 15
             selector = "LeaseBased"
             use_memory_store = false
 
@@ -231,6 +229,7 @@ mod tests {
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
         assert_eq!("127.0.0.1:3002".to_string(), options.server_addr);
         assert_eq!("127.0.0.1:2379".to_string(), options.store_addr);
+        assert_eq!(15, options.datanode_lease_secs);
         assert_eq!(SelectorType::LeaseBased, options.selector);
         assert_eq!("debug", options.logging.level.as_ref().unwrap());
         assert_eq!("/tmp/greptimedb/test/logs".to_string(), options.logging.dir);
@@ -267,7 +266,7 @@ mod tests {
             selector = "LeaseBased"
             use_memory_store = false
 
-            [http]
+            [http_options]
             addr = "127.0.0.1:4000"
 
             [logging]
@@ -290,10 +289,10 @@ mod tests {
                     Some("127.0.0.1:13002"),
                 ),
                 (
-                    // http.addr = 127.0.0.1:24000
+                    // http_options.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
-                        "http".to_uppercase(),
+                        "http_options".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -321,7 +320,7 @@ mod tests {
                 assert_eq!(opts.server_addr, "127.0.0.1:3002");
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.http.addr, "127.0.0.1:14000");
+                assert_eq!(opts.http_opts.addr, "127.0.0.1:14000");
 
                 // Should be default value.
                 assert_eq!(opts.store_addr, "127.0.0.1:2379");

src/cmd/src/options.rs

@@ -12,27 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_config::KvStoreConfig;
 use common_telemetry::logging::LoggingOptions;
 use config::{Config, Environment, File, FileFormat};
-use datanode::config::{DatanodeOptions, ProcedureConfig};
+use datanode::datanode::DatanodeOptions;
 use frontend::frontend::FrontendOptions;
 use meta_srv::metasrv::MetaSrvOptions;
 use serde::{Deserialize, Serialize};
 use snafu::ResultExt;
 
-use crate::error::{LoadLayeredConfigSnafu, Result, SerdeJsonSnafu};
+use crate::error::{LoadLayeredConfigSnafu, Result};
 
 pub const ENV_VAR_SEP: &str = "__";
 pub const ENV_LIST_SEP: &str = ",";
 
-/// Options mixed up from datanode, frontend and metasrv.
 pub struct MixOptions {
-    pub data_home: String,
+    pub fe_opts: FrontendOptions,
-    pub procedure: ProcedureConfig,
+    pub dn_opts: DatanodeOptions,
-    pub kv_store: KvStoreConfig,
-    pub frontend: FrontendOptions,
-    pub datanode: DatanodeOptions,
     pub logging: LoggingOptions,
 }
 
@@ -94,16 +89,9 @@ impl Options {
                 .ignore_empty(true)
         };
 
-        // Workaround: Replacement for `Config::try_from(&default_opts)` due to
-        // `ConfigSerializer` cannot handle the case of an empty struct contained
-        // within an iterative structure.
-        // See: https://github.com/mehcode/config-rs/issues/461
-        let json_str = serde_json::to_string(&default_opts).context(SerdeJsonSnafu)?;
-        let default_config = File::from_str(&json_str, FileFormat::Json);
-
         // Add default values and environment variables as the sources of the configuration.
         let mut layered_config = Config::builder()
-            .add_source(default_config)
+            .add_source(Config::try_from(&default_opts).context(LoadLayeredConfigSnafu)?)
             .add_source(env_source);
 
         // Add config file as the source of the configuration if it is specified.
@@ -127,7 +115,7 @@ mod tests {
     use std::time::Duration;
 
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::config::{DatanodeOptions, ObjectStoreConfig};
+    use datanode::datanode::{DatanodeOptions, ObjectStoreConfig};
 
     use super::*;
 
@@ -143,7 +131,7 @@ mod tests {
             mysql_addr = "127.0.0.1:4406"
             mysql_runtime_size = 2
 
-            [meta_client]
+            [meta_client_options]
             timeout_millis = 3000
             connect_timeout_millis = 5000
             tcp_nodelay = true
@@ -224,10 +212,10 @@ mod tests {
                     Some("/other/wal/dir"),
                 ),
                 (
-                    // meta_client.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
                     [
                         env_prefix.to_string(),
-                        "meta_client".to_uppercase(),
+                        "meta_client_options".to_uppercase(),
                         "metasrv_addrs".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -255,7 +243,7 @@ mod tests {
                     Some(Duration::from_secs(42))
                 );
                 assert_eq!(
-                    opts.meta_client.unwrap().metasrv_addrs,
+                    opts.meta_client_options.unwrap().metasrv_addrs,
                     vec![
                         "127.0.0.1:3001".to_string(),
                         "127.0.0.1:3002".to_string(),
@@ -263,6 +251,9 @@ mod tests {
                     ]
                 );
 
+                // Should be the values from config file, not environment variables.
+                assert_eq!(opts.wal.dir.unwrap(), "/tmp/greptimedb/wal");
+
                 // Should be default values.
                 assert_eq!(opts.node_id, None);
             },

src/cmd/src/standalone.rs

@@ -14,26 +14,17 @@
 
 use std::sync::Arc;
 
-use catalog::kvbackend::KvBackendCatalogManager;
-use catalog::CatalogManagerRef;
 use clap::Parser;
 use common_base::Plugins;
-use common_config::{metadata_store_dir, KvStoreConfig, WalConfig};
-use common_meta::cache_invalidator::DummyKvCacheInvalidator;
-use common_meta::kv_backend::KvBackendRef;
-use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
-use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
+use datanode::datanode::{Datanode, DatanodeOptions, ProcedureConfig, StorageConfig, WalConfig};
-use datanode::datanode::{Datanode, DatanodeBuilder};
+use datanode::instance::InstanceRef;
-use datanode::region_server::RegionServer;
-use file_engine::config::EngineConfig as FileEngineConfig;
 use frontend::frontend::FrontendOptions;
-use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
+use frontend::instance::{FrontendInstance, Instance as FeInstance};
 use frontend::service_config::{
     GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
 };
-use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
@@ -41,9 +32,10 @@ use servers::Mode;
 use snafu::ResultExt;
 
 use crate::error::{
-    IllegalConfigSnafu, InitMetadataSnafu, Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu,
+    IllegalConfigSnafu, Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu,
-    StartDatanodeSnafu, StartFrontendSnafu,
+    StartFrontendSnafu,
 };
+use crate::frontend::load_frontend_plugins;
 use crate::options::{MixOptions, Options, TopLevelOptions};
 
 #[derive(Parser)]
@@ -53,8 +45,12 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self, opts: MixOptions) -> Result<Instance> {
+    pub async fn build(
-        self.subcmd.build(opts).await
+        self,
+        fe_opts: FrontendOptions,
+        dn_opts: DatanodeOptions,
+    ) -> Result<Instance> {
+        self.subcmd.build(fe_opts, dn_opts).await
     }
 
     pub fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
@@ -68,9 +64,9 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self, opts: MixOptions) -> Result<Instance> {
+    async fn build(self, fe_opts: FrontendOptions, dn_opts: DatanodeOptions) -> Result<Instance> {
         match self {
-            SubCommand::Start(cmd) => cmd.build(opts).await,
+            SubCommand::Start(cmd) => cmd.build(fe_opts, dn_opts).await,
         }
     }
 
@@ -85,44 +81,38 @@ impl SubCommand {
 #[serde(default)]
 pub struct StandaloneOptions {
     pub mode: Mode,
+    pub enable_memory_catalog: bool,
     pub enable_telemetry: bool,
-    pub http: HttpOptions,
+    pub http_options: Option<HttpOptions>,
-    pub grpc: GrpcOptions,
+    pub grpc_options: Option<GrpcOptions>,
-    pub mysql: MysqlOptions,
+    pub mysql_options: Option<MysqlOptions>,
-    pub postgres: PostgresOptions,
+    pub postgres_options: Option<PostgresOptions>,
-    pub opentsdb: OpentsdbOptions,
+    pub opentsdb_options: Option<OpentsdbOptions>,
-    pub influxdb: InfluxdbOptions,
+    pub influxdb_options: Option<InfluxdbOptions>,
-    pub prom_store: PromStoreOptions,
+    pub prom_store_options: Option<PromStoreOptions>,
     pub wal: WalConfig,
     pub storage: StorageConfig,
-    pub kv_store: KvStoreConfig,
     pub procedure: ProcedureConfig,
     pub logging: LoggingOptions,
-    /// Options for different store engines.
-    pub region_engine: Vec<RegionEngineConfig>,
 }
 
 impl Default for StandaloneOptions {
     fn default() -> Self {
         Self {
             mode: Mode::Standalone,
+            enable_memory_catalog: false,
             enable_telemetry: true,
-            http: HttpOptions::default(),
+            http_options: Some(HttpOptions::default()),
-            grpc: GrpcOptions::default(),
+            grpc_options: Some(GrpcOptions::default()),
-            mysql: MysqlOptions::default(),
+            mysql_options: Some(MysqlOptions::default()),
-            postgres: PostgresOptions::default(),
+            postgres_options: Some(PostgresOptions::default()),
-            opentsdb: OpentsdbOptions::default(),
+            opentsdb_options: Some(OpentsdbOptions::default()),
-            influxdb: InfluxdbOptions::default(),
+            influxdb_options: Some(InfluxdbOptions::default()),
-            prom_store: PromStoreOptions::default(),
+            prom_store_options: Some(PromStoreOptions::default()),
             wal: WalConfig::default(),
             storage: StorageConfig::default(),
-            kv_store: KvStoreConfig::default(),
             procedure: ProcedureConfig::default(),
             logging: LoggingOptions::default(),
-            region_engine: vec![
-                RegionEngineConfig::Mito(MitoConfig::default()),
-                RegionEngineConfig::File(FileEngineConfig::default()),
-            ],
         }
     }
 }
@@ -131,14 +121,14 @@ impl StandaloneOptions {
     fn frontend_options(self) -> FrontendOptions {
         FrontendOptions {
             mode: self.mode,
-            http: self.http,
+            http_options: self.http_options,
-            grpc: self.grpc,
+            grpc_options: self.grpc_options,
-            mysql: self.mysql,
+            mysql_options: self.mysql_options,
-            postgres: self.postgres,
+            postgres_options: self.postgres_options,
-            opentsdb: self.opentsdb,
+            opentsdb_options: self.opentsdb_options,
-            influxdb: self.influxdb,
+            influxdb_options: self.influxdb_options,
-            prom_store: self.prom_store,
+            prom_store_options: self.prom_store_options,
-            meta_client: None,
+            meta_client_options: None,
             logging: self.logging,
             ..Default::default()
         }
@@ -146,11 +136,11 @@ impl StandaloneOptions {
 
     fn datanode_options(self) -> DatanodeOptions {
         DatanodeOptions {
-            node_id: Some(0),
+            enable_memory_catalog: self.enable_memory_catalog,
             enable_telemetry: self.enable_telemetry,
             wal: self.wal,
             storage: self.storage,
-            region_engine: self.region_engine,
+            procedure: self.procedure,
             ..Default::default()
         }
     }
@@ -164,7 +154,10 @@ pub struct Instance {
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
         // Start datanode instance before starting services, to avoid requests come in before internal components are started.
-        self.datanode.start().await.context(StartDatanodeSnafu)?;
+        self.datanode
+            .start_instance()
+            .await
+            .context(StartDatanodeSnafu)?;
         info!("Datanode instance started");
 
         self.frontend.start().await.context(StartFrontendSnafu)?;
@@ -178,7 +171,7 @@ impl Instance {
             .context(ShutdownFrontendSnafu)?;
 
         self.datanode
-            .shutdown()
+            .shutdown_instance()
             .await
             .context(ShutdownDatanodeSnafu)?;
         info!("Datanode instance stopped.");
@@ -203,6 +196,8 @@ struct StartCommand {
     influxdb_enable: bool,
     #[clap(short, long)]
     config_file: Option<String>,
+    #[clap(short = 'm', long = "memory-catalog")]
+    enable_memory_catalog: bool,
     #[clap(long)]
     tls_mode: Option<TlsMode>,
     #[clap(long)]
@@ -223,6 +218,8 @@ impl StartCommand {
             None,
         )?;
 
+        opts.enable_memory_catalog = self.enable_memory_catalog;
+
         opts.mode = Mode::Standalone;
 
         if let Some(dir) = top_level_options.log_dir {
@@ -240,7 +237,9 @@ impl StartCommand {
         );
 
         if let Some(addr) = &self.http_addr {
-            opts.http.addr = addr.clone()
+            if let Some(http_opts) = &mut opts.http_options {
+                http_opts.addr = addr.clone()
+            }
         }
 
         if let Some(addr) = &self.rpc_addr {
@@ -254,55 +253,48 @@ impl StartCommand {
                 }
                 .fail();
             }
-            opts.grpc.addr = addr.clone()
+            if let Some(grpc_opts) = &mut opts.grpc_options {
+                grpc_opts.addr = addr.clone()
+            }
         }
 
         if let Some(addr) = &self.mysql_addr {
-            opts.mysql.enable = true;
+            if let Some(mysql_opts) = &mut opts.mysql_options {
-            opts.mysql.addr = addr.clone();
+                mysql_opts.addr = addr.clone();
-            opts.mysql.tls = tls_opts.clone();
+                mysql_opts.tls = tls_opts.clone();
+            }
         }
 
         if let Some(addr) = &self.postgres_addr {
-            opts.postgres.enable = true;
+            if let Some(postgres_opts) = &mut opts.postgres_options {
-            opts.postgres.addr = addr.clone();
+                postgres_opts.addr = addr.clone();
-            opts.postgres.tls = tls_opts;
+                postgres_opts.tls = tls_opts;
+            }
         }
 
         if let Some(addr) = &self.opentsdb_addr {
-            opts.opentsdb.enable = true;
+            if let Some(opentsdb_addr) = &mut opts.opentsdb_options {
-            opts.opentsdb.addr = addr.clone();
+                opentsdb_addr.addr = addr.clone();
+            }
         }
 
         if self.influxdb_enable {
-            opts.influxdb.enable = self.influxdb_enable;
+            opts.influxdb_options = Some(InfluxdbOptions { enable: true });
         }
-        let kv_store = opts.kv_store.clone();
+
-        let procedure = opts.procedure.clone();
+        let fe_opts = opts.clone().frontend_options();
-        let frontend = opts.clone().frontend_options();
         let logging = opts.logging.clone();
-        let datanode = opts.datanode_options();
+        let dn_opts = opts.datanode_options();
 
         Ok(Options::Standalone(Box::new(MixOptions {
-            procedure,
+            fe_opts,
-            kv_store,
+            dn_opts,
-            data_home: datanode.storage.data_home.to_string(),
-            frontend,
-            datanode,
             logging,
         })))
     }
 
-    #[allow(unreachable_code)]
+    async fn build(self, fe_opts: FrontendOptions, dn_opts: DatanodeOptions) -> Result<Instance> {
-    #[allow(unused_variables)]
+        let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
-    #[allow(clippy::diverging_sub_expression)]
-    async fn build(self, opts: MixOptions) -> Result<Instance> {
-        let mut fe_opts = opts.frontend;
-        let fe_plugins = plugins::setup_frontend_plugins(&mut fe_opts)
-            .await
-            .context(StartFrontendSnafu)?;
-
-        let dn_opts = opts.datanode;
 
         info!("Standalone start command: {:#?}", self);
         info!(
@@ -310,43 +302,11 @@ impl StartCommand {
             fe_opts, dn_opts
         );
 
-        let metadata_dir = metadata_store_dir(&opts.data_home);
+        let datanode = Datanode::new(dn_opts.clone(), Default::default())
-        let (kv_store, procedure_manager) = FeInstance::try_build_standalone_components(
-            metadata_dir,
-            opts.kv_store,
-            opts.procedure,
-        )
-        .await
-        .context(StartFrontendSnafu)?;
-
-        let datanode =
-            DatanodeBuilder::new(dn_opts.clone(), Some(kv_store.clone()), Default::default())
-                .build()
-                .await
-                .context(StartDatanodeSnafu)?;
-        let region_server = datanode.region_server();
-
-        let catalog_manager = KvBackendCatalogManager::new(
-            kv_store.clone(),
-            Arc::new(DummyKvCacheInvalidator),
-            Arc::new(StandaloneDatanodeManager(region_server.clone())),
-        );
-
-        catalog_manager
-            .table_metadata_manager_ref()
-            .init()
             .await
-            .context(InitMetadataSnafu)?;
+            .context(StartDatanodeSnafu)?;
 
-        // TODO: build frontend instance like in distributed mode
+        let mut frontend = build_frontend(plugins.clone(), datanode.get_instance()).await?;
-        let mut frontend = build_frontend(
-            fe_plugins,
-            kv_store,
-            procedure_manager,
-            catalog_manager,
-            region_server,
-        )
-        .await?;
 
         frontend
             .build_servers(&fe_opts)
@@ -359,21 +319,13 @@ impl StartCommand {
 
 /// Build frontend instance in standalone mode
 async fn build_frontend(
-    plugins: Plugins,
+    plugins: Arc<Plugins>,
-    kv_store: KvBackendRef,
+    datanode_instance: InstanceRef,
-    procedure_manager: ProcedureManagerRef,
-    catalog_manager: CatalogManagerRef,
-    region_server: RegionServer,
 ) -> Result<FeInstance> {
-    let frontend_instance = FeInstance::try_new_standalone(
+    let mut frontend_instance = FeInstance::try_new_standalone(datanode_instance.clone())
-        kv_store,
+        .await
-        procedure_manager,
+        .context(StartFrontendSnafu)?;
-        catalog_manager,
+    frontend_instance.set_plugins(plugins.clone());
-        plugins,
-        region_server,
-    )
-    .await
-    .context(StartFrontendSnafu)?;
     Ok(frontend_instance)
 }
 
@@ -393,13 +345,13 @@ mod tests {
 
     #[tokio::test]
     async fn test_try_from_start_command_to_anymap() {
-        let mut fe_opts = FrontendOptions {
+        let command = StartCommand {
             user_provider: Some("static_user_provider:cmd:test=test".to_string()),
             ..Default::default()
         };
 
-        let plugins = plugins::setup_frontend_plugins(&mut fe_opts).await.unwrap();
+        let plugins = load_frontend_plugins(&command.user_provider);
-
+        let plugins = plugins.unwrap();
         let provider = plugins.get::<UserProviderRef>().unwrap();
         let result = provider
             .authenticate(
@@ -447,9 +399,9 @@ mod tests {
             checkpoint_margin = 9
             gc_duration = '7s'
 
-            [http]
+            [http_options]
             addr = "127.0.0.1:4000"
-            timeout = "33s"
+            timeout = "30s"
             body_limit = "128MB"
 
             [logging]
@@ -467,22 +419,40 @@ mod tests {
         else {
             unreachable!()
         };
-        let fe_opts = options.frontend;
+        let fe_opts = options.fe_opts;
-        let dn_opts = options.datanode;
+        let dn_opts = options.dn_opts;
         let logging_opts = options.logging;
         assert_eq!(Mode::Standalone, fe_opts.mode);
-        assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
+        assert_eq!(
-        assert_eq!(Duration::from_secs(33), fe_opts.http.timeout);
+            "127.0.0.1:4000".to_string(),
-        assert_eq!(ReadableSize::mb(128), fe_opts.http.body_limit);
+            fe_opts.http_options.as_ref().unwrap().addr
-        assert_eq!("127.0.0.1:4001".to_string(), fe_opts.grpc.addr);
+        );
-        assert!(fe_opts.mysql.enable);
+        assert_eq!(
-        assert_eq!("127.0.0.1:4002", fe_opts.mysql.addr);
+            Duration::from_secs(30),
-        assert_eq!(2, fe_opts.mysql.runtime_size);
+            fe_opts.http_options.as_ref().unwrap().timeout
-        assert_eq!(None, fe_opts.mysql.reject_no_database);
+        );
-        assert!(fe_opts.influxdb.enable);
+        assert_eq!(
+            ReadableSize::mb(128),
+            fe_opts.http_options.as_ref().unwrap().body_limit
+        );
+        assert_eq!(
+            "127.0.0.1:4001".to_string(),
+            fe_opts.grpc_options.unwrap().addr
+        );
+        assert_eq!(
+            "127.0.0.1:4002",
+            fe_opts.mysql_options.as_ref().unwrap().addr
+        );
+        assert_eq!(2, fe_opts.mysql_options.as_ref().unwrap().runtime_size);
+        assert_eq!(
+            None,
+            fe_opts.mysql_options.as_ref().unwrap().reject_no_database
+        );
+        assert!(fe_opts.influxdb_options.as_ref().unwrap().enable);
 
+        assert_eq!("/tmp/greptimedb/test/wal", dn_opts.wal.dir.unwrap());
         match &dn_opts.storage.store {
-            datanode::config::ObjectStoreConfig::S3(s3_config) => {
+            datanode::datanode::ObjectStoreConfig::S3(s3_config) => {
                 assert_eq!(
                     "Secret([REDACTED alloc::string::String])".to_string(),
                     format!("{:?}", s3_config.access_key_id)
@@ -524,7 +494,7 @@ mod tests {
         let toml_str = r#"
             mode = "standalone"
 
-            [http]
+            [http_options]
             addr = "127.0.0.1:4000"
 
             [logging]
@@ -556,10 +526,10 @@ mod tests {
                     Some("info"),
                 ),
                 (
-                    // http.addr = 127.0.0.1:24000
+                    // http_options.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
-                        "http".to_uppercase(),
+                        "http_options".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -590,33 +560,21 @@ mod tests {
                 assert_eq!(opts.logging.level.as_ref().unwrap(), "debug");
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.frontend.http.addr, "127.0.0.1:14000");
+                assert_eq!(
-                assert_eq!(ReadableSize::mb(64), opts.frontend.http.body_limit);
+                    opts.fe_opts.http_options.as_ref().unwrap().addr,
+                    "127.0.0.1:14000"
+                );
+                assert_eq!(
+                    ReadableSize::mb(64),
+                    opts.fe_opts.http_options.as_ref().unwrap().body_limit
+                );
 
                 // Should be default value.
-                assert_eq!(opts.frontend.grpc.addr, GrpcOptions::default().addr);
+                assert_eq!(
+                    opts.fe_opts.grpc_options.unwrap().addr,
+                    GrpcOptions::default().addr
+                );
             },
         );
     }
-
-    #[test]
-    fn test_load_default_standalone_options() {
-        let options: StandaloneOptions =
-            Options::load_layered_options(None, "GREPTIMEDB_FRONTEND", None).unwrap();
-        let default_options = StandaloneOptions::default();
-        assert_eq!(options.mode, default_options.mode);
-        assert_eq!(options.enable_telemetry, default_options.enable_telemetry);
-        assert_eq!(options.http, default_options.http);
-        assert_eq!(options.grpc, default_options.grpc);
-        assert_eq!(options.mysql, default_options.mysql);
-        assert_eq!(options.postgres, default_options.postgres);
-        assert_eq!(options.opentsdb, default_options.opentsdb);
-        assert_eq!(options.influxdb, default_options.influxdb);
-        assert_eq!(options.prom_store, default_options.prom_store);
-        assert_eq!(options.wal, default_options.wal);
-        assert_eq!(options.kv_store, default_options.kv_store);
-        assert_eq!(options.procedure, default_options.procedure);
-        assert_eq!(options.logging, default_options.logging);
-        assert_eq!(options.region_engine, default_options.region_engine);
-    }
 }

src/common/base/Cargo.toml

@@ -9,7 +9,6 @@ anymap = "1.0.0-beta.2"
 bitvec = "1.0"
 bytes = { version = "1.1", features = ["serde"] }
 common-error = { workspace = true }
-common-macro = { workspace = true }
 paste = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 snafu.workspace = true

src/common/base/src/buffer.rs

@@ -17,13 +17,11 @@ use std::io::{Read, Write};
 
 use bytes::{Buf, BufMut, BytesMut};
 use common_error::ext::ErrorExt;
-use common_macro::stack_trace_debug;
 use paste::paste;
 use snafu::{ensure, Location, ResultExt, Snafu};
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
     #[snafu(display(
         "Destination buffer overflow, src_len: {}, dst_len: {}",
@@ -39,10 +37,9 @@ pub enum Error {
     #[snafu(display("Buffer underflow"))]
     Underflow { location: Location },
 
-    #[snafu(display("IO operation reach EOF"))]
+    #[snafu(display("IO operation reach EOF, source: {}", source))]
     Eof {
-        #[snafu(source)]
+        source: std::io::Error,
-        error: std::io::Error,
         location: Location,
     },
 }

src/common/base/src/lib.rs

@@ -15,6 +15,7 @@
 pub mod bit_vec;
 pub mod buffer;
 pub mod bytes;
+pub mod paths;
 #[allow(clippy::all)]
 pub mod readable_size;
 
@@ -23,8 +24,6 @@ use std::sync::{Arc, Mutex, MutexGuard};
 
 pub use bit_vec::BitVec;
 
-/// [`Plugins`] is a wrapper of Arc contents.
-/// Make it Cloneable and we can treat it like an Arc struct.
 #[derive(Default, Clone)]
 pub struct Plugins {
     inner: Arc<Mutex<anymap::Map<dyn Any + Send + Sync>>>,

src/common/base/src/paths.rs

@@ -12,14 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_base::Plugins;
+//! Path constants for table engines, cluster states and WAL
-use datanode::config::DatanodeOptions;
+/// All paths relative to data_home(file storage) or root path(S3, OSS etc).
-use datanode::error::Result;
 
-pub async fn setup_datanode_plugins(_opts: &mut DatanodeOptions) -> Result<Plugins> {
+/// WAL dir for local file storage
-    Ok(Plugins::new())
+pub const WAL_DIR: &str = "wal/";
-}
 
-pub async fn start_datanode_plugins(_plugins: Plugins) -> Result<()> {
+/// Data dir for table engines
-    Ok(())
+pub const DATA_DIR: &str = "data/";
-}
+
+/// Cluster state dir
+pub const CLUSTER_DIR: &str = "cluster/";

src/common/catalog/Cargo.toml

@@ -6,7 +6,6 @@ license.workspace = true
 
 [dependencies]
 common-error = { workspace = true }
-common-macro = { workspace = true }
 serde.workspace = true
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }

src/common/catalog/src/consts.rs

@@ -41,7 +41,7 @@ pub fn default_engine() -> &'static str {
     MITO_ENGINE
 }
 
-pub const FILE_ENGINE: &str = "file";
+pub const IMMUTABLE_FILE_ENGINE: &str = "file";
 
 pub const SEMANTIC_TYPE_PRIMARY_KEY: &str = "TAG";
 pub const SEMANTIC_TYPE_FIELD: &str = "FIELD";

src/common/catalog/src/error.rs

@@ -16,24 +16,34 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Invalid full table name: {}", table_name))]
+    #[snafu(display("Invalid catalog info: {}", key))]
-    InvalidFullTableName {
+    InvalidCatalog { key: String, location: Location },
-        table_name: String,
+
+    #[snafu(display("Failed to deserialize catalog entry value: {}", raw))]
+    DeserializeCatalogEntryValue {
+        raw: String,
         location: Location,
+        source: serde_json::error::Error,
+    },
+
+    #[snafu(display("Failed to serialize catalog entry value"))]
+    SerializeCatalogEntryValue {
+        location: Location,
+        source: serde_json::error::Error,
     },
 }
 
 impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         match self {
-            Error::InvalidFullTableName { .. } => StatusCode::Unexpected,
+            Error::InvalidCatalog { .. }
+            | Error::DeserializeCatalogEntryValue { .. }
+            | Error::SerializeCatalogEntryValue { .. } => StatusCode::Unexpected,
         }
     }
 

src/common/config/Cargo.toml

@@ -1,10 +0,0 @@
-[package]
-name = "common-config"
-version.workspace = true
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-common-base.workspace = true
-humantime-serde.workspace = true
-serde.workspace = true

src/common/config/src/lib.rs

@@ -1,70 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::time::Duration;
-
-use common_base::readable_size::ReadableSize;
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(default)]
-pub struct WalConfig {
-    // wal file size in bytes
-    pub file_size: ReadableSize,
-    // wal purge threshold in bytes
-    pub purge_threshold: ReadableSize,
-    // purge interval in seconds
-    #[serde(with = "humantime_serde")]
-    pub purge_interval: Duration,
-    // read batch size
-    pub read_batch_size: usize,
-    // whether to sync log file after every write
-    pub sync_write: bool,
-}
-
-impl Default for WalConfig {
-    fn default() -> Self {
-        Self {
-            file_size: ReadableSize::mb(256),     // log file size 256MB
-            purge_threshold: ReadableSize::gb(4), // purge threshold 4GB
-            purge_interval: Duration::from_secs(600),
-            read_batch_size: 128,
-            sync_write: false,
-        }
-    }
-}
-
-pub fn metadata_store_dir(store_dir: &str) -> String {
-    format!("{store_dir}/metadata")
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(default)]
-pub struct KvStoreConfig {
-    // Kv file size in bytes
-    pub file_size: ReadableSize,
-    // Kv purge threshold in bytes
-    pub purge_threshold: ReadableSize,
-}
-
-impl Default for KvStoreConfig {
-    fn default() -> Self {
-        Self {
-            // log file size 256MB
-            file_size: ReadableSize::mb(256),
-            // purge threshold 4GB
-            purge_threshold: ReadableSize::gb(4),
-        }
-    }
-}

src/common/datasource/Cargo.toml

@@ -18,17 +18,14 @@ async-compression = { version = "0.3", features = [
 async-trait.workspace = true
 bytes = "1.1"
 common-error = { workspace = true }
-common-macro = { workspace = true }
 common-runtime = { workspace = true }
 datafusion.workspace = true
 derive_builder.workspace = true
 futures.workspace = true
-lazy_static.workspace = true
 object-store = { workspace = true }
 orc-rust = "0.2"
 paste = "1.0"
 regex = "1.7"
-serde.workspace = true
 snafu.workspace = true
 strum.workspace = true
 tokio-util.workspace = true

src/common/datasource/src/compression.rs

@@ -20,13 +20,12 @@ use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZstdD
 use async_compression::tokio::write;
 use bytes::Bytes;
 use futures::Stream;
-use serde::{Deserialize, Serialize};
 use strum::EnumIter;
 use tokio::io::{AsyncRead, AsyncWriteExt, BufReader};
 use tokio_util::io::{ReaderStream, StreamReader};
 
 use crate::error::{self, Error, Result};
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumIter, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumIter)]
 pub enum CompressionType {
     /// Gzip-ed file
     Gzip,

src/common/datasource/src/error.rs

@@ -17,14 +17,12 @@ use std::any::Any;
 use arrow_schema::ArrowError;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use datafusion::parquet::errors::ParquetError;
 use snafu::{Location, Snafu};
 use url::ParseError;
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unsupported compression type: {}", compression_type))]
     UnsupportedCompressionType {
@@ -32,11 +30,10 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Unsupported backend protocol: {}, url: {}", protocol, url))]
+    #[snafu(display("Unsupported backend protocol: {}", protocol))]
     UnsupportedBackendProtocol {
         protocol: String,
         location: Location,
-        url: String,
     },
 
     #[snafu(display("Unsupported format protocol: {}", format))]
@@ -45,109 +42,95 @@ pub enum Error {
     #[snafu(display("empty host: {}", url))]
     EmptyHostPath { url: String, location: Location },
 
-    #[snafu(display("Invalid url: {}", url))]
+    #[snafu(display("Invalid url: {}, error :{}", url, source))]
     InvalidUrl {
         url: String,
-        #[snafu(source)]
+        source: ParseError,
-        error: ParseError,
         location: Location,
     },
 
-    #[snafu(display("Failed to build backend"))]
+    #[snafu(display("Failed to build backend, source: {}", source))]
     BuildBackend {
-        #[snafu(source)]
+        source: object_store::Error,
-        error: object_store::Error,
         location: Location,
     },
 
-    #[snafu(display("Failed to build orc reader"))]
+    #[snafu(display("Failed to build orc reader, source: {}", source))]
     OrcReader {
         location: Location,
-        #[snafu(source)]
+        source: orc_rust::error::Error,
-        error: orc_rust::error::Error,
     },
 
-    #[snafu(display("Failed to read object from path: {}", path))]
+    #[snafu(display("Failed to read object from path: {}, source: {}", path, source))]
     ReadObject {
         path: String,
         location: Location,
-        #[snafu(source)]
+        source: object_store::Error,
-        error: object_store::Error,
     },
 
-    #[snafu(display("Failed to write object to path: {}", path))]
+    #[snafu(display("Failed to write object to path: {}, source: {}", path, source))]
     WriteObject {
         path: String,
         location: Location,
-        #[snafu(source)]
+        source: object_store::Error,
-        error: object_store::Error,
     },
 
-    #[snafu(display("Failed to write"))]
+    #[snafu(display("Failed to write: {}", source))]
     AsyncWrite {
-        #[snafu(source)]
+        source: std::io::Error,
-        error: std::io::Error,
         location: Location,
     },
 
-    #[snafu(display("Failed to write record batch"))]
+    #[snafu(display("Failed to write record batch: {}", source))]
     WriteRecordBatch {
         location: Location,
-        #[snafu(source)]
+        source: ArrowError,
-        error: ArrowError,
     },
 
-    #[snafu(display("Failed to encode record batch"))]
+    #[snafu(display("Failed to encode record batch: {}", source))]
     EncodeRecordBatch {
         location: Location,
-        #[snafu(source)]
+        source: ParquetError,
-        error: ParquetError,
     },
 
-    #[snafu(display("Failed to read record batch"))]
+    #[snafu(display("Failed to read record batch: {}", source))]
     ReadRecordBatch {
         location: Location,
-        #[snafu(source)]
+        source: datafusion::error::DataFusionError,
-        error: datafusion::error::DataFusionError,
     },
 
-    #[snafu(display("Failed to read parquet"))]
+    #[snafu(display("Failed to read parquet source: {}", source))]
     ReadParquetSnafu {
         location: Location,
-        #[snafu(source)]
+        source: datafusion::parquet::errors::ParquetError,
-        error: datafusion::parquet::errors::ParquetError,
     },
 
-    #[snafu(display("Failed to convert parquet to schema"))]
+    #[snafu(display("Failed to convert parquet to schema: {}", source))]
     ParquetToSchema {
         location: Location,
-        #[snafu(source)]
+        source: datafusion::parquet::errors::ParquetError,
-        error: datafusion::parquet::errors::ParquetError,
     },
 
-    #[snafu(display("Failed to infer schema from file"))]
+    #[snafu(display("Failed to infer schema from file, source: {}", source))]
     InferSchema {
         location: Location,
-        #[snafu(source)]
+        source: arrow_schema::ArrowError,
-        error: arrow_schema::ArrowError,
     },
 
-    #[snafu(display("Failed to list object in path: {}", path))]
+    #[snafu(display("Failed to list object in path: {}, source: {}", path, source))]
     ListObjects {
         path: String,
         location: Location,
-        #[snafu(source)]
+        source: object_store::Error,
-        error: object_store::Error,
     },
 
     #[snafu(display("Invalid connection: {}", msg))]
     InvalidConnection { msg: String, location: Location },
 
-    #[snafu(display("Failed to join handle"))]
+    #[snafu(display("Failed to join handle: {}", source))]
     JoinHandle {
         location: Location,
-        #[snafu(source)]
+        source: tokio::task::JoinError,
-        error: tokio::task::JoinError,
     },
 
     #[snafu(display("Failed to parse format {} with value: {}", key, value))]
@@ -157,10 +140,9 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to merge schema"))]
+    #[snafu(display("Failed to merge schema: {}", source))]
     MergeSchema {
-        #[snafu(source)]
+        source: arrow_schema::ArrowError,
-        error: arrow_schema::ArrowError,
         location: Location,
     },
 

src/common/datasource/src/lister.rs

@@ -27,7 +27,7 @@ pub enum Source {
 pub struct Lister {
     object_store: ObjectStore,
     source: Source,
-    root: String,
+    path: String,
     regex: Option<Regex>,
 }
 
@@ -35,13 +35,13 @@ impl Lister {
     pub fn new(
         object_store: ObjectStore,
         source: Source,
-        root: String,
+        path: String,
         regex: Option<Regex>,
     ) -> Self {
         Lister {
             object_store,
             source,
-            root,
+            path,
             regex,
         }
     }
@@ -51,9 +51,9 @@ impl Lister {
             Source::Dir => {
                 let streamer = self
                     .object_store
-                    .lister_with("/")
+                    .list(&self.path)
                     .await
-                    .context(error::ListObjectsSnafu { path: &self.root })?;
+                    .context(error::ListObjectsSnafu { path: &self.path })?;
 
                 streamer
                     .try_filter(|f| {
@@ -66,25 +66,17 @@ impl Lister {
                     })
                     .try_collect::<Vec<_>>()
                     .await
-                    .context(error::ListObjectsSnafu { path: &self.root })
+                    .context(error::ListObjectsSnafu { path: &self.path })
             }
             Source::Filename(filename) => {
                 // make sure this file exists
-                let _ = self.object_store.stat(filename).await.with_context(|_| {
+                let file_full_path = format!("{}{}", self.path, filename);
+                let _ = self.object_store.stat(&file_full_path).await.context(
                     error::ListObjectsSnafu {
-                        path: format!("{}{}", &self.root, filename),
+                        path: &file_full_path,
-                    }
+                    },
-                })?;
+                )?;
-
+                Ok(vec![Entry::new(&file_full_path)])
-                Ok(self
-                    .object_store
-                    .list_with("/")
-                    .await
-                    .context(error::ListObjectsSnafu { path: &self.root })?
-                    .into_iter()
-                    .find(|f| f.name() == filename)
-                    .map(|f| vec![f])
-                    .unwrap_or_default())
             }
         }
     }

src/common/datasource/src/object_store.rs

@@ -16,29 +16,19 @@ pub mod fs;
 pub mod s3;
 use std::collections::HashMap;
 
-use lazy_static::lazy_static;
 use object_store::ObjectStore;
-use regex::Regex;
 use snafu::{OptionExt, ResultExt};
 use url::{ParseError, Url};
 
 use self::fs::build_fs_backend;
 use self::s3::build_s3_backend;
 use crate::error::{self, Result};
-use crate::util::find_dir_and_filename;
 
 pub const FS_SCHEMA: &str = "FS";
 pub const S3_SCHEMA: &str = "S3";
 
-/// Returns `(schema, Option<host>, path)`
+/// Returns (schema, Option<host>, path)
 pub fn parse_url(url: &str) -> Result<(String, Option<String>, String)> {
-    #[cfg(windows)]
-    {
-        // On Windows, the url may start with `C:/`.
-        if let Some(_) = handle_windows_path(url) {
-            return Ok((FS_SCHEMA.to_string(), None, url.to_string()));
-        }
-    }
     let parsed_url = Url::parse(url);
     match parsed_url {
         Ok(url) => Ok((
@@ -54,47 +44,17 @@ pub fn parse_url(url: &str) -> Result<(String, Option<String>, String)> {
 }
 
 pub fn build_backend(url: &str, connection: &HashMap<String, String>) -> Result<ObjectStore> {
-    let (schema, host, path) = parse_url(url)?;
+    let (schema, host, _path) = parse_url(url)?;
-    let (root, _) = find_dir_and_filename(&path);
 
     match schema.to_uppercase().as_str() {
         S3_SCHEMA => {
             let host = host.context(error::EmptyHostPathSnafu {
                 url: url.to_string(),
             })?;
-            Ok(build_s3_backend(&host, &root, connection)?)
+            Ok(build_s3_backend(&host, "/", connection)?)
         }
-        FS_SCHEMA => Ok(build_fs_backend(&root)?),
+        FS_SCHEMA => Ok(build_fs_backend("/")?),
 
-        _ => error::UnsupportedBackendProtocolSnafu {
+        _ => error::UnsupportedBackendProtocolSnafu { protocol: schema }.fail(),
-            protocol: schema,
-            url,
-        }
-        .fail(),
-    }
-}
-
-lazy_static! {
-    static ref DISK_SYMBOL_PATTERN: Regex = Regex::new("^([A-Za-z]:/)").unwrap();
-}
-
-pub fn handle_windows_path(url: &str) -> Option<String> {
-    DISK_SYMBOL_PATTERN
-        .captures(url)
-        .map(|captures| captures[0].to_string())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::handle_windows_path;
-
-    #[test]
-    fn test_handle_windows_path() {
-        assert_eq!(
-            handle_windows_path("C:/to/path/file"),
-            Some("C:/".to_string())
-        );
-        assert_eq!(handle_windows_path("https://google.com"), None);
-        assert_eq!(handle_windows_path("s3://bucket/path/to"), None);
     }
 }

src/common/datasource/src/object_store/s3.rs

@@ -27,15 +27,6 @@ const SESSION_TOKEN: &str = "session_token";
 const REGION: &str = "region";
 const ENABLE_VIRTUAL_HOST_STYLE: &str = "enable_virtual_host_style";
 
-pub fn is_supported_in_s3(key: &str) -> bool {
-    key == ENDPOINT
-        || key == ACCESS_KEY_ID
-        || key == SECRET_ACCESS_KEY
-        || key == SESSION_TOKEN
-        || key == REGION
-        || key == ENABLE_VIRTUAL_HOST_STYLE
-}
-
 pub fn build_s3_backend(
     host: &str,
     path: &str,
@@ -84,18 +75,3 @@ pub fn build_s3_backend(
         .context(error::BuildBackendSnafu)?
         .finish())
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    #[test]
-    fn test_is_supported_in_s3() {
-        assert!(is_supported_in_s3(ENDPOINT));
-        assert!(is_supported_in_s3(ACCESS_KEY_ID));
-        assert!(is_supported_in_s3(SECRET_ACCESS_KEY));
-        assert!(is_supported_in_s3(SESSION_TOKEN));
-        assert!(is_supported_in_s3(REGION));
-        assert!(is_supported_in_s3(ENABLE_VIRTUAL_HOST_STYLE));
-        assert!(!is_supported_in_s3("foo"))
-    }
-}

src/common/error/src/ext.rs

@@ -13,12 +13,11 @@
 // limitations under the License.
 
 use std::any::Any;
-use std::sync::Arc;
 
 use crate::status_code::StatusCode;
 
 /// Extension to [`Error`](std::error::Error) in std.
-pub trait ErrorExt: StackError {
+pub trait ErrorExt: std::error::Error {
     /// Map this error to [StatusCode].
     fn status_code(&self) -> StatusCode {
         StatusCode::Unknown
@@ -34,63 +33,6 @@ pub trait ErrorExt: StackError {
     /// Returns the error as [Any](std::any::Any) so that it can be
     /// downcast to a specific implementation.
     fn as_any(&self) -> &dyn Any;
-
-    fn output_msg(&self) -> String
-    where
-        Self: Sized,
-    {
-        let error = self.last();
-        if let Some(external_error) = error.source() {
-            let external_root = external_error.sources().last().unwrap();
-
-            if error.to_string().is_empty() {
-                format!("{external_root}")
-            } else {
-                format!("{error}: {external_root}")
-            }
-        } else {
-            format!("{error}")
-        }
-    }
-}
-
-pub trait StackError: std::error::Error {
-    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>);
-
-    fn next(&self) -> Option<&dyn StackError>;
-
-    fn last(&self) -> &dyn StackError
-    where
-        Self: Sized,
-    {
-        let Some(mut result) = self.next() else {
-            return self;
-        };
-        while let Some(err) = result.next() {
-            result = err;
-        }
-        result
-    }
-}
-
-impl<T: ?Sized + StackError> StackError for Arc<T> {
-    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
-        self.as_ref().debug_fmt(layer, buf)
-    }
-
-    fn next(&self) -> Option<&dyn StackError> {
-        self.as_ref().next()
-    }
-}
-
-impl<T: StackError> StackError for Box<T> {
-    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
-        self.as_ref().debug_fmt(layer, buf)
-    }
-
-    fn next(&self) -> Option<&dyn StackError> {
-        self.as_ref().next()
-    }
 }
 
 /// An opaque boxed error based on errors that implement [ErrorExt] trait.
@@ -148,16 +90,6 @@ impl crate::snafu::ErrorCompat for BoxedError {
     }
 }
 
-impl StackError for BoxedError {
-    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
-        self.inner.debug_fmt(layer, buf)
-    }
-
-    fn next(&self) -> Option<&dyn StackError> {
-        self.inner.next()
-    }
-}
-
 /// Error type with plain error message
 #[derive(Debug)]
 pub struct PlainError {
@@ -196,13 +128,3 @@ impl crate::ext::ErrorExt for PlainError {
         self as _
     }
 }
-
-impl StackError for PlainError {
-    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
-        buf.push(format!("{}: {}", layer, self.msg))
-    }
-
-    fn next(&self) -> Option<&dyn StackError> {
-        None
-    }
-}

src/common/error/src/format.rs

@@ -50,7 +50,6 @@ mod tests {
     use snafu::{GenerateImplicitData, Location};
 
     use super::*;
-    use crate::ext::StackError;
 
     #[derive(Debug, Snafu)]
     #[snafu(display("This is a leaf error"))]
@@ -66,14 +65,6 @@ mod tests {
         }
     }
 
-    impl StackError for Leaf {
-        fn debug_fmt(&self, _: usize, _: &mut Vec<String>) {}
-
-        fn next(&self) -> Option<&dyn StackError> {
-            None
-        }
-    }
-
     #[derive(Debug, Snafu)]
     #[snafu(display("This is a leaf with location"))]
     struct LeafWithLocation {
@@ -90,14 +81,6 @@ mod tests {
         }
     }
 
-    impl StackError for LeafWithLocation {
-        fn debug_fmt(&self, _: usize, _: &mut Vec<String>) {}
-
-        fn next(&self) -> Option<&dyn StackError> {
-            None
-        }
-    }
-
     #[derive(Debug, Snafu)]
     #[snafu(display("Internal error"))]
     struct Internal {
@@ -116,17 +99,6 @@ mod tests {
         }
     }
 
-    impl StackError for Internal {
-        fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
-            buf.push(format!("{}: Internal error, at {}", layer, self.location));
-            self.source.debug_fmt(layer + 1, buf);
-        }
-
-        fn next(&self) -> Option<&dyn StackError> {
-            Some(&self.source)
-        }
-    }
-
     #[test]
     fn test_debug_format() {
         let err = Leaf;

src/common/error/src/lib.rs

@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-#![feature(error_iter)]
 
 pub mod ext;
 pub mod format;

src/common/error/src/mock.rs

@@ -19,7 +19,7 @@ use std::fmt;
 
 use snafu::Location;
 
-use crate::ext::{ErrorExt, StackError};
+use crate::ext::ErrorExt;
 use crate::status_code::StatusCode;
 
 /// A mock error mainly for test.
@@ -69,11 +69,3 @@ impl ErrorExt for MockError {
         self
     }
 }
-
-impl StackError for MockError {
-    fn debug_fmt(&self, _: usize, _: &mut Vec<String>) {}
-
-    fn next(&self) -> Option<&dyn StackError> {
-        None
-    }
-}

src/common/error/src/status_code.rs

@@ -56,9 +56,6 @@ pub enum StatusCode {
     TableColumnNotFound = 4002,
     TableColumnExists = 4003,
     DatabaseNotFound = 4004,
-    RegionNotFound = 4005,
-    RegionAlreadyExists = 4006,
-    RegionReadonly = 4007,
     // ====== End of catalog related status code =======
 
     // ====== Begin of storage related status code =====
@@ -116,9 +113,6 @@ impl StatusCode {
             | StatusCode::EngineExecuteQuery
             | StatusCode::TableAlreadyExists
             | StatusCode::TableNotFound
-            | StatusCode::RegionNotFound
-            | StatusCode::RegionAlreadyExists
-            | StatusCode::RegionReadonly
             | StatusCode::TableColumnNotFound
             | StatusCode::TableColumnExists
             | StatusCode::DatabaseNotFound
@@ -151,9 +145,6 @@ impl StatusCode {
             | StatusCode::InvalidSyntax
             | StatusCode::TableAlreadyExists
             | StatusCode::TableNotFound
-            | StatusCode::RegionNotFound
-            | StatusCode::RegionAlreadyExists
-            | StatusCode::RegionReadonly
             | StatusCode::TableColumnNotFound
             | StatusCode::TableColumnExists
             | StatusCode::DatabaseNotFound
@@ -182,11 +173,6 @@ impl StatusCode {
             v if v == StatusCode::EngineExecuteQuery as u32 => Some(StatusCode::EngineExecuteQuery),
             v if v == StatusCode::TableAlreadyExists as u32 => Some(StatusCode::TableAlreadyExists),
             v if v == StatusCode::TableNotFound as u32 => Some(StatusCode::TableNotFound),
-            v if v == StatusCode::RegionNotFound as u32 => Some(StatusCode::RegionNotFound),
-            v if v == StatusCode::RegionAlreadyExists as u32 => {
-                Some(StatusCode::RegionAlreadyExists)
-            }
-            v if v == StatusCode::RegionReadonly as u32 => Some(StatusCode::RegionReadonly),
             v if v == StatusCode::TableColumnNotFound as u32 => {
                 Some(StatusCode::TableColumnNotFound)
             }

src/common/function-macro/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "common-macro"
+name = "common-function-macro"
 version.workspace = true
 edition.workspace = true
 license.workspace = true
@@ -13,15 +13,6 @@ common-telemetry = { workspace = true }
 proc-macro2 = "1.0.66"
 quote = "1.0"
 syn = "1.0"
-syn2 = { version = "2.0", package = "syn", features = [
-    "derive",
-    "parsing",
-    "printing",
-    "clone-impls",
-    "proc-macro",
-    "extra-traits",
-    "full",
-] }
 
 [dev-dependencies]
 arc-swap = "1.0"

src/common/function-macro/src/lib.rs

@@ -0,0 +1,226 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod range_fn;
+
+use proc_macro::TokenStream;
+use quote::{quote, quote_spanned, ToTokens};
+use range_fn::process_range_fn;
+use syn::parse::Parser;
+use syn::spanned::Spanned;
+use syn::{
+    parse_macro_input, AttributeArgs, DeriveInput, ItemFn, ItemStruct, Lit, Meta, NestedMeta,
+};
+
+/// Make struct implemented trait [AggrFuncTypeStore], which is necessary when writing UDAF.
+/// This derive macro is expect to be used along with attribute macro [as_aggr_func_creator].
+#[proc_macro_derive(AggrFuncTypeStore)]
+pub fn aggr_func_type_store_derive(input: TokenStream) -> TokenStream {
+    let ast = parse_macro_input!(input as DeriveInput);
+    impl_aggr_func_type_store(&ast)
+}
+
+fn impl_aggr_func_type_store(ast: &DeriveInput) -> TokenStream {
+    let name = &ast.ident;
+    let gen = quote! {
+        use common_query::logical_plan::accumulator::AggrFuncTypeStore;
+        use common_query::error::{InvalidInputStateSnafu, Error as QueryError};
+        use datatypes::prelude::ConcreteDataType;
+
+        impl AggrFuncTypeStore for #name {
+            fn input_types(&self) -> std::result::Result<Vec<ConcreteDataType>, QueryError> {
+                let input_types = self.input_types.load();
+                snafu::ensure!(input_types.is_some(), InvalidInputStateSnafu);
+                Ok(input_types.as_ref().unwrap().as_ref().clone())
+            }
+
+            fn set_input_types(&self, input_types: Vec<ConcreteDataType>) -> std::result::Result<(), QueryError> {
+                let old = self.input_types.swap(Some(std::sync::Arc::new(input_types.clone())));
+                if let Some(old) = old {
+                    snafu::ensure!(old.len() == input_types.len(), InvalidInputStateSnafu);
+                    for (x, y) in old.iter().zip(input_types.iter()) {
+                        snafu::ensure!(x == y, InvalidInputStateSnafu);
+                    }
+                }
+                Ok(())
+            }
+        }
+    };
+    gen.into()
+}
+
+/// A struct can be used as a creator for aggregate function if it has been annotated with this
+/// attribute first. This attribute add a necessary field which is intended to store the input
+/// data's types to the struct.
+/// This attribute is expected to be used along with derive macro [AggrFuncTypeStore].
+#[proc_macro_attribute]
+pub fn as_aggr_func_creator(_args: TokenStream, input: TokenStream) -> TokenStream {
+    let mut item_struct = parse_macro_input!(input as ItemStruct);
+    if let syn::Fields::Named(ref mut fields) = item_struct.fields {
+        let result = syn::Field::parse_named.parse2(quote! {
+            input_types: arc_swap::ArcSwapOption<Vec<ConcreteDataType>>
+        });
+        match result {
+            Ok(field) => fields.named.push(field),
+            Err(e) => return e.into_compile_error().into(),
+        }
+    } else {
+        return quote_spanned!(
+            item_struct.fields.span() => compile_error!(
+                "This attribute macro needs to add fields to the its annotated struct, \
+                so the struct must have \"{}\".")
+        )
+        .into();
+    }
+    quote! {
+        #item_struct
+    }
+    .into()
+}
+
+/// Attribute macro to convert an arithimetic function to a range function. The annotated function
+/// should accept servaral arrays as input and return a single value as output. This procedure
+/// macro can works on any number of input parameters. Return type can be either primitive type
+/// or wrapped in `Option`.
+///
+/// # Example
+/// Take `count_over_time()` in PromQL as an example:
+/// ```rust, ignore
+/// /// The count of all values in the specified interval.
+/// #[range_fn(
+///     name = "CountOverTime",
+///     ret = "Float64Array",
+///     display_name = "prom_count_over_time"
+/// )]
+/// pub fn count_over_time(_: &TimestampMillisecondArray, values: &Float64Array) -> f64 {
+///      values.len() as f64
+/// }
+/// ```
+///
+/// # Arguments
+/// - `name`: The name of the generated [ScalarUDF] struct.
+/// - `ret`: The return type of the generated UDF function.
+/// - `display_name`: The display name of the generated UDF function.
+#[proc_macro_attribute]
+pub fn range_fn(args: TokenStream, input: TokenStream) -> TokenStream {
+    process_range_fn(args, input)
+}
+
+/// Attribute macro to print the caller to the annotated function.
+/// The caller is printed as its filename and the call site line number.
+///
+/// This macro works like this: inject the tracking codes as the first statement to the annotated
+/// function body. The tracking codes use [backtrace-rs](https://crates.io/crates/backtrace) to get
+/// the callers. So you must dependent on the `backtrace-rs` crate.
+///
+/// # Arguments
+/// - `depth`: The max depth of call stack to print. Optional, defaults to 1.
+///
+/// # Example
+/// ```rust, ignore
+///
+/// #[print_caller(depth = 3)]
+/// fn foo() {}
+/// ```
+#[proc_macro_attribute]
+pub fn print_caller(args: TokenStream, input: TokenStream) -> TokenStream {
+    let mut depth = 1;
+
+    let args = parse_macro_input!(args as AttributeArgs);
+    for meta in args.iter() {
+        if let NestedMeta::Meta(Meta::NameValue(name_value)) = meta {
+            let ident = name_value
+                .path
+                .get_ident()
+                .expect("Expected an ident!")
+                .to_string();
+            if ident == "depth" {
+                let Lit::Int(i) = &name_value.lit else {
+                    panic!("Expected 'depth' to be a valid int!")
+                };
+                depth = i.base10_parse::<usize>().expect("Invalid 'depth' value");
+                break;
+            }
+        }
+    }
+
+    let tokens: TokenStream = quote! {
+        {
+            let curr_file = file!();
+
+            let bt = backtrace::Backtrace::new();
+            let call_stack = bt
+                .frames()
+                .iter()
+                .skip_while(|f| {
+                    !f.symbols().iter().any(|s| {
+                        s.filename()
+                            .map(|p| p.ends_with(curr_file))
+                            .unwrap_or(false)
+                    })
+                })
+                .skip(1)
+                .take(#depth);
+
+            let call_stack = call_stack
+                .map(|f| {
+                    f.symbols()
+                        .iter()
+                        .map(|s| {
+                            let filename = s
+                                .filename()
+                                .map(|p| format!("{:?}", p))
+                                .unwrap_or_else(|| "unknown".to_string());
+
+                            let lineno = s
+                                .lineno()
+                                .map(|l| format!("{}", l))
+                                .unwrap_or_else(|| "unknown".to_string());
+
+                            format!("filename: {}, lineno: {}", filename, lineno)
+                        })
+                        .collect::<Vec<String>>()
+                        .join(", ")
+                })
+                .collect::<Vec<_>>();
+
+            match call_stack.len() {
+                0 => common_telemetry::info!("unable to find call stack"),
+                1 => common_telemetry::info!("caller: {}", call_stack[0]),
+                _ => {
+                    let mut s = String::new();
+                    s.push_str("[\n");
+                    for e in call_stack {
+                        s.push_str("\t");
+                        s.push_str(&e);
+                        s.push_str("\n");
+                    }
+                    s.push_str("]");
+                    common_telemetry::info!("call stack: {}", s)
+                }
+            }
+        }
+    }
+    .into();
+
+    let stmt = match syn::parse(tokens) {
+        Ok(stmt) => stmt,
+        Err(e) => return e.into_compile_error().into(),
+    };
+
+    let mut item = parse_macro_input!(input as ItemFn);
+    item.block.stmts.insert(0, stmt);
+
+    item.into_token_stream().into()
+}

src/common/function-macro/tests/test_derive.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_function_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use static_assertions::{assert_fields, assert_impl_all};
 
 #[as_aggr_func_creator]

src/common/function/Cargo.toml

@@ -8,7 +8,7 @@ license.workspace = true
 arc-swap = "1.0"
 chrono-tz = "0.6"
 common-error = { workspace = true }
-common-macro = { workspace = true }
+common-function-macro = { workspace = true }
 common-query = { workspace = true }
 common-time = { workspace = true }
 datafusion.workspace = true

src/common/function/src/scalars/aggregate/argmax.rs

@@ -15,7 +15,7 @@
 use std::cmp::Ordering;
 use std::sync::Arc;
 
-use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_function_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{BadAccumulatorImplSnafu, CreateAccumulatorSnafu, Result};
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::*;

src/common/function/src/scalars/aggregate/argmin.rs

@@ -15,7 +15,7 @@
 use std::cmp::Ordering;
 use std::sync::Arc;
 
-use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_function_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{BadAccumulatorImplSnafu, CreateAccumulatorSnafu, Result};
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::*;

src/common/function/src/scalars/aggregate/diff.rs

@@ -15,7 +15,7 @@
 use std::marker::PhantomData;
 use std::sync::Arc;
 
-use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_function_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
     CreateAccumulatorSnafu, DowncastVectorSnafu, FromScalarValueSnafu, Result,
 };

src/common/function/src/scalars/aggregate/mean.rs

@@ -15,7 +15,7 @@
 use std::marker::PhantomData;
 use std::sync::Arc;
 
-use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_function_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
     BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu, Result,
 };

src/common/function/src/scalars/aggregate/percentile.rs

@@ -16,7 +16,7 @@ use std::cmp::Reverse;
 use std::collections::BinaryHeap;
 use std::sync::Arc;
 
-use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_function_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
     self, BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu,
     FromScalarValueSnafu, InvalidInputColSnafu, Result,

src/common/function/src/scalars/aggregate/polyval.rs

@@ -15,7 +15,7 @@
 use std::marker::PhantomData;
 use std::sync::Arc;
 
-use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_function_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
     self, BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu,
     FromScalarValueSnafu, InvalidInputColSnafu, Result,

src/common/function/src/scalars/aggregate/scipy_stats_norm_cdf.rs

@@ -14,7 +14,7 @@
 
 use std::sync::Arc;
 
-use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_function_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
     self, BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu,
     FromScalarValueSnafu, GenerateFunctionSnafu, InvalidInputColSnafu, Result,

src/common/function/src/scalars/aggregate/scipy_stats_norm_pdf.rs

@@ -14,7 +14,7 @@
 
 use std::sync::Arc;
 
-use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_function_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
     self, BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu,
     FromScalarValueSnafu, GenerateFunctionSnafu, InvalidInputColSnafu, Result,

src/common/function/src/scalars/numpy/interp.rs

@@ -24,7 +24,7 @@ use datatypes::vectors::{Float64Vector, Vector, VectorRef};
 use datatypes::with_match_primitive_type_id;
 use snafu::{ensure, ResultExt};
 
-/// search the biggest number that smaller than x in xp
+/* search the biggest number that smaller than x in xp */
 fn linear_search_ascending_vector(x: Value, xp: &Float64Vector) -> usize {
     for i in 0..xp.len() {
         if x < xp.get(i) {
@@ -34,7 +34,7 @@ fn linear_search_ascending_vector(x: Value, xp: &Float64Vector) -> usize {
     xp.len() - 1
 }
 
-/// search the biggest number that smaller than x in xp
+/* search the biggest number that smaller than x in xp */
 fn binary_search_ascending_vector(key: Value, xp: &Float64Vector) -> usize {
     let mut left = 0;
     let mut right = xp.len();
@@ -67,8 +67,7 @@ fn concrete_type_to_primitive_vector(arg: &VectorRef) -> Result<Float64Vector> {
     })
 }
 
-/// One-dimensional linear interpolation for monotonically increasing sample points. Refers to
+/// https://github.com/numpy/numpy/blob/b101756ac02e390d605b2febcded30a1da50cc2c/numpy/core/src/multiarray/compiled_base.c#L491
-/// <https://github.com/numpy/numpy/blob/b101756ac02e390d605b2febcded30a1da50cc2c/numpy/core/src/multiarray/compiled_base.c#L491>
 #[allow(unused)]
 pub fn interp(args: &[VectorRef]) -> Result<VectorRef> {
     let mut left = None;

src/common/function/src/scalars/timestamp.rs

@@ -12,10 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 use std::sync::Arc;
-mod greatest;
 mod to_unixtime;
 
-use greatest::GreatestFunction;
 use to_unixtime::ToUnixtimeFunction;
 
 use crate::scalars::function_registry::FunctionRegistry;
@@ -25,6 +23,5 @@ pub(crate) struct TimestampFunction;
 impl TimestampFunction {
     pub fn register(registry: &FunctionRegistry) {
         registry.register(Arc::new(ToUnixtimeFunction));
-        registry.register(Arc::new(GreatestFunction));
     }
 }

src/common/function/src/scalars/timestamp/greatest.rs

@@ -1,175 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::fmt::{self};
-
-use common_query::error::{
-    self, ArrowComputeSnafu, InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu,
-};
-use common_query::prelude::{Signature, Volatility};
-use datatypes::arrow::array::AsArray;
-use datatypes::arrow::compute::cast;
-use datatypes::arrow::compute::kernels::comparison::gt_dyn;
-use datatypes::arrow::compute::kernels::zip;
-use datatypes::arrow::datatypes::{DataType as ArrowDataType, Date32Type};
-use datatypes::prelude::ConcreteDataType;
-use datatypes::vectors::{Helper, VectorRef};
-use snafu::{ensure, ResultExt};
-
-use crate::scalars::function::{Function, FunctionContext};
-
-#[derive(Clone, Debug, Default)]
-pub struct GreatestFunction;
-
-const NAME: &str = "greatest";
-
-impl Function for GreatestFunction {
-    fn name(&self) -> &str {
-        NAME
-    }
-
-    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
-        Ok(ConcreteDataType::date_datatype())
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::uniform(
-            2,
-            vec![
-                ConcreteDataType::string_datatype(),
-                ConcreteDataType::date_datatype(),
-            ],
-            Volatility::Immutable,
-        )
-    }
-
-    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 2,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect exactly two, have: {}",
-                    columns.len()
-                ),
-            }
-        );
-        match columns[0].data_type() {
-            ConcreteDataType::String(_) => {
-                let column1 = cast(&columns[0].to_arrow_array(), &ArrowDataType::Date32)
-                    .context(ArrowComputeSnafu)?;
-                let column1 = column1.as_primitive::<Date32Type>();
-                let column2 = cast(&columns[1].to_arrow_array(), &ArrowDataType::Date32)
-                    .context(ArrowComputeSnafu)?;
-                let column2 = column2.as_primitive::<Date32Type>();
-                let boolean_array = gt_dyn(&column1, &column2).context(ArrowComputeSnafu)?;
-                let result =
-                    zip::zip(&boolean_array, &column1, &column2).context(ArrowComputeSnafu)?;
-                Ok(Helper::try_into_vector(&result).context(error::FromArrowArraySnafu)?)
-            }
-            ConcreteDataType::Date(_) => {
-                let column1 = columns[0].to_arrow_array();
-                let column1 = column1.as_primitive::<Date32Type>();
-                let column2 = columns[1].to_arrow_array();
-                let column2 = column2.as_primitive::<Date32Type>();
-                let boolean_array = gt_dyn(&column1, &column2).context(ArrowComputeSnafu)?;
-                let result =
-                    zip::zip(&boolean_array, &column1, &column2).context(ArrowComputeSnafu)?;
-                Ok(Helper::try_into_vector(&result).context(error::FromArrowArraySnafu)?)
-            }
-            _ => UnsupportedInputDataTypeSnafu {
-                function: NAME,
-                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
-            }
-            .fail(),
-        }
-    }
-}
-
-impl fmt::Display for GreatestFunction {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "GREATEST")
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::str::FromStr;
-    use std::sync::Arc;
-
-    use common_time::Date;
-    use datatypes::prelude::ConcreteDataType;
-    use datatypes::types::DateType;
-    use datatypes::value::Value;
-    use datatypes::vectors::{DateVector, StringVector, Vector};
-
-    use super::GreatestFunction;
-    use crate::scalars::function::FunctionContext;
-    use crate::scalars::Function;
-
-    #[test]
-    fn test_greatest_takes_string_vector() {
-        let function = GreatestFunction;
-        assert_eq!(
-            function.return_type(&[]).unwrap(),
-            ConcreteDataType::Date(DateType)
-        );
-        let columns = vec![
-            Arc::new(StringVector::from(vec![
-                "1970-01-01".to_string(),
-                "2012-12-23".to_string(),
-            ])) as _,
-            Arc::new(StringVector::from(vec![
-                "2001-02-01".to_string(),
-                "1999-01-01".to_string(),
-            ])) as _,
-        ];
-
-        let result = function.eval(FunctionContext::default(), &columns).unwrap();
-        let result = result.as_any().downcast_ref::<DateVector>().unwrap();
-        assert_eq!(result.len(), 2);
-        assert_eq!(
-            result.get(0),
-            Value::Date(Date::from_str("2001-02-01").unwrap())
-        );
-        assert_eq!(
-            result.get(1),
-            Value::Date(Date::from_str("2012-12-23").unwrap())
-        );
-    }
-
-    #[test]
-    fn test_greatest_takes_date_vector() {
-        let function = GreatestFunction;
-        assert_eq!(
-            function.return_type(&[]).unwrap(),
-            ConcreteDataType::Date(DateType)
-        );
-        let columns = vec![
-            Arc::new(DateVector::from_slice(vec![-1, 2])) as _,
-            Arc::new(DateVector::from_slice(vec![0, 1])) as _,
-        ];
-
-        let result = function.eval(FunctionContext::default(), &columns).unwrap();
-        let result = result.as_any().downcast_ref::<DateVector>().unwrap();
-        assert_eq!(result.len(), 2);
-        assert_eq!(
-            result.get(0),
-            Value::Date(Date::from_str("1970-01-01").unwrap())
-        );
-        assert_eq!(
-            result.get(1),
-            Value::Date(Date::from_str("1970-01-03").unwrap())
-        );
-    }
-}