Revert "build(deps): upgrade opendal to 0.46 (#4037)"

This reverts commit f9db5ff0d6.

.github/actions/build-greptime-binary/action.yml

@@ -24,14 +24,6 @@ inputs:
     description: Build android artifacts
     required: false
     default: 'false'
-  image-namespace:
-    description: Image Namespace
-    required: false
-    default: 'greptime'
-  image-registry:
-    description: Image Registry
-    required: false
-    default: 'docker.io'
 runs:
   using: composite
   steps:
@@ -43,9 +35,7 @@ runs:
         make build-by-dev-builder \
           CARGO_PROFILE=${{ inputs.cargo-profile }} \
           FEATURES=${{ inputs.features }} \
-          BASE_IMAGE=${{ inputs.base-image }} \
+          BASE_IMAGE=${{ inputs.base-image }}
-          IMAGE_NAMESPACE=${{ inputs.image-namespace }} \
-          IMAGE_REGISTRY=${{ inputs.image-registry }}
 
     - name: Upload artifacts
       uses: ./.github/actions/upload-artifacts
@@ -63,9 +53,7 @@ runs:
       shell: bash
       if: ${{ inputs.build-android-artifacts == 'true' }}
       run: |
-        cd ${{ inputs.working-dir }} && make strip-android-bin \
+        cd ${{ inputs.working-dir }} && make strip-android-bin
-          IMAGE_NAMESPACE=${{ inputs.image-namespace }} \
-          IMAGE_REGISTRY=${{ inputs.image-registry }}
 
     - name: Upload android artifacts
       uses: ./.github/actions/upload-artifacts

.github/actions/build-linux-artifacts/action.yml

@@ -30,9 +30,7 @@ runs:
       # NOTE: If the BUILD_JOBS > 4, it's always OOM in EC2 instance.
       run: |
         cd ${{ inputs.working-dir }} && \
-        make run-it-in-container BUILD_JOBS=4 \
+        make run-it-in-container BUILD_JOBS=4
-        IMAGE_NAMESPACE=i8k6a5e1/greptime \
-        IMAGE_REGISTRY=public.ecr.aws
 
     - name: Upload sqlness logs
       if: ${{ failure() && inputs.disable-run-tests == 'false' }} # Only upload logs when the integration tests failed.
@@ -51,8 +49,6 @@ runs:
         artifacts-dir: greptime-linux-${{ inputs.arch }}-pyo3-${{ inputs.version }}
         version: ${{ inputs.version }}
         working-dir: ${{ inputs.working-dir }}
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime        
 
     - name: Build greptime without pyo3
       if: ${{ inputs.dev-mode == 'false' }}
@@ -64,8 +60,6 @@ runs:
         artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
         version: ${{ inputs.version }}
         working-dir: ${{ inputs.working-dir }}
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime
 
     - name: Clean up the target directory # Clean up the target directory for the centos7 base image, or it will still use the objects of last build.
       shell: bash
@@ -82,8 +76,6 @@ runs:
         artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
         version: ${{ inputs.version }}
         working-dir: ${{ inputs.working-dir }}
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime
 
     - name: Build greptime on android base image
       uses: ./.github/actions/build-greptime-binary
@@ -94,5 +86,3 @@ runs:
         version: ${{ inputs.version }}
         working-dir: ${{ inputs.working-dir }}
         build-android-artifacts: true
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime

.github/actions/build-macos-artifacts/action.yml

@@ -59,15 +59,9 @@ runs:
       if: ${{ inputs.disable-run-tests == 'false' }}
       uses: taiki-e/install-action@nextest
 
-    # Get proper backtraces in mac Sonoma. Currently there's an issue with the new
-    # linker that prevents backtraces from getting printed correctly.
-    #
-    # <https://github.com/rust-lang/rust/issues/113783> 
     - name: Run integration tests
       if: ${{ inputs.disable-run-tests == 'false' }}
       shell: bash
-      env: 
-        CARGO_BUILD_RUSTFLAGS: "-Clink-arg=-Wl,-ld_classic"
       run: |
         make test sqlness-test
 
@@ -81,8 +75,6 @@ runs:
 
     - name: Build greptime binary
       shell: bash
-      env: 
-        CARGO_BUILD_RUSTFLAGS: "-Clink-arg=-Wl,-ld_classic"
       run: |
         make build \
         CARGO_PROFILE=${{ inputs.cargo-profile }} \

.github/actions/setup-greptimedb-cluster/action.yml

@@ -22,9 +22,6 @@ inputs:
   etcd-endpoints:
     default: "etcd.etcd-cluster.svc.cluster.local:2379"
     description: "Etcd endpoints"
-  values-filename:
-    default: "with-minio.yaml"
-    
 
 runs:
   using: composite
@@ -60,7 +57,6 @@ runs:
         greptime/greptimedb-cluster \
         --create-namespace \
         -n my-greptimedb \
-        --values ./.github/actions/setup-greptimedb-cluster/${{ inputs.values-filename }} \
         --wait \
         --wait-for-jobs
   - name: Wait for GreptimeDB

.github/actions/setup-greptimedb-cluster/with-disk.yaml

@@ -1,18 +0,0 @@
-meta:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-datanode:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-frontend:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8

.github/actions/setup-greptimedb-cluster/with-minio-and-cache.yaml

@@ -1,38 +0,0 @@
-meta:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-    
-    [datanode]
-    [datanode.client]
-    timeout = "60s"
-datanode:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-    
-    [storage]
-    cache_path = "/data/greptimedb/s3cache"
-    cache_capacity = "256MB"
-frontend:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-
-    [meta_client]
-    ddl_timeout = "60s"
-objectStorage:
-  s3:
-    bucket: default
-    region: us-west-2
-    root: test-root
-    endpoint: http://minio.minio.svc.cluster.local 
-  credentials:
-    accessKeyId: rootuser
-    secretAccessKey: rootpass123

.github/actions/setup-greptimedb-cluster/with-minio.yaml

@@ -1,34 +0,0 @@
-meta:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-    
-    [datanode]
-    [datanode.client]
-    timeout = "60s"
-datanode:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-frontend:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-
-    [meta_client]
-    ddl_timeout = "60s"
-objectStorage:
-  s3:
-    bucket: default
-    region: us-west-2
-    root: test-root
-    endpoint: http://minio.minio.svc.cluster.local 
-  credentials:
-    accessKeyId: rootuser
-    secretAccessKey: rootpass123

.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml

@@ -1,45 +0,0 @@
-meta:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-    
-    [wal]
-    provider = "kafka"
-    broker_endpoints = ["kafka.kafka-cluster.svc.cluster.local:9092"]
-    num_topics = 3
-
-        
-    [datanode]
-    [datanode.client]
-    timeout = "60s"
-datanode:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-
-    [wal]
-    provider = "kafka"
-    broker_endpoints = ["kafka.kafka-cluster.svc.cluster.local:9092"]
-    linger = "2ms"
-frontend:
-  config: |-
-    [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-
-    [meta_client]
-    ddl_timeout = "60s"
-objectStorage:
-  s3:
-    bucket: default
-    region: us-west-2
-    root: test-root
-    endpoint: http://minio.minio.svc.cluster.local 
-  credentials:
-    accessKeyId: rootuser
-    secretAccessKey: rootpass123

.github/actions/setup-kafka-cluster/action.yml

@@ -1,24 +0,0 @@
-name: Setup Kafka cluster
-description: Deploy Kafka cluster on Kubernetes
-inputs:
-  controller-replicas:
-    default: 3
-    description: "Kafka controller replicas"
-  namespace:
-    default: "kafka-cluster"
-
-runs:
-  using: composite
-  steps:
-  - name: Install Kafka cluster
-    shell: bash
-    run: | 
-      helm upgrade \
-        --install kafka oci://registry-1.docker.io/bitnamicharts/kafka \
-        --set controller.replicaCount=${{ inputs.controller-replicas }} \
-        --set controller.resources.requests.cpu=50m \
-        --set controller.resources.requests.memory=128Mi \
-        --set listeners.controller.protocol=PLAINTEXT \
-        --set listeners.client.protocol=PLAINTEXT \
-        --create-namespace \
-        -n ${{ inputs.namespace }}

.github/actions/setup-minio/action.yml

@@ -1,24 +0,0 @@
-name: Setup Minio cluster
-description: Deploy Minio cluster on Kubernetes
-inputs:
-  replicas:
-    default: 1
-    description: "replicas"
-
-runs:
-  using: composite
-  steps:
-  - name: Install Etcd cluster
-    shell: bash
-    run: | 
-      helm repo add minio https://charts.min.io/
-      helm upgrade --install minio \
-      --set resources.requests.memory=128Mi \
-      --set replicas=${{ inputs.replicas }} \
-      --set mode=standalone \
-      --set rootUser=rootuser,rootPassword=rootpass123 \
-      --set buckets[0].name=default \
-      --set service.port=80,service.targetPort=9000 \
-      minio/minio \
-      --create-namespace \
-      -n minio

.github/workflows/develop.yml

@@ -160,16 +160,14 @@ jobs:
         run: |
           sudo apt-get install -y libfuzzer-14-dev
           rustup install nightly
-          cargo +nightly install cargo-fuzz cargo-gc-bin
+          cargo +nightly install cargo-fuzz
       - name: Download pre-built binaries
         uses: actions/download-artifact@v4
         with:
           name: bins
           path: .
       - name: Unzip binaries
-        run: | 
+        run: tar -xvf ./bins.tar.gz
-          tar -xvf ./bins.tar.gz
-          rm ./bins.tar.gz
       - name: Run GreptimeDB
         run: |
           ./bins/greptime standalone start&
@@ -184,7 +182,7 @@ jobs:
 
   unstable-fuzztest:
     name: Unstable Fuzz Test
-    needs: build-greptime-ci
+    needs: build
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -206,22 +204,27 @@ jobs:
         shell: bash
         run: |
           sudo apt update && sudo apt install -y libfuzzer-14-dev
-          cargo install cargo-fuzz cargo-gc-bin
+          cargo install cargo-fuzz
-      - name: Download pre-built binariy
+      - name: Download pre-built binaries
         uses: actions/download-artifact@v4
         with:
-          name: bin
+          name: bins
           path: .
-      - name: Unzip bianry
+      - name: Unzip binaries
+        run: tar -xvf ./bins.tar.gz
+      - name: Build Fuzz Test
+        shell: bash
         run: |
-          tar -xvf ./bin.tar.gz
+          cd tests-fuzz &
-          rm ./bin.tar.gz
+          cargo install cargo-gc-bin &
+          cargo gc &
+          cd ..
       - name: Run Fuzz Test
         uses: ./.github/actions/fuzz-test
         env:
           CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
           GT_MYSQL_ADDR: 127.0.0.1:4002
-          GT_FUZZ_BINARY_PATH: ./bin/greptime
+          GT_FUZZ_BINARY_PATH: ./bins/greptime
           GT_FUZZ_INSTANCE_ROOT_DIR: /tmp/unstable-greptime/
         with:
           target: ${{ matrix.target }}
@@ -260,7 +263,7 @@ jobs:
       - name: Build greptime bianry
         shell: bash
         # `cargo gc` will invoke `cargo build` with specified args
-        run: cargo gc --profile ci -- --bin greptime 
+        run: cargo build --bin greptime --profile ci
       - name: Pack greptime binary
         shell: bash
         run: |
@@ -275,39 +278,16 @@ jobs:
           version: current
 
   distributed-fuzztest: 
-    name: Fuzz Test (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
+    name: Fuzz Test (Distributed, Disk)
     runs-on: ubuntu-latest
     needs:  build-greptime-ci
     strategy:
       matrix:
         target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
-        mode: 
-          - name: "Disk"
-            minio: false
-            kafka: false
-            values: "with-disk.yaml"
-          - name: "Minio"
-            minio: true
-            kafka: false
-            values: "with-minio.yaml"
-          - name: "Minio with Cache"
-            minio: true
-            kafka: false
-            values: "with-minio-and-cache.yaml"
-          - name: "Remote WAL"
-            minio: true
-            kafka: true
-            values: "with-remote-wal.yaml"
     steps:
       - uses: actions/checkout@v4
       - name: Setup Kind
         uses: ./.github/actions/setup-kind
-      - if: matrix.mode.minio
-        name: Setup Minio
-        uses: ./.github/actions/setup-minio
-      - if: matrix.mode.kafka
-        name: Setup Kafka cluser
-        uses: ./.github/actions/setup-kafka-cluster
       - name: Setup Etcd cluser
         uses: ./.github/actions/setup-etcd-cluster
       # Prepares for fuzz tests
@@ -327,7 +307,7 @@ jobs:
         run: |
           sudo apt-get install -y libfuzzer-14-dev
           rustup install nightly
-          cargo +nightly install cargo-fuzz cargo-gc-bin
+          cargo +nightly install cargo-fuzz
       # Downloads ci image
       - name: Download pre-built binariy
         uses: actions/download-artifact@v4
@@ -335,9 +315,7 @@ jobs:
           name: bin
           path: .
       - name: Unzip binary
-        run: |
+        run: tar -xvf ./bin.tar.gz
-          tar -xvf ./bin.tar.gz
-          rm ./bin.tar.gz
       - name: Build and push GreptimeDB image
         uses: ./.github/actions/build-and-push-ci-image
       - name: Wait for etcd
@@ -347,22 +325,6 @@ jobs:
             pod -l app.kubernetes.io/instance=etcd \
             --timeout=120s \
             -n etcd-cluster
-      - if: matrix.mode.minio
-        name: Wait for minio
-        run: |
-          kubectl wait \
-            --for=condition=Ready \
-            pod -l app=minio \
-            --timeout=120s \
-            -n minio
-      - if: matrix.mode.kafka
-        name: Wait for kafka
-        run: |
-          kubectl wait \
-            --for=condition=Ready \
-            pod -l app.kubernetes.io/instance=kafka \
-            --timeout=120s \
-            -n kafka-cluster
       - name: Print etcd info
         shell: bash
         run: kubectl get all --show-labels -n etcd-cluster
@@ -371,7 +333,6 @@ jobs:
         uses: ./.github/actions/setup-greptimedb-cluster
         with:
           image-registry: localhost:5001
-          values-filename: ${{ matrix.mode.values }}
       - name: Port forward (mysql)
         run: |
           kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
@@ -397,32 +358,18 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: fuzz-tests-kind-logs-${{ matrix.mode.name }}-${{ matrix.target }}
+          name: fuzz-tests-kind-logs-${{ matrix.target }}
           path: /tmp/kind
           retention-days: 3
-      - name: Delete cluster
+        
-        if: success()
-        shell: bash
-        run: | 
-          kind delete cluster
-          docker stop $(docker ps -a -q)
-          docker rm $(docker ps -a -q)
-          docker system prune -f
 
   sqlness:
-    name: Sqlness Test (${{ matrix.mode.name }})
+    name: Sqlness Test
     needs: build
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ ubuntu-20.04 ]
-        mode:
-          - name: "Basic"
-            opts: ""
-            kafka: false
-          - name: "Remote WAL"
-            opts: "-w kafka -k 127.0.0.1:9092"
-            kafka: true
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
@@ -433,17 +380,43 @@ jobs:
           path: .
       - name: Unzip binaries
         run: tar -xvf ./bins.tar.gz
-      - if: matrix.mode.kafka
+      - name: Run sqlness
-        name: Setup kafka server
+        run: RUST_BACKTRACE=1 ./bins/sqlness-runner -c ./tests/cases --bins-dir ./bins --preserve-state
+      - name: Upload sqlness logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sqlness-logs
+          path: /tmp/sqlness*
+          retention-days: 3
+
+  sqlness-kafka-wal:
+    name: Sqlness Test with Kafka Wal
+    needs: build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-20.04 ]
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download pre-built binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: bins
+          path: .
+      - name: Unzip binaries
+        run: tar -xvf ./bins.tar.gz
+      - name: Setup kafka server
         working-directory: tests-integration/fixtures/kafka
         run: docker compose -f docker-compose-standalone.yml up -d --wait
       - name: Run sqlness
-        run: RUST_BACKTRACE=1 ./bins/sqlness-runner ${{ matrix.mode.opts }} -c ./tests/cases --bins-dir ./bins --preserve-state
+        run: RUST_BACKTRACE=1 ./bins/sqlness-runner -w kafka -k 127.0.0.1:9092 -c ./tests/cases --bins-dir ./bins --preserve-state
       - name: Upload sqlness logs
-        if: failure()
+        if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: sqlness-logs-${{ matrix.mode.name }}
+          name: sqlness-logs-with-kafka-wal
           path: /tmp/sqlness*
           retention-days: 3
 
@@ -532,9 +505,6 @@ jobs:
       - name: Setup kafka server
         working-directory: tests-integration/fixtures/kafka
         run: docker compose -f docker-compose-standalone.yml up -d --wait
-      - name: Setup minio
-        working-directory: tests-integration/fixtures/minio
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
       - name: Run nextest cases
         run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
         env:
@@ -545,11 +515,6 @@ jobs:
           GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
           GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
           GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
-          GT_MINIO_BUCKET: greptime
-          GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
-          GT_MINIO_ACCESS_KEY: superpower_password
-          GT_MINIO_REGION: us-west-2
-          GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
           GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
           GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
           UNITTEST_LOG_DIR: "__unittest_logs"

Cargo.toml

@@ -64,7 +64,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.8.2"
+version = "0.8.1"
 edition = "2021"
 license = "Apache-2.0"
 
@@ -104,15 +104,15 @@ clap = { version = "4.4", features = ["derive"] }
 config = "0.13.0"
 crossbeam-utils = "0.8"
 dashmap = "5.4"
-datafusion = { git = "https://github.com/apache/datafusion.git", rev = "08e19f4956d32164be6fc66eb5a4c080eb0023d1" }
+datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
-datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "08e19f4956d32164be6fc66eb5a4c080eb0023d1" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
-datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "08e19f4956d32164be6fc66eb5a4c080eb0023d1" }
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
-datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "08e19f4956d32164be6fc66eb5a4c080eb0023d1" }
+datafusion-functions = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
-datafusion-optimizer = { git = "https://github.com/apache/datafusion.git", rev = "08e19f4956d32164be6fc66eb5a4c080eb0023d1" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
-datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "08e19f4956d32164be6fc66eb5a4c080eb0023d1" }
+datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
-datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "08e19f4956d32164be6fc66eb5a4c080eb0023d1" }
+datafusion-physical-plan = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
-datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "08e19f4956d32164be6fc66eb5a4c080eb0023d1" }
+datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
-datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "08e19f4956d32164be6fc66eb5a4c080eb0023d1" }
+datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
 derive_builder = "0.12"
 dotenv = "0.15"
 # TODO(LFC): Wait for https://github.com/etcdv3/etcd-client/pull/76
@@ -146,15 +146,13 @@ raft-engine = { version = "0.4.1", default-features = false }
 rand = "0.8"
 regex = "1.8"
 regex-automata = { version = "0.4" }
-reqwest = { version = "0.12", default-features = false, features = [
+reqwest = { version = "0.11", default-features = false, features = [
     "json",
     "rustls-tls-native-roots",
     "stream",
     "multipart",
 ] }
 rskafka = "0.5"
-rstest = "0.21"
-rstest_reuse = "0.7"
 rust_decimal = "1.33"
 schemars = "0.8"
 serde = { version = "1.0", features = ["derive"] }
@@ -164,7 +162,7 @@ smallvec = { version = "1", features = ["serde"] }
 snafu = "0.8"
 sysinfo = "0.30"
 # on branch v0.44.x
-sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "54a267ac89c09b11c0c88934690530807185d3e7", features = [
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "e4e496b8d62416ad50ce70a1b460c7313610cf5d", features = [
     "visitor",
 ] }
 strum = { version = "0.25", features = ["derive"] }
@@ -252,12 +250,9 @@ incremental = false
 
 [profile.ci]
 inherits = "dev"
+debug = false
 strip = true
 
 [profile.dev.package.sqlness-runner]
 debug = false
 strip = true
-
-[profile.dev.package.tests-fuzz]
-debug = false
-strip = true

Makefile

@@ -163,13 +163,6 @@ nextest: ## Install nextest tools.
 sqlness-test: ## Run sqlness test.
 	cargo sqlness
 
-# Run fuzz test ${FUZZ_TARGET}.
-RUNS ?= 1
-FUZZ_TARGET ?= fuzz_alter_table
-.PHONY: fuzz
-fuzz:
-	cargo fuzz run ${FUZZ_TARGET} --fuzz-dir tests-fuzz -D -s none -- -runs=${RUNS}
-
 .PHONY: check
 check: ## Cargo check all the targets.
 	cargo check --workspace --all-targets --all-features
@@ -201,10 +194,6 @@ run-it-in-container: start-etcd ## Run integration tests in dev-builder.
 	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
 	make test sqlness-test BUILD_JOBS=${BUILD_JOBS}
 
-.PHONY: run-cluster-with-etcd
-run-cluster-with-etcd: ## Run greptime cluster with etcd in docker-compose.
-	 docker compose -f ./docker/docker-compose/cluster-with-etcd.yaml up
-
 ##@ Docs
 config-docs: ## Generate configuration documentation from toml files.
 	docker run --rm \

benchmarks/Cargo.toml

@@ -12,6 +12,7 @@ api.workspace = true
 arrow.workspace = true
 chrono.workspace = true
 clap.workspace = true
+client = { workspace = true, features = ["testing"] }
 common-base.workspace = true
 common-telemetry.workspace = true
 common-wal.workspace = true

config/config.md

@@ -13,10 +13,6 @@
 | `mode` | String | `standalone` | The running mode of the datanode. It can be `standalone` or `distributed`. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. |
 | `default_timezone` | String | `None` | The default timezone of the server. |
-| `runtime` | -- | -- | The runtime options. |
-| `runtime.read_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
-| `runtime.write_rt_size` | Integer | `8` | The number of threads to execute the runtime for global write operations. |
-| `runtime.bg_rt_size` | Integer | `8` | The number of threads to execute the runtime for global background operations. |
 | `http` | -- | -- | The HTTP server options. |
 | `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
 | `http.timeout` | String | `30s` | HTTP request timeout. |
@@ -158,10 +154,6 @@
 | --- | -----| ------- | ----------- |
 | `mode` | String | `standalone` | The running mode of the datanode. It can be `standalone` or `distributed`. |
 | `default_timezone` | String | `None` | The default timezone of the server. |
-| `runtime` | -- | -- | The runtime options. |
-| `runtime.read_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
-| `runtime.write_rt_size` | Integer | `8` | The number of threads to execute the runtime for global write operations. |
-| `runtime.bg_rt_size` | Integer | `8` | The number of threads to execute the runtime for global background operations. |
 | `heartbeat` | -- | -- | The heartbeat options. |
 | `heartbeat.interval` | String | `18s` | Interval for sending heartbeat messages to the metasrv. |
 | `heartbeat.retry_interval` | String | `3s` | Interval for retrying to send heartbeat messages to the metasrv. |
@@ -248,10 +240,6 @@
 | `use_memory_store` | Bool | `false` | Store data in memory. |
 | `enable_telemetry` | Bool | `true` | Whether to enable greptimedb telemetry. |
 | `store_key_prefix` | String | `""` | If it's not empty, the metasrv will store all data with this key prefix. |
-| `runtime` | -- | -- | The runtime options. |
-| `runtime.read_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
-| `runtime.write_rt_size` | Integer | `8` | The number of threads to execute the runtime for global write operations. |
-| `runtime.bg_rt_size` | Integer | `8` | The number of threads to execute the runtime for global background operations. |
 | `procedure` | -- | -- | Procedure storage options. |
 | `procedure.max_retry_times` | Integer | `12` | Procedure max retry time. |
 | `procedure.retry_delay` | String | `500ms` | Initial retry delay of procedures, increases exponentially |
@@ -306,17 +294,12 @@
 | `node_id` | Integer | `None` | The datanode identifier and should be unique in the cluster. |
 | `require_lease_before_startup` | Bool | `false` | Start services after regions have obtained leases.<br/>It will block the datanode start if it can't receive leases in the heartbeat from metasrv. |
 | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.<br/>By default, it provides services after all regions have been initialized. |
-| `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
 | `rpc_addr` | String | `127.0.0.1:3001` | The gRPC address of the datanode. |
 | `rpc_hostname` | String | `None` | The hostname of the datanode. |
 | `rpc_runtime_size` | Integer | `8` | The number of gRPC server worker threads. |
 | `rpc_max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
 | `rpc_max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. |
-| `runtime` | -- | -- | The runtime options. |
-| `runtime.read_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
-| `runtime.write_rt_size` | Integer | `8` | The number of threads to execute the runtime for global write operations. |
-| `runtime.bg_rt_size` | Integer | `8` | The number of threads to execute the runtime for global background operations. |
 | `heartbeat` | -- | -- | The heartbeat options. |
 | `heartbeat.interval` | String | `3s` | Interval for sending heartbeat messages to the metasrv. |
 | `heartbeat.retry_interval` | String | `3s` | Interval for retrying to send heartbeat messages to the metasrv. |

config/datanode.example.toml

@@ -13,9 +13,6 @@ require_lease_before_startup = false
 ## By default, it provides services after all regions have been initialized.
 init_regions_in_background = false
 
-## Parallelism of initializing regions.
-init_regions_parallelism = 16
-
 ## The gRPC address of the datanode.
 rpc_addr = "127.0.0.1:3001"
 
@@ -35,15 +32,6 @@ rpc_max_send_message_size = "512MB"
 ## Enable telemetry to collect anonymous usage data.
 enable_telemetry = true
 
-## The runtime options.
-[runtime]
-## The number of threads to execute the runtime for global read operations.
-read_rt_size = 8
-## The number of threads to execute the runtime for global write operations.
-write_rt_size = 8
-## The number of threads to execute the runtime for global background operations.
-bg_rt_size = 8
-
 ## The heartbeat options.
 [heartbeat]
 ## Interval for sending heartbeat messages to the metasrv.

config/frontend.example.toml

@@ -5,15 +5,6 @@ mode = "standalone"
 ## +toml2docs:none-default
 default_timezone = "UTC"
 
-## The runtime options.
-[runtime]
-## The number of threads to execute the runtime for global read operations.
-read_rt_size = 8
-## The number of threads to execute the runtime for global write operations.
-write_rt_size = 8
-## The number of threads to execute the runtime for global background operations.
-bg_rt_size = 8
-
 ## The heartbeat options.
 [heartbeat]
 ## Interval for sending heartbeat messages to the metasrv.

config/metasrv.example.toml

@@ -25,15 +25,6 @@ enable_telemetry = true
 ## If it's not empty, the metasrv will store all data with this key prefix.
 store_key_prefix = ""
 
-## The runtime options.
-[runtime]
-## The number of threads to execute the runtime for global read operations.
-read_rt_size = 8
-## The number of threads to execute the runtime for global write operations.
-write_rt_size = 8
-## The number of threads to execute the runtime for global background operations.
-bg_rt_size = 8
-
 ## Procedure storage options.
 [procedure]
 

config/standalone.example.toml

@@ -8,15 +8,6 @@ enable_telemetry = true
 ## +toml2docs:none-default
 default_timezone = "UTC"
 
-## The runtime options.
-[runtime]
-## The number of threads to execute the runtime for global read operations.
-read_rt_size = 8
-## The number of threads to execute the runtime for global write operations.
-write_rt_size = 8
-## The number of threads to execute the runtime for global background operations.
-bg_rt_size = 8
-
 ## The HTTP server options.
 [http]
 ## The address to bind the HTTP server.

docker/docker-compose/cluster-with-etcd.yaml

@@ -1,102 +0,0 @@
-x-custom:
-  initial_cluster_token: &initial_cluster_token "--initial-cluster-token=etcd-cluster"
-  common_settings: &common_settings
-    image: quay.io/coreos/etcd:v3.5.10
-    entrypoint: /usr/local/bin/etcd
-
-services:
-  etcd0:
-    <<: *common_settings
-    container_name: etcd0
-    ports:
-      - 2379:2379
-      - 2380:2380
-    command:
-      - --name=etcd0
-      - --data-dir=/var/lib/etcd
-      - --initial-advertise-peer-urls=http://etcd0:2380
-      - --listen-peer-urls=http://0.0.0.0:2380
-      - --listen-client-urls=http://0.0.0.0:2379
-      - --advertise-client-urls=http://etcd0:2379
-      - --heartbeat-interval=250
-      - --election-timeout=1250
-      - --initial-cluster=etcd0=http://etcd0:2380
-      - --initial-cluster-state=new
-      - *initial_cluster_token
-    volumes:
-      - /tmp/greptimedb-cluster-docker-compose/etcd0:/var/lib/etcd
-    healthcheck:
-      test: [ "CMD", "etcdctl", "--endpoints=http://etcd0:2379", "endpoint", "health" ]
-      interval: 5s
-      timeout: 3s
-      retries: 5
-    networks:
-      - greptimedb
-
-  metasrv:
-    image: docker.io/greptime/greptimedb:latest
-    container_name: metasrv
-    ports:
-      - 3002:3002
-    command:
-      - metasrv
-      - start
-      - --bind-addr=0.0.0.0:3002
-      - --server-addr=metasrv:3002
-      - --store-addrs=etcd0:2379
-    healthcheck:
-      test: [ "CMD", "curl", "-f", "http://metasrv:3002/health" ]
-      interval: 5s
-      timeout: 3s
-      retries: 5
-    depends_on:
-      etcd0:
-        condition: service_healthy
-    networks:
-      - greptimedb
-
-  datanode0:
-    image: docker.io/greptime/greptimedb:latest
-    container_name: datanode0
-    ports:
-      - 3001:3001
-    command:
-      - datanode
-      - start
-      - --node-id=0
-      - --rpc-addr=0.0.0.0:3001
-      - --rpc-hostname=datanode0:3001
-      - --metasrv-addr=metasrv:3002
-    volumes:
-      - /tmp/greptimedb-cluster-docker-compose/datanode0:/tmp/greptimedb
-    depends_on:
-      metasrv:
-        condition: service_healthy
-    networks:
-      - greptimedb
-
-  frontend0:
-    image: docker.io/greptime/greptimedb:latest
-    container_name: frontend0
-    ports:
-      - 4000:4000
-      - 4001:4001
-      - 4002:4002
-      - 4003:4003
-    command:
-      - frontend
-      - start
-      - --metasrv-addrs=metasrv:3002
-      - --http-addr=0.0.0.0:4000
-      - --rpc-addr=0.0.0.0:4001
-      - --mysql-addr=0.0.0.0:4002
-      - --postgres-addr=0.0.0.0:4003
-    depends_on:
-      metasrv:
-        condition: service_healthy
-    networks:
-      - greptimedb
-
-networks:
-  greptimedb:
-    name: greptimedb

src/catalog/src/information_schema/tables.rs

@@ -105,9 +105,7 @@ impl InformationTable for InformationSchemaTables {
                     .make_tables(Some(request))
                     .await
                     .map(|x| x.into_df_record_batch())
-                    .map_err(|err| {
+                    .map_err(Into::into)
-                        datafusion::error::DataFusionError::External(format!("{err:?}").into())
-                    })
             }),
         ));
         Ok(Box::pin(

src/client/Cargo.toml

@@ -31,11 +31,9 @@ moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
 prometheus.workspace = true
 prost.workspace = true
-query.workspace = true
 rand.workspace = true
 serde_json.workspace = true
 snafu.workspace = true
-substrait.workspace = true
 tokio.workspace = true
 tokio-stream = { workspace = true, features = ["net"] }
 tonic.workspace = true
@@ -44,6 +42,7 @@ tonic.workspace = true
 common-grpc-expr.workspace = true
 datanode.workspace = true
 derive-new = "0.5"
+substrait.workspace = true
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 

src/client/src/error.rs

@@ -192,9 +192,6 @@ impl Error {
             } | Self::RegionServer {
                 code: Code::Unavailable,
                 ..
-            } | Self::RegionServer {
-                code: Code::Unknown,
-                ..
             }
         )
     }

src/client/src/region.rs

@@ -15,7 +15,7 @@
 use std::sync::Arc;
 
 use api::region::RegionResponse;
-use api::v1::region::RegionRequest;
+use api::v1::region::{QueryRequest, RegionRequest};
 use api::v1::ResponseHeader;
 use arc_swap::ArcSwapOption;
 use arrow_flight::Ticket;
@@ -26,15 +26,12 @@ use common_error::status_code::StatusCode;
 use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_meta::error::{self as meta_error, Result as MetaResult};
 use common_meta::node_manager::Datanode;
-use common_query::request::QueryRequest;
 use common_recordbatch::error::ExternalSnafu;
 use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
 use common_telemetry::error;
 use common_telemetry::tracing_context::TracingContext;
 use prost::Message;
-use query::query_engine::DefaultSerializer;
 use snafu::{location, Location, OptionExt, ResultExt};
-use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
 use tokio_stream::StreamExt;
 
 use crate::error::{
@@ -66,17 +63,6 @@ impl Datanode for RegionRequester {
     }
 
     async fn handle_query(&self, request: QueryRequest) -> MetaResult<SendableRecordBatchStream> {
-        let plan = DFLogicalSubstraitConvertor
-            .encode(&request.plan, DefaultSerializer)
-            .map_err(BoxedError::new)
-            .context(meta_error::ExternalSnafu)?
-            .to_vec();
-        let request = api::v1::region::QueryRequest {
-            header: request.header,
-            region_id: request.region_id.as_u64(),
-            plan,
-        };
-
         let ticket = Ticket {
             ticket: request.encode_to_vec().into(),
         };

src/cmd/src/datanode.rs

@@ -23,6 +23,7 @@ use common_telemetry::info;
 use common_telemetry::logging::TracingOptions;
 use common_version::{short_version, version};
 use common_wal::config::DatanodeWalConfig;
+use datanode::config::DatanodeOptions;
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use datanode::service::DatanodeServiceBuilder;
 use meta_client::MetaClientOptions;
@@ -33,13 +34,11 @@ use tracing_appender::non_blocking::WorkerGuard;
 use crate::error::{
     LoadLayeredConfigSnafu, MissingConfigSnafu, Result, ShutdownDatanodeSnafu, StartDatanodeSnafu,
 };
-use crate::options::{GlobalOptions, GreptimeOptions};
+use crate::options::GlobalOptions;
 use crate::{log_versions, App};
 
 pub const APP_NAME: &str = "greptime-datanode";
 
-type DatanodeOptions = GreptimeOptions<datanode::config::DatanodeOptions>;
-
 pub struct Instance {
     datanode: Datanode,
 
@@ -98,9 +97,7 @@ impl Command {
     }
 
     pub fn load_options(&self, global_options: &GlobalOptions) -> Result<DatanodeOptions> {
-        match &self.subcmd {
+        self.subcmd.load_options(global_options)
-            SubCommand::Start(cmd) => cmd.load_options(global_options),
-        }
     }
 }
 
@@ -115,6 +112,12 @@ impl SubCommand {
             SubCommand::Start(cmd) => cmd.build(opts).await,
         }
     }
+
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<DatanodeOptions> {
+        match self {
+            SubCommand::Start(cmd) => cmd.load_options(global_options),
+        }
+    }
 }
 
 #[derive(Debug, Parser, Default)]
@@ -143,25 +146,22 @@ struct StartCommand {
 
 impl StartCommand {
     fn load_options(&self, global_options: &GlobalOptions) -> Result<DatanodeOptions> {
-        let mut opts = DatanodeOptions::load_layered_options(
+        self.merge_with_cli_options(
-            self.config_file.as_deref(),
+            global_options,
-            self.env_prefix.as_ref(),
+            DatanodeOptions::load_layered_options(
+                self.config_file.as_deref(),
+                self.env_prefix.as_ref(),
+            )
+            .context(LoadLayeredConfigSnafu)?,
         )
-        .context(LoadLayeredConfigSnafu)?;
-
-        self.merge_with_cli_options(global_options, &mut opts)?;
-
-        Ok(opts)
     }
 
     // The precedence order is: cli > config file > environment variables > default values.
     fn merge_with_cli_options(
         &self,
         global_options: &GlobalOptions,
-        opts: &mut DatanodeOptions,
+        mut opts: DatanodeOptions,
-    ) -> Result<()> {
+    ) -> Result<DatanodeOptions> {
-        let opts = &mut opts.component;
-
         if let Some(dir) = &global_options.log_dir {
             opts.logging.dir.clone_from(dir);
         }
@@ -231,28 +231,25 @@ impl StartCommand {
         // Disable dashboard in datanode.
         opts.http.disable_dashboard = true;
 
-        Ok(())
+        Ok(opts)
     }
 
-    async fn build(&self, opts: DatanodeOptions) -> Result<Instance> {
+    async fn build(&self, mut opts: DatanodeOptions) -> Result<Instance> {
-        common_runtime::init_global_runtimes(&opts.runtime);
-
         let guard = common_telemetry::init_global_logging(
             APP_NAME,
-            &opts.component.logging,
+            &opts.logging,
-            &opts.component.tracing,
+            &opts.tracing,
-            opts.component.node_id.map(|x| x.to_string()),
+            opts.node_id.map(|x| x.to_string()),
         );
         log_versions(version!(), short_version!());
 
-        info!("Datanode start command: {:#?}", self);
-        info!("Datanode options: {:#?}", opts);
-
-        let mut opts = opts.component;
         let plugins = plugins::setup_datanode_plugins(&mut opts)
             .await
             .context(StartDatanodeSnafu)?;
 
+        info!("Datanode start command: {:#?}", self);
+        info!("Datanode options: {:#?}", opts);
+
         let node_id = opts
             .node_id
             .context(MissingConfigSnafu { msg: "'node_id'" })?;
@@ -356,7 +353,7 @@ mod tests {
             ..Default::default()
         };
 
-        let options = cmd.load_options(&Default::default()).unwrap().component;
+        let options = cmd.load_options(&GlobalOptions::default()).unwrap();
 
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
         assert_eq!(Some(42), options.node_id);
@@ -417,8 +414,7 @@ mod tests {
     fn test_try_from_cmd() {
         let opt = StartCommand::default()
             .load_options(&GlobalOptions::default())
-            .unwrap()
+            .unwrap();
-            .component;
         assert_eq!(Mode::Standalone, opt.mode);
 
         let opt = (StartCommand {
@@ -427,8 +423,7 @@ mod tests {
             ..Default::default()
         })
         .load_options(&GlobalOptions::default())
-        .unwrap()
+        .unwrap();
-        .component;
         assert_eq!(Mode::Distributed, opt.mode);
 
         assert!((StartCommand {
@@ -459,8 +454,7 @@ mod tests {
                 #[cfg(feature = "tokio-console")]
                 tokio_console_addr: None,
             })
-            .unwrap()
+            .unwrap();
-            .component;
 
         let logging_opt = options.logging;
         assert_eq!("/tmp/greptimedb/test/logs", logging_opt.dir);
@@ -542,7 +536,7 @@ mod tests {
                     ..Default::default()
                 };
 
-                let opts = command.load_options(&Default::default()).unwrap().component;
+                let opts = command.load_options(&GlobalOptions::default()).unwrap();
 
                 // Should be read from env, env > default values.
                 let DatanodeWalConfig::RaftEngine(raft_engine_config) = opts.wal else {
@@ -568,10 +562,7 @@ mod tests {
                 assert_eq!(raft_engine_config.dir.unwrap(), "/other/wal/dir");
 
                 // Should be default value.
-                assert_eq!(
+                assert_eq!(opts.http.addr, DatanodeOptions::default().http.addr);
-                    opts.http.addr,
-                    DatanodeOptions::default().component.http.addr
-                );
             },
         );
     }

src/cmd/src/frontend.rs

@@ -29,6 +29,7 @@ use common_telemetry::info;
 use common_telemetry::logging::TracingOptions;
 use common_time::timezone::set_default_timezone;
 use common_version::{short_version, version};
+use frontend::frontend::FrontendOptions;
 use frontend::heartbeat::handler::invalidate_table_cache::InvalidateTableCacheHandler;
 use frontend::heartbeat::HeartbeatTask;
 use frontend::instance::builder::FrontendBuilder;
@@ -43,11 +44,9 @@ use tracing_appender::non_blocking::WorkerGuard;
 use crate::error::{
     self, InitTimezoneSnafu, LoadLayeredConfigSnafu, MissingConfigSnafu, Result, StartFrontendSnafu,
 };
-use crate::options::{GlobalOptions, GreptimeOptions};
+use crate::options::GlobalOptions;
 use crate::{log_versions, App};
 
-type FrontendOptions = GreptimeOptions<frontend::frontend::FrontendOptions>;
-
 pub struct Instance {
     frontend: FeInstance,
 
@@ -165,25 +164,22 @@ pub struct StartCommand {
 
 impl StartCommand {
     fn load_options(&self, global_options: &GlobalOptions) -> Result<FrontendOptions> {
-        let mut opts = FrontendOptions::load_layered_options(
+        self.merge_with_cli_options(
-            self.config_file.as_deref(),
+            global_options,
-            self.env_prefix.as_ref(),
+            FrontendOptions::load_layered_options(
+                self.config_file.as_deref(),
+                self.env_prefix.as_ref(),
+            )
+            .context(LoadLayeredConfigSnafu)?,
         )
-        .context(LoadLayeredConfigSnafu)?;
-
-        self.merge_with_cli_options(global_options, &mut opts)?;
-
-        Ok(opts)
     }
 
     // The precedence order is: cli > config file > environment variables > default values.
     fn merge_with_cli_options(
         &self,
         global_options: &GlobalOptions,
-        opts: &mut FrontendOptions,
+        mut opts: FrontendOptions,
-    ) -> Result<()> {
+    ) -> Result<FrontendOptions> {
-        let opts = &mut opts.component;
-
         if let Some(dir) = &global_options.log_dir {
             opts.logging.dir.clone_from(dir);
         }
@@ -246,29 +242,26 @@ impl StartCommand {
 
         opts.user_provider.clone_from(&self.user_provider);
 
-        Ok(())
+        Ok(opts)
     }
 
-    async fn build(&self, opts: FrontendOptions) -> Result<Instance> {
+    async fn build(&self, mut opts: FrontendOptions) -> Result<Instance> {
-        common_runtime::init_global_runtimes(&opts.runtime);
-
         let guard = common_telemetry::init_global_logging(
             APP_NAME,
-            &opts.component.logging,
+            &opts.logging,
-            &opts.component.tracing,
+            &opts.tracing,
-            opts.component.node_id.clone(),
+            opts.node_id.clone(),
         );
         log_versions(version!(), short_version!());
 
-        info!("Frontend start command: {:#?}", self);
-        info!("Frontend options: {:#?}", opts);
-
-        let mut opts = opts.component;
         #[allow(clippy::unnecessary_mut_passed)]
         let plugins = plugins::setup_frontend_plugins(&mut opts)
             .await
             .context(StartFrontendSnafu)?;
 
+        info!("Frontend start command: {:#?}", self);
+        info!("Frontend options: {:#?}", opts);
+
         set_default_timezone(opts.default_timezone.as_deref()).context(InitTimezoneSnafu)?;
 
         let meta_client_options = opts.meta_client.as_ref().context(MissingConfigSnafu {
@@ -387,14 +380,14 @@ mod tests {
             ..Default::default()
         };
 
-        let opts = command.load_options(&Default::default()).unwrap().component;
+        let opts = command.load_options(&GlobalOptions::default()).unwrap();
 
         assert_eq!(opts.http.addr, "127.0.0.1:1234");
         assert_eq!(ReadableSize::mb(64), opts.http.body_limit);
         assert_eq!(opts.mysql.addr, "127.0.0.1:5678");
         assert_eq!(opts.postgres.addr, "127.0.0.1:5432");
 
-        let default_opts = FrontendOptions::default().component;
+        let default_opts = FrontendOptions::default();
 
         assert_eq!(opts.grpc.addr, default_opts.grpc.addr);
         assert!(opts.mysql.enable);
@@ -435,8 +428,7 @@ mod tests {
             ..Default::default()
         };
 
-        let fe_opts = command.load_options(&Default::default()).unwrap().component;
+        let fe_opts = command.load_options(&GlobalOptions::default()).unwrap();
-
         assert_eq!(Mode::Distributed, fe_opts.mode);
         assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
         assert_eq!(Duration::from_secs(30), fe_opts.http.timeout);
@@ -450,7 +442,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_try_from_start_command_to_anymap() {
-        let mut fe_opts = frontend::frontend::FrontendOptions {
+        let mut fe_opts = FrontendOptions {
             http: HttpOptions {
                 disable_dashboard: false,
                 ..Default::default()
@@ -487,8 +479,7 @@ mod tests {
                 #[cfg(feature = "tokio-console")]
                 tokio_console_addr: None,
             })
-            .unwrap()
+            .unwrap();
-            .component;
 
         let logging_opt = options.logging;
         assert_eq!("/tmp/greptimedb/test/logs", logging_opt.dir);
@@ -566,7 +557,7 @@ mod tests {
                     ..Default::default()
                 };
 
-                let fe_opts = command.load_options(&Default::default()).unwrap().component;
+                let fe_opts = command.load_options(&GlobalOptions::default()).unwrap();
 
                 // Should be read from env, env > default values.
                 assert_eq!(fe_opts.mysql.runtime_size, 11);

src/cmd/src/metasrv.rs

@@ -21,15 +21,14 @@ use common_telemetry::info;
 use common_telemetry::logging::TracingOptions;
 use common_version::{short_version, version};
 use meta_srv::bootstrap::MetasrvInstance;
+use meta_srv::metasrv::MetasrvOptions;
 use snafu::ResultExt;
 use tracing_appender::non_blocking::WorkerGuard;
 
 use crate::error::{self, LoadLayeredConfigSnafu, Result, StartMetaServerSnafu};
-use crate::options::{GlobalOptions, GreptimeOptions};
+use crate::options::GlobalOptions;
 use crate::{log_versions, App};
 
-type MetasrvOptions = GreptimeOptions<meta_srv::metasrv::MetasrvOptions>;
-
 pub const APP_NAME: &str = "greptime-metasrv";
 
 pub struct Instance {
@@ -140,25 +139,22 @@ struct StartCommand {
 
 impl StartCommand {
     fn load_options(&self, global_options: &GlobalOptions) -> Result<MetasrvOptions> {
-        let mut opts = MetasrvOptions::load_layered_options(
+        self.merge_with_cli_options(
-            self.config_file.as_deref(),
+            global_options,
-            self.env_prefix.as_ref(),
+            MetasrvOptions::load_layered_options(
+                self.config_file.as_deref(),
+                self.env_prefix.as_ref(),
+            )
+            .context(LoadLayeredConfigSnafu)?,
         )
-        .context(LoadLayeredConfigSnafu)?;
-
-        self.merge_with_cli_options(global_options, &mut opts)?;
-
-        Ok(opts)
     }
 
     // The precedence order is: cli > config file > environment variables > default values.
     fn merge_with_cli_options(
         &self,
         global_options: &GlobalOptions,
-        opts: &mut MetasrvOptions,
+        mut opts: MetasrvOptions,
-    ) -> Result<()> {
+    ) -> Result<MetasrvOptions> {
-        let opts = &mut opts.component;
-
         if let Some(dir) = &global_options.log_dir {
             opts.logging.dir.clone_from(dir);
         }
@@ -221,28 +217,21 @@ impl StartCommand {
         // Disable dashboard in metasrv.
         opts.http.disable_dashboard = true;
 
-        Ok(())
+        Ok(opts)
     }
 
-    async fn build(&self, opts: MetasrvOptions) -> Result<Instance> {
+    async fn build(&self, mut opts: MetasrvOptions) -> Result<Instance> {
-        common_runtime::init_global_runtimes(&opts.runtime);
+        let guard =
-
+            common_telemetry::init_global_logging(APP_NAME, &opts.logging, &opts.tracing, None);
-        let guard = common_telemetry::init_global_logging(
-            APP_NAME,
-            &opts.component.logging,
-            &opts.component.tracing,
-            None,
-        );
         log_versions(version!(), short_version!());
 
-        info!("Metasrv start command: {:#?}", self);
-        info!("Metasrv options: {:#?}", opts);
-
-        let mut opts = opts.component;
         let plugins = plugins::setup_metasrv_plugins(&mut opts)
             .await
             .context(StartMetaServerSnafu)?;
 
+        info!("Metasrv start command: {:#?}", self);
+        info!("Metasrv options: {:#?}", opts);
+
         let builder = meta_srv::bootstrap::metasrv_builder(&opts, plugins.clone(), None)
             .await
             .context(error::BuildMetaServerSnafu)?;
@@ -277,7 +266,7 @@ mod tests {
             ..Default::default()
         };
 
-        let options = cmd.load_options(&Default::default()).unwrap().component;
+        let options = cmd.load_options(&GlobalOptions::default()).unwrap();
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
         assert_eq!(vec!["127.0.0.1:2380".to_string()], options.store_addrs);
         assert_eq!(SelectorType::LoadBased, options.selector);
@@ -310,7 +299,7 @@ mod tests {
             ..Default::default()
         };
 
-        let options = cmd.load_options(&Default::default()).unwrap().component;
+        let options = cmd.load_options(&GlobalOptions::default()).unwrap();
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
         assert_eq!("127.0.0.1:3002".to_string(), options.server_addr);
         assert_eq!(vec!["127.0.0.1:2379".to_string()], options.store_addrs);
@@ -360,8 +349,7 @@ mod tests {
                 #[cfg(feature = "tokio-console")]
                 tokio_console_addr: None,
             })
-            .unwrap()
+            .unwrap();
-            .component;
 
         let logging_opt = options.logging;
         assert_eq!("/tmp/greptimedb/test/logs", logging_opt.dir);
@@ -418,7 +406,7 @@ mod tests {
                     ..Default::default()
                 };
 
-                let opts = command.load_options(&Default::default()).unwrap().component;
+                let opts = command.load_options(&GlobalOptions::default()).unwrap();
 
                 // Should be read from env, env > default values.
                 assert_eq!(opts.bind_addr, "127.0.0.1:14002");

src/cmd/src/options.rs

@@ -13,9 +13,6 @@
 // limitations under the License.
 
 use clap::Parser;
-use common_config::Configurable;
-use common_runtime::global::RuntimeOptions;
-use serde::{Deserialize, Serialize};
 
 #[derive(Parser, Default, Debug, Clone)]
 pub struct GlobalOptions {
@@ -32,22 +29,3 @@ pub struct GlobalOptions {
     #[arg(global = true)]
     pub tokio_console_addr: Option<String>,
 }
-
-// TODO(LFC): Move logging and tracing options into global options, like the runtime options.
-/// All the options of GreptimeDB.
-#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
-#[serde(default)]
-pub struct GreptimeOptions<T> {
-    /// The runtime options.
-    pub runtime: RuntimeOptions,
-
-    /// The options of each component (like Datanode or Standalone) of GreptimeDB.
-    #[serde(flatten)]
-    pub component: T,
-}
-
-impl<T: Configurable> Configurable for GreptimeOptions<T> {
-    fn env_list_keys() -> Option<&'static [&'static str]> {
-        T::env_list_keys()
-    }
-}

src/cmd/src/standalone.rs

@@ -67,7 +67,7 @@ use crate::error::{
     ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu, StartProcedureManagerSnafu,
     StartWalOptionsAllocatorSnafu, StopProcedureManagerSnafu,
 };
-use crate::options::{GlobalOptions, GreptimeOptions};
+use crate::options::GlobalOptions;
 use crate::{log_versions, App};
 
 pub const APP_NAME: &str = "greptime-standalone";
@@ -79,14 +79,11 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(&self, opts: GreptimeOptions<StandaloneOptions>) -> Result<Instance> {
+    pub async fn build(&self, opts: StandaloneOptions) -> Result<Instance> {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(
+    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<StandaloneOptions> {
-        &self,
-        global_options: &GlobalOptions,
-    ) -> Result<GreptimeOptions<StandaloneOptions>> {
         self.subcmd.load_options(global_options)
     }
 }
@@ -97,23 +94,20 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(&self, opts: GreptimeOptions<StandaloneOptions>) -> Result<Instance> {
+    async fn build(&self, opts: StandaloneOptions) -> Result<Instance> {
         match self {
             SubCommand::Start(cmd) => cmd.build(opts).await,
         }
     }
 
-    fn load_options(
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<StandaloneOptions> {
-        &self,
-        global_options: &GlobalOptions,
-    ) -> Result<GreptimeOptions<StandaloneOptions>> {
         match self {
             SubCommand::Start(cmd) => cmd.load_options(global_options),
         }
     }
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(default)]
 pub struct StandaloneOptions {
     pub mode: Mode,
@@ -167,7 +161,7 @@ impl Default for StandaloneOptions {
     }
 }
 
-impl Configurable for StandaloneOptions {
+impl Configurable<'_> for StandaloneOptions {
     fn env_list_keys() -> Option<&'static [&'static str]> {
         Some(&["wal.broker_endpoints"])
     }
@@ -297,27 +291,23 @@ pub struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<StandaloneOptions> {
-        &self,
+        self.merge_with_cli_options(
-        global_options: &GlobalOptions,
+            global_options,
-    ) -> Result<GreptimeOptions<StandaloneOptions>> {
+            StandaloneOptions::load_layered_options(
-        let mut opts = GreptimeOptions::<StandaloneOptions>::load_layered_options(
+                self.config_file.as_deref(),
-            self.config_file.as_deref(),
+                self.env_prefix.as_ref(),
-            self.env_prefix.as_ref(),
+            )
+            .context(LoadLayeredConfigSnafu)?,
         )
-        .context(LoadLayeredConfigSnafu)?;
-
-        self.merge_with_cli_options(global_options, &mut opts.component)?;
-
-        Ok(opts)
     }
 
     // The precedence order is: cli > config file > environment variables > default values.
     pub fn merge_with_cli_options(
         &self,
         global_options: &GlobalOptions,
-        opts: &mut StandaloneOptions,
+        mut opts: StandaloneOptions,
-    ) -> Result<()> {
+    ) -> Result<StandaloneOptions> {
         // Should always be standalone mode.
         opts.mode = Mode::Standalone;
 
@@ -379,27 +369,20 @@ impl StartCommand {
 
         opts.user_provider.clone_from(&self.user_provider);
 
-        Ok(())
+        Ok(opts)
     }
 
     #[allow(unreachable_code)]
     #[allow(unused_variables)]
     #[allow(clippy::diverging_sub_expression)]
-    async fn build(&self, opts: GreptimeOptions<StandaloneOptions>) -> Result<Instance> {
+    async fn build(&self, opts: StandaloneOptions) -> Result<Instance> {
-        common_runtime::init_global_runtimes(&opts.runtime);
+        let guard =
-
+            common_telemetry::init_global_logging(APP_NAME, &opts.logging, &opts.tracing, None);
-        let guard = common_telemetry::init_global_logging(
-            APP_NAME,
-            &opts.component.logging,
-            &opts.component.tracing,
-            None,
-        );
         log_versions(version!(), short_version!());
 
         info!("Standalone start command: {:#?}", self);
-        info!("Standalone options: {opts:#?}");
+        info!("Building standalone instance with {opts:#?}");
 
-        let opts = opts.component;
         let mut fe_opts = opts.frontend_options();
         #[allow(clippy::unnecessary_mut_passed)]
         let fe_plugins = plugins::setup_frontend_plugins(&mut fe_opts) // mut ref is MUST, DO NOT change it
@@ -454,11 +437,9 @@ impl StartCommand {
         );
         let flownode = Arc::new(flow_builder.build().await);
 
-        let datanode = DatanodeBuilder::new(dn_opts, fe_plugins.clone())
+        let builder =
-            .with_kv_backend(kv_backend.clone())
+            DatanodeBuilder::new(dn_opts, fe_plugins.clone()).with_kv_backend(kv_backend.clone());
-            .build()
+        let datanode = builder.build().await.context(StartDatanodeSnafu)?;
-            .await
-            .context(StartDatanodeSnafu)?;
 
         let node_manager = Arc::new(StandaloneDatanodeManager {
             region_server: datanode.region_server(),
@@ -683,10 +664,7 @@ mod tests {
             ..Default::default()
         };
 
-        let options = cmd
+        let options = cmd.load_options(&GlobalOptions::default()).unwrap();
-            .load_options(&GlobalOptions::default())
-            .unwrap()
-            .component;
         let fe_opts = options.frontend_options();
         let dn_opts = options.datanode_options();
         let logging_opts = options.logging;
@@ -747,8 +725,7 @@ mod tests {
                 #[cfg(feature = "tokio-console")]
                 tokio_console_addr: None,
             })
-            .unwrap()
+            .unwrap();
-            .component;
 
         assert_eq!("/tmp/greptimedb/test/logs", opts.logging.dir);
         assert_eq!("debug", opts.logging.level.unwrap());
@@ -810,7 +787,7 @@ mod tests {
                     ..Default::default()
                 };
 
-                let opts = command.load_options(&Default::default()).unwrap().component;
+                let opts = command.load_options(&GlobalOptions::default()).unwrap();
 
                 // Should be read from env, env > default values.
                 assert_eq!(opts.logging.dir, "/other/log/dir");

src/cmd/tests/load_config_test.rs

@@ -1,231 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::time::Duration;
-
-use cmd::options::GreptimeOptions;
-use cmd::standalone::StandaloneOptions;
-use common_base::readable_size::ReadableSize;
-use common_config::Configurable;
-use common_runtime::global::RuntimeOptions;
-use common_telemetry::logging::LoggingOptions;
-use common_wal::config::raft_engine::RaftEngineConfig;
-use common_wal::config::{DatanodeWalConfig, StandaloneWalConfig};
-use datanode::config::{DatanodeOptions, RegionEngineConfig, StorageConfig};
-use frontend::frontend::FrontendOptions;
-use frontend::service_config::datanode::DatanodeClientOptions;
-use meta_client::MetaClientOptions;
-use meta_srv::metasrv::MetasrvOptions;
-use meta_srv::selector::SelectorType;
-use mito2::config::MitoConfig;
-use servers::export_metrics::ExportMetricsOption;
-
-#[test]
-fn test_load_datanode_example_config() {
-    let example_config = common_test_util::find_workspace_path("config/datanode.example.toml");
-    let options =
-        GreptimeOptions::<DatanodeOptions>::load_layered_options(example_config.to_str(), "")
-            .unwrap();
-
-    let expected = GreptimeOptions::<DatanodeOptions> {
-        runtime: RuntimeOptions {
-            read_rt_size: 8,
-            write_rt_size: 8,
-            bg_rt_size: 8,
-        },
-        component: DatanodeOptions {
-            node_id: Some(42),
-            rpc_hostname: Some("127.0.0.1".to_string()),
-            meta_client: Some(MetaClientOptions {
-                metasrv_addrs: vec!["127.0.0.1:3002".to_string()],
-                timeout: Duration::from_secs(3),
-                heartbeat_timeout: Duration::from_millis(500),
-                ddl_timeout: Duration::from_secs(10),
-                connect_timeout: Duration::from_secs(1),
-                tcp_nodelay: true,
-                metadata_cache_max_capacity: 100000,
-                metadata_cache_ttl: Duration::from_secs(600),
-                metadata_cache_tti: Duration::from_secs(300),
-            }),
-            wal: DatanodeWalConfig::RaftEngine(RaftEngineConfig {
-                dir: Some("/tmp/greptimedb/wal".to_string()),
-                sync_period: Some(Duration::from_secs(10)),
-                ..Default::default()
-            }),
-            storage: StorageConfig {
-                data_home: "/tmp/greptimedb/".to_string(),
-                ..Default::default()
-            },
-            region_engine: vec![RegionEngineConfig::Mito(MitoConfig {
-                num_workers: 8,
-                auto_flush_interval: Duration::from_secs(3600),
-                scan_parallelism: 0,
-                global_write_buffer_size: ReadableSize::gb(1),
-                global_write_buffer_reject_size: ReadableSize::gb(2),
-                sst_meta_cache_size: ReadableSize::mb(128),
-                vector_cache_size: ReadableSize::mb(512),
-                page_cache_size: ReadableSize::mb(512),
-                max_background_jobs: 4,
-                ..Default::default()
-            })],
-            logging: LoggingOptions {
-                level: Some("info".to_string()),
-                otlp_endpoint: Some("".to_string()),
-                tracing_sample_ratio: Some(Default::default()),
-                ..Default::default()
-            },
-            export_metrics: ExportMetricsOption {
-                self_import: Some(Default::default()),
-                remote_write: Some(Default::default()),
-                ..Default::default()
-            },
-            ..Default::default()
-        },
-    };
-
-    assert_eq!(options, expected);
-}
-
-#[test]
-fn test_load_frontend_example_config() {
-    let example_config = common_test_util::find_workspace_path("config/frontend.example.toml");
-    let options =
-        GreptimeOptions::<FrontendOptions>::load_layered_options(example_config.to_str(), "")
-            .unwrap();
-    let expected = GreptimeOptions::<FrontendOptions> {
-        runtime: RuntimeOptions {
-            read_rt_size: 8,
-            write_rt_size: 8,
-            bg_rt_size: 8,
-        },
-        component: FrontendOptions {
-            default_timezone: Some("UTC".to_string()),
-            meta_client: Some(MetaClientOptions {
-                metasrv_addrs: vec!["127.0.0.1:3002".to_string()],
-                timeout: Duration::from_secs(3),
-                heartbeat_timeout: Duration::from_millis(500),
-                ddl_timeout: Duration::from_secs(10),
-                connect_timeout: Duration::from_secs(1),
-                tcp_nodelay: true,
-                metadata_cache_max_capacity: 100000,
-                metadata_cache_ttl: Duration::from_secs(600),
-                metadata_cache_tti: Duration::from_secs(300),
-            }),
-            logging: LoggingOptions {
-                level: Some("info".to_string()),
-                otlp_endpoint: Some("".to_string()),
-                tracing_sample_ratio: Some(Default::default()),
-                ..Default::default()
-            },
-            datanode: frontend::service_config::DatanodeOptions {
-                client: DatanodeClientOptions {
-                    connect_timeout: Duration::from_secs(10),
-                    tcp_nodelay: true,
-                },
-            },
-            export_metrics: ExportMetricsOption {
-                self_import: Some(Default::default()),
-                remote_write: Some(Default::default()),
-                ..Default::default()
-            },
-            ..Default::default()
-        },
-    };
-    assert_eq!(options, expected);
-}
-
-#[test]
-fn test_load_metasrv_example_config() {
-    let example_config = common_test_util::find_workspace_path("config/metasrv.example.toml");
-    let options =
-        GreptimeOptions::<MetasrvOptions>::load_layered_options(example_config.to_str(), "")
-            .unwrap();
-    let expected = GreptimeOptions::<MetasrvOptions> {
-        runtime: RuntimeOptions {
-            read_rt_size: 8,
-            write_rt_size: 8,
-            bg_rt_size: 8,
-        },
-        component: MetasrvOptions {
-            selector: SelectorType::LeaseBased,
-            data_home: "/tmp/metasrv/".to_string(),
-            logging: LoggingOptions {
-                dir: "/tmp/greptimedb/logs".to_string(),
-                level: Some("info".to_string()),
-                otlp_endpoint: Some("".to_string()),
-                tracing_sample_ratio: Some(Default::default()),
-                ..Default::default()
-            },
-            export_metrics: ExportMetricsOption {
-                self_import: Some(Default::default()),
-                remote_write: Some(Default::default()),
-                ..Default::default()
-            },
-            ..Default::default()
-        },
-    };
-    assert_eq!(options, expected);
-}
-
-#[test]
-fn test_load_standalone_example_config() {
-    let example_config = common_test_util::find_workspace_path("config/standalone.example.toml");
-    let options =
-        GreptimeOptions::<StandaloneOptions>::load_layered_options(example_config.to_str(), "")
-            .unwrap();
-    let expected = GreptimeOptions::<StandaloneOptions> {
-        runtime: RuntimeOptions {
-            read_rt_size: 8,
-            write_rt_size: 8,
-            bg_rt_size: 8,
-        },
-        component: StandaloneOptions {
-            default_timezone: Some("UTC".to_string()),
-            wal: StandaloneWalConfig::RaftEngine(RaftEngineConfig {
-                dir: Some("/tmp/greptimedb/wal".to_string()),
-                sync_period: Some(Duration::from_secs(10)),
-                ..Default::default()
-            }),
-            region_engine: vec![RegionEngineConfig::Mito(MitoConfig {
-                num_workers: 8,
-                auto_flush_interval: Duration::from_secs(3600),
-                scan_parallelism: 0,
-                global_write_buffer_size: ReadableSize::gb(1),
-                global_write_buffer_reject_size: ReadableSize::gb(2),
-                sst_meta_cache_size: ReadableSize::mb(128),
-                vector_cache_size: ReadableSize::mb(512),
-                page_cache_size: ReadableSize::mb(512),
-                max_background_jobs: 4,
-                ..Default::default()
-            })],
-            storage: StorageConfig {
-                data_home: "/tmp/greptimedb/".to_string(),
-                ..Default::default()
-            },
-            logging: LoggingOptions {
-                level: Some("info".to_string()),
-                otlp_endpoint: Some("".to_string()),
-                tracing_sample_ratio: Some(Default::default()),
-                ..Default::default()
-            },
-            export_metrics: ExportMetricsOption {
-                self_import: Some(Default::default()),
-                remote_write: Some(Default::default()),
-                ..Default::default()
-            },
-            ..Default::default()
-        },
-    };
-    assert_eq!(options, expected);
-}

src/common/config/src/config.rs

@@ -13,8 +13,7 @@
 // limitations under the License.
 
 use config::{Environment, File, FileFormat};
-use serde::de::DeserializeOwned;
+use serde::{Deserialize, Serialize};
-use serde::Serialize;
 use snafu::ResultExt;
 
 use crate::error::{LoadLayeredConfigSnafu, Result, SerdeJsonSnafu, TomlFormatSnafu};
@@ -26,7 +25,7 @@ pub const ENV_VAR_SEP: &str = "__";
 pub const ENV_LIST_SEP: &str = ",";
 
 /// Configuration trait defines the common interface for configuration that can be loaded from multiple sources and serialized to TOML.
-pub trait Configurable: Serialize + DeserializeOwned + Default + Sized {
+pub trait Configurable<'de>: Serialize + Deserialize<'de> + Default + Sized {
     /// Load the configuration from multiple sources and merge them.
     /// The precedence order is: config file > environment variables > default values.
     /// `env_prefix` is the prefix of environment variables, e.g. "FRONTEND__xxx".
@@ -129,7 +128,7 @@ mod tests {
         }
     }
 
-    impl Configurable for TestDatanodeConfig {
+    impl Configurable<'_> for TestDatanodeConfig {
         fn env_list_keys() -> Option<&'static [&'static str]> {
             Some(&["meta_client.metasrv_addrs"])
         }

src/common/datasource/Cargo.toml

@@ -20,7 +20,6 @@ async-compression = { version = "0.3", features = [
 ] }
 async-trait.workspace = true
 bytes.workspace = true
-common-base.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 common-recordbatch.workspace = true
@@ -34,7 +33,6 @@ object-store.workspace = true
 orc-rust = { git = "https://github.com/datafusion-contrib/datafusion-orc.git", rev = "502217315726314c4008808fe169764529640599" }
 parquet.workspace = true
 paste = "1.0"
-rand.workspace = true
 regex = "1.7"
 serde.workspace = true
 snafu.workspace = true
@@ -44,7 +42,4 @@ tokio-util.workspace = true
 url = "2.3"
 
 [dev-dependencies]
-common-telemetry.workspace = true
 common-test-util.workspace = true
-dotenv.workspace = true
-uuid.workspace = true

src/common/datasource/src/compression.rs

@@ -92,44 +92,34 @@ impl CompressionType {
 macro_rules! impl_compression_type {
     ($(($enum_item:ident, $prefix:ident)),*) => {
         paste::item! {
-            use bytes::{Buf, BufMut, BytesMut};
-
             impl CompressionType {
-                pub async fn encode<B: Buf>(&self, mut content: B) -> io::Result<Vec<u8>> {
+                pub async fn encode(&self, content: impl AsRef<[u8]>) -> io::Result<Vec<u8>> {
                     match self {
                         $(
                             CompressionType::$enum_item => {
-                                let mut buffer = Vec::with_capacity(content.remaining());
+                                let mut buffer = Vec::with_capacity(content.as_ref().len());
                                 let mut encoder = write::[<$prefix Encoder>]::new(&mut buffer);
-                                encoder.write_all_buf(&mut content).await?;
+                                encoder.write_all(content.as_ref()).await?;
                                 encoder.shutdown().await?;
                                 Ok(buffer)
                             }
                         )*
-                        CompressionType::Uncompressed => {
+                        CompressionType::Uncompressed => Ok(content.as_ref().to_vec()),
-                            let mut bs = BytesMut::with_capacity(content.remaining());
-                            bs.put(content);
-                            Ok(bs.to_vec())
-                        },
                     }
                 }
 
-                pub async fn decode<B: Buf>(&self, mut content: B) -> io::Result<Vec<u8>> {
+                pub async fn decode(&self, content: impl AsRef<[u8]>) -> io::Result<Vec<u8>> {
                     match self {
                         $(
                             CompressionType::$enum_item => {
-                                let mut buffer = Vec::with_capacity(content.remaining() * 2);
+                                let mut buffer = Vec::with_capacity(content.as_ref().len() * 2);
                                 let mut encoder = write::[<$prefix Decoder>]::new(&mut buffer);
-                                encoder.write_all_buf(&mut content).await?;
+                                encoder.write_all(content.as_ref()).await?;
                                 encoder.shutdown().await?;
                                 Ok(buffer)
                             }
                         )*
-                        CompressionType::Uncompressed => {
+                        CompressionType::Uncompressed => Ok(content.as_ref().to_vec()),
-                            let mut bs = BytesMut::with_capacity(content.remaining());
-                            bs.put(content);
-                            Ok(bs.to_vec())
-                        },
                     }
                 }
 
@@ -161,13 +151,13 @@ macro_rules! impl_compression_type {
                 $(
                 #[tokio::test]
                 async fn [<test_ $enum_item:lower _compression>]() {
-                    let string = "foo_bar".as_bytes();
+                    let string = "foo_bar".as_bytes().to_vec();
                     let compress = CompressionType::$enum_item
-                        .encode(string)
+                        .encode(&string)
                         .await
                         .unwrap();
                     let decompress = CompressionType::$enum_item
-                        .decode(compress.as_slice())
+                        .decode(&compress)
                         .await
                         .unwrap();
                     assert_eq!(decompress, string);
@@ -175,13 +165,13 @@ macro_rules! impl_compression_type {
 
                 #[tokio::test]
                 async fn test_uncompression() {
-                    let string = "foo_bar".as_bytes();
+                    let string = "foo_bar".as_bytes().to_vec();
                     let compress = CompressionType::Uncompressed
-                        .encode(string)
+                        .encode(&string)
                         .await
                         .unwrap();
                     let decompress = CompressionType::Uncompressed
-                        .decode(compress.as_slice())
+                        .decode(&compress)
                         .await
                         .unwrap();
                     assert_eq!(decompress, string);

src/common/datasource/src/file_format.rs

@@ -36,7 +36,6 @@ use datafusion::physical_plan::SendableRecordBatchStream;
 use futures::StreamExt;
 use object_store::ObjectStore;
 use snafu::ResultExt;
-use tokio_util::compat::FuturesAsyncWriteCompatExt;
 
 use self::csv::CsvFormat;
 use self::json::JsonFormat;
@@ -46,7 +45,6 @@ use crate::buffered_writer::{DfRecordBatchEncoder, LazyBufferedWriter};
 use crate::compression::CompressionType;
 use crate::error::{self, Result};
 use crate::share_buffer::SharedBuffer;
-use crate::DEFAULT_WRITE_BUFFER_SIZE;
 
 pub const FORMAT_COMPRESSION_TYPE: &str = "compression_type";
 pub const FORMAT_DELIMITER: &str = "delimiter";
@@ -148,8 +146,7 @@ pub fn open_with_decoder<T: ArrowDecoder, F: Fn() -> DataFusionResult<T>>(
         let reader = object_store
             .reader(&path)
             .await
-            .map_err(|e| DataFusionError::External(Box::new(e)))?
+            .map_err(|e| DataFusionError::External(Box::new(e)))?;
-            .into_bytes_stream(..);
 
         let mut upstream = compression_type.convert_stream(reader).fuse();
 
@@ -205,9 +202,7 @@ pub async fn stream_to_file<T: DfRecordBatchEncoder, U: Fn(SharedBuffer) -> T>(
         store
             .writer_with(&path)
             .concurrent(concurrency)
-            .chunk(DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize)
             .await
-            .map(|v| v.into_futures_async_write().compat_write())
             .context(error::WriteObjectSnafu { path })
     });
 

src/common/datasource/src/file_format/csv.rs

@@ -29,7 +29,6 @@ use datafusion::physical_plan::SendableRecordBatchStream;
 use derive_builder::Builder;
 use object_store::ObjectStore;
 use snafu::ResultExt;
-use tokio_util::compat::FuturesAsyncReadCompatExt;
 use tokio_util::io::SyncIoBridge;
 
 use super::stream_to_file;
@@ -165,16 +164,10 @@ impl FileOpener for CsvOpener {
 #[async_trait]
 impl FileFormat for CsvFormat {
     async fn infer_schema(&self, store: &ObjectStore, path: &str) -> Result<Schema> {
-        let meta = store
-            .stat(path)
-            .await
-            .context(error::ReadObjectSnafu { path })?;
         let reader = store
             .reader(path)
             .await
-            .context(error::ReadObjectSnafu { path })?
+            .context(error::ReadObjectSnafu { path })?;
-            .into_futures_async_read(0..meta.content_length())
-            .compat();
 
         let decoded = self.compression_type.convert_async_read(reader);
 

src/common/datasource/src/file_format/json.rs

@@ -31,7 +31,6 @@ use datafusion::error::{DataFusionError, Result as DataFusionResult};
 use datafusion::physical_plan::SendableRecordBatchStream;
 use object_store::ObjectStore;
 use snafu::ResultExt;
-use tokio_util::compat::FuturesAsyncReadCompatExt;
 use tokio_util::io::SyncIoBridge;
 
 use super::stream_to_file;
@@ -83,16 +82,10 @@ impl Default for JsonFormat {
 #[async_trait]
 impl FileFormat for JsonFormat {
     async fn infer_schema(&self, store: &ObjectStore, path: &str) -> Result<Schema> {
-        let meta = store
-            .stat(path)
-            .await
-            .context(error::ReadObjectSnafu { path })?;
         let reader = store
             .reader(path)
             .await
-            .context(error::ReadObjectSnafu { path })?
+            .context(error::ReadObjectSnafu { path })?;
-            .into_futures_async_read(0..meta.content_length())
-            .compat();
 
         let decoded = self.compression_type.convert_async_read(reader);
 

src/common/datasource/src/file_format/orc.rs

@@ -16,17 +16,15 @@ use std::sync::Arc;
 
 use arrow_schema::{ArrowError, Schema, SchemaRef};
 use async_trait::async_trait;
-use bytes::Bytes;
 use common_recordbatch::adapter::RecordBatchStreamTypeAdapter;
 use datafusion::datasource::physical_plan::{FileMeta, FileOpenFuture, FileOpener};
 use datafusion::error::{DataFusionError, Result as DfResult};
-use futures::future::BoxFuture;
+use futures::{StreamExt, TryStreamExt};
-use futures::{FutureExt, StreamExt, TryStreamExt};
 use object_store::ObjectStore;
 use orc_rust::arrow_reader::ArrowReaderBuilder;
 use orc_rust::async_arrow_reader::ArrowStreamReader;
-use orc_rust::reader::AsyncChunkReader;
 use snafu::ResultExt;
+use tokio::io::{AsyncRead, AsyncSeek};
 
 use crate::error::{self, Result};
 use crate::file_format::FileFormat;
@@ -34,49 +32,18 @@ use crate::file_format::FileFormat;
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub struct OrcFormat;
 
-#[derive(Clone)]
+pub async fn new_orc_stream_reader<R: AsyncRead + AsyncSeek + Unpin + Send + 'static>(
-pub struct ReaderAdapter {
+    reader: R,
-    reader: object_store::Reader,
+) -> Result<ArrowStreamReader<R>> {
-    len: u64,
-}
-
-impl ReaderAdapter {
-    pub fn new(reader: object_store::Reader, len: u64) -> Self {
-        Self { reader, len }
-    }
-}
-
-impl AsyncChunkReader for ReaderAdapter {
-    fn len(&mut self) -> BoxFuture<'_, std::io::Result<u64>> {
-        async move { Ok(self.len) }.boxed()
-    }
-
-    fn get_bytes(
-        &mut self,
-        offset_from_start: u64,
-        length: u64,
-    ) -> BoxFuture<'_, std::io::Result<Bytes>> {
-        async move {
-            let bytes = self
-                .reader
-                .read(offset_from_start..offset_from_start + length)
-                .await?;
-            Ok(bytes.to_bytes())
-        }
-        .boxed()
-    }
-}
-
-pub async fn new_orc_stream_reader(
-    reader: ReaderAdapter,
-) -> Result<ArrowStreamReader<ReaderAdapter>> {
     let reader_build = ArrowReaderBuilder::try_new_async(reader)
         .await
         .context(error::OrcReaderSnafu)?;
     Ok(reader_build.build_async())
 }
 
-pub async fn infer_orc_schema(reader: ReaderAdapter) -> Result<Schema> {
+pub async fn infer_orc_schema<R: AsyncRead + AsyncSeek + Unpin + Send + 'static>(
+    reader: R,
+) -> Result<Schema> {
     let reader = new_orc_stream_reader(reader).await?;
     Ok(reader.schema().as_ref().clone())
 }
@@ -84,15 +51,13 @@ pub async fn infer_orc_schema(reader: ReaderAdapter) -> Result<Schema> {
 #[async_trait]
 impl FileFormat for OrcFormat {
     async fn infer_schema(&self, store: &ObjectStore, path: &str) -> Result<Schema> {
-        let meta = store
-            .stat(path)
-            .await
-            .context(error::ReadObjectSnafu { path })?;
         let reader = store
             .reader(path)
             .await
             .context(error::ReadObjectSnafu { path })?;
-        let schema = infer_orc_schema(ReaderAdapter::new(reader, meta.content_length())).await?;
+
+        let schema = infer_orc_schema(reader).await?;
+
         Ok(schema)
     }
 }
@@ -132,22 +97,14 @@ impl FileOpener for OrcOpener {
         };
         let projection = self.projection.clone();
         Ok(Box::pin(async move {
-            let path = meta.location().to_string();
-
-            let meta = object_store
-                .stat(&path)
-                .await
-                .map_err(|e| DataFusionError::External(Box::new(e)))?;
-
             let reader = object_store
-                .reader(&path)
+                .reader(meta.location().to_string().as_str())
                 .await
                 .map_err(|e| DataFusionError::External(Box::new(e)))?;
 
-            let stream_reader =
+            let stream_reader = new_orc_stream_reader(reader)
-                new_orc_stream_reader(ReaderAdapter::new(reader, meta.content_length()))
+                .await
-                    .await
+                .map_err(|e| DataFusionError::External(Box::new(e)))?;
-                    .map_err(|e| DataFusionError::External(Box::new(e)))?;
 
             let stream =
                 RecordBatchStreamTypeAdapter::new(projected_schema, stream_reader, projection);

src/common/datasource/src/file_format/parquet.rs

@@ -16,7 +16,7 @@ use std::result;
 use std::sync::Arc;
 
 use arrow::record_batch::RecordBatch;
-use arrow_schema::Schema;
+use arrow_schema::{Schema, SchemaRef};
 use async_trait::async_trait;
 use datafusion::datasource::physical_plan::{FileMeta, ParquetFileReaderFactory};
 use datafusion::error::Result as DatafusionResult;
@@ -29,18 +29,15 @@ use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
 use datafusion::physical_plan::SendableRecordBatchStream;
 use futures::future::BoxFuture;
 use futures::StreamExt;
-use object_store::{FuturesAsyncReader, ObjectStore};
+use object_store::{ObjectStore, Reader, Writer};
-use parquet::arrow::AsyncArrowWriter;
 use parquet::basic::{Compression, ZstdLevel};
 use parquet::file::properties::WriterProperties;
 use snafu::ResultExt;
-use tokio_util::compat::{Compat, FuturesAsyncReadCompatExt, FuturesAsyncWriteCompatExt};
 
-use crate::buffered_writer::{ArrowWriterCloser, DfRecordBatchEncoder};
+use crate::buffered_writer::{ArrowWriterCloser, DfRecordBatchEncoder, LazyBufferedWriter};
-use crate::error::{self, Result, WriteObjectSnafu, WriteParquetSnafu};
+use crate::error::{self, Result};
 use crate::file_format::FileFormat;
 use crate::share_buffer::SharedBuffer;
-use crate::DEFAULT_WRITE_BUFFER_SIZE;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub struct ParquetFormat {}
@@ -48,16 +45,10 @@ pub struct ParquetFormat {}
 #[async_trait]
 impl FileFormat for ParquetFormat {
     async fn infer_schema(&self, store: &ObjectStore, path: &str) -> Result<Schema> {
-        let meta = store
-            .stat(path)
-            .await
-            .context(error::ReadObjectSnafu { path })?;
         let mut reader = store
             .reader(path)
             .await
-            .context(error::ReadObjectSnafu { path })?
+            .context(error::ReadObjectSnafu { path })?;
-            .into_futures_async_read(0..meta.content_length())
-            .compat();
 
         let metadata = reader
             .get_metadata()
@@ -107,7 +98,7 @@ impl ParquetFileReaderFactory for DefaultParquetFileReaderFactory {
 
 pub struct LazyParquetFileReader {
     object_store: ObjectStore,
-    reader: Option<Compat<FuturesAsyncReader>>,
+    reader: Option<Reader>,
     path: String,
 }
 
@@ -123,13 +114,7 @@ impl LazyParquetFileReader {
     /// Must initialize the reader, or throw an error from the future.
     async fn maybe_initialize(&mut self) -> result::Result<(), object_store::Error> {
         if self.reader.is_none() {
-            let meta = self.object_store.stat(&self.path).await?;
+            let reader = self.object_store.reader(&self.path).await?;
-            let reader = self
-                .object_store
-                .reader(&self.path)
-                .await?
-                .into_futures_async_read(0..meta.content_length())
-                .compat();
             self.reader = Some(reader);
         }
 
@@ -175,6 +160,72 @@ impl ArrowWriterCloser for ArrowWriter<SharedBuffer> {
     }
 }
 
+/// Parquet writer that buffers row groups in memory and writes buffered data to an underlying
+/// storage by chunks to reduce memory consumption.
+pub struct BufferedWriter {
+    inner: InnerBufferedWriter,
+}
+
+type InnerBufferedWriter = LazyBufferedWriter<
+    object_store::Writer,
+    ArrowWriter<SharedBuffer>,
+    impl Fn(String) -> BoxFuture<'static, Result<Writer>>,
+>;
+
+impl BufferedWriter {
+    fn make_write_factory(
+        store: ObjectStore,
+        concurrency: usize,
+    ) -> impl Fn(String) -> BoxFuture<'static, Result<Writer>> {
+        move |path| {
+            let store = store.clone();
+            Box::pin(async move {
+                store
+                    .writer_with(&path)
+                    .concurrent(concurrency)
+                    .await
+                    .context(error::WriteObjectSnafu { path })
+            })
+        }
+    }
+
+    pub async fn try_new(
+        path: String,
+        store: ObjectStore,
+        arrow_schema: SchemaRef,
+        props: Option<WriterProperties>,
+        buffer_threshold: usize,
+        concurrency: usize,
+    ) -> error::Result<Self> {
+        let buffer = SharedBuffer::with_capacity(buffer_threshold);
+
+        let arrow_writer = ArrowWriter::try_new(buffer.clone(), arrow_schema.clone(), props)
+            .context(error::WriteParquetSnafu { path: &path })?;
+
+        Ok(Self {
+            inner: LazyBufferedWriter::new(
+                buffer_threshold,
+                buffer,
+                arrow_writer,
+                &path,
+                Self::make_write_factory(store, concurrency),
+            ),
+        })
+    }
+
+    /// Write a record batch to stream writer.
+    pub async fn write(&mut self, arrow_batch: &RecordBatch) -> error::Result<()> {
+        self.inner.write(arrow_batch).await
+    }
+
+    /// Close parquet writer.
+    ///
+    /// Return file metadata and bytes written.
+    pub async fn close(self) -> error::Result<(FileMetaData, u64)> {
+        self.inner.close_with_arrow_writer().await
+    }
+}
+
 /// Output the stream to a parquet file.
 ///
 /// Returns number of rows written.
@@ -182,33 +233,29 @@ pub async fn stream_to_parquet(
     mut stream: SendableRecordBatchStream,
     store: ObjectStore,
     path: &str,
+    threshold: usize,
     concurrency: usize,
 ) -> Result<usize> {
     let write_props = WriterProperties::builder()
         .set_compression(Compression::ZSTD(ZstdLevel::default()))
         .build();
     let schema = stream.schema();
-    let inner_writer = store
+    let mut buffered_writer = BufferedWriter::try_new(
-        .writer_with(path)
+        path.to_string(),
-        .concurrent(concurrency)
+        store,
-        .chunk(DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize)
+        schema,
-        .await
+        Some(write_props),
-        .map(|w| w.into_futures_async_write().compat_write())
+        threshold,
-        .context(WriteObjectSnafu { path })?;
+        concurrency,
-
+    )
-    let mut writer = AsyncArrowWriter::try_new(inner_writer, schema, Some(write_props))
+    .await?;
-        .context(WriteParquetSnafu { path })?;
     let mut rows_written = 0;
-
     while let Some(batch) = stream.next().await {
         let batch = batch.context(error::ReadRecordBatchSnafu)?;
-        writer
+        buffered_writer.write(&batch).await?;
-            .write(&batch)
-            .await
-            .context(WriteParquetSnafu { path })?;
         rows_written += batch.num_rows();
     }
-    writer.close().await.context(WriteParquetSnafu { path })?;
+    buffered_writer.close().await?;
     Ok(rows_written)
 }
 

src/common/datasource/src/lib.rs

@@ -27,8 +27,3 @@ pub mod test_util;
 #[cfg(test)]
 pub mod tests;
 pub mod util;
-
-use common_base::readable_size::ReadableSize;
-
-/// Default write buffer size, it should be greater than the default minimum upload part of S3 (5mb).
-pub const DEFAULT_WRITE_BUFFER_SIZE: ReadableSize = ReadableSize::mb(8);

src/common/datasource/src/test_util.rs

@@ -120,7 +120,7 @@ pub async fn setup_stream_to_json_test(origin_path: &str, threshold: impl Fn(usi
 
     let written = tmp_store.read(&output_path).await.unwrap();
     let origin = store.read(origin_path).await.unwrap();
-    assert_eq_lines(written.to_vec(), origin.to_vec());
+    assert_eq_lines(written, origin);
 }
 
 pub async fn setup_stream_to_csv_test(origin_path: &str, threshold: impl Fn(usize) -> usize) {
@@ -158,7 +158,7 @@ pub async fn setup_stream_to_csv_test(origin_path: &str, threshold: impl Fn(usiz
 
     let written = tmp_store.read(&output_path).await.unwrap();
     let origin = store.read(origin_path).await.unwrap();
-    assert_eq_lines(written.to_vec(), origin.to_vec());
+    assert_eq_lines(written, origin);
 }
 
 // Ignore the CRLF difference across operating systems.

src/common/error/Cargo.toml

@@ -10,4 +10,3 @@ workspace = true
 [dependencies]
 snafu.workspace = true
 strum.workspace = true
-tonic.workspace = true

src/common/error/src/status_code.rs

@@ -15,7 +15,6 @@
 use std::fmt;
 
 use strum::{AsRefStr, EnumIter, EnumString, FromRepr};
-use tonic::Code;
 
 /// Common status code for public API.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, EnumString, AsRefStr, EnumIter, FromRepr)]
@@ -203,75 +202,6 @@ impl fmt::Display for StatusCode {
     }
 }
 
-#[macro_export]
-macro_rules! define_into_tonic_status {
-    ($Error: ty) => {
-        impl From<$Error> for tonic::Status {
-            fn from(err: $Error) -> Self {
-                use tonic::codegen::http::{HeaderMap, HeaderValue};
-                use tonic::metadata::MetadataMap;
-                use $crate::GREPTIME_DB_HEADER_ERROR_CODE;
-
-                let mut headers = HeaderMap::<HeaderValue>::with_capacity(2);
-
-                // If either of the status_code or error msg cannot convert to valid HTTP header value
-                // (which is a very rare case), just ignore. Client will use Tonic status code and message.
-                let status_code = err.status_code();
-                headers.insert(
-                    GREPTIME_DB_HEADER_ERROR_CODE,
-                    HeaderValue::from(status_code as u32),
-                );
-                let root_error = err.output_msg();
-
-                let metadata = MetadataMap::from_headers(headers);
-                tonic::Status::with_metadata(
-                    $crate::status_code::status_to_tonic_code(status_code),
-                    root_error,
-                    metadata,
-                )
-            }
-        }
-    };
-}
-
-/// Returns the tonic [Code] of a [StatusCode].
-pub fn status_to_tonic_code(status_code: StatusCode) -> Code {
-    match status_code {
-        StatusCode::Success => Code::Ok,
-        StatusCode::Unknown => Code::Unknown,
-        StatusCode::Unsupported => Code::Unimplemented,
-        StatusCode::Unexpected
-        | StatusCode::Internal
-        | StatusCode::PlanQuery
-        | StatusCode::EngineExecuteQuery => Code::Internal,
-        StatusCode::InvalidArguments | StatusCode::InvalidSyntax | StatusCode::RequestOutdated => {
-            Code::InvalidArgument
-        }
-        StatusCode::Cancelled => Code::Cancelled,
-        StatusCode::TableAlreadyExists
-        | StatusCode::TableColumnExists
-        | StatusCode::RegionAlreadyExists
-        | StatusCode::FlowAlreadyExists => Code::AlreadyExists,
-        StatusCode::TableNotFound
-        | StatusCode::RegionNotFound
-        | StatusCode::TableColumnNotFound
-        | StatusCode::DatabaseNotFound
-        | StatusCode::UserNotFound
-        | StatusCode::FlowNotFound => Code::NotFound,
-        StatusCode::StorageUnavailable | StatusCode::RegionNotReady => Code::Unavailable,
-        StatusCode::RuntimeResourcesExhausted
-        | StatusCode::RateLimited
-        | StatusCode::RegionBusy => Code::ResourceExhausted,
-        StatusCode::UnsupportedPasswordType
-        | StatusCode::UserPasswordMismatch
-        | StatusCode::AuthHeaderNotFound
-        | StatusCode::InvalidAuthHeader => Code::Unauthenticated,
-        StatusCode::AccessDenied | StatusCode::PermissionDenied | StatusCode::RegionReadonly => {
-            Code::PermissionDenied
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use strum::IntoEnumIterator;

src/common/function/src/system/procedure_state.rs

@@ -44,10 +44,10 @@ struct ProcedureStateJson {
 /// A function to query procedure state by its id.
 /// Such as `procedure_state(pid)`.
 #[admin_fn(
-    name = ProcedureStateFunction,
+    name = "ProcedureStateFunction",
-    display_name = procedure_state,
+    display_name = "procedure_state",
-    sig_fn = signature,
+    sig_fn = "signature",
-    ret = string
+    ret = "string"
 )]
 pub(crate) async fn procedure_state(
     procedure_service_handler: &ProcedureServiceHandlerRef,

src/common/function/src/table/flush_compact_region.rs

@@ -35,7 +35,7 @@ use crate::helper::cast_u64;
 macro_rules! define_region_function {
     ($name: expr, $display_name_str: expr, $display_name: ident) => {
         /// A function to $display_name
-        #[admin_fn(name = $name, display_name = $display_name_str, sig_fn = signature, ret = uint64)]
+        #[admin_fn(name = $name, display_name = $display_name_str, sig_fn = "signature", ret = "uint64")]
         pub(crate) async fn $display_name(
             table_mutation_handler: &TableMutationHandlerRef,
             query_ctx: &QueryContextRef,
@@ -53,7 +53,7 @@ macro_rules! define_region_function {
 
             let Some(region_id) = cast_u64(&params[0])? else {
                 return UnsupportedInputDataTypeSnafu {
-                    function: stringify!($display_name_str),
+                    function: $display_name_str,
                     datatypes: params.iter().map(|v| v.data_type()).collect::<Vec<_>>(),
                 }
                 .fail();
@@ -68,9 +68,9 @@ macro_rules! define_region_function {
     };
 }
 
-define_region_function!(FlushRegionFunction, flush_region, flush_region);
+define_region_function!("FlushRegionFunction", "flush_region", flush_region);
 
-define_region_function!(CompactRegionFunction, compact_region, compact_region);
+define_region_function!("CompactRegionFunction", "compact_region", compact_region);
 
 fn signature() -> Signature {
     Signature::uniform(1, ConcreteDataType::numerics(), Volatility::Immutable)

src/common/function/src/table/flush_compact_table.rs

@@ -40,10 +40,10 @@ use crate::handlers::TableMutationHandlerRef;
 const COMPACT_TYPE_STRICT_WINDOW: &str = "strict_window";
 
 #[admin_fn(
-    name = FlushTableFunction,
+    name = "FlushTableFunction",
-    display_name = flush_table,
+    display_name = "flush_table",
-    sig_fn = flush_signature,
+    sig_fn = "flush_signature",
-    ret = uint64
+    ret = "uint64"
 )]
 pub(crate) async fn flush_table(
     table_mutation_handler: &TableMutationHandlerRef,
@@ -87,10 +87,10 @@ pub(crate) async fn flush_table(
 }
 
 #[admin_fn(
-    name = CompactTableFunction,
+    name = "CompactTableFunction",
-    display_name = compact_table,
+    display_name = "compact_table",
-    sig_fn = compact_signature,
+    sig_fn = "compact_signature",
-    ret = uint64
+    ret = "uint64"
 )]
 pub(crate) async fn compact_table(
     table_mutation_handler: &TableMutationHandlerRef,

src/common/function/src/table/migrate_region.rs

@@ -46,10 +46,10 @@ const DEFAULT_REPLAY_TIMEOUT_SECS: u64 = 10;
 /// - `from_peer`:  the source peer id
 /// - `to_peer`:  the target peer id
 #[admin_fn(
-    name = MigrateRegionFunction,
+    name = "MigrateRegionFunction",
-    display_name = migrate_region,
+    display_name = "migrate_region",
-    sig_fn = signature,
+    sig_fn = "signature",
-    ret = string
+    ret = "string"
 )]
 pub(crate) async fn migrate_region(
     procedure_service_handler: &ProcedureServiceHandlerRef,

src/common/macro/Cargo.toml

@@ -13,7 +13,13 @@ workspace = true
 [dependencies]
 proc-macro2 = "1.0.66"
 quote = "1.0"
-syn = { version = "2.0", features = [
+syn = "1.0"
+syn2 = { version = "2.0", package = "syn", features = [
+    "derive",
+    "parsing",
+    "printing",
+    "clone-impls",
+    "proc-macro",
     "extra-traits",
     "full",
 ] }

src/common/macro/src/admin_fn.rs

@@ -16,11 +16,11 @@ use proc_macro::TokenStream;
 use quote::quote;
 use syn::spanned::Spanned;
 use syn::{
-    parse_macro_input, Attribute, Ident, ItemFn, Signature, Type, TypePath, TypeReference,
+    parse_macro_input, Attribute, AttributeArgs, Ident, ItemFn, Signature, Type, TypePath,
-    Visibility,
+    TypeReference, Visibility,
 };
 
-use crate::utils::extract_input_types;
+use crate::utils::{extract_arg_map, extract_input_types, get_ident};
 
 /// Internal util macro to early return on error.
 macro_rules! ok {
@@ -40,31 +40,12 @@ macro_rules! error {
 }
 
 pub(crate) fn process_admin_fn(args: TokenStream, input: TokenStream) -> TokenStream {
-    let mut name: Option<Ident> = None;
+    let mut result = TokenStream::new();
-    let mut display_name: Option<Ident> = None;
-    let mut sig_fn: Option<Ident> = None;
-    let mut ret: Option<Ident> = None;
-
-    let parser = syn::meta::parser(|meta| {
-        if meta.path.is_ident("name") {
-            name = Some(meta.value()?.parse()?);
-            Ok(())
-        } else if meta.path.is_ident("display_name") {
-            display_name = Some(meta.value()?.parse()?);
-            Ok(())
-        } else if meta.path.is_ident("sig_fn") {
-            sig_fn = Some(meta.value()?.parse()?);
-            Ok(())
-        } else if meta.path.is_ident("ret") {
-            ret = Some(meta.value()?.parse()?);
-            Ok(())
-        } else {
-            Err(meta.error("unsupported property"))
-        }
-    });
 
     // extract arg map
-    parse_macro_input!(args with parser);
+    let arg_pairs = parse_macro_input!(args as AttributeArgs);
+    let arg_span = arg_pairs[0].span();
+    let arg_map = ok!(extract_arg_map(arg_pairs));
 
     // decompose the fn block
     let compute_fn = parse_macro_input!(input as ItemFn);
@@ -91,17 +72,16 @@ pub(crate) fn process_admin_fn(args: TokenStream, input: TokenStream) -> TokenSt
     }
     let handler_type = ok!(extract_handler_type(&arg_types));
 
-    let mut result = TokenStream::new();
     // build the struct and its impl block
     // only do this when `display_name` is specified
-    if let Some(display_name) = display_name {
+    if let Ok(display_name) = get_ident(&arg_map, "display_name", arg_span) {
         let struct_code = build_struct(
             attrs,
             vis,
             fn_name,
-            name.expect("name required"),
+            ok!(get_ident(&arg_map, "name", arg_span)),
-            sig_fn.expect("sig_fn required"),
+            ok!(get_ident(&arg_map, "sig_fn", arg_span)),
-            ret.expect("ret required"),
+            ok!(get_ident(&arg_map, "ret", arg_span)),
             handler_type,
             display_name,
         );

src/common/macro/src/print_caller.rs

@@ -14,24 +14,28 @@
 
 use proc_macro::TokenStream;
 use quote::{quote, ToTokens};
-use syn::{parse_macro_input, ItemFn, LitInt};
+use syn::{parse_macro_input, AttributeArgs, ItemFn, Lit, Meta, NestedMeta};
 
 pub(crate) fn process_print_caller(args: TokenStream, input: TokenStream) -> TokenStream {
     let mut depth = 1;
-    let parser = syn::meta::parser(|meta| {
-        if meta.path.is_ident("depth") {
-            depth = meta
-                .value()?
-                .parse::<LitInt>()
-                .and_then(|v| v.base10_parse::<usize>())
-                .expect("Invalid 'depth' value");
-            Ok(())
-        } else {
-            Err(meta.error("unsupported property"))
-        }
-    });
 
-    parse_macro_input!(args with parser);
+    let args = parse_macro_input!(args as AttributeArgs);
+    for meta in args.iter() {
+        if let NestedMeta::Meta(Meta::NameValue(name_value)) = meta {
+            let ident = name_value
+                .path
+                .get_ident()
+                .expect("Expected an ident!")
+                .to_string();
+            if ident == "depth" {
+                let Lit::Int(i) = &name_value.lit else {
+                    panic!("Expected 'depth' to be a valid int!")
+                };
+                depth = i.base10_parse::<usize>().expect("Invalid 'depth' value");
+                break;
+            }
+        }
+    }
 
     let tokens: TokenStream = quote! {
         {

src/common/macro/src/range_fn.rs

@@ -16,10 +16,11 @@ use proc_macro::TokenStream;
 use quote::quote;
 use syn::spanned::Spanned;
 use syn::{
-    parse_macro_input, Attribute, Ident, ItemFn, Signature, Type, TypeReference, Visibility,
+    parse_macro_input, Attribute, AttributeArgs, Ident, ItemFn, Signature, Type, TypeReference,
+    Visibility,
 };
 
-use crate::utils::extract_input_types;
+use crate::utils::{extract_arg_map, extract_input_types, get_ident};
 
 macro_rules! ok {
     ($item:expr) => {
@@ -31,27 +32,12 @@ macro_rules! ok {
 }
 
 pub(crate) fn process_range_fn(args: TokenStream, input: TokenStream) -> TokenStream {
-    let mut name: Option<Ident> = None;
+    let mut result = TokenStream::new();
-    let mut display_name: Option<Ident> = None;
-    let mut ret: Option<Ident> = None;
-
-    let parser = syn::meta::parser(|meta| {
-        if meta.path.is_ident("name") {
-            name = Some(meta.value()?.parse()?);
-            Ok(())
-        } else if meta.path.is_ident("display_name") {
-            display_name = Some(meta.value()?.parse()?);
-            Ok(())
-        } else if meta.path.is_ident("ret") {
-            ret = Some(meta.value()?.parse()?);
-            Ok(())
-        } else {
-            Err(meta.error("unsupported property"))
-        }
-    });
 
     // extract arg map
-    parse_macro_input!(args with parser);
+    let arg_pairs = parse_macro_input!(args as AttributeArgs);
+    let arg_span = arg_pairs[0].span();
+    let arg_map = ok!(extract_arg_map(arg_pairs));
 
     // decompose the fn block
     let compute_fn = parse_macro_input!(input as ItemFn);
@@ -82,27 +68,25 @@ pub(crate) fn process_range_fn(args: TokenStream, input: TokenStream) -> TokenSt
         })
         .collect::<Vec<_>>();
 
-    let mut result = TokenStream::new();
-
     // build the struct and its impl block
     // only do this when `display_name` is specified
-    if let Some(display_name) = display_name {
+    if let Ok(display_name) = get_ident(&arg_map, "display_name", arg_span) {
         let struct_code = build_struct(
             attrs,
             vis,
-            name.clone().expect("name required"),
+            ok!(get_ident(&arg_map, "name", arg_span)),
             display_name,
             array_types,
-            ret.clone().expect("ret required"),
+            ok!(get_ident(&arg_map, "ret", arg_span)),
         );
         result.extend(struct_code);
     }
 
     let calc_fn_code = build_calc_fn(
-        name.expect("name required"),
+        ok!(get_ident(&arg_map, "name", arg_span)),
         arg_types,
         fn_name.clone(),
-        ret.expect("ret required"),
+        ok!(get_ident(&arg_map, "ret", arg_span)),
     );
     // preserve this fn, but remove its `pub` modifier
     let input_fn_code: TokenStream = quote! {

src/common/macro/src/stack_trace_debug.rs

@@ -16,13 +16,13 @@
 
 use proc_macro2::{Span, TokenStream as TokenStream2};
 use quote::{quote, quote_spanned};
-use syn::spanned::Spanned;
+use syn2::spanned::Spanned;
-use syn::{parenthesized, Attribute, Ident, ItemEnum, Variant};
+use syn2::{parenthesized, Attribute, Ident, ItemEnum, Variant};
 
 pub fn stack_trace_style_impl(args: TokenStream2, input: TokenStream2) -> TokenStream2 {
     let input_cloned: TokenStream2 = input.clone();
 
-    let error_enum_definition: ItemEnum = syn::parse2(input_cloned).unwrap();
+    let error_enum_definition: ItemEnum = syn2::parse2(input_cloned).unwrap();
     let enum_name = error_enum_definition.ident;
 
     let mut variants = vec![];

src/common/macro/src/utils.rs

@@ -12,10 +12,48 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
+
+use proc_macro2::Span;
 use syn::punctuated::Punctuated;
 use syn::spanned::Spanned;
 use syn::token::Comma;
-use syn::{FnArg, Type};
+use syn::{FnArg, Ident, Meta, MetaNameValue, NestedMeta, Type};
+
+/// Extract a String <-> Ident map from the attribute args.
+pub(crate) fn extract_arg_map(args: Vec<NestedMeta>) -> Result<HashMap<String, Ident>, syn::Error> {
+    args.into_iter()
+        .map(|meta| {
+            if let NestedMeta::Meta(Meta::NameValue(MetaNameValue { path, lit, .. })) = meta {
+                let name = path.get_ident().unwrap().to_string();
+                let ident = match lit {
+                    syn::Lit::Str(lit_str) => lit_str.parse::<Ident>(),
+                    _ => Err(syn::Error::new(
+                        lit.span(),
+                        "Unexpected attribute format. Expected `name = \"value\"`",
+                    )),
+                }?;
+                Ok((name, ident))
+            } else {
+                Err(syn::Error::new(
+                    meta.span(),
+                    "Unexpected attribute format. Expected `name = \"value\"`",
+                ))
+            }
+        })
+        .collect::<Result<HashMap<String, Ident>, syn::Error>>()
+}
+
+/// Helper function to get an Ident from the previous arg map.
+pub(crate) fn get_ident(
+    map: &HashMap<String, Ident>,
+    key: &str,
+    span: Span,
+) -> Result<Ident, syn::Error> {
+    map.get(key)
+        .cloned()
+        .ok_or_else(|| syn::Error::new(span, format!("Expect attribute {key} but not found")))
+}
 
 /// Extract the argument list from the annotated function.
 pub(crate) fn extract_input_types(

src/common/meta/Cargo.toml

@@ -25,13 +25,11 @@ common-grpc-expr.workspace = true
 common-macro.workspace = true
 common-procedure.workspace = true
 common-procedure-test.workspace = true
-common-query.workspace = true
 common-recordbatch.workspace = true
 common-telemetry.workspace = true
 common-time.workspace = true
 common-wal.workspace = true
 datafusion-common.workspace = true
-datafusion-expr.workspace = true
 datatypes.workspace = true
 derive_builder.workspace = true
 etcd-client.workspace = true

src/common/meta/src/ddl/drop_database/executor.rs

@@ -131,10 +131,9 @@ mod tests {
     use std::sync::Arc;
 
     use api::region::RegionResponse;
-    use api::v1::region::RegionRequest;
+    use api::v1::region::{QueryRequest, RegionRequest};
     use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
     use common_error::ext::BoxedError;
-    use common_query::request::QueryRequest;
     use common_recordbatch::SendableRecordBatchStream;
     use table::table_name::TableName;
 

src/common/meta/src/ddl/test_util/datanode_handler.rs

@@ -13,10 +13,9 @@
 // limitations under the License.
 
 use api::region::RegionResponse;
-use api::v1::region::RegionRequest;
+use api::v1::region::{QueryRequest, RegionRequest};
 use common_error::ext::{BoxedError, ErrorExt, StackError};
 use common_error::status_code::StatusCode;
-use common_query::request::QueryRequest;
 use common_recordbatch::SendableRecordBatchStream;
 use common_telemetry::debug;
 use snafu::{ResultExt, Snafu};

src/common/meta/src/key/flow.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod flow_info;
+pub(crate) mod flow_info;
 pub(crate) mod flow_name;
 pub(crate) mod flownode_flow;
 pub(crate) mod table_flow;

src/common/meta/src/key/flow/flow_info.rs

@@ -141,26 +141,6 @@ impl FlowInfoValue {
     pub fn source_table_ids(&self) -> &[TableId] {
         &self.source_table_ids
     }
-
-    pub fn flow_name(&self) -> &String {
-        &self.flow_name
-    }
-
-    pub fn sink_table_name(&self) -> &TableName {
-        &self.sink_table_name
-    }
-
-    pub fn raw_sql(&self) -> &String {
-        &self.raw_sql
-    }
-
-    pub fn expire_after(&self) -> Option<i64> {
-        self.expire_after
-    }
-
-    pub fn comment(&self) -> &String {
-        &self.comment
-    }
 }
 
 pub type FlowInfoManagerRef = Arc<FlowInfoManager>;

src/common/meta/src/node_manager.rs

@@ -16,9 +16,8 @@ use std::sync::Arc;
 
 use api::region::RegionResponse;
 use api::v1::flow::{FlowRequest, FlowResponse};
-use api::v1::region::{InsertRequests, RegionRequest};
+use api::v1::region::{InsertRequests, QueryRequest, RegionRequest};
 pub use common_base::AffectedRows;
-use common_query::request::QueryRequest;
 use common_recordbatch::SendableRecordBatchStream;
 
 use crate::error::Result;

src/common/meta/src/test_util.rs

@@ -16,9 +16,8 @@ use std::sync::Arc;
 
 use api::region::RegionResponse;
 use api::v1::flow::{FlowRequest, FlowResponse};
-use api::v1::region::{InsertRequests, RegionRequest};
+use api::v1::region::{InsertRequests, QueryRequest, RegionRequest};
 pub use common_base::AffectedRows;
-use common_query::request::QueryRequest;
 use common_recordbatch::SendableRecordBatchStream;
 
 use crate::cache_invalidator::DummyCacheInvalidator;

src/common/procedure/src/store/state_store.rs

@@ -179,7 +179,7 @@ impl StateStore for ObjectStateStore {
                             ))
                         })
                         .context(ListStateSnafu { path: key })?;
-                    yield (key.into(), value.to_vec());
+                    yield (key.into(), value);
                 }
             }
         });

src/common/query/Cargo.toml

@@ -27,7 +27,6 @@ snafu.workspace = true
 sqlparser.workspace = true
 sqlparser_derive = "0.1"
 statrs = "0.16"
-store-api.workspace = true
 
 [dev-dependencies]
 common-base.workspace = true

src/common/query/src/lib.rs

@@ -17,7 +17,6 @@ pub mod error;
 mod function;
 pub mod logical_plan;
 pub mod prelude;
-pub mod request;
 mod signature;
 #[cfg(any(test, feature = "testing"))]
 pub mod test_util;

src/common/query/src/logical_plan/udaf.rs

@@ -22,7 +22,7 @@ use std::sync::Arc;
 
 use datafusion::arrow::datatypes::Field;
 use datafusion_common::Result;
-use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
+use datafusion_expr::function::AccumulatorArgs;
 use datafusion_expr::{
     Accumulator, AccumulatorFactoryFunction, AggregateUDF as DfAggregateUdf, AggregateUDFImpl,
 };
@@ -129,13 +129,13 @@ impl AggregateUDFImpl for DfUdafAdapter {
         (self.accumulator)(acc_args)
     }
 
-    fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<Field>> {
+    fn state_fields(&self, name: &str, _: ArrowDataType, _: Vec<Field>) -> Result<Vec<Field>> {
         let state_types = self.creator.state_types()?;
         let fields = state_types
             .into_iter()
             .enumerate()
             .map(|(i, t)| {
-                let name = format!("{}_{i}", args.name);
+                let name = format!("{name}_{i}");
                 Field::new(name, t.as_arrow_type(), true)
             })
             .collect::<Vec<_>>();

src/common/query/src/logical_plan/udf.rs

@@ -108,10 +108,6 @@ impl ScalarUDFImpl for DfUdfAdapter {
     fn invoke(&self, args: &[DfColumnarValue]) -> datafusion_common::Result<DfColumnarValue> {
         (self.fun)(args)
     }
-
-    fn invoke_no_args(&self, number_rows: usize) -> datafusion_common::Result<DfColumnarValue> {
-        Ok((self.fun)(&[])?.into_array(number_rows)?.into())
-    }
 }
 
 impl From<ScalarUdf> for DfScalarUDF {

src/common/query/src/request.rs

@@ -1,29 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use api::v1::region::RegionRequestHeader;
-use datafusion_expr::LogicalPlan;
-use store_api::storage::RegionId;
-
-/// The query request to be handled by the RegionServer (Datanode).
-pub struct QueryRequest {
-    /// The header of this request. Often to store some context of the query. None means all to defaults.
-    pub header: Option<RegionRequestHeader>,
-
-    /// The id of the region to be queried.
-    pub region_id: RegionId,
-
-    /// The form of the query: a logical plan.
-    pub plan: LogicalPlan,
-}

src/common/query/src/signature.rs

@@ -27,6 +27,10 @@ pub enum TypeSignature {
     /// arbitrary number of arguments of an common type out of a list of valid types
     // A function such as `concat` is `Variadic(vec![ConcreteDataType::String, ConcreteDataType::String])`
     Variadic(Vec<ConcreteDataType>),
+    /// arbitrary number of arguments of an arbitrary but equal type
+    // A function such as `array` is `VariadicEqual`
+    // The first argument decides the type used for coercion
+    VariadicEqual,
     /// One or more arguments with arbitrary types
     VariadicAny,
     /// fixed number of arguments of an arbitrary but equal type out of a list of valid types
@@ -63,7 +67,6 @@ impl Signature {
             volatility,
         }
     }
-
     /// variadic - Creates a variadic signature that represents an arbitrary number of arguments all from a type in common_types.
     pub fn variadic(common_types: Vec<ConcreteDataType>, volatility: Volatility) -> Self {
         Self {
@@ -71,6 +74,13 @@ impl Signature {
             volatility,
         }
     }
+    /// variadic_equal - Creates a variadic signature that represents an arbitrary number of arguments of the same type.
+    pub fn variadic_equal(volatility: Volatility) -> Self {
+        Self {
+            type_signature: TypeSignature::VariadicEqual,
+            volatility,
+        }
+    }
 
     /// variadic_any - Creates a variadic signature that represents an arbitrary number of arguments of any type.
     pub fn variadic_any(volatility: Volatility) -> Self {
@@ -121,6 +131,7 @@ impl From<TypeSignature> for DfTypeSignature {
             TypeSignature::Variadic(types) => {
                 DfTypeSignature::Variadic(concrete_types_to_arrow_types(types))
             }
+            TypeSignature::VariadicEqual => DfTypeSignature::VariadicEqual,
             TypeSignature::Uniform(n, types) => {
                 DfTypeSignature::Uniform(n, concrete_types_to_arrow_types(types))
             }

src/common/query/src/test_util.rs

@@ -20,7 +20,7 @@ use datafusion::logical_expr::LogicalPlan;
 use crate::error::Result;
 use crate::logical_plan::SubstraitPlanDecoder;
 
-/// Dummy [`SubstraitPlanDecoder`] for test.
+/// Dummy `[SubstraitPlanDecoder]` for test.
 pub struct DummyDecoder;
 
 impl DummyDecoder {

src/common/recordbatch/src/adapter.rs

@@ -292,7 +292,7 @@ impl ExecutionPlanVisitor for MetricCollector {
         // skip if no metric available
         let Some(metric) = plan.metrics() else {
             self.record_batch_metrics.plan_metrics.push(PlanMetrics {
-                plan: std::any::type_name::<Self>().to_string(),
+                plan: plan.name().to_string(),
                 level: self.current_level,
                 metrics: vec![],
             });

src/common/runtime/Cargo.toml

@@ -13,15 +13,13 @@ common-error.workspace = true
 common-macro.workspace = true
 common-telemetry.workspace = true
 lazy_static.workspace = true
-num_cpus.workspace = true
 once_cell.workspace = true
 paste.workspace = true
 prometheus.workspace = true
-serde.workspace = true
 snafu.workspace = true
 tokio.workspace = true
 tokio-metrics = "0.3"
-tokio-metrics-collector = { git = "https://github.com/MichaelScofield/tokio-metrics-collector.git", rev = "89d692d5753d28564a7aac73c6ac5aba22243ba0" }
+tokio-metrics-collector = "0.2"
 tokio-util.workspace = true
 
 [dev-dependencies]

src/common/runtime/src/global.rs

@@ -19,7 +19,6 @@ use std::sync::{Mutex, Once};
 use common_telemetry::info;
 use once_cell::sync::Lazy;
 use paste::paste;
-use serde::{Deserialize, Serialize};
 
 use crate::{Builder, JoinHandle, Runtime};
 
@@ -27,28 +26,6 @@ const READ_WORKERS: usize = 8;
 const WRITE_WORKERS: usize = 8;
 const BG_WORKERS: usize = 8;
 
-/// The options for the global runtimes.
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
-pub struct RuntimeOptions {
-    /// The number of threads to execute the runtime for global read operations.
-    pub read_rt_size: usize,
-    /// The number of threads to execute the runtime for global write operations.
-    pub write_rt_size: usize,
-    /// The number of threads to execute the runtime for global background operations.
-    pub bg_rt_size: usize,
-}
-
-impl Default for RuntimeOptions {
-    fn default() -> Self {
-        let cpus = num_cpus::get();
-        Self {
-            read_rt_size: cpus,
-            write_rt_size: cpus,
-            bg_rt_size: cpus,
-        }
-    }
-}
-
 pub fn create_runtime(runtime_name: &str, thread_name: &str, worker_threads: usize) -> Runtime {
     info!("Creating runtime with runtime_name: {runtime_name}, thread_name: {thread_name}, work_threads: {worker_threads}.");
     Builder::default()
@@ -135,26 +112,18 @@ static CONFIG_RUNTIMES: Lazy<Mutex<ConfigRuntimes>> =
 /// # Panics
 /// Panics when the global runtimes are already initialized.
 /// You should call this function before using any runtime functions.
-pub fn init_global_runtimes(options: &RuntimeOptions) {
+pub fn init_global_runtimes(
+    read: Option<Runtime>,
+    write: Option<Runtime>,
+    background: Option<Runtime>,
+) {
     static START: Once = Once::new();
     START.call_once(move || {
         let mut c = CONFIG_RUNTIMES.lock().unwrap();
         assert!(!c.already_init, "Global runtimes already initialized");
-        c.read_runtime = Some(create_runtime(
+        c.read_runtime = read;
-            "global-read",
+        c.write_runtime = write;
-            "global-read-worker",
+        c.bg_runtime = background;
-            options.read_rt_size,
-        ));
-        c.write_runtime = Some(create_runtime(
-            "global-write",
-            "global-write-worker",
-            options.write_rt_size,
-        ));
-        c.bg_runtime = Some(create_runtime(
-            "global-bg",
-            "global-bg-worker",
-            options.bg_rt_size,
-        ));
     });
 }
 

src/common/runtime/src/lib.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 pub mod error;
-pub mod global;
+mod global;
 mod metrics;
 mod repeated_task;
 pub mod runtime;

src/common/test-util/Cargo.toml

@@ -8,7 +8,7 @@ license.workspace = true
 workspace = true
 
 [dependencies]
-client = { workspace = true, features = ["testing"] }
+client.workspace = true
 common-query.workspace = true
 common-recordbatch.workspace = true
 once_cell.workspace = true

src/common/test-util/src/recordbatch.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use client::Database;
 use common_query::OutputData;
 use common_recordbatch::util;
 
@@ -30,25 +29,3 @@ pub async fn check_output_stream(output: OutputData, expected: &str) {
     let pretty_print = recordbatches.pretty_print().unwrap();
     assert_eq!(pretty_print, expected, "actual: \n{}", pretty_print);
 }
-
-pub async fn execute_and_check_output(db: &Database, sql: &str, expected: ExpectedOutput<'_>) {
-    let output = db.sql(sql).await.unwrap();
-    let output = output.data;
-
-    match (&output, expected) {
-        (OutputData::AffectedRows(x), ExpectedOutput::AffectedRows(y)) => {
-            assert_eq!(
-                *x, y,
-                r#"
-expected: {y}
-actual: {x}
-"#
-            )
-        }
-        (OutputData::RecordBatches(_), ExpectedOutput::QueryResult(x))
-        | (OutputData::Stream(_), ExpectedOutput::QueryResult(x)) => {
-            check_output_stream(output, x).await
-        }
-        _ => panic!(),
-    }
-}

src/common/time/src/timestamp.rs

@@ -14,7 +14,7 @@
 
 use core::default::Default;
 use std::cmp::Ordering;
-use std::fmt::{self, Display, Formatter, Write};
+use std::fmt::{Display, Formatter, Write};
 use std::hash::{Hash, Hasher};
 use std::time::Duration;
 
@@ -41,7 +41,7 @@ use crate::{error, Interval};
 /// # Note:
 /// For values out of range, you can still store these timestamps, but while performing arithmetic
 /// or formatting operations, it will return an error or just overflow.
-#[derive(Clone, Default, Copy, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default, Copy, Serialize, Deserialize)]
 pub struct Timestamp {
     value: i64,
     unit: TimeUnit,
@@ -498,12 +498,6 @@ impl From<Timestamp> for serde_json::Value {
     }
 }
 
-impl fmt::Debug for Timestamp {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        write!(f, "{}::{}", self.value, self.unit)
-    }
-}
-
 #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
 pub enum TimeUnit {
     Second,
@@ -1388,24 +1382,4 @@ mod tests {
             Timestamp::MAX_SECOND.to_timezone_aware_string(Some(&Timezone::Named(Tz::UTC)))
         );
     }
-
-    #[test]
-    fn test_debug_timestamp() {
-        assert_eq!(
-            "1000::Second",
-            format!("{:?}", Timestamp::new(1000, TimeUnit::Second))
-        );
-        assert_eq!(
-            "1001::Millisecond",
-            format!("{:?}", Timestamp::new(1001, TimeUnit::Millisecond))
-        );
-        assert_eq!(
-            "1002::Microsecond",
-            format!("{:?}", Timestamp::new(1002, TimeUnit::Microsecond))
-        );
-        assert_eq!(
-            "1003::Nanosecond",
-            format!("{:?}", Timestamp::new(1003, TimeUnit::Nanosecond))
-        );
-    }
 }

src/datanode/Cargo.toml

@@ -57,7 +57,6 @@ servers.workspace = true
 session.workspace = true
 snafu.workspace = true
 store-api.workspace = true
-substrait.workspace = true
 table.workspace = true
 tokio.workspace = true
 toml.workspace = true

src/datanode/src/config.rs

@@ -15,7 +15,7 @@
 //! Datanode configurations
 
 use common_base::readable_size::ReadableSize;
-use common_base::secrets::{ExposeSecret, SecretString};
+use common_base::secrets::SecretString;
 use common_config::Configurable;
 use common_grpc::channel_manager::{
     DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
@@ -38,7 +38,7 @@ pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize::mb(256);
 const DEFAULT_DATA_HOME: &str = "/tmp/greptimedb";
 
 /// Object storage config
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(tag = "type")]
 pub enum ObjectStoreConfig {
     File(FileConfig),
@@ -61,7 +61,7 @@ impl ObjectStoreConfig {
 }
 
 /// Storage engine config
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(default)]
 pub struct StorageConfig {
     /// The working directory of database
@@ -85,7 +85,7 @@ impl Default for StorageConfig {
 #[serde(default)]
 pub struct FileConfig {}
 
-#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 #[serde(default)]
 pub struct ObjectStorageCacheConfig {
     /// The local file cache directory
@@ -109,18 +109,6 @@ pub struct S3Config {
     pub cache: ObjectStorageCacheConfig,
 }
 
-impl PartialEq for S3Config {
-    fn eq(&self, other: &Self) -> bool {
-        self.bucket == other.bucket
-            && self.root == other.root
-            && self.access_key_id.expose_secret() == other.access_key_id.expose_secret()
-            && self.secret_access_key.expose_secret() == other.secret_access_key.expose_secret()
-            && self.endpoint == other.endpoint
-            && self.region == other.region
-            && self.cache == other.cache
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(default)]
 pub struct OssConfig {
@@ -135,17 +123,6 @@ pub struct OssConfig {
     pub cache: ObjectStorageCacheConfig,
 }
 
-impl PartialEq for OssConfig {
-    fn eq(&self, other: &Self) -> bool {
-        self.bucket == other.bucket
-            && self.root == other.root
-            && self.access_key_id.expose_secret() == other.access_key_id.expose_secret()
-            && self.access_key_secret.expose_secret() == other.access_key_secret.expose_secret()
-            && self.endpoint == other.endpoint
-            && self.cache == other.cache
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(default)]
 pub struct AzblobConfig {
@@ -161,18 +138,6 @@ pub struct AzblobConfig {
     pub cache: ObjectStorageCacheConfig,
 }
 
-impl PartialEq for AzblobConfig {
-    fn eq(&self, other: &Self) -> bool {
-        self.container == other.container
-            && self.root == other.root
-            && self.account_name.expose_secret() == other.account_name.expose_secret()
-            && self.account_key.expose_secret() == other.account_key.expose_secret()
-            && self.endpoint == other.endpoint
-            && self.sas_token == other.sas_token
-            && self.cache == other.cache
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(default)]
 pub struct GcsConfig {
@@ -186,17 +151,6 @@ pub struct GcsConfig {
     pub cache: ObjectStorageCacheConfig,
 }
 
-impl PartialEq for GcsConfig {
-    fn eq(&self, other: &Self) -> bool {
-        self.root == other.root
-            && self.bucket == other.bucket
-            && self.scope == other.scope
-            && self.credential_path.expose_secret() == other.credential_path.expose_secret()
-            && self.endpoint == other.endpoint
-            && self.cache == other.cache
-    }
-}
-
 impl Default for S3Config {
     fn default() -> Self {
         Self {
@@ -257,14 +211,13 @@ impl Default for ObjectStoreConfig {
     }
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(default)]
 pub struct DatanodeOptions {
     pub mode: Mode,
     pub node_id: Option<u64>,
     pub require_lease_before_startup: bool,
     pub init_regions_in_background: bool,
-    pub init_regions_parallelism: usize,
     pub rpc_addr: String,
     pub rpc_hostname: Option<String>,
     pub rpc_runtime_size: usize,
@@ -292,7 +245,6 @@ impl Default for DatanodeOptions {
             node_id: None,
             require_lease_before_startup: false,
             init_regions_in_background: false,
-            init_regions_parallelism: 16,
             rpc_addr: "127.0.0.1:3001".to_string(),
             rpc_hostname: None,
             rpc_runtime_size: 8,
@@ -315,7 +267,7 @@ impl Default for DatanodeOptions {
     }
 }
 
-impl Configurable for DatanodeOptions {
+impl Configurable<'_> for DatanodeOptions {
     fn env_list_keys() -> Option<&'static [&'static str]> {
         Some(&["meta_client.metasrv_addrs", "wal.broker_endpoints"])
     }

src/datanode/src/datanode.rs

@@ -31,6 +31,7 @@ use common_wal::config::kafka::DatanodeKafkaConfig;
 use common_wal::config::raft_engine::RaftEngineConfig;
 use common_wal::config::DatanodeWalConfig;
 use file_engine::engine::FileRegionEngine;
+use futures_util::future::try_join_all;
 use futures_util::TryStreamExt;
 use log_store::kafka::log_store::KafkaLogStore;
 use log_store::raft_engine::log_store::RaftEngineLogStore;
@@ -44,17 +45,17 @@ use query::QueryEngineFactory;
 use servers::export_metrics::ExportMetricsTask;
 use servers::server::ServerHandlers;
 use servers::Mode;
-use snafu::{ensure, OptionExt, ResultExt};
+use snafu::{OptionExt, ResultExt};
 use store_api::path_utils::{region_dir, WAL_DIR};
 use store_api::region_engine::RegionEngineRef;
-use store_api::region_request::RegionOpenRequest;
+use store_api::region_request::{RegionOpenRequest, RegionRequest};
 use store_api::storage::RegionId;
 use tokio::fs;
 use tokio::sync::Notify;
 
 use crate::config::{DatanodeOptions, RegionEngineConfig};
 use crate::error::{
-    self, BuildMitoEngineSnafu, CreateDirSnafu, GetMetadataSnafu, MissingKvBackendSnafu,
+    BuildMitoEngineSnafu, CreateDirSnafu, GetMetadataSnafu, MissingKvBackendSnafu,
     MissingNodeIdSnafu, OpenLogStoreSnafu, Result, RuntimeResourceSnafu, ShutdownInstanceSnafu,
     ShutdownServerSnafu, StartServerSnafu,
 };
@@ -67,6 +68,8 @@ use crate::heartbeat::HeartbeatTask;
 use crate::region_server::{DummyTableProviderFactory, RegionServer};
 use crate::store;
 
+const OPEN_REGION_PARALLELISM: usize = 16;
+
 /// Datanode service.
 pub struct Datanode {
     services: ServerHandlers,
@@ -216,12 +219,8 @@ impl DatanodeBuilder {
             .await
             .context(GetMetadataSnafu)?;
 
-        let open_all_regions = open_all_regions(
+        let open_all_regions =
-            region_server.clone(),
+            open_all_regions(region_server.clone(), table_values, !controlled_by_metasrv);
-            table_values,
-            !controlled_by_metasrv,
-            self.opts.init_regions_parallelism,
-        );
 
         if self.opts.init_regions_in_background {
             // Opens regions in background.
@@ -287,13 +286,7 @@ impl DatanodeBuilder {
             .await
             .context(GetMetadataSnafu)?;
 
-        open_all_regions(
+        open_all_regions(region_server.clone(), table_values, open_with_writable).await
-            region_server.clone(),
-            table_values,
-            open_with_writable,
-            self.opts.init_regions_parallelism,
-        )
-        .await
     }
 
     async fn new_region_server(
@@ -454,7 +447,6 @@ async fn open_all_regions(
     region_server: RegionServer,
     table_values: Vec<DatanodeTableValue>,
     open_with_writable: bool,
-    init_regions_parallelism: usize,
 ) -> Result<()> {
     let mut regions = vec![];
     for table_value in table_values {
@@ -475,46 +467,40 @@ async fn open_all_regions(
             ));
         }
     }
-    let num_regions = regions.len();
+    info!("going to open {} region(s)", regions.len());
-    info!("going to open {} region(s)", num_regions);
+    let semaphore = Arc::new(tokio::sync::Semaphore::new(OPEN_REGION_PARALLELISM));
+    let mut tasks = vec![];
 
-    let mut region_requests = Vec::with_capacity(regions.len());
+    let region_server_ref = &region_server;
     for (region_id, engine, store_path, options) in regions {
         let region_dir = region_dir(&store_path, region_id);
-        region_requests.push((
+        let semaphore_moved = semaphore.clone();
-            region_id,
-            RegionOpenRequest {
-                engine,
-                region_dir,
-                options,
-                skip_wal_replay: false,
-            },
-        ));
-    }
 
-    let open_regions = region_server
+        tasks.push(async move {
-        .handle_batch_open_requests(init_regions_parallelism, region_requests)
+            let _permit = semaphore_moved.acquire().await;
-        .await?;
+            region_server_ref
-    ensure!(
+                .handle_request(
-        open_regions.len() == num_regions,
+                    region_id,
-        error::UnexpectedSnafu {
+                    RegionRequest::Open(RegionOpenRequest {
-            violated: format!(
+                        engine: engine.clone(),
-                "Expected to open {} of regions, only {} of regions has opened",
+                        region_dir,
-                num_regions,
+                        options,
-                open_regions.len()
+                        skip_wal_replay: false,
-            )
+                    }),
-        }
+                )
-    );
+                .await?;
-
+            if open_with_writable {
-    for region_id in open_regions {
+                if let Err(e) = region_server_ref.set_writable(region_id, true) {
-        if open_with_writable {
+                    error!(
-            if let Err(e) = region_server.set_writable(region_id, true) {
+                        e; "failed to set writable for region {region_id}"
-                error!(
+                    );
-                    e; "failed to set writable for region {region_id}"
+                }
-                );
             }
-        }
+            Ok(())
+        });
     }
+    let _ = try_join_all(tasks).await?;
+
     info!("all regions are opened");
 
     Ok(())

src/datanode/src/error.rs

@@ -15,10 +15,10 @@
 use std::any::Any;
 use std::sync::Arc;
 
-use common_error::define_into_tonic_status;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
+use servers::define_into_tonic_status;
 use snafu::{Location, Snafu};
 use store_api::storage::RegionId;
 use table::error::Error as TableError;
@@ -292,13 +292,6 @@ pub enum Error {
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to open batch regions"))]
-    HandleBatchOpenRequest {
-        #[snafu(implicit)]
-        location: Location,
-        source: BoxedError,
-    },
-
     #[snafu(display("RegionId {} not found", region_id))]
     RegionNotFound {
         region_id: RegionId,
@@ -394,14 +387,6 @@ pub enum Error {
         location: Location,
         source: BoxedError,
     },
-
-    #[snafu(display("DataFusion"))]
-    DataFusion {
-        #[snafu(source)]
-        error: datafusion::error::DataFusionError,
-        #[snafu(implicit)]
-        location: Location,
-    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -454,8 +439,7 @@ impl ErrorExt for Error {
             | IncorrectInternalState { .. }
             | ShutdownInstance { .. }
             | RegionEngineNotFound { .. }
-            | UnsupportedOutput { .. }
+            | UnsupportedOutput { .. } => StatusCode::Internal,
-            | DataFusion { .. } => StatusCode::Internal,
 
             RegionNotFound { .. } => StatusCode::RegionNotFound,
             RegionNotReady { .. } => StatusCode::RegionNotReady,
@@ -471,9 +455,9 @@ impl ErrorExt for Error {
             TableIdProviderNotFound { .. } | UnsupportedGrpcRequest { .. } => {
                 StatusCode::Unsupported
             }
-            HandleRegionRequest { source, .. }
+            HandleRegionRequest { source, .. } | GetRegionMetadata { source, .. } => {
-            | GetRegionMetadata { source, .. }
+                source.status_code()
-            | HandleBatchOpenRequest { source, .. } => source.status_code(),
+            }
             StopRegionEngine { source, .. } => source.status_code(),
 
             FindLogicalRegions { source, .. } => source.status_code(),

src/datanode/src/region_server.rs

@@ -18,25 +18,20 @@ use std::ops::Deref;
 use std::sync::{Arc, RwLock};
 
 use api::region::RegionResponse;
-use api::v1::region::{region_request, RegionResponse as RegionResponseV1};
+use api::v1::region::{region_request, QueryRequest, RegionResponse as RegionResponseV1};
 use api::v1::{ResponseHeader, Status};
 use arrow_flight::{FlightData, Ticket};
 use async_trait::async_trait;
 use bytes::Bytes;
 use common_error::ext::BoxedError;
 use common_error::status_code::StatusCode;
-use common_query::request::QueryRequest;
 use common_query::OutputData;
 use common_recordbatch::SendableRecordBatchStream;
 use common_runtime::Runtime;
 use common_telemetry::tracing::{self, info_span};
 use common_telemetry::tracing_context::{FutureExt, TracingContext};
-use common_telemetry::{error, info, warn};
+use common_telemetry::{info, warn};
 use dashmap::DashMap;
-use datafusion::datasource::{provider_as_source, TableProvider};
-use datafusion::error::Result as DfResult;
-use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRewriter};
-use datafusion_expr::{LogicalPlan, TableSource};
 use futures_util::future::try_join_all;
 use metric_engine::engine::MetricEngine;
 use mito2::engine::MITO_ENGINE_NAME;
@@ -49,22 +44,20 @@ use servers::error::{self as servers_error, ExecuteGrpcRequestSnafu, Result as S
 use servers::grpc::flight::{FlightCraft, FlightRecordBatchStream, TonicStream};
 use servers::grpc::region_server::RegionServerHandler;
 use session::context::{QueryContextBuilder, QueryContextRef};
-use snafu::{ensure, OptionExt, ResultExt};
+use snafu::{OptionExt, ResultExt};
 use store_api::metric_engine_consts::{
     FILE_ENGINE_NAME, LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME,
 };
 use store_api::region_engine::{RegionEngineRef, RegionRole, SetReadonlyResponse};
-use store_api::region_request::{
+use store_api::region_request::{AffectedRows, RegionCloseRequest, RegionRequest};
-    AffectedRows, RegionCloseRequest, RegionOpenRequest, RegionRequest,
-};
 use store_api::storage::RegionId;
 use tonic::{Request, Response, Result as TonicResult};
 
 use crate::error::{
-    self, BuildRegionRequestsSnafu, DataFusionSnafu, DecodeLogicalPlanSnafu,
+    self, BuildRegionRequestsSnafu, DecodeLogicalPlanSnafu, ExecuteLogicalPlanSnafu,
-    ExecuteLogicalPlanSnafu, FindLogicalRegionsSnafu, HandleBatchOpenRequestSnafu,
+    FindLogicalRegionsSnafu, HandleRegionRequestSnafu, NewPlanDecoderSnafu,
-    HandleRegionRequestSnafu, NewPlanDecoderSnafu, RegionEngineNotFoundSnafu, RegionNotFoundSnafu,
+    RegionEngineNotFoundSnafu, RegionNotFoundSnafu, Result, StopRegionEngineSnafu, UnexpectedSnafu,
-    RegionNotReadySnafu, Result, StopRegionEngineSnafu, UnexpectedSnafu, UnsupportedOutputSnafu,
+    UnsupportedOutputSnafu,
 };
 use crate::event_listener::RegionServerEventListenerRef;
 
@@ -123,17 +116,6 @@ impl RegionServer {
             })
     }
 
-    #[tracing::instrument(skip_all)]
-    pub async fn handle_batch_open_requests(
-        &self,
-        parallelism: usize,
-        requests: Vec<(RegionId, RegionOpenRequest)>,
-    ) -> Result<Vec<RegionId>> {
-        self.inner
-            .handle_batch_open_requests(parallelism, requests)
-            .await
-    }
-
     #[tracing::instrument(skip_all, fields(request_type = request.request_type()))]
     pub async fn handle_request(
         &self,
@@ -143,94 +125,9 @@ impl RegionServer {
         self.inner.handle_request(region_id, request).await
     }
 
-    async fn table_provider(&self, region_id: RegionId) -> Result<Arc<dyn TableProvider>> {
-        let status = self
-            .inner
-            .region_map
-            .get(&region_id)
-            .context(RegionNotFoundSnafu { region_id })?
-            .clone();
-        ensure!(
-            matches!(status, RegionEngineWithStatus::Ready(_)),
-            RegionNotReadySnafu { region_id }
-        );
-
-        self.inner
-            .table_provider_factory
-            .create(region_id, status.into_engine())
-            .await
-            .context(ExecuteLogicalPlanSnafu)
-    }
-
-    /// Handle reads from remote. They're often query requests received by our Arrow Flight service.
-    pub async fn handle_remote_read(
-        &self,
-        request: api::v1::region::QueryRequest,
-    ) -> Result<SendableRecordBatchStream> {
-        let region_id = RegionId::from_u64(request.region_id);
-        let provider = self.table_provider(region_id).await?;
-        let catalog_list = Arc::new(DummyCatalogList::with_table_provider(provider));
-
-        let query_ctx: QueryContextRef = request
-            .header
-            .as_ref()
-            .map(|h| Arc::new(h.into()))
-            .unwrap_or_else(|| Arc::new(QueryContextBuilder::default().build()));
-
-        let decoder = self
-            .inner
-            .query_engine
-            .engine_context(query_ctx)
-            .new_plan_decoder()
-            .context(NewPlanDecoderSnafu)?;
-
-        let plan = decoder
-            .decode(Bytes::from(request.plan), catalog_list, false)
-            .await
-            .context(DecodeLogicalPlanSnafu)?;
-
-        self.inner
-            .handle_read(QueryRequest {
-                header: request.header,
-                region_id,
-                plan,
-            })
-            .await
-    }
-
     #[tracing::instrument(skip_all)]
     pub async fn handle_read(&self, request: QueryRequest) -> Result<SendableRecordBatchStream> {
-        let provider = self.table_provider(request.region_id).await?;
+        self.inner.handle_read(request).await
-
-        struct RegionDataSourceInjector {
-            source: Arc<dyn TableSource>,
-        }
-
-        impl TreeNodeRewriter for RegionDataSourceInjector {
-            type Node = LogicalPlan;
-
-            fn f_up(&mut self, node: Self::Node) -> DfResult<Transformed<Self::Node>> {
-                Ok(match node {
-                    LogicalPlan::TableScan(mut scan) => {
-                        scan.source = self.source.clone();
-                        Transformed::yes(LogicalPlan::TableScan(scan))
-                    }
-                    _ => Transformed::no(node),
-                })
-            }
-        }
-
-        let plan = request
-            .plan
-            .rewrite(&mut RegionDataSourceInjector {
-                source: provider_as_source(provider),
-            })
-            .context(DataFusionSnafu)?
-            .data;
-
-        self.inner
-            .handle_read(QueryRequest { plan, ..request })
-            .await
     }
 
     /// Returns all opened and reportable regions.
@@ -392,7 +289,7 @@ impl FlightCraft for RegionServer {
         request: Request<Ticket>,
     ) -> TonicResult<Response<TonicStream<FlightData>>> {
         let ticket = request.into_inner().ticket;
-        let request = api::v1::region::QueryRequest::decode(ticket.as_ref())
+        let request = QueryRequest::decode(ticket.as_ref())
             .context(servers_error::InvalidFlightTicketSnafu)?;
         let tracing_context = request
             .header
@@ -401,7 +298,7 @@ impl FlightCraft for RegionServer {
             .unwrap_or_default();
 
         let result = self
-            .handle_remote_read(request)
+            .handle_read(request)
             .trace(tracing_context.attach(info_span!("RegionServer::handle_read")))
             .await?;
 
@@ -429,6 +326,10 @@ impl RegionEngineWithStatus {
             RegionEngineWithStatus::Ready(engine) => engine,
         }
     }
+
+    pub fn is_registering(&self) -> bool {
+        matches!(self, Self::Registering(_))
+    }
 }
 
 impl Deref for RegionEngineWithStatus {
@@ -553,113 +454,6 @@ impl RegionServerInner {
         Ok(CurrentEngine::Engine(engine))
     }
 
-    async fn handle_batch_open_requests_inner(
-        &self,
-        engine: RegionEngineRef,
-        parallelism: usize,
-        requests: Vec<(RegionId, RegionOpenRequest)>,
-    ) -> Result<Vec<RegionId>> {
-        let region_changes = requests
-            .iter()
-            .map(|(region_id, open)| {
-                let attribute = parse_region_attribute(&open.engine, &open.options)?;
-                Ok((*region_id, RegionChange::Register(attribute)))
-            })
-            .collect::<Result<HashMap<_, _>>>()?;
-
-        for (&region_id, region_change) in &region_changes {
-            self.set_region_status_not_ready(region_id, &engine, region_change)
-        }
-
-        let mut open_regions = Vec::with_capacity(requests.len());
-        let mut errors = vec![];
-        match engine
-            .handle_batch_open_requests(parallelism, requests)
-            .await
-            .with_context(|_| HandleBatchOpenRequestSnafu)
-        {
-            Ok(results) => {
-                for (region_id, result) in results {
-                    let region_change = &region_changes[&region_id];
-                    match result {
-                        Ok(_) => {
-                            if let Err(e) = self
-                                .set_region_status_ready(region_id, engine.clone(), *region_change)
-                                .await
-                            {
-                                error!(e; "Failed to set region to ready: {}", region_id);
-                                errors.push(BoxedError::new(e));
-                            } else {
-                                open_regions.push(region_id)
-                            }
-                        }
-                        Err(e) => {
-                            self.unset_region_status(region_id, *region_change);
-                            error!(e; "Failed to open region: {}", region_id);
-                            errors.push(e);
-                        }
-                    }
-                }
-            }
-            Err(e) => {
-                for (&region_id, region_change) in &region_changes {
-                    self.unset_region_status(region_id, *region_change);
-                }
-                error!(e; "Failed to open batch regions");
-                errors.push(BoxedError::new(e));
-            }
-        }
-
-        if !errors.is_empty() {
-            return error::UnexpectedSnafu {
-                // Returns the first error.
-                violated: format!("Failed to open batch regions: {:?}", errors[0]),
-            }
-            .fail();
-        }
-
-        Ok(open_regions)
-    }
-
-    pub async fn handle_batch_open_requests(
-        &self,
-        parallelism: usize,
-        requests: Vec<(RegionId, RegionOpenRequest)>,
-    ) -> Result<Vec<RegionId>> {
-        let mut engine_grouped_requests: HashMap<String, Vec<_>> =
-            HashMap::with_capacity(requests.len());
-        for (region_id, request) in requests {
-            if let Some(requests) = engine_grouped_requests.get_mut(&request.engine) {
-                requests.push((region_id, request));
-            } else {
-                engine_grouped_requests
-                    .insert(request.engine.to_string(), vec![(region_id, request)]);
-            }
-        }
-
-        let mut results = Vec::with_capacity(engine_grouped_requests.keys().len());
-        for (engine, requests) in engine_grouped_requests {
-            let engine = self
-                .engines
-                .read()
-                .unwrap()
-                .get(&engine)
-                .with_context(|| RegionEngineNotFoundSnafu { name: &engine })?
-                .clone();
-            results.push(
-                self.handle_batch_open_requests_inner(engine, parallelism, requests)
-                    .await,
-            )
-        }
-
-        Ok(results
-            .into_iter()
-            .collect::<Result<Vec<_>>>()?
-            .into_iter()
-            .flatten()
-            .collect::<Vec<_>>())
-    }
-
     pub async fn handle_request(
         &self,
         region_id: RegionId,
@@ -827,16 +621,51 @@ impl RegionServerInner {
     pub async fn handle_read(&self, request: QueryRequest) -> Result<SendableRecordBatchStream> {
         // TODO(ruihang): add metrics and set trace id
 
+        let QueryRequest {
+            header,
+            region_id,
+            plan,
+        } = request;
+        let region_id = RegionId::from_u64(region_id);
+
         // Build query context from gRPC header
-        let query_ctx: QueryContextRef = request
+        let ctx: QueryContextRef = header
-            .header
             .as_ref()
             .map(|h| Arc::new(h.into()))
             .unwrap_or_else(|| QueryContextBuilder::default().build().into());
 
+        // build dummy catalog list
+        let region_status = self
+            .region_map
+            .get(&region_id)
+            .with_context(|| RegionNotFoundSnafu { region_id })?
+            .clone();
+
+        if region_status.is_registering() {
+            return error::RegionNotReadySnafu { region_id }.fail();
+        }
+
+        let table_provider = self
+            .table_provider_factory
+            .create(region_id, region_status.into_engine())
+            .await
+            .context(ExecuteLogicalPlanSnafu)?;
+
+        let catalog_list = Arc::new(DummyCatalogList::with_table_provider(table_provider));
+        let query_engine_ctx = self.query_engine.engine_context(ctx.clone());
+        let plan_decoder = query_engine_ctx
+            .new_plan_decoder()
+            .context(NewPlanDecoderSnafu)?;
+
+        // decode substrait plan to logical plan and execute it
+        let logical_plan = plan_decoder
+            .decode(Bytes::from(plan), catalog_list, false)
+            .await
+            .context(DecodeLogicalPlanSnafu)?;
+
         let result = self
             .query_engine
-            .execute(request.plan.into(), query_ctx)
+            .execute(logical_plan.into(), ctx)
             .await
             .context(ExecuteLogicalPlanSnafu)?;
 
@@ -886,7 +715,6 @@ impl RegionServerInner {
     }
 }
 
-#[derive(Debug, Clone, Copy)]
 enum RegionChange {
     None,
     Register(RegionAttribute),
@@ -912,7 +740,6 @@ fn parse_region_attribute(
     }
 }
 
-#[derive(Debug, Clone, Copy)]
 enum RegionAttribute {
     Mito,
     Metric { physical: bool },

src/datanode/src/store.rs

@@ -20,6 +20,7 @@ mod gcs;
 mod oss;
 mod s3;
 
+use std::sync::Arc;
 use std::time::Duration;
 use std::{env, path};
 
@@ -28,7 +29,7 @@ use common_telemetry::info;
 use object_store::layers::{LruCacheLayer, RetryLayer};
 use object_store::services::Fs;
 use object_store::util::{join_dir, normalize_dir, with_instrument_layers};
-use object_store::{HttpClient, ObjectStore};
+use object_store::{HttpClient, ObjectStore, ObjectStoreBuilder};
 use snafu::prelude::*;
 
 use crate::config::{ObjectStoreConfig, DEFAULT_OBJECT_STORE_CACHE_SIZE};
@@ -106,13 +107,13 @@ async fn create_object_store_with_cache(
     if let Some(path) = cache_path {
         let atomic_temp_dir = join_dir(path, ".tmp/");
         clean_temp_dir(&atomic_temp_dir)?;
-        let mut builder = Fs::default();
+        let cache_store = Fs::default()
-        builder.root(path).atomic_write_dir(&atomic_temp_dir);
+            .root(path)
-        let cache_store = ObjectStore::new(builder)
+            .atomic_write_dir(&atomic_temp_dir)
-            .context(error::InitBackendSnafu)?
+            .build()
-            .finish();
+            .context(error::InitBackendSnafu)?;
 
-        let cache_layer = LruCacheLayer::new(cache_store, cache_capacity.0 as usize)
+        let cache_layer = LruCacheLayer::new(Arc::new(cache_store), cache_capacity.0 as usize)
             .await
             .context(error::InitBackendSnafu)?;
 

src/file-engine/src/manifest.rs

@@ -71,8 +71,7 @@ impl FileRegionManifest {
         let bs = object_store
             .read(path)
             .await
-            .context(LoadRegionManifestSnafu { region_id })?
+            .context(LoadRegionManifestSnafu { region_id })?;
-            .to_vec();
         Self::decode(bs.as_slice())
     }
 

src/flow/src/adapter.rs

@@ -64,6 +64,8 @@ mod table_source;
 
 use error::Error;
 
+pub const PER_REQ_MAX_ROW_CNT: usize = 8192;
+
 // TODO: replace this with `GREPTIME_TIMESTAMP` before v0.9
 pub const AUTO_CREATED_PLACEHOLDER_TS_COL: &str = "__ts_placeholder";
 
@@ -265,7 +267,7 @@ impl FlownodeManager {
             let ctx = Arc::new(QueryContext::with(&catalog, &schema));
             // TODO(discord9): instead of auto build table from request schema, actually build table
             // before `create flow` to be able to assign pk and ts etc.
-            let (primary_keys, schema, is_ts_placeholder) = if let Some(table_id) = self
+            let (primary_keys, schema, is_auto_create) = if let Some(table_id) = self
                 .table_info_source
                 .get_table_id_from_name(&table_name)
                 .await?
@@ -317,12 +319,7 @@ impl FlownodeManager {
                     .map(|v| {
                         v.column_indices
                             .iter()
-                            .map(|i| {
+                            .map(|i| format!("Col_{i}"))
-                                schema
-                                    .get_name(*i)
-                                    .clone()
-                                    .unwrap_or_else(|| format!("Col_{i}"))
-                            })
                             .collect_vec()
                     })
                     .unwrap_or_default();
@@ -331,8 +328,15 @@ impl FlownodeManager {
                     ConcreteDataType::timestamp_millisecond_datatype(),
                     true,
                 );
+                // TODO(discord9): bugged so we can't infer time index from flow plan, so we have to manually set one
+                let ts_col = ColumnSchema::new(
+                    AUTO_CREATED_PLACEHOLDER_TS_COL,
+                    ConcreteDataType::timestamp_millisecond_datatype(),
+                    true,
+                )
+                .with_time_index(true);
 
-                let original_schema = schema
+                let wout_ts = schema
                     .typ()
                     .column_types
                     .clone()
@@ -343,33 +347,16 @@ impl FlownodeManager {
                             .names
                             .get(idx)
                             .cloned()
-                            .flatten()
                             .unwrap_or(format!("Col_{}", idx));
-                        let ret = ColumnSchema::new(name, typ.scalar_type, typ.nullable);
+                        ColumnSchema::new(name, typ.scalar_type, typ.nullable)
-                        if schema.typ().time_index == Some(idx) {
-                            ret.with_time_index(true)
-                        } else {
-                            ret
-                        }
                     })
                     .collect_vec();
 
-                let mut with_auto_added_col = original_schema.clone();
+                let mut with_ts = wout_ts.clone();
-                with_auto_added_col.push(update_at);
+                with_ts.push(update_at);
+                with_ts.push(ts_col);
 
-                // if no time index, add one as placeholder
+                (primary_keys, with_ts, true)
-                let no_time_index = schema.typ().time_index.is_none();
-                if no_time_index {
-                    let ts_col = ColumnSchema::new(
-                        AUTO_CREATED_PLACEHOLDER_TS_COL,
-                        ConcreteDataType::timestamp_millisecond_datatype(),
-                        true,
-                    )
-                    .with_time_index(true);
-                    with_auto_added_col.push(ts_col);
-                }
-
-                (primary_keys, with_auto_added_col, no_time_index)
             };
             let schema_len = schema.len();
             let proto_schema = column_schemas_to_proto(schema, &primary_keys)?;
@@ -392,7 +379,7 @@ impl FlownodeManager {
                                     now,
                                 ))]);
                                 // ts col, if auto create
-                                if is_ts_placeholder {
+                                if is_auto_create {
                                     ensure!(
                                         row.len() == schema_len - 1,
                                         InternalSnafu {
@@ -523,13 +510,12 @@ impl FlownodeManager {
         debug!("Starting to run");
         loop {
             // TODO(discord9): only run when new inputs arrive or scheduled to
-            if let Err(err) = self.run_available(true).await {
+            debug!("call run_available in run every second");
-                common_telemetry::error!(err;"Run available errors");
+            self.run_available(true).await.unwrap();
-            }
+            debug!("call send_writeback_requests in run every second");
             // TODO(discord9): error handling
-            if let Err(err) = self.send_writeback_requests().await {
+            self.send_writeback_requests().await.unwrap();
-                common_telemetry::error!(err;"Send writeback request errors");
+            debug!("call log_all_errors in run every second");
-            };
             self.log_all_errors().await;
             tokio::time::sleep(std::time::Duration::from_secs(1)).await;
         }
@@ -612,7 +598,6 @@ impl FlownodeManager {
                 break;
             }
         }
-        self.node_context.write().await.remove_flow(flow_id);
         Ok(())
     }
 
@@ -659,9 +644,8 @@ impl FlownodeManager {
         node_ctx.query_context = query_ctx.map(Arc::new);
         // construct a active dataflow state with it
         let flow_plan = sql_to_flow_plan(&mut node_ctx, &self.query_engine, &sql).await?;
-
         debug!("Flow {:?}'s Plan is {:?}", flow_id, flow_plan);
-        node_ctx.assign_table_schema(&sink_table_name, flow_plan.schema.clone())?;
+        node_ctx.assign_table_schema(&sink_table_name, flow_plan.typ.clone())?;
 
         let _ = comment;
         let _ = flow_options;

src/flow/src/adapter/error.rs

@@ -16,12 +16,12 @@
 
 use std::any::Any;
 
-use common_error::define_into_tonic_status;
 use common_error::ext::BoxedError;
 use common_macro::stack_trace_debug;
 use common_telemetry::common_error::ext::ErrorExt;
 use common_telemetry::common_error::status_code::StatusCode;
 use datatypes::value::Value;
+use servers::define_into_tonic_status;
 use snafu::{Location, Snafu};
 
 use crate::adapter::FlowId;

src/flow/src/adapter/flownode_impl.rs

@@ -26,7 +26,6 @@ use itertools::Itertools;
 use snafu::{OptionExt, ResultExt};
 use store_api::storage::RegionId;
 
-use crate::adapter::error::InternalSnafu;
 use crate::adapter::FlownodeManager;
 use crate::repr::{self, DiffRow};
 
@@ -127,16 +126,6 @@ impl Flownode for FlownodeManager {
                     .context(UnexpectedSnafu {
                         err_msg: format!("Table not found: {}", table_id),
                     })?;
-                let table_col_names = table_col_names
-                    .iter().enumerate()
-                    .map(|(idx,name)| match name {
-                        Some(name) => Ok(name.clone()),
-                        None => InternalSnafu {
-                            reason: format!("Expect column {idx} of table id={table_id} to have name in table schema, found None"),
-                        }
-                        .fail().map_err(BoxedError::new).context(ExternalSnafu),
-                    })
-                    .collect::<Result<Vec<_>>>()?;
                 let name_to_col = HashMap::<_, _>::from_iter(
                     insert_schema
                         .iter()

src/flow/src/adapter/node_context.rs

@@ -27,7 +27,7 @@ use crate::adapter::error::{Error, EvalSnafu, TableNotFoundSnafu};
 use crate::adapter::{FlowId, TableName, TableSource};
 use crate::expr::error::InternalSnafu;
 use crate::expr::GlobalId;
-use crate::repr::{DiffRow, RelationDesc, BROADCAST_CAP};
+use crate::repr::{DiffRow, RelationDesc, RelationType, BROADCAST_CAP};
 
 /// A context that holds the information of the dataflow
 #[derive(Default, Debug)]
@@ -36,7 +36,6 @@ pub struct FlownodeContext {
     pub source_to_tasks: BTreeMap<TableId, BTreeSet<FlowId>>,
     /// mapping from task to sink table, useful for sending data back to the client when a task is done running
     pub flow_to_sink: BTreeMap<FlowId, TableName>,
-    pub sink_to_flow: BTreeMap<TableName, FlowId>,
     /// broadcast sender for source table, any incoming write request will be sent to the source table's corresponding sender
     ///
     /// Note that we are getting insert requests with table id, so we should use table id as the key
@@ -185,21 +184,7 @@ impl FlownodeContext {
         }
 
         self.add_sink_receiver(sink_table_name.clone());
-        self.flow_to_sink.insert(task_id, sink_table_name.clone());
+        self.flow_to_sink.insert(task_id, sink_table_name);
-        self.sink_to_flow.insert(sink_table_name, task_id);
-    }
-
-    /// remove flow from worker context
-    pub fn remove_flow(&mut self, task_id: FlowId) {
-        if let Some(sink_table_name) = self.flow_to_sink.remove(&task_id) {
-            self.sink_to_flow.remove(&sink_table_name);
-        }
-        for (source_table_id, tasks) in self.source_to_tasks.iter_mut() {
-            tasks.remove(&task_id);
-            if tasks.is_empty() {
-                self.source_sender.remove(source_table_id);
-            }
-        }
     }
 
     /// try add source sender, if already exist, do nothing
@@ -322,14 +307,14 @@ impl FlownodeContext {
     pub fn assign_table_schema(
         &mut self,
         table_name: &TableName,
-        schema: RelationDesc,
+        schema: RelationType,
     ) -> Result<(), Error> {
         let gid = self
             .table_repr
             .get_by_name(table_name)
             .map(|(_, gid)| gid)
             .unwrap();
-        self.schema.insert(gid, schema);
+        self.schema.insert(gid, schema.into_unnamed());
         Ok(())
     }
 

src/flow/src/adapter/table_source.rs

@@ -132,7 +132,7 @@ impl TableSource {
                         nullable: col.is_nullable(),
                         scalar_type: col.data_type,
                     },
-                    Some(col.name),
+                    col.name,
                 )
             })
             .unzip();

src/flow/src/adapter/worker.rs

@@ -514,7 +514,7 @@ mod test {
                 plan: Plan::Get {
                     id: Id::Global(GlobalId::User(1)),
                 },
-                schema: RelationType::new(vec![]).into_unnamed(),
+                typ: RelationType::new(vec![]),
             },
         );
         let create_reqs = Request::Create {

src/flow/src/compute/render.rs

@@ -111,7 +111,7 @@ impl<'referred, 'df> Context<'referred, 'df> {
                 input,
                 key_val_plan,
                 reduce_plan,
-            } => self.render_reduce(input, key_val_plan, reduce_plan, plan.schema.typ),
+            } => self.render_reduce(input, key_val_plan, reduce_plan, plan.typ),
             Plan::Join { .. } => NotImplementedSnafu {
                 reason: "Join is still WIP",
             }

src/flow/src/compute/render/map.rs

@@ -113,21 +113,9 @@ fn mfp_subgraph(
     scheduler: &Scheduler,
     send: &PortCtx<SEND, Toff>,
 ) {
-    // all updates that should be send immediately
-    let mut output_now = vec![];
     let run_mfp = || {
-        let mut all_updates = eval_mfp_core(input, mfp_plan, now, err_collector);
+        let all_updates = eval_mfp_core(input, mfp_plan, now, err_collector);
-        all_updates.retain(|(kv, ts, d)| {
+        arrange.write().apply_updates(now, all_updates)?;
-            if *ts > now {
-                true
-            } else {
-                output_now.push((kv.clone(), *ts, *d));
-                false
-            }
-        });
-        let future_updates = all_updates;
-
-        arrange.write().apply_updates(now, future_updates)?;
         Ok(())
     };
     err_collector.run(run_mfp);
@@ -142,19 +130,13 @@ fn mfp_subgraph(
         std::ops::Bound::Excluded(from),
         std::ops::Bound::Included(now),
     );
-
-    // find all updates that need to be send from arrangement
     let output_kv = arrange.read().get_updates_in_range(range);
-
     // the output is expected to be key -> empty val
     let output = output_kv
         .into_iter()
-        .chain(output_now) // chain previous immediately send updates
         .map(|((key, _v), ts, diff)| (key, ts, diff))
         .collect_vec();
-    // send output
     send.give(output);
-
     let run_compaction = || {
         arrange.write().compact_to(now)?;
         Ok(())
@@ -252,7 +234,7 @@ mod test {
             .unwrap();
 
         let bundle = ctx
-            .render_mfp(Box::new(input_plan.with_types(typ.into_unnamed())), mfp)
+            .render_mfp(Box::new(input_plan.with_types(typ)), mfp)
             .unwrap();
         let output = get_output_handle(&mut ctx, bundle);
         // drop ctx here to simulate actual process of compile first, run later scenario
@@ -312,7 +294,7 @@ mod test {
             )])
             .unwrap();
         let bundle = ctx
-            .render_mfp(Box::new(input_plan.with_types(typ.into_unnamed())), mfp)
+            .render_mfp(Box::new(input_plan.with_types(typ)), mfp)
             .unwrap();
 
         let output = get_output_handle(&mut ctx, bundle);
@@ -323,42 +305,4 @@ mod test {
         ]);
         run_and_check(&mut state, &mut df, 1..5, expected, output);
     }
-
-    /// test if mfp operator can run multiple times within same tick
-    #[test]
-    fn test_render_mfp_multiple_times() {
-        let mut df = Hydroflow::new();
-        let mut state = DataflowState::default();
-        let mut ctx = harness_test_ctx(&mut df, &mut state);
-
-        let (sender, recv) = tokio::sync::broadcast::channel(1000);
-        let collection = ctx.render_source(recv).unwrap();
-        ctx.insert_global(GlobalId::User(1), collection);
-        let input_plan = Plan::Get {
-            id: expr::Id::Global(GlobalId::User(1)),
-        };
-        let typ = RelationType::new(vec![ColumnType::new_nullable(
-            ConcreteDataType::int64_datatype(),
-        )]);
-        // filter: col(0)>1
-        let mfp = MapFilterProject::new(1)
-            .filter(vec![ScalarExpr::Column(0).call_binary(
-                ScalarExpr::literal(1.into(), ConcreteDataType::int32_datatype()),
-                BinaryFunc::Gt,
-            )])
-            .unwrap();
-        let bundle = ctx
-            .render_mfp(Box::new(input_plan.with_types(typ.into_unnamed())), mfp)
-            .unwrap();
-
-        let output = get_output_handle(&mut ctx, bundle);
-        drop(ctx);
-        sender.send((Row::new(vec![2.into()]), 0, 1)).unwrap();
-        state.run_available_with_schedule(&mut df);
-        assert_eq!(output.borrow().len(), 1);
-        output.borrow_mut().clear();
-        sender.send((Row::new(vec![3.into()]), 0, 1)).unwrap();
-        state.run_available_with_schedule(&mut df);
-        assert_eq!(output.borrow().len(), 1);
-    }
 }

src/flow/src/compute/render/reduce.rs

@@ -813,12 +813,11 @@ mod test {
             distinct: false,
         };
         let expected = TypedPlan {
-            schema: RelationType::new(vec![
+            typ: RelationType::new(vec![
                 ColumnType::new(CDT::uint64_datatype(), true), // sum(number)
                 ColumnType::new(CDT::datetime_datatype(), false), // window start
                 ColumnType::new(CDT::datetime_datatype(), false), // window end
-            ])
+            ]),
-            .into_unnamed(),
             // TODO(discord9): mfp indirectly ref to key columns
             /*
             .with_key(vec![1])
@@ -830,13 +829,10 @@ mod test {
                             Plan::Get {
                                 id: crate::expr::Id::Global(GlobalId::User(1)),
                             }
-                            .with_types(
+                            .with_types(RelationType::new(vec![
-                                RelationType::new(vec![
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::uint32_datatype(), false),
+                                ColumnType::new(ConcreteDataType::datetime_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                            ])),
-                                ])
-                                .into_unnamed(),
-                            ),
                         ),
                         key_val_plan: KeyValPlan {
                             key_plan: MapFilterProject::new(2)
@@ -884,8 +880,7 @@ mod test {
                             ColumnType::new(CDT::uint64_datatype(), true),    //sum(number)
                         ])
                         .with_key(vec![1])
-                        .with_time_index(Some(0))
+                        .with_time_index(Some(0)),
-                        .into_unnamed(),
                     ),
                 ),
                 mfp: MapFilterProject::new(3)
@@ -982,8 +977,7 @@ mod test {
             els: Box::new(ScalarExpr::Literal(Value::Null, CDT::uint64_datatype())),
         };
         let expected = TypedPlan {
-            schema: RelationType::new(vec![ColumnType::new(CDT::uint64_datatype(), true)])
+            typ: RelationType::new(vec![ColumnType::new(CDT::uint64_datatype(), true)]),
-                .into_unnamed(),
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Reduce {
@@ -991,13 +985,9 @@ mod test {
                             Plan::Get {
                                 id: crate::expr::Id::Global(GlobalId::User(1)),
                             }
-                            .with_types(
+                            .with_types(RelationType::new(vec![
-                                RelationType::new(vec![ColumnType::new(
+                                ColumnType::new(ConcreteDataType::int64_datatype(), false),
-                                    ConcreteDataType::int64_datatype(),
+                            ])),
-                                    false,
-                                )])
-                                .into_unnamed(),
-                            ),
                         ),
                         key_val_plan: KeyValPlan {
                             key_plan: MapFilterProject::new(1)
@@ -1018,13 +1008,10 @@ mod test {
                             distinct_aggrs: vec![],
                         }),
                     }
-                    .with_types(
+                    .with_types(RelationType::new(vec![
-                        RelationType::new(vec![
+                        ColumnType::new(ConcreteDataType::uint32_datatype(), true),
-                            ColumnType::new(ConcreteDataType::uint32_datatype(), true),
+                        ColumnType::new(ConcreteDataType::int64_datatype(), true),
-                            ColumnType::new(ConcreteDataType::int64_datatype(), true),
+                    ])),
-                        ])
-                        .into_unnamed(),
-                    ),
                 ),
                 mfp: MapFilterProject::new(2)
                     .map(vec![
@@ -1081,7 +1068,7 @@ mod test {
         let reduce_plan = ReducePlan::Distinct;
         let bundle = ctx
             .render_reduce(
-                Box::new(input_plan.with_types(typ.into_unnamed())),
+                Box::new(input_plan.with_types(typ)),
                 key_val_plan,
                 reduce_plan,
                 RelationType::empty(),
@@ -1156,7 +1143,7 @@ mod test {
         let reduce_plan = ReducePlan::Accumulable(accum_plan);
         let bundle = ctx
             .render_reduce(
-                Box::new(input_plan.with_types(typ.into_unnamed())),
+                Box::new(input_plan.with_types(typ)),
                 key_val_plan,
                 reduce_plan,
                 RelationType::empty(),
@@ -1237,7 +1224,7 @@ mod test {
         let reduce_plan = ReducePlan::Accumulable(accum_plan);
         let bundle = ctx
             .render_reduce(
-                Box::new(input_plan.with_types(typ.into_unnamed())),
+                Box::new(input_plan.with_types(typ)),
                 key_val_plan,
                 reduce_plan,
                 RelationType::empty(),
@@ -1314,7 +1301,7 @@ mod test {
         let reduce_plan = ReducePlan::Accumulable(accum_plan);
         let bundle = ctx
             .render_reduce(
-                Box::new(input_plan.with_types(typ.into_unnamed())),
+                Box::new(input_plan.with_types(typ)),
                 key_val_plan,
                 reduce_plan,
                 RelationType::empty(),
@@ -1406,7 +1393,7 @@ mod test {
         let reduce_plan = ReducePlan::Accumulable(accum_plan);
         let bundle = ctx
             .render_reduce(
-                Box::new(input_plan.with_types(typ.into_unnamed())),
+                Box::new(input_plan.with_types(typ)),
                 key_val_plan,
                 reduce_plan,
                 RelationType::empty(),

src/flow/src/compute/render/src_sink.rs

@@ -62,6 +62,7 @@ impl<'referred, 'df> Context<'referred, 'df> {
                 let arr = arranged.get_updates_in_range(..=now);
                 err_collector.run(|| arranged.compact_to(now));
 
+                debug!("Call source");
                 let prev_avail = arr.into_iter().map(|((k, _), t, d)| (k, t, d));
                 let mut to_send = Vec::new();
                 let mut to_arrange = Vec::new();

src/flow/src/expr/linear.rs

@@ -321,38 +321,6 @@ impl MapFilterProject {
     pub fn optimize(&mut self) {
         // TODO(discord9): optimize
     }
-    /// get the mapping of old columns to new columns after the mfp
-    pub fn get_old_to_new_mapping(&self) -> BTreeMap<usize, usize> {
-        BTreeMap::from_iter(
-            self.projection
-                .clone()
-                .into_iter()
-                .enumerate()
-                .map(|(new, old)| {
-                    // `projection` give the new -> old mapping
-                    let mut old = old;
-                    // trace back to the original column
-                    // since there maybe indirect ref to old columns like
-                    // col 2 <- expr=col(2) at pos col 4 <- expr=col(4) at pos col 6
-                    // ideally such indirect ref should be optimize away
-                    // TODO(discord9): refactor this after impl `optimize()`
-                    while let Some(ScalarExpr::Column(prev)) = if old >= self.input_arity {
-                        // get the correspond expr if not a original column
-                        self.expressions.get(old - self.input_arity)
-                    } else {
-                        // we don't care about non column ref case since only need old to new column mapping
-                        // in which case, the old->new mapping remain the same
-                        None
-                    } {
-                        old = *prev;
-                        if old < self.input_arity {
-                            break;
-                        }
-                    }
-                    (old, new)
-                }),
-        )
-    }
 
     /// Convert the `MapFilterProject` into a staged evaluation plan.
     ///

src/flow/src/plan.rs

@@ -30,22 +30,21 @@ use crate::expr::{
 };
 use crate::plan::join::JoinPlan;
 pub(crate) use crate::plan::reduce::{AccumulablePlan, AggrWithIndex, KeyValPlan, ReducePlan};
-use crate::repr::{ColumnType, DiffRow, RelationDesc, RelationType};
+use crate::repr::{ColumnType, DiffRow, RelationType};
 
 /// A plan for a dataflow component. But with type to indicate the output type of the relation.
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)]
 pub struct TypedPlan {
     /// output type of the relation
-    pub schema: RelationDesc,
+    pub typ: RelationType,
     /// The untyped plan.
     pub plan: Plan,
 }
 
 impl TypedPlan {
     /// directly apply a mfp to the plan
-    pub fn mfp(self, mfp: SafeMfpPlan) -> Result<Self, Error> {
+    pub fn mfp(self, mfp: MapFilterProject) -> Result<Self, Error> {
-        let new_type = self.schema.apply_mfp(&mfp)?;
+        let new_type = self.typ.apply_mfp(&mfp)?;
-        let mfp = mfp.mfp;
         let plan = match self.plan {
             Plan::Mfp {
                 input,
@@ -60,14 +59,14 @@ impl TypedPlan {
             },
         };
         Ok(TypedPlan {
-            schema: new_type,
+            typ: new_type,
             plan,
         })
     }
 
     /// project the plan to the given expressions
     pub fn projection(self, exprs: Vec<TypedExpr>) -> Result<Self, Error> {
-        let input_arity = self.schema.typ.column_types.len();
+        let input_arity = self.typ.column_types.len();
         let output_arity = exprs.len();
         let (exprs, _expr_typs): (Vec<_>, Vec<_>) = exprs
             .into_iter()
@@ -75,10 +74,8 @@ impl TypedPlan {
             .unzip();
         let mfp = MapFilterProject::new(input_arity)
             .map(exprs)?
-            .project(input_arity..input_arity + output_arity)?
+            .project(input_arity..input_arity + output_arity)?;
-            .into_safe();
+        let out_typ = self.typ.apply_mfp(&mfp)?;
-        let out_typ = self.schema.apply_mfp(&mfp)?;
-        let mfp = mfp.mfp;
         // special case for mfp to compose when the plan is already mfp
         let plan = match self.plan {
             Plan::Mfp {
@@ -93,15 +90,12 @@ impl TypedPlan {
                 mfp,
             },
         };
-        Ok(TypedPlan {
+        Ok(TypedPlan { typ: out_typ, plan })
-            schema: out_typ,
-            plan,
-        })
     }
 
     /// Add a new filter to the plan, will filter out the records that do not satisfy the filter
     pub fn filter(self, filter: TypedExpr) -> Result<Self, Error> {
-        let typ = self.schema.clone();
+        let typ = self.typ.clone();
         let plan = match self.plan {
             Plan::Mfp {
                 input,
@@ -112,10 +106,10 @@ impl TypedPlan {
             },
             _ => Plan::Mfp {
                 input: Box::new(self),
-                mfp: MapFilterProject::new(typ.typ.column_types.len()).filter(vec![filter.expr])?,
+                mfp: MapFilterProject::new(typ.column_types.len()).filter(vec![filter.expr])?,
             },
         };
-        Ok(TypedPlan { schema: typ, plan })
+        Ok(TypedPlan { typ, plan })
     }
 }
 
@@ -233,7 +227,7 @@ impl Plan {
 }
 
 impl Plan {
-    pub fn with_types(self, schema: RelationDesc) -> TypedPlan {
+    pub fn with_types(self, typ: RelationType) -> TypedPlan {
-        TypedPlan { schema, plan: self }
+        TypedPlan { typ, plan: self }
     }
 }

src/flow/src/repr.rs

@@ -53,8 +53,7 @@ pub type KeyValDiffRow = ((Row, Row), Timestamp, Diff);
 
 /// broadcast channel capacity, can be important to memory consumption, since this influence how many
 /// updates can be buffered in memory in the entire dataflow
-/// TODO(discord9): add config for this, so cpu&mem usage can be balanced and configured by this
+pub const BROADCAST_CAP: usize = 8192;
-pub const BROADCAST_CAP: usize = 65535;
 
 /// Convert a value that is or can be converted to Datetime to internal timestamp
 ///

src/flow/src/repr/relation.rs

@@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt};
 
 use crate::adapter::error::{InvalidQuerySnafu, Result, UnexpectedSnafu};
-use crate::expr::{MapFilterProject, SafeMfpPlan, ScalarExpr};
+use crate::expr::MapFilterProject;
 
 /// a set of column indices that are "keys" for the collection.
 #[derive(Default, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
@@ -111,8 +111,7 @@ impl RelationType {
     /// then new key=`[1]`, new time index=`[0]`
     ///
     /// note that this function will remove empty keys like key=`[]` will be removed
-    pub fn apply_mfp(&self, mfp: &SafeMfpPlan) -> Result<Self> {
+    pub fn apply_mfp(&self, mfp: &MapFilterProject) -> Result<Self> {
-        let mfp = &mfp.mfp;
         let mut all_types = self.column_types.clone();
         for expr in &mfp.expressions {
             let expr_typ = expr.typ(&self.column_types)?;
@@ -133,7 +132,13 @@ impl RelationType {
             })
             .try_collect()?;
 
-        let old_to_new_col = mfp.get_old_to_new_mapping();
+        let old_to_new_col = BTreeMap::from_iter(
+            mfp.projection
+                .clone()
+                .into_iter()
+                .enumerate()
+                .map(|(new, old)| (old, new)),
+        );
 
         // since it's just a mfp, we also try to preserve keys&time index information, if they survive mfp transform
         let keys = self
@@ -259,15 +264,15 @@ impl RelationType {
     }
 
     /// Return relation describe with column names
-    pub fn into_named(self, names: Vec<Option<ColumnName>>) -> RelationDesc {
+    pub fn into_named(self, names: Vec<ColumnName>) -> RelationDesc {
         RelationDesc { typ: self, names }
     }
 
     /// Return relation describe without column names
     pub fn into_unnamed(self) -> RelationDesc {
         RelationDesc {
-            names: vec![None; self.column_types.len()],
             typ: self,
+            names: vec![],
         }
     }
 }
@@ -331,36 +336,10 @@ fn return_true() -> bool {
 ///
 /// It bundles a [`RelationType`] with the name of each column in the relation.
 /// Individual column names are optional.
-#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Hash)]
 pub struct RelationDesc {
     pub typ: RelationType,
-    pub names: Vec<Option<ColumnName>>,
+    pub names: Vec<ColumnName>,
-}
-
-impl RelationDesc {
-    /// apply mfp, and also project col names for the projected columns
-    pub fn apply_mfp(&self, mfp: &SafeMfpPlan) -> Result<Self> {
-        // TODO: find a way to deduce name at best effect
-        let names = {
-            let mfp = &mfp.mfp;
-            let mut names = self.names.clone();
-            for expr in &mfp.expressions {
-                if let ScalarExpr::Column(i) = expr {
-                    names.push(self.names.get(*i).cloned().flatten());
-                } else {
-                    names.push(None);
-                }
-            }
-            mfp.projection
-                .iter()
-                .map(|i| names.get(*i).cloned().flatten())
-                .collect_vec()
-        };
-        Ok(Self {
-            typ: self.typ.apply_mfp(mfp)?,
-            names,
-        })
-    }
 }
 
 impl RelationDesc {
@@ -379,7 +358,7 @@ impl RelationDesc {
     pub fn try_new<I, N>(typ: RelationType, names: I) -> Result<Self>
     where
         I: IntoIterator<Item = N>,
-        N: Into<Option<ColumnName>>,
+        N: Into<ColumnName>,
     {
         let names: Vec<_> = names.into_iter().map(|name| name.into()).collect();
         ensure!(
@@ -404,7 +383,7 @@ impl RelationDesc {
     pub fn new_unchecked<I, N>(typ: RelationType, names: I) -> Self
     where
         I: IntoIterator<Item = N>,
-        N: Into<Option<ColumnName>>,
+        N: Into<ColumnName>,
     {
         let names: Vec<_> = names.into_iter().map(|name| name.into()).collect();
         assert_eq!(typ.arity(), names.len());
@@ -415,7 +394,7 @@ impl RelationDesc {
     where
         I: IntoIterator<Item = (N, T)>,
         T: Into<ColumnType>,
-        N: Into<Option<ColumnName>>,
+        N: Into<ColumnName>,
     {
         let (names, types): (Vec<_>, Vec<_>) = iter.into_iter().unzip();
         let types = types.into_iter().map(Into::into).collect();
@@ -441,7 +420,7 @@ impl RelationDesc {
     /// Appends a column with the specified name and type.
     pub fn with_column<N>(mut self, name: N, column_type: ColumnType) -> Self
     where
-        N: Into<Option<ColumnName>>,
+        N: Into<ColumnName>,
     {
         self.typ.column_types.push(column_type);
         self.names.push(name.into());
@@ -466,7 +445,7 @@ impl RelationDesc {
     pub fn try_with_names<I, N>(self, names: I) -> Result<Self>
     where
         I: IntoIterator<Item = N>,
-        N: Into<Option<ColumnName>>,
+        N: Into<ColumnName>,
     {
         Self::try_new(self.typ, names)
     }
@@ -482,7 +461,7 @@ impl RelationDesc {
     }
 
     /// Returns an iterator over the columns in this relation.
-    pub fn iter(&self) -> impl Iterator<Item = (&Option<ColumnName>, &ColumnType)> {
+    pub fn iter(&self) -> impl Iterator<Item = (&ColumnName, &ColumnType)> {
         self.iter_names().zip(self.iter_types())
     }
 
@@ -492,7 +471,7 @@ impl RelationDesc {
     }
 
     /// Returns an iterator over the names of the columns in this relation.
-    pub fn iter_names(&self) -> impl Iterator<Item = &Option<ColumnName>> {
+    pub fn iter_names(&self) -> impl Iterator<Item = &ColumnName> {
         self.names.iter()
     }
 
@@ -503,7 +482,7 @@ impl RelationDesc {
     /// specified name, the leftmost column is returned.
     pub fn get_by_name(&self, name: &ColumnName) -> Option<(usize, &ColumnType)> {
         self.iter_names()
-            .position(|n| n.as_ref() == Some(name))
+            .position(|n| n == name)
             .map(|i| (i, &self.typ.column_types[i]))
     }
 
@@ -512,7 +491,7 @@ impl RelationDesc {
     /// # Panics
     ///
     /// Panics if `i` is not a valid column index.
-    pub fn get_name(&self, i: usize) -> &Option<ColumnName> {
+    pub fn get_name(&self, i: usize) -> &ColumnName {
         &self.names[i]
     }
 
@@ -521,7 +500,7 @@ impl RelationDesc {
     /// # Panics
     ///
     /// Panics if `i` is not a valid column index.
-    pub fn get_name_mut(&mut self, i: usize) -> &mut Option<ColumnName> {
+    pub fn get_name_mut(&mut self, i: usize) -> &mut ColumnName {
         &mut self.names[i]
     }
 
@@ -536,7 +515,7 @@ impl RelationDesc {
     pub fn get_unambiguous_name(&self, i: usize) -> Option<&ColumnName> {
         let name = &self.names[i];
         if self.iter_names().filter(|n| *n == name).count() == 1 {
-            name.as_ref()
+            Some(name)
         } else {
             None
         }

src/flow/src/transform.rs

@@ -212,7 +212,7 @@ mod test {
             let schema = RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)]);
 
             tri_map.insert(Some(name.clone()), Some(1024), gid);
-            schemas.insert(gid, schema.into_named(vec![Some("number".to_string())]));
+            schemas.insert(gid, schema.into_unnamed());
         }
 
         {
@@ -226,10 +226,7 @@ mod test {
                 ColumnType::new(CDT::uint32_datatype(), false),
                 ColumnType::new(CDT::datetime_datatype(), false),
             ]);
-            schemas.insert(
+            schemas.insert(gid, schema.into_unnamed());
-                gid,
-                schema.into_named(vec![Some("number".to_string()), Some("ts".to_string())]),
-            );
             tri_map.insert(Some(name.clone()), Some(1025), gid);
         }
 

src/flow/src/transform/aggr.rs

@@ -285,20 +285,6 @@ impl KeyValPlan {
     }
 }
 
-/// find out the column that should be time index in group exprs(which is all columns that should be keys)
-/// TODO(discord9): better ways to assign time index
-fn find_time_index_in_group_exprs(group_exprs: &[TypedExpr]) -> Option<usize> {
-    group_exprs.iter().position(|expr| {
-        matches!(
-            &expr.expr,
-            ScalarExpr::CallUnary {
-                func: UnaryFunc::TumbleWindowFloor { .. },
-                expr: _
-            }
-        )
-    })
-}
-
 impl TypedPlan {
     /// Convert AggregateRel into Flow's TypedPlan
     ///
@@ -320,57 +306,44 @@ impl TypedPlan {
             let group_exprs = TypedExpr::from_substrait_agg_grouping(
                 ctx,
                 &agg.groupings,
-                &input.schema.typ,
+                &input.typ,
                 extensions,
             )?;
 
-            TypedExpr::expand_multi_value(&input.schema.typ, &group_exprs)?
+            TypedExpr::expand_multi_value(&input.typ, &group_exprs)?
         };
 
-        let time_index = find_time_index_in_group_exprs(&group_exprs);
+        let time_index = group_exprs.iter().position(|expr| {
+            matches!(
+                &expr.expr,
+                ScalarExpr::CallUnary {
+                    func: UnaryFunc::TumbleWindowFloor { .. },
+                    expr: _
+                }
+            )
+        });
 
-        let (mut aggr_exprs, post_mfp) = AggregateExpr::from_substrait_agg_measures(
+        let (mut aggr_exprs, post_mfp) =
-            ctx,
+            AggregateExpr::from_substrait_agg_measures(ctx, &agg.measures, &input.typ, extensions)?;
-            &agg.measures,
-            &input.schema.typ,
-            extensions,
-        )?;
 
         let key_val_plan = KeyValPlan::from_substrait_gen_key_val_plan(
             &mut aggr_exprs,
             &group_exprs,
-            input.schema.typ.column_types.len(),
+            input.typ.column_types.len(),
         )?;
 
         // output type is group_exprs + aggr_exprs
         let output_type = {
             let mut output_types = Vec::new();
-            // give best effort to get column name
-            let mut output_names = Vec::new();
             // first append group_expr as key, then aggr_expr as value
             for expr in &group_exprs {
                 output_types.push(expr.typ.clone());
-                let col_name = match &expr.expr {
-                    ScalarExpr::CallUnary {
-                        func: UnaryFunc::TumbleWindowFloor { .. },
-                        ..
-                    } => Some("window_start".to_string()),
-                    ScalarExpr::CallUnary {
-                        func: UnaryFunc::TumbleWindowCeiling { .. },
-                        ..
-                    } => Some("window_end".to_string()),
-                    ScalarExpr::Column(col) => input.schema.get_name(*col).clone(),
-                    _ => None,
-                };
-                output_names.push(col_name)
             }
 
             for aggr in &aggr_exprs {
                 output_types.push(ColumnType::new_nullable(
                     aggr.func.signature().output.clone(),
                 ));
-                // TODO(discord9): find a clever way to name them?
-                output_names.push(None);
             }
             // TODO(discord9): try best to get time
             if group_exprs.is_empty() {
@@ -378,9 +351,8 @@ impl TypedPlan {
             } else {
                 RelationType::new(output_types).with_key((0..group_exprs.len()).collect_vec())
             }
-            .with_time_index(time_index)
+        }
-            .into_named(output_names)
+        .with_time_index(time_index);
-        };
 
         // copy aggr_exprs to full_aggrs, and split them into simple_aggrs and distinct_aggrs
         // also set them input/output column
@@ -418,13 +390,13 @@ impl TypedPlan {
         // FIX(discord9): deal with key first
         if post_mfp.is_identity() {
             Ok(TypedPlan {
-                schema: output_type,
+                typ: output_type,
                 plan,
             })
         } else {
             // make post_mfp map identical mapping of keys
             let input = TypedPlan {
-                schema: output_type.clone(),
+                typ: output_type.clone(),
                 plan,
             };
             let key_arity = group_exprs.len();
@@ -442,7 +414,7 @@ impl TypedPlan {
                 .filter(f)?
                 .project(p)?;
             Ok(TypedPlan {
-                schema: output_type.apply_mfp(&post_mfp.clone().into_safe())?,
+                typ: output_type.apply_mfp(&post_mfp)?,
                 plan: Plan::Mfp {
                     input: Box::new(input),
                     mfp: post_mfp,
@@ -462,16 +434,7 @@ mod test {
     use crate::plan::{Plan, TypedPlan};
     use crate::repr::{self, ColumnType, RelationType};
     use crate::transform::test::{create_test_ctx, create_test_query_engine, sql_to_substrait};
-    /// TODO(discord9): add more illegal sql tests
-    #[tokio::test]
-    async fn tes_missing_key_check() {
-        let engine = create_test_query_engine();
-        let sql = "SELECT avg(number) FROM numbers_with_ts GROUP BY tumble(ts, '1 hour'), number";
-        let plan = sql_to_substrait(engine.clone(), sql).await;
 
-        let mut ctx = create_test_ctx();
-        assert!(TypedPlan::from_substrait_plan(&mut ctx, &plan).is_err());
-    }
     /// TODO(discord9): add more illegal sql tests
     #[tokio::test]
     async fn test_tumble_composite() {
@@ -507,6 +470,10 @@ mod test {
             els: Box::new(ScalarExpr::Literal(Value::Null, CDT::uint64_datatype())),
         };
         let expected = TypedPlan {
+            // TODO(discord9): mfp indirectly ref to key columns
+            /*
+            .with_key(vec![1])
+            .with_time_index(Some(0)),*/
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Reduce {
@@ -514,16 +481,10 @@ mod test {
                             Plan::Get {
                                 id: crate::expr::Id::Global(GlobalId::User(1)),
                             }
-                            .with_types(
+                            .with_types(RelationType::new(vec![
-                                RelationType::new(vec![
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::uint32_datatype(), false),
+                                ColumnType::new(ConcreteDataType::datetime_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                            ])),
-                                ])
-                                .into_named(vec![
-                                    Some("number".to_string()),
-                                    Some("ts".to_string()),
-                                ]),
-                            ),
                         ),
                         key_val_plan: KeyValPlan {
                             key_plan: MapFilterProject::new(2)
@@ -579,14 +540,7 @@ mod test {
                             ColumnType::new(CDT::int64_datatype(), true),  // avg.count(number)
                         ])
                         .with_key(vec![1, 2])
-                        .with_time_index(Some(0))
+                        .with_time_index(Some(0)),
-                        .into_named(vec![
-                            Some("window_start".to_string()),
-                            Some("window_end".to_string()),
-                            Some("number".to_string()),
-                            None,
-                            None,
-                        ]),
                     ),
                 ),
                 mfp: MapFilterProject::new(5)
@@ -601,19 +555,11 @@ mod test {
                     .project(vec![6, 7, 8, 9])
                     .unwrap(),
             },
-            schema: RelationType::new(vec![
+            typ: RelationType::new(vec![
                 ColumnType::new(CDT::uint32_datatype(), false), // number
                 ColumnType::new(CDT::uint64_datatype(), true),  // avg(number)
                 ColumnType::new(CDT::datetime_datatype(), false), // window start
                 ColumnType::new(CDT::datetime_datatype(), false), // window end
-            ])
-            .with_key(vec![0, 3])
-            .with_time_index(Some(2))
-            .into_named(vec![
-                Some("number".to_string()),
-                None,
-                Some("window_start".to_string()),
-                Some("window_end".to_string()),
             ]),
         };
         assert_eq!(flow_plan, expected);
@@ -634,17 +580,10 @@ mod test {
             distinct: false,
         };
         let expected = TypedPlan {
-            schema: RelationType::new(vec![
+            typ: RelationType::new(vec![
                 ColumnType::new(CDT::uint64_datatype(), true), // sum(number)
                 ColumnType::new(CDT::datetime_datatype(), false), // window start
                 ColumnType::new(CDT::datetime_datatype(), false), // window end
-            ])
-            .with_key(vec![2])
-            .with_time_index(Some(1))
-            .into_named(vec![
-                None,
-                Some("window_start".to_string()),
-                Some("window_end".to_string()),
             ]),
             // TODO(discord9): mfp indirectly ref to key columns
             /*
@@ -657,16 +596,10 @@ mod test {
                             Plan::Get {
                                 id: crate::expr::Id::Global(GlobalId::User(1)),
                             }
-                            .with_types(
+                            .with_types(RelationType::new(vec![
-                                RelationType::new(vec![
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::uint32_datatype(), false),
+                                ColumnType::new(ConcreteDataType::datetime_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                            ])),
-                                ])
-                                .into_named(vec![
-                                    Some("number".to_string()),
-                                    Some("ts".to_string()),
-                                ]),
-                            ),
                         ),
                         key_val_plan: KeyValPlan {
                             key_plan: MapFilterProject::new(2)
@@ -714,12 +647,7 @@ mod test {
                             ColumnType::new(CDT::uint64_datatype(), true),    //sum(number)
                         ])
                         .with_key(vec![1])
-                        .with_time_index(Some(0))
+                        .with_time_index(Some(0)),
-                        .into_named(vec![
-                            Some("window_start".to_string()),
-                            Some("window_end".to_string()),
-                            None,
-                        ]),
                     ),
                 ),
                 mfp: MapFilterProject::new(3)
@@ -752,18 +680,15 @@ mod test {
             distinct: false,
         };
         let expected = TypedPlan {
-            schema: RelationType::new(vec![
+            typ: RelationType::new(vec![
                 ColumnType::new(CDT::uint64_datatype(), true), // sum(number)
                 ColumnType::new(CDT::datetime_datatype(), false), // window start
                 ColumnType::new(CDT::datetime_datatype(), false), // window end
-            ])
-            .with_key(vec![2])
-            .with_time_index(Some(1))
-            .into_named(vec![
-                None,
-                Some("window_start".to_string()),
-                Some("window_end".to_string()),
             ]),
+            // TODO(discord9): mfp indirectly ref to key columns
+            /*
+            .with_key(vec![1])
+            .with_time_index(Some(0)),*/
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Reduce {
@@ -771,16 +696,10 @@ mod test {
                             Plan::Get {
                                 id: crate::expr::Id::Global(GlobalId::User(1)),
                             }
-                            .with_types(
+                            .with_types(RelationType::new(vec![
-                                RelationType::new(vec![
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::uint32_datatype(), false),
+                                ColumnType::new(ConcreteDataType::datetime_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                            ])),
-                                ])
-                                .into_named(vec![
-                                    Some("number".to_string()),
-                                    Some("ts".to_string()),
-                                ]),
-                            ),
                         ),
                         key_val_plan: KeyValPlan {
                             key_plan: MapFilterProject::new(2)
@@ -828,12 +747,7 @@ mod test {
                             ColumnType::new(CDT::uint64_datatype(), true),    //sum(number)
                         ])
                         .with_key(vec![1])
-                        .with_time_index(Some(0))
+                        .with_time_index(Some(0)),
-                        .into_named(vec![
-                            Some("window_start".to_string()),
-                            Some("window_end".to_string()),
-                            None,
-                        ]),
                     ),
                 ),
                 mfp: MapFilterProject::new(3)
@@ -884,12 +798,10 @@ mod test {
             els: Box::new(ScalarExpr::Literal(Value::Null, CDT::uint64_datatype())),
         };
         let expected = TypedPlan {
-            schema: RelationType::new(vec![
+            typ: RelationType::new(vec![
                 ColumnType::new(CDT::uint64_datatype(), true), // sum(number) -> u64
                 ColumnType::new(CDT::uint32_datatype(), false), // number
-            ])
+            ]),
-            .with_key(vec![1])
-            .into_named(vec![None, Some("number".to_string())]),
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Reduce {
@@ -897,13 +809,9 @@ mod test {
                             Plan::Get {
                                 id: crate::expr::Id::Global(GlobalId::User(0)),
                             }
-                            .with_types(
+                            .with_types(RelationType::new(vec![
-                                RelationType::new(vec![ColumnType::new(
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ConcreteDataType::uint32_datatype(),
+                            ])),
-                                    false,
-                                )])
-                                .into_named(vec![Some("number".to_string())]),
-                            ),
                         ),
                         key_val_plan: KeyValPlan {
                             key_plan: MapFilterProject::new(1)
@@ -932,12 +840,7 @@ mod test {
                             ColumnType::new(ConcreteDataType::uint64_datatype(), true),  // sum
                             ColumnType::new(ConcreteDataType::int64_datatype(), true),   // count
                         ])
-                        .with_key(vec![0])
+                        .with_key(vec![0]),
-                        .into_named(vec![
-                            Some("number".to_string()),
-                            None,
-                            None,
-                        ]),
                     ),
                 ),
                 mfp: MapFilterProject::new(3)
@@ -989,8 +892,7 @@ mod test {
             els: Box::new(ScalarExpr::Literal(Value::Null, CDT::uint64_datatype())),
         };
         let expected = TypedPlan {
-            schema: RelationType::new(vec![ColumnType::new(CDT::uint64_datatype(), true)])
+            typ: RelationType::new(vec![ColumnType::new(CDT::uint64_datatype(), true)]),
-                .into_named(vec![None]),
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Reduce {
@@ -998,13 +900,9 @@ mod test {
                             Plan::Get {
                                 id: crate::expr::Id::Global(GlobalId::User(0)),
                             }
-                            .with_types(
+                            .with_types(RelationType::new(vec![
-                                RelationType::new(vec![ColumnType::new(
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ConcreteDataType::uint32_datatype(),
+                            ])),
-                                    false,
-                                )])
-                                .into_named(vec![Some("number".to_string())]),
-                            ),
                         ),
                         key_val_plan: KeyValPlan {
                             key_plan: MapFilterProject::new(1)
@@ -1025,13 +923,10 @@ mod test {
                             distinct_aggrs: vec![],
                         }),
                     }
-                    .with_types(
+                    .with_types(RelationType::new(vec![
-                        RelationType::new(vec![
+                        ColumnType::new(ConcreteDataType::uint64_datatype(), true),
-                            ColumnType::new(ConcreteDataType::uint64_datatype(), true), // sum
+                        ColumnType::new(ConcreteDataType::int64_datatype(), true),
-                            ColumnType::new(ConcreteDataType::int64_datatype(), true),  // count
+                    ])),
-                        ])
-                        .into_named(vec![None, None]),
-                    ),
                 ),
                 mfp: MapFilterProject::new(2)
                     .map(vec![
@@ -1065,8 +960,7 @@ mod test {
             distinct: false,
         };
         let expected = TypedPlan {
-            schema: RelationType::new(vec![ColumnType::new(CDT::uint64_datatype(), true)])
+            typ: RelationType::new(vec![ColumnType::new(CDT::uint64_datatype(), true)]),
-                .into_unnamed(),
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Reduce {
@@ -1074,13 +968,9 @@ mod test {
                             Plan::Get {
                                 id: crate::expr::Id::Global(GlobalId::User(0)),
                             }
-                            .with_types(
+                            .with_types(RelationType::new(vec![
-                                RelationType::new(vec![ColumnType::new(
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ConcreteDataType::uint32_datatype(),
+                            ])),
-                                    false,
-                                )])
-                                .into_named(vec![Some("number".to_string())]),
-                            ),
                         ),
                         key_val_plan: KeyValPlan {
                             key_plan: MapFilterProject::new(1)
@@ -1098,7 +988,7 @@ mod test {
                             distinct_aggrs: vec![],
                         }),
                     }
-                    .with_types(typ.into_unnamed()),
+                    .with_types(typ),
                 ),
                 mfp: MapFilterProject::new(1)
                     .map(vec![ScalarExpr::Column(0), ScalarExpr::Column(1)])
@@ -1125,12 +1015,10 @@ mod test {
             distinct: false,
         };
         let expected = TypedPlan {
-            schema: RelationType::new(vec![
+            typ: RelationType::new(vec![
                 ColumnType::new(CDT::uint64_datatype(), true), // col sum(number)
                 ColumnType::new(CDT::uint32_datatype(), false), // col number
-            ])
+            ]),
-            .with_key(vec![1])
-            .into_named(vec![None, Some("number".to_string())]),
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Reduce {
@@ -1138,13 +1026,9 @@ mod test {
                             Plan::Get {
                                 id: crate::expr::Id::Global(GlobalId::User(0)),
                             }
-                            .with_types(
+                            .with_types(RelationType::new(vec![
-                                RelationType::new(vec![ColumnType::new(
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ConcreteDataType::uint32_datatype(),
+                            ])),
-                                    false,
-                                )])
-                                .into_named(vec![Some("number".to_string())]),
-                            ),
                         ),
                         key_val_plan: KeyValPlan {
                             key_plan: MapFilterProject::new(1)
@@ -1169,8 +1053,7 @@ mod test {
                             ColumnType::new(CDT::uint32_datatype(), false), // col number
                             ColumnType::new(CDT::uint64_datatype(), true),  // col sum(number)
                         ])
-                        .with_key(vec![0])
+                        .with_key(vec![0]),
-                        .into_named(vec![Some("number".to_string()), None]),
                     ),
                 ),
                 mfp: MapFilterProject::new(2)
@@ -1203,8 +1086,7 @@ mod test {
             distinct: false,
         };
         let expected = TypedPlan {
-            schema: RelationType::new(vec![ColumnType::new(CDT::uint64_datatype(), true)])
+            typ: RelationType::new(vec![ColumnType::new(CDT::uint64_datatype(), true)]),
-                .into_unnamed(),
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Reduce {
@@ -1212,13 +1094,9 @@ mod test {
                             Plan::Get {
                                 id: crate::expr::Id::Global(GlobalId::User(0)),
                             }
-                            .with_types(
+                            .with_types(RelationType::new(vec![
-                                RelationType::new(vec![ColumnType::new(
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ConcreteDataType::uint32_datatype(),
+                            ])),
-                                    false,
-                                )])
-                                .into_named(vec![Some("number".to_string())]),
-                            ),
                         ),
                         key_val_plan: KeyValPlan {
                             key_plan: MapFilterProject::new(1)
@@ -1239,10 +1117,10 @@ mod test {
                             distinct_aggrs: vec![],
                         }),
                     }
-                    .with_types(
+                    .with_types(RelationType::new(vec![ColumnType::new(
-                        RelationType::new(vec![ColumnType::new(CDT::uint64_datatype(), true)])
+                        CDT::uint64_datatype(),
-                            .into_unnamed(),
+                        true,
-                    ),
+                    )])),
                 ),
                 mfp: MapFilterProject::new(1)
                     .map(vec![ScalarExpr::Column(0), ScalarExpr::Column(1)])

src/flow/src/transform/expr.rs

@@ -363,20 +363,16 @@ mod test {
             ],
         };
         let expected = TypedPlan {
-            schema: RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)])
+            typ: RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)]),
-                .into_named(vec![Some("number".to_string())]),
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Get {
                         id: crate::expr::Id::Global(GlobalId::User(0)),
                     }
-                    .with_types(
+                    .with_types(RelationType::new(vec![ColumnType::new(
-                        RelationType::new(vec![ColumnType::new(
+                        ConcreteDataType::uint32_datatype(),
-                            ConcreteDataType::uint32_datatype(),
+                        false,
-                            false,
+                    )])),
-                        )])
-                        .into_named(vec![Some("number".to_string())]),
-                    ),
                 ),
                 mfp: MapFilterProject::new(1)
                     .map(vec![ScalarExpr::Column(0)])
@@ -401,8 +397,7 @@ mod test {
         let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan);
 
         let expected = TypedPlan {
-            schema: RelationType::new(vec![ColumnType::new(CDT::boolean_datatype(), true)])
+            typ: RelationType::new(vec![ColumnType::new(CDT::boolean_datatype(), true)]),
-                .into_unnamed(),
             plan: Plan::Constant {
                 rows: vec![(
                     repr::Row::new(vec![Value::from(true)]),
@@ -426,20 +421,16 @@ mod test {
         let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan);
 
         let expected = TypedPlan {
-            schema: RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), true)])
+            typ: RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), true)]),
-                .into_unnamed(),
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Get {
                         id: crate::expr::Id::Global(GlobalId::User(0)),
                     }
-                    .with_types(
+                    .with_types(RelationType::new(vec![ColumnType::new(
-                        RelationType::new(vec![ColumnType::new(
+                        ConcreteDataType::uint32_datatype(),
-                            ConcreteDataType::uint32_datatype(),
+                        false,
-                            false,
+                    )])),
-                        )])
-                        .into_named(vec![Some("number".to_string())]),
-                    ),
                 ),
                 mfp: MapFilterProject::new(1)
                     .map(vec![ScalarExpr::Column(0).call_binary(
@@ -464,20 +455,16 @@ mod test {
         let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan);
 
         let expected = TypedPlan {
-            schema: RelationType::new(vec![ColumnType::new(CDT::int16_datatype(), true)])
+            typ: RelationType::new(vec![ColumnType::new(CDT::int16_datatype(), true)]),
-                .into_unnamed(),
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Get {
                         id: crate::expr::Id::Global(GlobalId::User(0)),
                     }
-                    .with_types(
+                    .with_types(RelationType::new(vec![ColumnType::new(
-                        RelationType::new(vec![ColumnType::new(
+                        ConcreteDataType::uint32_datatype(),
-                            ConcreteDataType::uint32_datatype(),
+                        false,
-                            false,
+                    )])),
-                        )])
-                        .into_named(vec![Some("number".to_string())]),
-                    ),
                 ),
                 mfp: MapFilterProject::new(1)
                     .map(vec![ScalarExpr::Literal(
@@ -503,20 +490,16 @@ mod test {
         let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan);
 
         let expected = TypedPlan {
-            schema: RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), true)])
+            typ: RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), true)]),
-                .into_unnamed(),
             plan: Plan::Mfp {
                 input: Box::new(
                     Plan::Get {
                         id: crate::expr::Id::Global(GlobalId::User(0)),
                     }
-                    .with_types(
+                    .with_types(RelationType::new(vec![ColumnType::new(
-                        RelationType::new(vec![ColumnType::new(
+                        ConcreteDataType::uint32_datatype(),
-                            ConcreteDataType::uint32_datatype(),
+                        false,
-                            false,
+                    )])),
-                        )])
-                        .into_named(vec![Some("number".to_string())]),
-                    ),
                 ),
                 mfp: MapFilterProject::new(1)
                     .map(vec![ScalarExpr::Column(0)

src/flow/src/transform/literal.rs

@@ -175,8 +175,7 @@ mod test {
         let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan);
 
         let expected = TypedPlan {
-            schema: RelationType::new(vec![ColumnType::new(CDT::int64_datatype(), true)])
+            typ: RelationType::new(vec![ColumnType::new(CDT::int64_datatype(), true)]),
-                .into_unnamed(),
             plan: Plan::Constant {
                 rows: vec![(
                     repr::Row::new(vec![Value::Int64(1)]),