refactor: per reviews

* feat: enable tcp keepalive for http server

* chore: for enterprise's update

* resolve PR comments

.github/actions/build-and-push-ci-image/action.yml

@@ -0,0 +1,18 @@
+name: Build and push CI Docker image
+description: Build and push CI Docker image to local registry
+inputs:
+  binary_path:
+    default: "./bin"
+    description: "Binary path"
+runs:
+  using: composite
+  steps: 
+    - name: Build and push to local registry
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./docker/ci/ubuntu/Dockerfile.fuzztests
+        push: true
+        tags: localhost:5001/greptime/greptimedb:latest
+        build-args: |
+          BINARY_PATH=${{ inputs.binary_path }}

.github/actions/setup-etcd-cluster/action.yml

@@ -0,0 +1,25 @@
+name: Setup Etcd cluster
+description: Deploy Etcd cluster on Kubernetes
+inputs:
+  etcd-replicas:
+    default: 3
+    description: "Etcd replicas"
+  namespace:
+    default: "etcd-cluster"
+
+runs:
+  using: composite
+  steps:
+  - name: Install Etcd cluster
+    shell: bash
+    run: | 
+      helm upgrade \
+        --install etcd oci://registry-1.docker.io/bitnamicharts/etcd \
+        --set replicaCount=${{ inputs.etcd-replicas }} \
+        --set resources.requests.cpu=50m \
+        --set resources.requests.memory=128Mi \
+        --set auth.rbac.create=false \
+        --set auth.rbac.token.enabled=false \
+        --set persistence.size=2Gi \
+        --create-namespace \
+        -n ${{ inputs.namespace }}

.github/actions/setup-greptimedb-cluster/action.yml

@@ -0,0 +1,85 @@
+name: Setup GreptimeDB cluster
+description: Deploy GreptimeDB cluster on Kubernetes
+inputs:
+  frontend-replicas:
+    default: 2
+    description: "Number of Frontend replicas"
+  datanode-replicas:
+    default: 2
+    description: "Number of Datanode replicas"
+  meta-replicas:
+    default: 3
+    description: "Number of Metasrv replicas"
+  image-registry: 
+    default: "docker.io"
+    description: "Image registry"
+  image-repository: 
+    default: "greptime/greptimedb"
+    description: "Image repository"
+  image-tag: 
+    default: "latest"
+    description: 'Image tag'
+  etcd-endpoints:
+    default: "etcd.etcd-cluster.svc.cluster.local:2379"
+    description: "Etcd endpoints"
+
+runs:
+  using: composite
+  steps:
+  - name: Install GreptimeDB operator
+    shell: bash
+    run: |
+      helm repo add greptime https://greptimeteam.github.io/helm-charts/ 
+      helm repo update
+      helm upgrade \
+        --install \
+        --create-namespace \
+        greptimedb-operator greptime/greptimedb-operator \
+        -n greptimedb-admin \
+        --wait \
+        --wait-for-jobs
+  - name: Install GreptimeDB cluster
+    shell: bash
+    run: | 
+      helm upgrade \
+        --install my-greptimedb \
+        --set meta.etcdEndpoints=${{ inputs.etcd-endpoints }} \
+        --set image.registry=${{ inputs.image-registry }} \
+        --set image.repository=${{ inputs.image-repository }}  \
+        --set image.tag=${{ inputs.image-tag }} \
+        --set base.podTemplate.main.resources.requests.cpu=50m \
+        --set base.podTemplate.main.resources.requests.memory=256Mi \
+        --set base.podTemplate.main.resources.limits.cpu=1000m \
+        --set base.podTemplate.main.resources.limits.memory=2Gi \
+        --set frontend.replicas=${{ inputs.frontend-replicas }} \
+        --set datanode.replicas=${{ inputs.datanode-replicas }} \
+        --set meta.replicas=${{ inputs.meta-replicas }} \
+        greptime/greptimedb-cluster \
+        --create-namespace \
+        -n my-greptimedb \
+        --wait \
+        --wait-for-jobs
+  - name: Wait for GreptimeDB
+    shell: bash
+    run: |
+      while true; do 
+        PHASE=$(kubectl -n my-greptimedb get gtc my-greptimedb -o jsonpath='{.status.clusterPhase}')
+        if [ "$PHASE" == "Running" ]; then
+          echo "Cluster is ready"
+          break
+        else
+          echo "Cluster is not ready yet: Current phase: $PHASE"
+          kubectl get pods -n my-greptimedb
+          sleep 5 # wait for 5 seconds before check again.
+        fi
+      done
+  - name: Print GreptimeDB info
+    if: always()
+    shell: bash
+    run: | 
+      kubectl get all --show-labels -n my-greptimedb
+  - name: Describe Nodes
+    if: always()
+    shell: bash
+    run: | 
+      kubectl describe nodes

.github/actions/setup-kind/action.yml

@@ -0,0 +1,10 @@
+name: Setup Kind
+description: Deploy Kind
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v4
+  - name: Create kind cluster
+    shell: bash
+    run: |
+      ./.github/scripts/kind-with-registry.sh

.github/actions/sqlness-test/action.yml

@@ -57,3 +57,14 @@ runs:
         AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
       run: |
         aws s3 rm s3://${{ inputs.aws-ci-test-bucket }}/${{ inputs.data-root }} --recursive
+    - name: Export kind logs
+      if: failure()
+      shell: bash
+      run: kind export logs -n greptimedb-operator-e2e /tmp/kind
+    - name: Upload logs
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: kind-logs
+        path: /tmp/kind
+        retention-days: 3

.github/scripts/kind-with-registry.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+# 1. Create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \
+    registry:2
+fi
+
+# 2. Create kind cluster with containerd registry config dir enabled
+# TODO: kind will eventually enable this by default and this patch will
+# be unnecessary.
+#
+# See:
+# https://github.com/kubernetes-sigs/kind/issues/2875
+# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
+# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
+cat <<EOF | kind create cluster --wait 2m --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"
+EOF
+
+# 3. Add the registry config to the nodes
+#
+# This is necessary because localhost resolves to loopback addresses that are
+# network-namespace local.
+# In other words: localhost in the container is not localhost on the host.
+#
+# We want a consistent name that works from both ends, so we tell containerd to
+# alias localhost:${reg_port} to the registry container when pulling images
+REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
+for node in $(kind get nodes); do
+  docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+  cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${reg_name}:5000"]
+EOF
+done
+
+# 4. Connect the registry to the cluster network if not already connected
+# This allows kind to bootstrap the network but ensures they're on the same network
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+# 5. Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF

.github/workflows/dev-build.yml

@@ -82,6 +82,9 @@ env:
   # The source code will check out in the following path: '${WORKING_DIR}/dev/greptime'.
   CHECKOUT_GREPTIMEDB_PATH: dev/greptimedb
 
+permissions:
+  issues: write
+
 jobs:
   allocate-runners:
     name: Allocate runners
@@ -321,7 +324,7 @@ jobs:
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
 
   notification:
-    if: ${{ always() }} # Not requiring successful dependent jobs, always run.
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && always() }} # Not requiring successful dependent jobs, always run.
     name: Send notification to Greptime team
     needs: [
       release-images-to-dockerhub
@@ -330,16 +333,25 @@ jobs:
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:
-      - name: Notifiy dev build successful result
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-cyborg
+      - name: Report CI status
+        id: report-ci-status
+        working-directory: cyborg
+        run: pnpm tsx bin/report-ci-failure.ts
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CI_REPORT_STATUS: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }}
+      - name: Notify dev build successful result
         uses: slackapi/slack-github-action@v1.23.0
         if: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }}
         with:
           payload: |
             {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has completed successfully."}
 
-      - name: Notifiy dev build failed result
+      - name: Notify dev build failed result
         uses: slackapi/slack-github-action@v1.23.0
         if: ${{ needs.release-images-to-dockerhub.outputs.build-result != 'success' }}
         with:
           payload: |
-            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'."}
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check ${{ steps.report-ci-status.outputs.html_url }}."}

.github/workflows/develop.yml

@@ -231,6 +231,130 @@ jobs:
           path: /tmp/unstable-greptime/
           retention-days: 3
 
+  build-greptime-ci:
+    name: Build GreptimeDB binary (profile-CI)
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-20.04 ]
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # Shares across multiple jobs
+          shared-key: "build-greptime-ci"
+      - name: Install cargo-gc-bin
+        shell: bash
+        run: cargo install cargo-gc-bin
+      - name: Build greptime bianry
+        shell: bash
+        # `cargo gc` will invoke `cargo build` with specified args
+        run: cargo build --bin greptime --profile ci
+      - name: Pack greptime binary
+        shell: bash
+        run: |
+          mkdir bin && \
+          mv ./target/ci/greptime bin
+      - name: Print greptime binaries info
+        run: ls -lh bin
+      - name: Upload artifacts
+        uses: ./.github/actions/upload-artifacts
+        with:
+          artifacts-dir: bin
+          version: current
+
+  distributed-fuzztest: 
+    name: Fuzz Test (Distributed, Disk)
+    runs-on: ubuntu-latest
+    needs:  build-greptime-ci
+    strategy:
+      matrix:
+        target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Kind
+        uses: ./.github/actions/setup-kind
+      - name: Setup Etcd cluser
+        uses: ./.github/actions/setup-etcd-cluster
+      # Prepares for fuzz tests
+      - uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          # Shares across multiple jobs
+          shared-key: "fuzz-test-targets"
+      - name: Set Rust Fuzz
+        shell: bash
+        run: |
+          sudo apt-get install -y libfuzzer-14-dev
+          rustup install nightly
+          cargo +nightly install cargo-fuzz
+      # Downloads ci image
+      - name: Download pre-built binariy
+        uses: actions/download-artifact@v4
+        with:
+          name: bin
+          path: .
+      - name: Unzip binary
+        run: tar -xvf ./bin.tar.gz
+      - name: Build and push GreptimeDB image
+        uses: ./.github/actions/build-and-push-ci-image
+      - name: Wait for etcd
+        run: |
+          kubectl wait \
+            --for=condition=Ready \
+            pod -l app.kubernetes.io/instance=etcd \
+            --timeout=120s \
+            -n etcd-cluster
+      - name: Print etcd info
+        shell: bash
+        run: kubectl get all --show-labels -n etcd-cluster
+      # Setup cluster for test
+      - name: Setup GreptimeDB cluster
+        uses: ./.github/actions/setup-greptimedb-cluster
+        with:
+          image-registry: localhost:5001
+      - name: Port forward (mysql)
+        run: |
+          kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
+      - name: Fuzz Test
+        uses: ./.github/actions/fuzz-test
+        env:
+          CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
+          GT_MYSQL_ADDR: 127.0.0.1:4002
+        with:
+          target: ${{ matrix.target }}
+          max-total-time: 120
+      - name: Describe Nodes
+        if: failure()
+        shell: bash
+        run: | 
+          kubectl describe nodes      
+      - name: Export kind logs
+        if: failure()
+        shell: bash
+        run: | 
+          kind export logs /tmp/kind
+      - name: Upload logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: fuzz-tests-kind-logs-${{ matrix.target }}
+          path: /tmp/kind
+          retention-days: 3
+        
 
   sqlness:
     name: Sqlness Test

.github/workflows/nightly-build.yml

@@ -66,6 +66,13 @@ env:
 
   NIGHTLY_RELEASE_PREFIX: nightly
 
+  # Use the different image name to avoid conflict with the release images.
+  # The DockerHub image will be greptime/greptimedb-nightly.
+  IMAGE_NAME: greptimedb-nightly
+
+permissions:
+  issues: write
+
 jobs:
   allocate-runners:
     name: Allocate runners
@@ -188,6 +195,7 @@ jobs:
         with:
           image-registry: docker.io
           image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-name: ${{ env.IMAGE_NAME }}
           image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
           image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
           version: ${{ needs.allocate-runners.outputs.version }}
@@ -220,7 +228,7 @@ jobs:
         with:
           src-image-registry: docker.io
           src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          src-image-name: greptimedb
+          src-image-name: ${{ env.IMAGE_NAME }}
           dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
           dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
           dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
@@ -285,7 +293,7 @@ jobs:
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
 
   notification:
-    if: ${{ always() }} # Not requiring successful dependent jobs, always run.
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && always() }} # Not requiring successful dependent jobs, always run.
     name: Send notification to Greptime team
     needs: [
       release-images-to-dockerhub
@@ -294,16 +302,25 @@ jobs:
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:
-      - name: Notifiy nightly build successful result
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-cyborg
+      - name: Report CI status
+        id: report-ci-status
+        working-directory: cyborg
+        run: pnpm tsx bin/report-ci-failure.ts
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CI_REPORT_STATUS: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result == 'success' }}
+      - name: Notify nightly build successful result
         uses: slackapi/slack-github-action@v1.23.0
         if: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result == 'success' }}
         with:
           payload: |
             {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has completed successfully."}
 
-      - name: Notifiy nightly build failed result
+      - name: Notify nightly build failed result
         uses: slackapi/slack-github-action@v1.23.0
         if: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result != 'success' }}
         with:
           payload: |
-            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'."}
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check ${{ steps.report-ci-status.outputs.html_url }}."}

.github/workflows/nightly-ci.yml

@@ -12,6 +12,9 @@ concurrency:
 env:
   RUST_TOOLCHAIN: nightly-2024-04-20
 
+permissions:
+  issues: write
+
 jobs:
   sqlness-test:
     name: Run sqlness test
@@ -22,7 +25,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-
       - name: Run sqlness test
         uses: ./.github/actions/sqlness-test
         with:
@@ -39,6 +41,7 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-cyborg
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -49,14 +52,6 @@ jobs:
         uses: Swatinem/rust-cache@v2
       - name: Run sqlness
         run: cargo sqlness
-      - name: Notify slack if failed
-        if: failure()
-        uses: slackapi/slack-github-action@v1.23.0
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
-        with:
-          payload: |
-            {"text": "Nightly CI failed for sqlness tests"}
       - name: Upload sqlness logs
         if: always()
         uses: actions/upload-artifact@v4
@@ -73,6 +68,7 @@ jobs:
     steps:
       - run: git config --global core.autocrlf false
       - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-cyborg
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -110,11 +106,53 @@ jobs:
           GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
           GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
           UNITTEST_LOG_DIR: "__unittest_logs"
-      - name: Notify slack if failed
-        if: failure()
-        uses: slackapi/slack-github-action@v1.23.0
+
+  check-status:
+    name: Check status
+    needs: [
+      sqlness-test,
+      sqlness-windows,
+      test-on-windows,
+    ]
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-20.04
+    outputs:
+      check-result: ${{ steps.set-check-result.outputs.check-result }}
+    steps:
+      - name: Set check result
+        id: set-check-result
+        run: |
+          echo "check-result=success" >> $GITHUB_OUTPUT
+
+  notification:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && always() }} # Not requiring successful dependent jobs, always run.
+    name: Send notification to Greptime team
+    needs: [
+      check-status
+    ]
+    runs-on: ubuntu-20.04
+    env:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-cyborg
+      - name: Report CI status
+        id: report-ci-status
+        working-directory: cyborg
+        run: pnpm tsx bin/report-ci-failure.ts
         env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CI_REPORT_STATUS: ${{ needs.check-status.outputs.check-result == 'success' }}
+      - name: Notify dev build successful result
+        uses: slackapi/slack-github-action@v1.23.0
+        if: ${{ needs.check-status.outputs.check-result == 'success' }}
         with:
           payload: |
-            {"text": "Nightly CI failed for cargo test"}
+            {"text": "Nightly CI has completed successfully."}
+
+      - name: Notify dev build failed result
+        uses: slackapi/slack-github-action@v1.23.0
+        if: ${{ needs.check-status.outputs.check-result != 'success' }}
+        with:
+          payload: |
+            {"text": "Nightly CI failed has failed, please check ${{ steps.report-ci-status.outputs.html_url }}."}

.github/workflows/release.yml

@@ -91,7 +91,12 @@ env:
   # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
   NIGHTLY_RELEASE_PREFIX: nightly
   # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
-  NEXT_RELEASE_VERSION: v0.8.0
+  NEXT_RELEASE_VERSION: v0.9.0
+
+# Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+permissions:
+  issues: write # Allows the action to create issues for cyborg.
+  contents: write # Allows the action to create a release.
 
 jobs:
   allocate-runners:
@@ -245,7 +250,7 @@ jobs:
       - name: Set build macos result
         id: set-build-macos-result
         run: |
-          echo "build-macos-result=success" >> $GITHUB_OUTPUT    
+          echo "build-macos-result=success" >> $GITHUB_OUTPUT
 
   build-windows-artifacts:
     name: Build Windows artifacts
@@ -318,7 +323,7 @@ jobs:
       - name: Set build image result
         id: set-build-image-result
         run: |
-          echo "build-image-result=success" >> $GITHUB_OUTPUT    
+          echo "build-image-result=success" >> $GITHUB_OUTPUT
 
   release-cn-artifacts:
     name: Release artifacts to CN region
@@ -436,7 +441,7 @@ jobs:
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
 
   notification:
-    if: ${{ always() || github.repository == 'GreptimeTeam/greptimedb' }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || github.event_name == 'schedule') && always() }}
     name: Send notification to Greptime team
     needs: [
       release-images-to-dockerhub,
@@ -447,16 +452,25 @@ jobs:
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:
-      - name: Notifiy release successful result
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-cyborg
+      - name: Report CI status
+        id: report-ci-status
+        working-directory: cyborg
+        run: pnpm tsx bin/report-ci-failure.ts
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CI_REPORT_STATUS: ${{ needs.release-images-to-dockerhub.outputs.build-image-result == 'success' && needs.build-windows-artifacts.outputs.build-windows-result == 'success' && needs.build-macos-artifacts.outputs.build-macos-result == 'success' }}
+      - name: Notify release successful result
         uses: slackapi/slack-github-action@v1.25.0
         if: ${{ needs.release-images-to-dockerhub.outputs.build-image-result == 'success' && needs.build-windows-artifacts.outputs.build-windows-result == 'success' && needs.build-macos-artifacts.outputs.build-macos-result == 'success' }}
         with:
           payload: |
             {"text": "GreptimeDB's release version has completed successfully."}
 
-      - name: Notifiy release failed result
+      - name: Notify release failed result
         uses: slackapi/slack-github-action@v1.25.0
         if: ${{ needs.release-images-to-dockerhub.outputs.build-image-result != 'success' || needs.build-windows-artifacts.outputs.build-windows-result != 'success' || needs.build-macos-artifacts.outputs.build-macos-result != 'success' }}
         with:
           payload: |
-            {"text": "GreptimeDB's release version has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/release.yml'."}
+            {"text": "GreptimeDB's release version has failed, please check ${{ steps.report-ci-status.outputs.html_url }}."}

Cargo.toml

@@ -64,13 +64,14 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.7.2"
+version = "0.8.0"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.lints]
 clippy.print_stdout = "warn"
 clippy.print_stderr = "warn"
+clippy.dbg_macro = "warn"
 clippy.implicit_clone = "warn"
 clippy.readonly_write_lock = "allow"
 rust.unknown_lints = "deny"
@@ -109,6 +110,7 @@ datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev
 datafusion-functions = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
 datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
 datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
+datafusion-physical-plan = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
 datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
 datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" }
 derive_builder = "0.12"
@@ -118,7 +120,7 @@ etcd-client = { git = "https://github.com/MichaelScofield/etcd-client.git", rev
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "a11db14b8502f55ca5348917fd18e6fcf140f55e" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "902f75fdd170c572e90b1f640161d90995f20218" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
@@ -138,6 +140,7 @@ parquet = { version = "51.0.0", default-features = false, features = ["arrow", "
 paste = "1.0"
 pin-project = "1.0"
 prometheus = { version = "0.13.3", features = ["process"] }
+promql-parser = { version = "0.4" }
 prost = "0.12"
 raft-engine = { version = "0.4.1", default-features = false }
 rand = "0.8"
@@ -159,7 +162,7 @@ smallvec = { version = "1", features = ["serde"] }
 snafu = "0.8"
 sysinfo = "0.30"
 # on branch v0.44.x
-sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "c919990bf62ad38d2b0c0a3bc90b26ad919d51b0", features = [
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "e4e496b8d62416ad50ce70a1b460c7313610cf5d", features = [
     "visitor",
 ] }
 strum = { version = "0.25", features = ["derive"] }
@@ -168,7 +171,7 @@ tokio = { version = "1.36", features = ["full"] }
 tokio-stream = { version = "0.1" }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.8.8"
-tonic = { version = "0.11", features = ["tls"] }
+tonic = { version = "0.11", features = ["tls", "gzip", "zstd"] }
 uuid = { version = "1.7", features = ["serde", "v4", "fast-rng"] }
 zstd = "0.13"
 
@@ -246,6 +249,11 @@ lto = "thin"
 debug = false
 incremental = false
 
+[profile.ci]
+inherits = "dev"
+debug = false
+strip = true
+
 [profile.dev.package.sqlness-runner]
 debug = false
 strip = true

Makefile

@@ -199,7 +199,7 @@ config-docs: ## Generate configuration documentation from toml files.
 	docker run --rm \
     -v ${PWD}:/greptimedb \
     -w /greptimedb/config \
-    toml2docs/toml2docs:latest \
+    toml2docs/toml2docs:v0.1.1 \
     -p '##' \
     -t ./config-docs-template.md \
     -o ./config.md

config/config-docs-template.md

@@ -1,10 +1,16 @@
 # Configurations
 
+- [Standalone Mode](#standalone-mode)
+- [Distributed Mode](#distributed-mode)
+    - [Frontend](#frontend)
+    - [Metasrv](#metasrv)
+    - [Datanode](#datanode)
+
 ## Standalone Mode
 
 {{ toml2docs "./standalone.example.toml" }}
 
-## Cluster Mode
+## Distributed Mode
 
 ### Frontend
 

config/config.md

@@ -1,5 +1,11 @@
 # Configurations
 
+- [Standalone Mode](#standalone-mode)
+- [Distributed Mode](#distributed-mode)
+    - [Frontend](#frontend)
+    - [Metasrv](#metasrv)
+    - [Datanode](#datanode)
+
 ## Standalone Mode
 
 | Key | Type | Default | Descriptions |
@@ -14,6 +20,11 @@
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
+| `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
+| `grpc.tls.mode` | String | `disable` | TLS mode. |
+| `grpc.tls.cert_path` | String | `None` | Certificate file path. |
+| `grpc.tls.key_path` | String | `None` | Private key file path. |
+| `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.<br/>For now, gRPC tls config does not support auto reload. |
 | `mysql` | -- | -- | MySQL server options. |
 | `mysql.enable` | Bool | `true` | Whether to enable. |
 | `mysql.addr` | String | `127.0.0.1:4002` | The addr to bind the MySQL server. |
@@ -27,7 +38,7 @@
 | `postgres.enable` | Bool | `true` | Whether to enable |
 | `postgres.addr` | String | `127.0.0.1:4003` | The addr to bind the PostgresSQL server. |
 | `postgres.runtime_size` | Integer | `2` | The number of server worker threads. |
-| `postgres.tls` | -- | -- | PostgresSQL server TLS options, see `mysql_options.tls` section. |
+| `postgres.tls` | -- | -- | PostgresSQL server TLS options, see `mysql.tls` section. |
 | `postgres.tls.mode` | String | `disable` | TLS mode. |
 | `postgres.tls.cert_path` | String | `None` | Certificate file path. |
 | `postgres.tls.key_path` | String | `None` | Private key file path. |
@@ -96,6 +107,10 @@
 | `region_engine.mito.sst_meta_cache_size` | String | `128MB` | Cache size for SST metadata. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/32 of OS memory with a max limitation of 128MB. |
 | `region_engine.mito.vector_cache_size` | String | `512MB` | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.page_cache_size` | String | `512MB` | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
+| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache. |
+| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/write_cache`. |
+| `region_engine.mito.experimental_write_cache_size` | String | `512MB` | Capacity for write cache. |
+| `region_engine.mito.experimental_write_cache_ttl` | String | `1h` | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.scan_parallelism` | Integer | `0` | Parallelism to scan a region (default: 1/4 of cpu cores).<br/>- `0`: using the default value (1/4 of cpu cores).<br/>- `1`: scan in current thread.<br/>- `n`: scan in parallelism n. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |
@@ -131,7 +146,7 @@
 | `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
 
 
-## Cluster Mode
+## Distributed Mode
 
 ### Frontend
 
@@ -149,6 +164,11 @@
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
+| `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
+| `grpc.tls.mode` | String | `disable` | TLS mode. |
+| `grpc.tls.cert_path` | String | `None` | Certificate file path. |
+| `grpc.tls.key_path` | String | `None` | Private key file path. |
+| `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.<br/>For now, gRPC tls config does not support auto reload. |
 | `mysql` | -- | -- | MySQL server options. |
 | `mysql.enable` | Bool | `true` | Whether to enable. |
 | `mysql.addr` | String | `127.0.0.1:4002` | The addr to bind the MySQL server. |
@@ -162,7 +182,7 @@
 | `postgres.enable` | Bool | `true` | Whether to enable |
 | `postgres.addr` | String | `127.0.0.1:4003` | The addr to bind the PostgresSQL server. |
 | `postgres.runtime_size` | Integer | `2` | The number of server worker threads. |
-| `postgres.tls` | -- | -- | PostgresSQL server TLS options, see `mysql_options.tls` section. |
+| `postgres.tls` | -- | -- | PostgresSQL server TLS options, see `mysql.tls` section. |
 | `postgres.tls.mode` | String | `disable` | TLS mode. |
 | `postgres.tls.cert_path` | String | `None` | Certificate file path. |
 | `postgres.tls.key_path` | String | `None` | Private key file path. |
@@ -344,6 +364,10 @@
 | `region_engine.mito.sst_meta_cache_size` | String | `128MB` | Cache size for SST metadata. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/32 of OS memory with a max limitation of 128MB. |
 | `region_engine.mito.vector_cache_size` | String | `512MB` | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.page_cache_size` | String | `512MB` | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
+| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache. |
+| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/write_cache`. |
+| `region_engine.mito.experimental_write_cache_size` | String | `512MB` | Capacity for write cache. |
+| `region_engine.mito.experimental_write_cache_ttl` | String | `1h` | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.scan_parallelism` | Integer | `0` | Parallelism to scan a region (default: 1/4 of cpu cores).<br/>- `0`: using the default value (1/4 of cpu cores).<br/>- `1`: scan in current thread.<br/>- `n`: scan in parallelism n. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |

config/datanode.example.toml

@@ -324,6 +324,18 @@ vector_cache_size = "512MB"
 ## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
 page_cache_size = "512MB"
 
+## Whether to enable the experimental write cache.
+enable_experimental_write_cache = false
+
+## File system path for write cache, defaults to `{data_home}/write_cache`.
+experimental_write_cache_path = ""
+
+## Capacity for write cache.
+experimental_write_cache_size = "512MB"
+
+## TTL for write cache.
+experimental_write_cache_ttl = "1h"
+
 ## Buffer size for SST writing.
 sst_write_buffer_size = "8MB"
 

config/frontend.example.toml

@@ -30,6 +30,23 @@ addr = "127.0.0.1:4001"
 ## The number of server worker threads.
 runtime_size = 8
 
+## gRPC server TLS options, see `mysql.tls` section.
+[grpc.tls]
+## TLS mode.
+mode = "disable"
+
+## Certificate file path.
+## +toml2docs:none-default
+cert_path = ""
+
+## Private key file path.
+## +toml2docs:none-default
+key_path = ""
+
+## Watch for Certificate and key file change and auto reload.
+## For now, gRPC tls config does not support auto reload.
+watch = false
+
 ## MySQL server options.
 [mysql]
 ## Whether to enable.
@@ -70,7 +87,7 @@ addr = "127.0.0.1:4003"
 ## The number of server worker threads.
 runtime_size = 2
 
-## PostgresSQL server TLS options, see `mysql_options.tls` section.
+## PostgresSQL server TLS options, see `mysql.tls` section.
 [postgres.tls]
 ## TLS mode.
 mode = "disable"

config/standalone.example.toml

@@ -25,6 +25,23 @@ addr = "127.0.0.1:4001"
 ## The number of server worker threads.
 runtime_size = 8
 
+## gRPC server TLS options, see `mysql.tls` section.
+[grpc.tls]
+## TLS mode.
+mode = "disable"
+
+## Certificate file path.
+## +toml2docs:none-default
+cert_path = ""
+
+## Private key file path.
+## +toml2docs:none-default
+key_path = ""
+
+## Watch for Certificate and key file change and auto reload.
+## For now, gRPC tls config does not support auto reload.
+watch = false
+
 ## MySQL server options.
 [mysql]
 ## Whether to enable.
@@ -65,7 +82,7 @@ addr = "127.0.0.1:4003"
 ## The number of server worker threads.
 runtime_size = 2
 
-## PostgresSQL server TLS options, see `mysql_options.tls` section.
+## PostgresSQL server TLS options, see `mysql.tls` section.
 [postgres.tls]
 ## TLS mode.
 mode = "disable"
@@ -367,6 +384,18 @@ vector_cache_size = "512MB"
 ## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
 page_cache_size = "512MB"
 
+## Whether to enable the experimental write cache.
+enable_experimental_write_cache = false
+
+## File system path for write cache, defaults to `{data_home}/write_cache`.
+experimental_write_cache_path = ""
+
+## Capacity for write cache.
+experimental_write_cache_size = "512MB"
+
+## TTL for write cache.
+experimental_write_cache_ttl = "1h"
+
 ## Buffer size for SST writing.
 sst_write_buffer_size = "8MB"
 

cyborg/bin/report-ci-failure.ts

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2023 Greptime Team
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as core from '@actions/core'
+import {handleError, obtainClient} from "@/common"
+import {context} from "@actions/github"
+import _ from "lodash"
+
+async function main() {
+    const success = process.env["CI_REPORT_STATUS"] === "true"
+    core.info(`CI_REPORT_STATUS=${process.env["CI_REPORT_STATUS"]}, resolved to ${success}`)
+
+    const client = obtainClient("GITHUB_TOKEN")
+    const title = `Workflow run '${context.workflow}' failed`
+    const url = `${process.env["GITHUB_SERVER_URL"]}/${process.env["GITHUB_REPOSITORY"]}/actions/runs/${process.env["GITHUB_RUN_ID"]}`
+    const failure_comment = `@GreptimeTeam/db-approver\nNew failure: ${url} `
+    const success_comment = `@GreptimeTeam/db-approver\nBack to success: ${url}`
+
+    const {owner, repo} = context.repo
+    const labels = ['O-ci-failure']
+
+    const issues = await client.paginate(client.rest.issues.listForRepo, {
+        owner,
+        repo,
+        labels: labels.join(','),
+        state: "open",
+        sort: "created",
+        direction: "desc",
+    });
+    const issue = _.find(issues, (i) => i.title === title);
+
+    if (issue) { // exist issue
+        core.info(`Found previous issue ${issue.html_url}`)
+        if (!success) {
+            await client.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number: issue.number,
+                body: failure_comment,
+            })
+        } else {
+            await client.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number: issue.number,
+                body: success_comment,
+            })
+            await client.rest.issues.update({
+                owner,
+                repo,
+                issue_number: issue.number,
+                state: "closed",
+                state_reason: "completed",
+            })
+        }
+        core.setOutput("html_url", issue.html_url)
+    } else if (!success) { // create new issue for failure
+        const issue = await client.rest.issues.create({
+            owner,
+            repo,
+            title,
+            labels,
+            body: failure_comment,
+        })
+        core.info(`Created issue ${issue.data.html_url}`)
+        core.setOutput("html_url", issue.data.html_url)
+    }
+}
+
+main().catch(handleError)

docker/ci/ubuntu/Dockerfile.fuzztests

@@ -0,0 +1,16 @@
+FROM ubuntu:22.04
+
+# The binary name of GreptimeDB executable.
+# Defaults to "greptime", but sometimes in other projects it might be different.
+ARG TARGET_BIN=greptime
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    ca-certificates \
+    curl
+
+ARG BINARY_PATH
+ADD $BINARY_PATH/$TARGET_BIN /greptime/bin/
+
+ENV PATH /greptime/bin/:$PATH
+
+ENTRYPOINT ["greptime"]

docker/dev-builder/android/Dockerfile

@@ -34,7 +34,7 @@ RUN rustup toolchain install ${RUST_TOOLCHAIN}
 RUN rustup target add aarch64-linux-android
 
 # Install cargo-ndk
-RUN cargo install cargo-ndk
+RUN cargo install cargo-ndk@3.5.4
 ENV ANDROID_NDK_HOME $NDK_ROOT
 
 # Builder entrypoint.

docs/benchmarks/tsbs/v0.8.0.md

@@ -0,0 +1,58 @@
+# TSBS benchmark - v0.8.0
+
+## Environment
+
+### Local
+
+|        |                                    |
+| ------ | ---------------------------------- |
+| CPU    | AMD Ryzen 7 7735HS (8 core 3.2GHz) |
+| Memory | 32GB                               |
+| Disk   | SOLIDIGM SSDPFKNU010TZ             |
+| OS     | Ubuntu 22.04.2 LTS                 |
+
+### Amazon EC2
+
+|         |                |
+| ------- | -------------- |
+| Machine | c5d.2xlarge    |
+| CPU     | 8 core         |
+| Memory  | 16GB           |
+| Disk    | 50GB (GP3)     |
+| OS      | Ubuntu 22.04.1 |
+
+## Write performance
+
+| Environment     | Ingest rate (rows/s) |
+| --------------- | -------------------- |
+| Local           | 315369.66            |
+| EC2 c5d.2xlarge | 222148.56            |
+
+## Query performance
+
+| Query type            | Local (ms) | EC2 c5d.2xlarge (ms) |
+| --------------------- | ---------- | -------------------- |
+| cpu-max-all-1         | 24.63      | 15.29                |
+| cpu-max-all-8         | 51.69      | 33.53                |
+| double-groupby-1      | 673.51     | 1295.38              |
+| double-groupby-5      | 1244.93    | 1993.91              |
+| double-groupby-all    | 2215.44    | 3056.77              |
+| groupby-orderby-limit | 754.50     | 1546.49              |
+| high-cpu-1            | 19.62      | 11.58                |
+| high-cpu-all          | 5402.31    | 8011.43              |
+| lastpoint             | 6756.12    | 9312.67              |
+| single-groupby-1-1-1  | 15.70      | 7.67                 |
+| single-groupby-1-1-12 | 16.72      | 9.29                 |
+| single-groupby-1-8-1  | 26.72      | 17.97                |
+| single-groupby-5-1-1  | 18.17      | 10.09                |
+| single-groupby-5-1-12 | 20.04      | 12.37                |
+| single-groupby-5-8-1  | 35.63      | 23.13                |
+
+`single-groupby-1-1-1` query throughput
+
+| Environment     | Client concurrency | mean time (ms) | qps (queries/sec) |
+| --------------- | ------------------ | -------------- | ----------------- |
+| Local           | 50                 | 42.87          | 1165.73           |
+| Local           | 100                | 89.29          | 1119.38           |
+| EC2 c5d.2xlarge | 50                 | 69.25          | 721.73            |
+| EC2 c5d.2xlarge | 100                | 140.93         | 709.35            |

src/catalog/src/information_schema/predicate.rs

@@ -14,10 +14,9 @@
 
 use arrow::array::StringArray;
 use arrow::compute::kernels::comparison;
-use common_query::logical_plan::DfExpr;
 use datafusion::common::ScalarValue;
 use datafusion::logical_expr::expr::Like;
-use datafusion::logical_expr::Operator;
+use datafusion::logical_expr::{Expr, Operator};
 use datatypes::value::Value;
 use store_api::storage::ScanRequest;
 
@@ -118,12 +117,12 @@ impl Predicate {
     }
 
     /// Try to create a predicate from datafusion [`Expr`], return None if fails.
-    fn from_expr(expr: DfExpr) -> Option<Predicate> {
+    fn from_expr(expr: Expr) -> Option<Predicate> {
         match expr {
             // NOT expr
-            DfExpr::Not(expr) => Some(Predicate::Not(Box::new(Self::from_expr(*expr)?))),
+            Expr::Not(expr) => Some(Predicate::Not(Box::new(Self::from_expr(*expr)?))),
             // expr LIKE pattern
-            DfExpr::Like(Like {
+            Expr::Like(Like {
                 negated,
                 expr,
                 pattern,
@@ -131,10 +130,10 @@ impl Predicate {
                 ..
             }) if is_column(&expr) && is_string_literal(&pattern) => {
                 // Safety: ensured by gurad
-                let DfExpr::Column(c) = *expr else {
+                let Expr::Column(c) = *expr else {
                     unreachable!();
                 };
-                let DfExpr::Literal(ScalarValue::Utf8(Some(pattern))) = *pattern else {
+                let Expr::Literal(ScalarValue::Utf8(Some(pattern))) = *pattern else {
                     unreachable!();
                 };
 
@@ -147,10 +146,10 @@ impl Predicate {
                 }
             }
             // left OP right
-            DfExpr::BinaryExpr(bin) => match (*bin.left, bin.op, *bin.right) {
+            Expr::BinaryExpr(bin) => match (*bin.left, bin.op, *bin.right) {
                 // left == right
-                (DfExpr::Literal(scalar), Operator::Eq, DfExpr::Column(c))
-                | (DfExpr::Column(c), Operator::Eq, DfExpr::Literal(scalar)) => {
+                (Expr::Literal(scalar), Operator::Eq, Expr::Column(c))
+                | (Expr::Column(c), Operator::Eq, Expr::Literal(scalar)) => {
                     let Ok(v) = Value::try_from(scalar) else {
                         return None;
                     };
@@ -158,8 +157,8 @@ impl Predicate {
                     Some(Predicate::Eq(c.name, v))
                 }
                 // left != right
-                (DfExpr::Literal(scalar), Operator::NotEq, DfExpr::Column(c))
-                | (DfExpr::Column(c), Operator::NotEq, DfExpr::Literal(scalar)) => {
+                (Expr::Literal(scalar), Operator::NotEq, Expr::Column(c))
+                | (Expr::Column(c), Operator::NotEq, Expr::Literal(scalar)) => {
                     let Ok(v) = Value::try_from(scalar) else {
                         return None;
                     };
@@ -183,14 +182,14 @@ impl Predicate {
                 _ => None,
             },
             // [NOT] IN (LIST)
-            DfExpr::InList(list) => {
+            Expr::InList(list) => {
                 match (*list.expr, list.list, list.negated) {
                     // column [NOT] IN (v1, v2, v3, ...)
-                    (DfExpr::Column(c), list, negated) if is_all_scalars(&list) => {
+                    (Expr::Column(c), list, negated) if is_all_scalars(&list) => {
                         let mut values = Vec::with_capacity(list.len());
                         for scalar in list {
                             // Safety: checked by `is_all_scalars`
-                            let DfExpr::Literal(scalar) = scalar else {
+                            let Expr::Literal(scalar) = scalar else {
                                 unreachable!();
                             };
 
@@ -237,12 +236,12 @@ fn like_utf8(s: &str, pattern: &str, case_insensitive: &bool) -> Option<bool> {
     Some(booleans.value(0))
 }
 
-fn is_string_literal(expr: &DfExpr) -> bool {
-    matches!(expr, DfExpr::Literal(ScalarValue::Utf8(Some(_))))
+fn is_string_literal(expr: &Expr) -> bool {
+    matches!(expr, Expr::Literal(ScalarValue::Utf8(Some(_))))
 }
 
-fn is_column(expr: &DfExpr) -> bool {
-    matches!(expr, DfExpr::Column(_))
+fn is_column(expr: &Expr) -> bool {
+    matches!(expr, Expr::Column(_))
 }
 
 /// A list of predicate
@@ -257,7 +256,7 @@ impl Predicates {
             let mut predicates = Vec::with_capacity(request.filters.len());
 
             for filter in &request.filters {
-                if let Some(predicate) = Predicate::from_expr(filter.df_expr().clone()) {
+                if let Some(predicate) = Predicate::from_expr(filter.clone()) {
                     predicates.push(predicate);
                 }
             }
@@ -286,8 +285,8 @@ impl Predicates {
 }
 
 /// Returns true when the values are all [`DfExpr::Literal`].
-fn is_all_scalars(list: &[DfExpr]) -> bool {
-    list.iter().all(|v| matches!(v, DfExpr::Literal(_)))
+fn is_all_scalars(list: &[Expr]) -> bool {
+    list.iter().all(|v| matches!(v, Expr::Literal(_)))
 }
 
 #[cfg(test)]
@@ -376,7 +375,7 @@ mod tests {
     #[test]
     fn test_predicate_like() {
         // case insensitive
-        let expr = DfExpr::Like(Like {
+        let expr = Expr::Like(Like {
             negated: false,
             expr: Box::new(column("a")),
             pattern: Box::new(string_literal("%abc")),
@@ -403,7 +402,7 @@ mod tests {
         assert!(p.eval(&[]).is_none());
 
         // case sensitive
-        let expr = DfExpr::Like(Like {
+        let expr = Expr::Like(Like {
             negated: false,
             expr: Box::new(column("a")),
             pattern: Box::new(string_literal("%abc")),
@@ -423,7 +422,7 @@ mod tests {
         assert!(p.eval(&[]).is_none());
 
         // not like
-        let expr = DfExpr::Like(Like {
+        let expr = Expr::Like(Like {
             negated: true,
             expr: Box::new(column("a")),
             pattern: Box::new(string_literal("%abc")),
@@ -437,15 +436,15 @@ mod tests {
         assert!(p.eval(&[]).is_none());
     }
 
-    fn column(name: &str) -> DfExpr {
-        DfExpr::Column(Column {
+    fn column(name: &str) -> Expr {
+        Expr::Column(Column {
             relation: None,
             name: name.to_string(),
         })
     }
 
-    fn string_literal(v: &str) -> DfExpr {
-        DfExpr::Literal(ScalarValue::Utf8(Some(v.to_string())))
+    fn string_literal(v: &str) -> Expr {
+        Expr::Literal(ScalarValue::Utf8(Some(v.to_string())))
     }
 
     fn match_string_value(v: &Value, expected: &str) -> bool {
@@ -463,14 +462,14 @@ mod tests {
         result
     }
 
-    fn mock_exprs() -> (DfExpr, DfExpr) {
-        let expr1 = DfExpr::BinaryExpr(BinaryExpr {
+    fn mock_exprs() -> (Expr, Expr) {
+        let expr1 = Expr::BinaryExpr(BinaryExpr {
             left: Box::new(column("a")),
             op: Operator::Eq,
             right: Box::new(string_literal("a_value")),
         });
 
-        let expr2 = DfExpr::BinaryExpr(BinaryExpr {
+        let expr2 = Expr::BinaryExpr(BinaryExpr {
             left: Box::new(column("b")),
             op: Operator::NotEq,
             right: Box::new(string_literal("b_value")),
@@ -491,17 +490,17 @@ mod tests {
         assert!(matches!(&p2, Predicate::NotEq(column, v) if column == "b"
                          && match_string_value(v, "b_value")));
 
-        let and_expr = DfExpr::BinaryExpr(BinaryExpr {
+        let and_expr = Expr::BinaryExpr(BinaryExpr {
             left: Box::new(expr1.clone()),
             op: Operator::And,
             right: Box::new(expr2.clone()),
         });
-        let or_expr = DfExpr::BinaryExpr(BinaryExpr {
+        let or_expr = Expr::BinaryExpr(BinaryExpr {
             left: Box::new(expr1.clone()),
             op: Operator::Or,
             right: Box::new(expr2.clone()),
         });
-        let not_expr = DfExpr::Not(Box::new(expr1.clone()));
+        let not_expr = Expr::Not(Box::new(expr1.clone()));
 
         let and_p = Predicate::from_expr(and_expr).unwrap();
         assert!(matches!(and_p, Predicate::And(left, right) if *left == p1 && *right == p2));
@@ -510,7 +509,7 @@ mod tests {
         let not_p = Predicate::from_expr(not_expr).unwrap();
         assert!(matches!(not_p, Predicate::Not(p) if *p == p1));
 
-        let inlist_expr = DfExpr::InList(InList {
+        let inlist_expr = Expr::InList(InList {
             expr: Box::new(column("a")),
             list: vec![string_literal("a1"), string_literal("a2")],
             negated: false,
@@ -520,7 +519,7 @@ mod tests {
         assert!(matches!(&inlist_p, Predicate::InList(c, values) if c == "a"
                          && match_string_values(values, &["a1", "a2"])));
 
-        let inlist_expr = DfExpr::InList(InList {
+        let inlist_expr = Expr::InList(InList {
             expr: Box::new(column("a")),
             list: vec![string_literal("a1"), string_literal("a2")],
             negated: true,
@@ -540,7 +539,7 @@ mod tests {
         let (expr1, expr2) = mock_exprs();
 
         let request = ScanRequest {
-            filters: vec![expr1.into(), expr2.into()],
+            filters: vec![expr1, expr2],
             ..Default::default()
         };
         let predicates = Predicates::from_scan_request(&Some(request));
@@ -578,7 +577,7 @@ mod tests {
 
         let (expr1, expr2) = mock_exprs();
         let request = ScanRequest {
-            filters: vec![expr1.into(), expr2.into()],
+            filters: vec![expr1, expr2],
             ..Default::default()
         };
         let predicates = Predicates::from_scan_request(&Some(request));

src/client/src/client.rs

@@ -19,9 +19,10 @@ use api::v1::prometheus_gateway_client::PrometheusGatewayClient;
 use api::v1::region::region_client::RegionClient as PbRegionClient;
 use api::v1::HealthCheckRequest;
 use arrow_flight::flight_service_client::FlightServiceClient;
-use common_grpc::channel_manager::ChannelManager;
+use common_grpc::channel_manager::{ChannelConfig, ChannelManager, ClientTlsOption};
 use parking_lot::RwLock;
 use snafu::{OptionExt, ResultExt};
+use tonic::codec::CompressionEncoding;
 use tonic::transport::Channel;
 
 use crate::load_balance::{LoadBalance, Loadbalancer};
@@ -86,6 +87,17 @@ impl Client {
         Self::with_manager_and_urls(ChannelManager::new(), urls)
     }
 
+    pub fn with_tls_and_urls<U, A>(urls: A, client_tls: ClientTlsOption) -> Result<Self>
+    where
+        U: AsRef<str>,
+        A: AsRef<[U]>,
+    {
+        let channel_config = ChannelConfig::default().client_tls_config(client_tls);
+        let channel_manager = ChannelManager::with_tls_config(channel_config)
+            .context(error::CreateTlsChannelSnafu)?;
+        Ok(Self::with_manager_and_urls(channel_manager, urls))
+    }
+
     pub fn with_manager_and_urls<U, A>(channel_manager: ChannelManager, urls: A) -> Self
     where
         U: AsRef<str>,
@@ -151,24 +163,34 @@ impl Client {
 
     pub fn make_flight_client(&self) -> Result<FlightClient> {
         let (addr, channel) = self.find_channel()?;
-        Ok(FlightClient {
-            addr,
-            client: FlightServiceClient::new(channel)
-                .max_decoding_message_size(self.max_grpc_recv_message_size())
-                .max_encoding_message_size(self.max_grpc_send_message_size()),
-        })
+
+        let client = FlightServiceClient::new(channel)
+            .max_decoding_message_size(self.max_grpc_recv_message_size())
+            .max_encoding_message_size(self.max_grpc_send_message_size())
+            .accept_compressed(CompressionEncoding::Zstd)
+            .send_compressed(CompressionEncoding::Zstd);
+
+        Ok(FlightClient { addr, client })
     }
 
     pub(crate) fn raw_region_client(&self) -> Result<PbRegionClient<Channel>> {
         let (_, channel) = self.find_channel()?;
-        Ok(PbRegionClient::new(channel)
+        let client = PbRegionClient::new(channel)
             .max_decoding_message_size(self.max_grpc_recv_message_size())
-            .max_encoding_message_size(self.max_grpc_send_message_size()))
+            .max_encoding_message_size(self.max_grpc_send_message_size())
+            .accept_compressed(CompressionEncoding::Zstd)
+            .send_compressed(CompressionEncoding::Zstd);
+        Ok(client)
     }
 
     pub fn make_prometheus_gateway_client(&self) -> Result<PrometheusGatewayClient<Channel>> {
         let (_, channel) = self.find_channel()?;
-        Ok(PrometheusGatewayClient::new(channel))
+        let client = PrometheusGatewayClient::new(channel)
+            .accept_compressed(CompressionEncoding::Gzip)
+            .accept_compressed(CompressionEncoding::Zstd)
+            .send_compressed(CompressionEncoding::Gzip)
+            .send_compressed(CompressionEncoding::Zstd);
+        Ok(client)
     }
 
     pub async fn health_check(&self) -> Result<()> {

src/client/src/error.rs

@@ -18,7 +18,7 @@ use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_error::{GREPTIME_DB_HEADER_ERROR_CODE, GREPTIME_DB_HEADER_ERROR_MSG};
 use common_macro::stack_trace_debug;
-use snafu::{Location, Snafu};
+use snafu::{location, Location, Snafu};
 use tonic::{Code, Status};
 
 #[derive(Snafu)]
@@ -82,15 +82,36 @@ pub enum Error {
         source: common_grpc::error::Error,
     },
 
+    #[snafu(display("Failed to create Tls channel manager"))]
+    CreateTlsChannel {
+        #[snafu(implicit)]
+        location: Location,
+        source: common_grpc::error::Error,
+    },
+
     #[snafu(display("Failed to request RegionServer, code: {}", code))]
-    RegionServer { code: Code, source: BoxedError },
+    RegionServer {
+        code: Code,
+        source: BoxedError,
+        #[snafu(implicit)]
+        location: Location,
+    },
 
     // Server error carried in Tonic Status's metadata.
     #[snafu(display("{}", msg))]
-    Server { code: StatusCode, msg: String },
+    Server {
+        code: StatusCode,
+        msg: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 
     #[snafu(display("Illegal Database response: {err_msg}"))]
-    IllegalDatabaseResponse { err_msg: String },
+    IllegalDatabaseResponse {
+        err_msg: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 
     #[snafu(display("Failed to send request with streaming: {}", err_msg))]
     ClientStreaming {
@@ -115,9 +136,9 @@ impl ErrorExt for Error {
             Error::FlightGet { source, .. }
             | Error::HandleRequest { source, .. }
             | Error::RegionServer { source, .. } => source.status_code(),
-            Error::CreateChannel { source, .. } | Error::ConvertFlightData { source, .. } => {
-                source.status_code()
-            }
+            Error::CreateChannel { source, .. }
+            | Error::ConvertFlightData { source, .. }
+            | Error::CreateTlsChannel { source, .. } => source.status_code(),
             Error::IllegalGrpcClientState { .. } => StatusCode::Unexpected,
         }
     }
@@ -148,7 +169,11 @@ impl From<Status> for Error {
         let msg = get_metadata_value(&e, GREPTIME_DB_HEADER_ERROR_MSG)
             .unwrap_or_else(|| e.message().to_string());
 
-        Self::Server { code, msg }
+        Self::Server {
+            code,
+            msg,
+            location: location!(),
+        }
     }
 }
 

src/client/src/region.rs

@@ -189,6 +189,7 @@ impl RegionRequester {
                 error::Error::RegionServer {
                     code,
                     source: BoxedError::new(err),
+                    location: location!(),
                 }
             })?
             .into_inner();
@@ -272,7 +273,7 @@ mod test {
                 err_msg: "blabla".to_string(),
             }),
         }));
-        let Server { code, msg } = result.unwrap_err() else {
+        let Server { code, msg, .. } = result.unwrap_err() else {
             unreachable!()
         };
         assert_eq!(code, StatusCode::Internal);

src/cmd/Cargo.toml

@@ -74,6 +74,7 @@ substrait.workspace = true
 table.workspace = true
 tokio.workspace = true
 toml.workspace = true
+tracing-appender = "0.2"
 
 [target.'cfg(not(windows))'.dependencies]
 tikv-jemallocator = "0.5"

src/cmd/src/bin/greptime.rs

@@ -14,13 +14,11 @@
 
 #![doc = include_str!("../../../../README.md")]
 
-use std::fmt;
-
 use clap::{Parser, Subcommand};
 use cmd::error::Result;
-use cmd::options::{GlobalOptions, Options};
-use cmd::{cli, datanode, frontend, log_versions, metasrv, standalone, start_app, App};
-use common_version::{short_version, version};
+use cmd::options::GlobalOptions;
+use cmd::{cli, datanode, frontend, metasrv, standalone, App};
+use common_version::version;
 
 #[derive(Parser)]
 #[command(name = "greptime", author, version, long_version = version!(), about)]
@@ -56,58 +54,6 @@ enum SubCommand {
     Cli(cli::Command),
 }
 
-impl SubCommand {
-    async fn build(self, opts: Options) -> Result<Box<dyn App>> {
-        let app: Box<dyn App> = match (self, opts) {
-            (SubCommand::Datanode(cmd), Options::Datanode(dn_opts)) => {
-                let app = cmd.build(*dn_opts).await?;
-                Box::new(app) as _
-            }
-            (SubCommand::Frontend(cmd), Options::Frontend(fe_opts)) => {
-                let app = cmd.build(*fe_opts).await?;
-                Box::new(app) as _
-            }
-            (SubCommand::Metasrv(cmd), Options::Metasrv(meta_opts)) => {
-                let app = cmd.build(*meta_opts).await?;
-                Box::new(app) as _
-            }
-            (SubCommand::Standalone(cmd), Options::Standalone(opts)) => {
-                let app = cmd.build(*opts).await?;
-                Box::new(app) as _
-            }
-            (SubCommand::Cli(cmd), Options::Cli(_)) => {
-                let app = cmd.build().await?;
-                Box::new(app) as _
-            }
-
-            _ => unreachable!(),
-        };
-        Ok(app)
-    }
-
-    fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
-        match self {
-            SubCommand::Datanode(cmd) => cmd.load_options(global_options),
-            SubCommand::Frontend(cmd) => cmd.load_options(global_options),
-            SubCommand::Metasrv(cmd) => cmd.load_options(global_options),
-            SubCommand::Standalone(cmd) => cmd.load_options(global_options),
-            SubCommand::Cli(cmd) => cmd.load_options(global_options),
-        }
-    }
-}
-
-impl fmt::Display for SubCommand {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            SubCommand::Datanode(..) => write!(f, "greptime-datanode"),
-            SubCommand::Frontend(..) => write!(f, "greptime-frontend"),
-            SubCommand::Metasrv(..) => write!(f, "greptime-metasrv"),
-            SubCommand::Standalone(..) => write!(f, "greptime-standalone"),
-            SubCommand::Cli(_) => write!(f, "greptime-cli"),
-        }
-    }
-}
-
 #[cfg(not(windows))]
 #[global_allocator]
 static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
@@ -119,24 +65,38 @@ async fn main() -> Result<()> {
 }
 
 async fn start(cli: Command) -> Result<()> {
-    let subcmd = cli.subcmd;
-
-    let app_name = subcmd.to_string();
-
-    let opts = subcmd.load_options(&cli.global_options)?;
-
-    let _guard = common_telemetry::init_global_logging(
-        &app_name,
-        opts.logging_options(),
-        &cli.global_options.tracing_options(),
-        opts.node_id(),
-    );
-
-    log_versions(version!(), short_version!());
-
-    let app = subcmd.build(opts).await?;
-
-    start_app(app).await
+    match cli.subcmd {
+        SubCommand::Datanode(cmd) => {
+            cmd.build(cmd.load_options(&cli.global_options)?)
+                .await?
+                .run()
+                .await
+        }
+        SubCommand::Frontend(cmd) => {
+            cmd.build(cmd.load_options(&cli.global_options)?)
+                .await?
+                .run()
+                .await
+        }
+        SubCommand::Metasrv(cmd) => {
+            cmd.build(cmd.load_options(&cli.global_options)?)
+                .await?
+                .run()
+                .await
+        }
+        SubCommand::Standalone(cmd) => {
+            cmd.build(cmd.load_options(&cli.global_options)?)
+                .await?
+                .run()
+                .await
+        }
+        SubCommand::Cli(cmd) => {
+            cmd.build(cmd.load_options(&cli.global_options)?)
+                .await?
+                .run()
+                .await
+        }
+    }
 }
 
 fn setup_human_panic() {

src/cmd/src/cli.rs

@@ -30,15 +30,18 @@ mod upgrade;
 use async_trait::async_trait;
 use bench::BenchTableMetadataCommand;
 use clap::Parser;
-use common_telemetry::logging::LoggingOptions;
+use common_telemetry::logging::{LoggingOptions, TracingOptions};
+use tracing_appender::non_blocking::WorkerGuard;
 // pub use repl::Repl;
 use upgrade::UpgradeCommand;
 
 use self::export::ExportCommand;
 use crate::error::Result;
-use crate::options::{GlobalOptions, Options};
+use crate::options::GlobalOptions;
 use crate::App;
 
+pub const APP_NAME: &str = "greptime-cli";
+
 #[async_trait]
 pub trait Tool: Send + Sync {
     async fn do_work(&self) -> Result<()>;
@@ -46,18 +49,24 @@ pub trait Tool: Send + Sync {
 
 pub struct Instance {
     tool: Box<dyn Tool>,
+
+    // Keep the logging guard to prevent the worker from being dropped.
+    _guard: Vec<WorkerGuard>,
 }
 
 impl Instance {
-    fn new(tool: Box<dyn Tool>) -> Self {
-        Self { tool }
+    fn new(tool: Box<dyn Tool>, guard: Vec<WorkerGuard>) -> Self {
+        Self {
+            tool,
+            _guard: guard,
+        }
     }
 }
 
 #[async_trait]
 impl App for Instance {
     fn name(&self) -> &str {
-        "greptime-cli"
+        APP_NAME
     }
 
     async fn start(&mut self) -> Result<()> {
@@ -80,11 +89,18 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self) -> Result<Instance> {
-        self.cmd.build().await
+    pub async fn build(&self, opts: LoggingOptions) -> Result<Instance> {
+        let guard = common_telemetry::init_global_logging(
+            APP_NAME,
+            &opts,
+            &TracingOptions::default(),
+            None,
+        );
+
+        self.cmd.build(guard).await
     }
 
-    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
+    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<LoggingOptions> {
         let mut logging_opts = LoggingOptions::default();
 
         if let Some(dir) = &global_options.log_dir {
@@ -93,7 +109,7 @@ impl Command {
 
         logging_opts.level.clone_from(&global_options.log_level);
 
-        Ok(Options::Cli(Box::new(logging_opts)))
+        Ok(logging_opts)
     }
 }
 
@@ -106,12 +122,12 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self) -> Result<Instance> {
+    async fn build(&self, guard: Vec<WorkerGuard>) -> Result<Instance> {
         match self {
             // SubCommand::Attach(cmd) => cmd.build().await,
-            SubCommand::Upgrade(cmd) => cmd.build().await,
-            SubCommand::Bench(cmd) => cmd.build().await,
-            SubCommand::Export(cmd) => cmd.build().await,
+            SubCommand::Upgrade(cmd) => cmd.build(guard).await,
+            SubCommand::Bench(cmd) => cmd.build(guard).await,
+            SubCommand::Export(cmd) => cmd.build(guard).await,
         }
     }
 }

src/cmd/src/cli/bench.rs

@@ -30,6 +30,7 @@ use datatypes::schema::{ColumnSchema, RawSchema};
 use rand::Rng;
 use store_api::storage::RegionNumber;
 use table::metadata::{RawTableInfo, RawTableMeta, TableId, TableIdent, TableType};
+use tracing_appender::non_blocking::WorkerGuard;
 
 use self::metadata::TableMetadataBencher;
 use crate::cli::{Instance, Tool};
@@ -61,7 +62,7 @@ pub struct BenchTableMetadataCommand {
 }
 
 impl BenchTableMetadataCommand {
-    pub async fn build(&self) -> Result<Instance> {
+    pub async fn build(&self, guard: Vec<WorkerGuard>) -> Result<Instance> {
         let etcd_store = EtcdStore::with_endpoints([&self.etcd_addr], 128)
             .await
             .unwrap();
@@ -72,7 +73,7 @@ impl BenchTableMetadataCommand {
             table_metadata_manager,
             count: self.count,
         };
-        Ok(Instance::new(Box::new(tool)))
+        Ok(Instance::new(Box::new(tool), guard))
     }
 }
 

src/cmd/src/cli/export.rs

@@ -30,6 +30,7 @@ use tokio::fs::File;
 use tokio::io::{AsyncWriteExt, BufWriter};
 use tokio::sync::Semaphore;
 use tokio::time::Instant;
+use tracing_appender::non_blocking::WorkerGuard;
 
 use crate::cli::{Instance, Tool};
 use crate::error::{
@@ -80,7 +81,7 @@ pub struct ExportCommand {
 }
 
 impl ExportCommand {
-    pub async fn build(&self) -> Result<Instance> {
+    pub async fn build(&self, guard: Vec<WorkerGuard>) -> Result<Instance> {
         let (catalog, schema) = split_database(&self.database)?;
 
         let auth_header = if let Some(basic) = &self.auth_basic {
@@ -90,15 +91,18 @@ impl ExportCommand {
             None
         };
 
-        Ok(Instance::new(Box::new(Export {
-            addr: self.addr.clone(),
-            catalog,
-            schema,
-            output_dir: self.output_dir.clone(),
-            parallelism: self.export_jobs,
-            target: self.target.clone(),
-            auth_header,
-        })))
+        Ok(Instance::new(
+            Box::new(Export {
+                addr: self.addr.clone(),
+                catalog,
+                schema,
+                output_dir: self.output_dir.clone(),
+                parallelism: self.export_jobs,
+                target: self.target.clone(),
+                auth_header,
+            }),
+            guard,
+        ))
     }
 }
 
@@ -176,8 +180,12 @@ impl Export {
     }
 
     /// Return a list of [`TableReference`] to be exported.
-    /// Includes all tables under the given `catalog` and `schema`
-    async fn get_table_list(&self, catalog: &str, schema: &str) -> Result<Vec<TableReference>> {
+    /// Includes all tables under the given `catalog` and `schema`.
+    async fn get_table_list(
+        &self,
+        catalog: &str,
+        schema: &str,
+    ) -> Result<(Vec<TableReference>, Vec<TableReference>)> {
         // Puts all metric table first
         let sql = format!(
             "select table_catalog, table_schema, table_name from \
@@ -214,7 +222,7 @@ impl Export {
         debug!("Fetched table list: {:?}", records);
 
         if records.is_empty() {
-            return Ok(vec![]);
+            return Ok((vec![], vec![]));
         }
 
         let mut remaining_tables = Vec::with_capacity(records.len());
@@ -232,11 +240,11 @@ impl Export {
                 remaining_tables.push(table);
             }
         }
-        let mut tables = Vec::with_capacity(metric_physical_tables.len() + remaining_tables.len());
-        tables.extend(metric_physical_tables.into_iter());
-        tables.extend(remaining_tables);
 
-        Ok(tables)
+        Ok((
+            metric_physical_tables.into_iter().collect(),
+            remaining_tables,
+        ))
     }
 
     async fn show_create_table(&self, catalog: &str, schema: &str, table: &str) -> Result<String> {
@@ -265,15 +273,16 @@ impl Export {
             let semaphore_moved = semaphore.clone();
             tasks.push(async move {
                 let _permit = semaphore_moved.acquire().await.unwrap();
-                let table_list = self.get_table_list(&catalog, &schema).await?;
-                let table_count = table_list.len();
+                let (metric_physical_tables, remaining_tables) =
+                    self.get_table_list(&catalog, &schema).await?;
+                let table_count = metric_physical_tables.len() + remaining_tables.len();
                 tokio::fs::create_dir_all(&self.output_dir)
                     .await
                     .context(FileIoSnafu)?;
                 let output_file =
                     Path::new(&self.output_dir).join(format!("{catalog}-{schema}.sql"));
                 let mut file = File::create(output_file).await.context(FileIoSnafu)?;
-                for (c, s, t) in table_list {
+                for (c, s, t) in metric_physical_tables.into_iter().chain(remaining_tables) {
                     match self.show_create_table(&c, &s, &t).await {
                         Err(e) => {
                             error!(e; r#"Failed to export table "{}"."{}"."{}""#, c, s, t)
@@ -322,15 +331,25 @@ impl Export {
                     .await
                     .context(FileIoSnafu)?;
                 let output_dir = Path::new(&self.output_dir).join(format!("{catalog}-{schema}/"));
-
-                // copy database to
-                let sql = format!(
-                    "copy database {} to '{}' with (format='parquet');",
-                    schema,
-                    output_dir.to_str().unwrap()
-                );
-                self.sql(&sql).await?;
-                info!("finished exporting {catalog}.{schema} data");
+                // Ignores metric physical tables
+                let (metrics_tables, table_list) = self.get_table_list(&catalog, &schema).await?;
+                for (_, _, table_name) in metrics_tables {
+                    warn!("Ignores metric physical table: {table_name}");
+                }
+                for (catalog_name, schema_name, table_name) in table_list {
+                    // copy table to
+                    let sql = format!(
+                        r#"Copy "{}"."{}"."{}" TO '{}{}.parquet' WITH (format='parquet');"#,
+                        catalog_name,
+                        schema_name,
+                        table_name,
+                        output_dir.to_str().unwrap(),
+                        table_name,
+                    );
+                    info!("Executing sql: {sql}");
+                    self.sql(&sql).await?;
+                }
+                info!("Finished exporting {catalog}.{schema} data");
 
                 // export copy from sql
                 let dir_filenames = match output_dir.read_dir() {

src/cmd/src/cli/upgrade.rs

@@ -40,6 +40,7 @@ use etcd_client::Client;
 use futures::TryStreamExt;
 use prost::Message;
 use snafu::ResultExt;
+use tracing_appender::non_blocking::WorkerGuard;
 use v1_helper::{CatalogKey as v1CatalogKey, SchemaKey as v1SchemaKey, TableGlobalValue};
 
 use crate::cli::{Instance, Tool};
@@ -63,7 +64,7 @@ pub struct UpgradeCommand {
 }
 
 impl UpgradeCommand {
-    pub async fn build(&self) -> Result<Instance> {
+    pub async fn build(&self, guard: Vec<WorkerGuard>) -> Result<Instance> {
         let client = Client::connect([&self.etcd_addr], None)
             .await
             .context(ConnectEtcdSnafu {
@@ -77,7 +78,7 @@ impl UpgradeCommand {
             skip_schema_keys: self.skip_schema_keys,
             skip_table_route_keys: self.skip_table_route_keys,
         };
-        Ok(Instance::new(Box::new(tool)))
+        Ok(Instance::new(Box::new(tool), guard))
     }
 }
 

src/cmd/src/datanode.rs

@@ -21,6 +21,7 @@ use clap::Parser;
 use common_config::Configurable;
 use common_telemetry::info;
 use common_telemetry::logging::TracingOptions;
+use common_version::{short_version, version};
 use common_wal::config::DatanodeWalConfig;
 use datanode::config::DatanodeOptions;
 use datanode::datanode::{Datanode, DatanodeBuilder};
@@ -28,20 +29,29 @@ use datanode::service::DatanodeServiceBuilder;
 use meta_client::MetaClientOptions;
 use servers::Mode;
 use snafu::{OptionExt, ResultExt};
+use tracing_appender::non_blocking::WorkerGuard;
 
 use crate::error::{
     LoadLayeredConfigSnafu, MissingConfigSnafu, Result, ShutdownDatanodeSnafu, StartDatanodeSnafu,
 };
-use crate::options::{GlobalOptions, Options};
-use crate::App;
+use crate::options::GlobalOptions;
+use crate::{log_versions, App};
+
+pub const APP_NAME: &str = "greptime-datanode";
 
 pub struct Instance {
     datanode: Datanode,
+
+    // Keep the logging guard to prevent the worker from being dropped.
+    _guard: Vec<WorkerGuard>,
 }
 
 impl Instance {
-    pub fn new(datanode: Datanode) -> Self {
-        Self { datanode }
+    pub fn new(datanode: Datanode, guard: Vec<WorkerGuard>) -> Self {
+        Self {
+            datanode,
+            _guard: guard,
+        }
     }
 
     pub fn datanode_mut(&mut self) -> &mut Datanode {
@@ -56,7 +66,7 @@ impl Instance {
 #[async_trait]
 impl App for Instance {
     fn name(&self) -> &str {
-        "greptime-datanode"
+        APP_NAME
     }
 
     async fn start(&mut self) -> Result<()> {
@@ -82,11 +92,11 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self, opts: DatanodeOptions) -> Result<Instance> {
+    pub async fn build(&self, opts: DatanodeOptions) -> Result<Instance> {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
+    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<DatanodeOptions> {
         self.subcmd.load_options(global_options)
     }
 }
@@ -97,13 +107,13 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self, opts: DatanodeOptions) -> Result<Instance> {
+    async fn build(&self, opts: DatanodeOptions) -> Result<Instance> {
         match self {
             SubCommand::Start(cmd) => cmd.build(opts).await,
         }
     }
 
-    fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<DatanodeOptions> {
         match self {
             SubCommand::Start(cmd) => cmd.load_options(global_options),
         }
@@ -135,17 +145,15 @@ struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
-        Ok(Options::Datanode(Box::new(
-            self.merge_with_cli_options(
-                global_options,
-                DatanodeOptions::load_layered_options(
-                    self.config_file.as_deref(),
-                    self.env_prefix.as_ref(),
-                )
-                .context(LoadLayeredConfigSnafu)?,
-            )?,
-        )))
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<DatanodeOptions> {
+        self.merge_with_cli_options(
+            global_options,
+            DatanodeOptions::load_layered_options(
+                self.config_file.as_deref(),
+                self.env_prefix.as_ref(),
+            )
+            .context(LoadLayeredConfigSnafu)?,
+        )
     }
 
     // The precedence order is: cli > config file > environment variables > default values.
@@ -226,7 +234,15 @@ impl StartCommand {
         Ok(opts)
     }
 
-    async fn build(self, mut opts: DatanodeOptions) -> Result<Instance> {
+    async fn build(&self, mut opts: DatanodeOptions) -> Result<Instance> {
+        let guard = common_telemetry::init_global_logging(
+            APP_NAME,
+            &opts.logging,
+            &opts.tracing,
+            opts.node_id.map(|x| x.to_string()),
+        );
+        log_versions(version!(), short_version!());
+
         let plugins = plugins::setup_datanode_plugins(&mut opts)
             .await
             .context(StartDatanodeSnafu)?;
@@ -265,7 +281,7 @@ impl StartCommand {
             .context(StartDatanodeSnafu)?;
         datanode.setup_services(services);
 
-        Ok(Instance::new(datanode))
+        Ok(Instance::new(datanode, guard))
     }
 }
 
@@ -337,10 +353,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Datanode(options) = cmd.load_options(&GlobalOptions::default()).unwrap()
-        else {
-            unreachable!()
-        };
+        let options = cmd.load_options(&GlobalOptions::default()).unwrap();
 
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
         assert_eq!(Some(42), options.node_id);
@@ -399,23 +412,19 @@ mod tests {
 
     #[test]
     fn test_try_from_cmd() {
-        if let Options::Datanode(opt) = StartCommand::default()
+        let opt = StartCommand::default()
             .load_options(&GlobalOptions::default())
-            .unwrap()
-        {
-            assert_eq!(Mode::Standalone, opt.mode)
-        }
+            .unwrap();
+        assert_eq!(Mode::Standalone, opt.mode);
 
-        if let Options::Datanode(opt) = (StartCommand {
+        let opt = (StartCommand {
             node_id: Some(42),
             metasrv_addrs: Some(vec!["127.0.0.1:3002".to_string()]),
             ..Default::default()
         })
         .load_options(&GlobalOptions::default())
-        .unwrap()
-        {
-            assert_eq!(Mode::Distributed, opt.mode)
-        }
+        .unwrap();
+        assert_eq!(Mode::Distributed, opt.mode);
 
         assert!((StartCommand {
             metasrv_addrs: Some(vec!["127.0.0.1:3002".to_string()]),
@@ -447,7 +456,7 @@ mod tests {
             })
             .unwrap();
 
-        let logging_opt = options.logging_options();
+        let logging_opt = options.logging;
         assert_eq!("/tmp/greptimedb/test/logs", logging_opt.dir);
         assert_eq!("debug", logging_opt.level.as_ref().unwrap());
     }
@@ -527,11 +536,7 @@ mod tests {
                     ..Default::default()
                 };
 
-                let Options::Datanode(opts) =
-                    command.load_options(&GlobalOptions::default()).unwrap()
-                else {
-                    unreachable!()
-                };
+                let opts = command.load_options(&GlobalOptions::default()).unwrap();
 
                 // Should be read from env, env > default values.
                 let DatanodeWalConfig::RaftEngine(raft_engine_config) = opts.wal else {

src/cmd/src/frontend.rs

@@ -31,6 +31,7 @@ use common_meta::heartbeat::handler::HandlerGroupExecutor;
 use common_telemetry::info;
 use common_telemetry::logging::TracingOptions;
 use common_time::timezone::set_default_timezone;
+use common_version::{short_version, version};
 use frontend::frontend::FrontendOptions;
 use frontend::heartbeat::handler::invalidate_table_cache::InvalidateTableCacheHandler;
 use frontend::heartbeat::HeartbeatTask;
@@ -41,20 +42,29 @@ use meta_client::MetaClientOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
 use snafu::{OptionExt, ResultExt};
+use tracing_appender::non_blocking::WorkerGuard;
 
 use crate::error::{
     self, InitTimezoneSnafu, LoadLayeredConfigSnafu, MissingConfigSnafu, Result, StartFrontendSnafu,
 };
-use crate::options::{GlobalOptions, Options};
-use crate::App;
+use crate::options::GlobalOptions;
+use crate::{log_versions, App};
 
 pub struct Instance {
     frontend: FeInstance,
+
+    // Keep the logging guard to prevent the worker from being dropped.
+    _guard: Vec<WorkerGuard>,
 }
 
+pub const APP_NAME: &str = "greptime-frontend";
+
 impl Instance {
-    pub fn new(frontend: FeInstance) -> Self {
-        Self { frontend }
+    pub fn new(frontend: FeInstance, guard: Vec<WorkerGuard>) -> Self {
+        Self {
+            frontend,
+            _guard: guard,
+        }
     }
 
     pub fn mut_inner(&mut self) -> &mut FeInstance {
@@ -69,7 +79,7 @@ impl Instance {
 #[async_trait]
 impl App for Instance {
     fn name(&self) -> &str {
-        "greptime-frontend"
+        APP_NAME
     }
 
     async fn start(&mut self) -> Result<()> {
@@ -95,11 +105,11 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self, opts: FrontendOptions) -> Result<Instance> {
+    pub async fn build(&self, opts: FrontendOptions) -> Result<Instance> {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
+    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<FrontendOptions> {
         self.subcmd.load_options(global_options)
     }
 }
@@ -110,13 +120,13 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self, opts: FrontendOptions) -> Result<Instance> {
+    async fn build(&self, opts: FrontendOptions) -> Result<Instance> {
         match self {
             SubCommand::Start(cmd) => cmd.build(opts).await,
         }
     }
 
-    fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<FrontendOptions> {
         match self {
             SubCommand::Start(cmd) => cmd.load_options(global_options),
         }
@@ -156,17 +166,15 @@ pub struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
-        Ok(Options::Frontend(Box::new(
-            self.merge_with_cli_options(
-                global_options,
-                FrontendOptions::load_layered_options(
-                    self.config_file.as_deref(),
-                    self.env_prefix.as_ref(),
-                )
-                .context(LoadLayeredConfigSnafu)?,
-            )?,
-        )))
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<FrontendOptions> {
+        self.merge_with_cli_options(
+            global_options,
+            FrontendOptions::load_layered_options(
+                self.config_file.as_deref(),
+                self.env_prefix.as_ref(),
+            )
+            .context(LoadLayeredConfigSnafu)?,
+        )
     }
 
     // The precedence order is: cli > config file > environment variables > default values.
@@ -208,6 +216,7 @@ impl StartCommand {
 
         if let Some(addr) = &self.rpc_addr {
             opts.grpc.addr.clone_from(addr);
+            opts.grpc.tls = tls_opts.clone();
         }
 
         if let Some(addr) = &self.mysql_addr {
@@ -239,7 +248,15 @@ impl StartCommand {
         Ok(opts)
     }
 
-    async fn build(self, mut opts: FrontendOptions) -> Result<Instance> {
+    async fn build(&self, mut opts: FrontendOptions) -> Result<Instance> {
+        let guard = common_telemetry::init_global_logging(
+            APP_NAME,
+            &opts.logging,
+            &opts.tracing,
+            opts.node_id.clone(),
+        );
+        log_versions(version!(), short_version!());
+
         #[allow(clippy::unnecessary_mut_passed)]
         let plugins = plugins::setup_frontend_plugins(&mut opts)
             .await
@@ -349,7 +366,7 @@ impl StartCommand {
             .build_servers(opts, servers)
             .context(StartFrontendSnafu)?;
 
-        Ok(Instance::new(instance))
+        Ok(Instance::new(instance, guard))
     }
 }
 
@@ -379,10 +396,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Frontend(opts) = command.load_options(&GlobalOptions::default()).unwrap()
-        else {
-            unreachable!()
-        };
+        let opts = command.load_options(&GlobalOptions::default()).unwrap();
 
         assert_eq!(opts.http.addr, "127.0.0.1:1234");
         assert_eq!(ReadableSize::mb(64), opts.http.body_limit);
@@ -430,10 +444,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Frontend(fe_opts) = command.load_options(&GlobalOptions::default()).unwrap()
-        else {
-            unreachable!()
-        };
+        let fe_opts = command.load_options(&GlobalOptions::default()).unwrap();
         assert_eq!(Mode::Distributed, fe_opts.mode);
         assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
         assert_eq!(Duration::from_secs(30), fe_opts.http.timeout);
@@ -486,7 +497,7 @@ mod tests {
             })
             .unwrap();
 
-        let logging_opt = options.logging_options();
+        let logging_opt = options.logging;
         assert_eq!("/tmp/greptimedb/test/logs", logging_opt.dir);
         assert_eq!("debug", logging_opt.level.as_ref().unwrap());
     }
@@ -562,11 +573,7 @@ mod tests {
                     ..Default::default()
                 };
 
-                let Options::Frontend(fe_opts) =
-                    command.load_options(&GlobalOptions::default()).unwrap()
-                else {
-                    unreachable!()
-                };
+                let fe_opts = command.load_options(&GlobalOptions::default()).unwrap();
 
                 // Should be read from env, env > default values.
                 assert_eq!(fe_opts.mysql.runtime_size, 11);

src/cmd/src/lib.rs

@@ -17,6 +17,8 @@
 use async_trait::async_trait;
 use common_telemetry::{error, info};
 
+use crate::error::Result;
+
 pub mod cli;
 pub mod datanode;
 pub mod error;
@@ -35,39 +37,39 @@ pub trait App: Send {
     fn name(&self) -> &str;
 
     /// A hook for implementor to make something happened before actual startup. Defaults to no-op.
-    async fn pre_start(&mut self) -> error::Result<()> {
+    async fn pre_start(&mut self) -> Result<()> {
         Ok(())
     }
 
-    async fn start(&mut self) -> error::Result<()>;
+    async fn start(&mut self) -> Result<()>;
 
     /// Waits the quit signal by default.
     fn wait_signal(&self) -> bool {
         true
     }
 
-    async fn stop(&self) -> error::Result<()>;
-}
+    async fn stop(&self) -> Result<()>;
 
-pub async fn start_app(mut app: Box<dyn App>) -> error::Result<()> {
-    info!("Starting app: {}", app.name());
+    async fn run(&mut self) -> Result<()> {
+        info!("Starting app: {}", self.name());
 
-    app.pre_start().await?;
+        self.pre_start().await?;
 
-    app.start().await?;
+        self.start().await?;
 
-    if app.wait_signal() {
-        if let Err(e) = tokio::signal::ctrl_c().await {
-            error!("Failed to listen for ctrl-c signal: {}", e);
-            // It's unusual to fail to listen for ctrl-c signal, maybe there's something unexpected in
-            // the underlying system. So we stop the app instead of running nonetheless to let people
-            // investigate the issue.
+        if self.wait_signal() {
+            if let Err(e) = tokio::signal::ctrl_c().await {
+                error!(e; "Failed to listen for ctrl-c signal");
+                // It's unusual to fail to listen for ctrl-c signal, maybe there's something unexpected in
+                // the underlying system. So we stop the app instead of running nonetheless to let people
+                // investigate the issue.
+            }
         }
-    }
 
-    app.stop().await?;
-    info!("Goodbye!");
-    Ok(())
+        self.stop().await?;
+        info!("Goodbye!");
+        Ok(())
+    }
 }
 
 /// Log the versions of the application, and the arguments passed to the cli.

src/cmd/src/metasrv.rs

@@ -19,28 +19,38 @@ use clap::Parser;
 use common_config::Configurable;
 use common_telemetry::info;
 use common_telemetry::logging::TracingOptions;
+use common_version::{short_version, version};
 use meta_srv::bootstrap::MetasrvInstance;
 use meta_srv::metasrv::MetasrvOptions;
 use snafu::ResultExt;
+use tracing_appender::non_blocking::WorkerGuard;
 
 use crate::error::{self, LoadLayeredConfigSnafu, Result, StartMetaServerSnafu};
-use crate::options::{GlobalOptions, Options};
-use crate::App;
+use crate::options::GlobalOptions;
+use crate::{log_versions, App};
+
+pub const APP_NAME: &str = "greptime-metasrv";
 
 pub struct Instance {
     instance: MetasrvInstance,
+
+    // Keep the logging guard to prevent the worker from being dropped.
+    _guard: Vec<WorkerGuard>,
 }
 
 impl Instance {
-    fn new(instance: MetasrvInstance) -> Self {
-        Self { instance }
+    fn new(instance: MetasrvInstance, guard: Vec<WorkerGuard>) -> Self {
+        Self {
+            instance,
+            _guard: guard,
+        }
     }
 }
 
 #[async_trait]
 impl App for Instance {
     fn name(&self) -> &str {
-        "greptime-metasrv"
+        APP_NAME
     }
 
     async fn start(&mut self) -> Result<()> {
@@ -66,11 +76,11 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self, opts: MetasrvOptions) -> Result<Instance> {
+    pub async fn build(&self, opts: MetasrvOptions) -> Result<Instance> {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
+    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<MetasrvOptions> {
         self.subcmd.load_options(global_options)
     }
 }
@@ -81,13 +91,13 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self, opts: MetasrvOptions) -> Result<Instance> {
+    async fn build(&self, opts: MetasrvOptions) -> Result<Instance> {
         match self {
             SubCommand::Start(cmd) => cmd.build(opts).await,
         }
     }
 
-    fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<MetasrvOptions> {
         match self {
             SubCommand::Start(cmd) => cmd.load_options(global_options),
         }
@@ -128,17 +138,15 @@ struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
-        Ok(Options::Metasrv(Box::new(
-            self.merge_with_cli_options(
-                global_options,
-                MetasrvOptions::load_layered_options(
-                    self.config_file.as_deref(),
-                    self.env_prefix.as_ref(),
-                )
-                .context(LoadLayeredConfigSnafu)?,
-            )?,
-        )))
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<MetasrvOptions> {
+        self.merge_with_cli_options(
+            global_options,
+            MetasrvOptions::load_layered_options(
+                self.config_file.as_deref(),
+                self.env_prefix.as_ref(),
+            )
+            .context(LoadLayeredConfigSnafu)?,
+        )
     }
 
     // The precedence order is: cli > config file > environment variables > default values.
@@ -212,7 +220,11 @@ impl StartCommand {
         Ok(opts)
     }
 
-    async fn build(self, mut opts: MetasrvOptions) -> Result<Instance> {
+    async fn build(&self, mut opts: MetasrvOptions) -> Result<Instance> {
+        let guard =
+            common_telemetry::init_global_logging(APP_NAME, &opts.logging, &opts.tracing, None);
+        log_versions(version!(), short_version!());
+
         let plugins = plugins::setup_metasrv_plugins(&mut opts)
             .await
             .context(StartMetaServerSnafu)?;
@@ -229,7 +241,7 @@ impl StartCommand {
             .await
             .context(error::BuildMetaServerSnafu)?;
 
-        Ok(Instance::new(instance))
+        Ok(Instance::new(instance, guard))
     }
 }
 
@@ -254,9 +266,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Metasrv(options) = cmd.load_options(&GlobalOptions::default()).unwrap() else {
-            unreachable!()
-        };
+        let options = cmd.load_options(&GlobalOptions::default()).unwrap();
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
         assert_eq!(vec!["127.0.0.1:2380".to_string()], options.store_addrs);
         assert_eq!(SelectorType::LoadBased, options.selector);
@@ -289,9 +299,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Metasrv(options) = cmd.load_options(&GlobalOptions::default()).unwrap() else {
-            unreachable!()
-        };
+        let options = cmd.load_options(&GlobalOptions::default()).unwrap();
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
         assert_eq!("127.0.0.1:3002".to_string(), options.server_addr);
         assert_eq!(vec!["127.0.0.1:2379".to_string()], options.store_addrs);
@@ -343,7 +351,7 @@ mod tests {
             })
             .unwrap();
 
-        let logging_opt = options.logging_options();
+        let logging_opt = options.logging;
         assert_eq!("/tmp/greptimedb/test/logs", logging_opt.dir);
         assert_eq!("debug", logging_opt.level.as_ref().unwrap());
     }
@@ -398,11 +406,7 @@ mod tests {
                     ..Default::default()
                 };
 
-                let Options::Metasrv(opts) =
-                    command.load_options(&GlobalOptions::default()).unwrap()
-                else {
-                    unreachable!()
-                };
+                let opts = command.load_options(&GlobalOptions::default()).unwrap();
 
                 // Should be read from env, env > default values.
                 assert_eq!(opts.bind_addr, "127.0.0.1:14002");

src/cmd/src/options.rs

@@ -13,20 +13,6 @@
 // limitations under the License.
 
 use clap::Parser;
-use common_telemetry::logging::{LoggingOptions, TracingOptions};
-use datanode::config::DatanodeOptions;
-use frontend::frontend::FrontendOptions;
-use meta_srv::metasrv::MetasrvOptions;
-
-use crate::standalone::StandaloneOptions;
-
-pub enum Options {
-    Datanode(Box<DatanodeOptions>),
-    Frontend(Box<FrontendOptions>),
-    Metasrv(Box<MetasrvOptions>),
-    Standalone(Box<StandaloneOptions>),
-    Cli(Box<LoggingOptions>),
-}
 
 #[derive(Parser, Default, Debug, Clone)]
 pub struct GlobalOptions {
@@ -43,32 +29,3 @@ pub struct GlobalOptions {
     #[arg(global = true)]
     pub tokio_console_addr: Option<String>,
 }
-
-impl GlobalOptions {
-    pub fn tracing_options(&self) -> TracingOptions {
-        TracingOptions {
-            #[cfg(feature = "tokio-console")]
-            tokio_console_addr: self.tokio_console_addr.clone(),
-        }
-    }
-}
-
-impl Options {
-    pub fn logging_options(&self) -> &LoggingOptions {
-        match self {
-            Options::Datanode(opts) => &opts.logging,
-            Options::Frontend(opts) => &opts.logging,
-            Options::Metasrv(opts) => &opts.logging,
-            Options::Standalone(opts) => &opts.logging,
-            Options::Cli(opts) => opts,
-        }
-    }
-
-    pub fn node_id(&self) -> Option<String> {
-        match self {
-            Options::Metasrv(_) | Options::Cli(_) | Options::Standalone(_) => None,
-            Options::Datanode(opt) => opt.node_id.map(|x| x.to_string()),
-            Options::Frontend(opt) => opt.node_id.clone(),
-        }
-    }
-}

src/cmd/src/standalone.rs

@@ -41,6 +41,7 @@ use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::{LoggingOptions, TracingOptions};
 use common_time::timezone::set_default_timezone;
+use common_version::{short_version, version};
 use common_wal::config::StandaloneWalConfig;
 use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
 use datanode::datanode::{Datanode, DatanodeBuilder};
@@ -61,6 +62,7 @@ use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
 use snafu::{OptionExt, ResultExt};
+use tracing_appender::non_blocking::WorkerGuard;
 
 use crate::error::{
     BuildCacheRegistrySnafu, CacheRequiredSnafu, CreateDirSnafu, IllegalConfigSnafu,
@@ -68,8 +70,10 @@ use crate::error::{
     ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu,
     StartProcedureManagerSnafu, StartWalOptionsAllocatorSnafu, StopProcedureManagerSnafu,
 };
-use crate::options::{GlobalOptions, Options};
-use crate::App;
+use crate::options::GlobalOptions;
+use crate::{log_versions, App};
+
+pub const APP_NAME: &str = "greptime-standalone";
 
 #[derive(Parser)]
 pub struct Command {
@@ -78,11 +82,11 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self, opts: StandaloneOptions) -> Result<Instance> {
+    pub async fn build(&self, opts: StandaloneOptions) -> Result<Instance> {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
+    pub fn load_options(&self, global_options: &GlobalOptions) -> Result<StandaloneOptions> {
         self.subcmd.load_options(global_options)
     }
 }
@@ -93,13 +97,13 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self, opts: StandaloneOptions) -> Result<Instance> {
+    async fn build(&self, opts: StandaloneOptions) -> Result<Instance> {
         match self {
             SubCommand::Start(cmd) => cmd.build(opts).await,
         }
     }
 
-    fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<StandaloneOptions> {
         match self {
             SubCommand::Start(cmd) => cmd.load_options(global_options),
         }
@@ -207,12 +211,15 @@ pub struct Instance {
     frontend: FeInstance,
     procedure_manager: ProcedureManagerRef,
     wal_options_allocator: WalOptionsAllocatorRef,
+
+    // Keep the logging guard to prevent the worker from being dropped.
+    _guard: Vec<WorkerGuard>,
 }
 
 #[async_trait]
 impl App for Instance {
     fn name(&self) -> &str {
-        "greptime-standalone"
+        APP_NAME
     }
 
     async fn start(&mut self) -> Result<()> {
@@ -287,21 +294,19 @@ pub struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
-        Ok(Options::Standalone(Box::new(
-            self.merge_with_cli_options(
-                global_options,
-                StandaloneOptions::load_layered_options(
-                    self.config_file.as_deref(),
-                    self.env_prefix.as_ref(),
-                )
-                .context(LoadLayeredConfigSnafu)?,
-            )?,
-        )))
+    fn load_options(&self, global_options: &GlobalOptions) -> Result<StandaloneOptions> {
+        self.merge_with_cli_options(
+            global_options,
+            StandaloneOptions::load_layered_options(
+                self.config_file.as_deref(),
+                self.env_prefix.as_ref(),
+            )
+            .context(LoadLayeredConfigSnafu)?,
+        )
     }
 
     // The precedence order is: cli > config file > environment variables > default values.
-    fn merge_with_cli_options(
+    pub fn merge_with_cli_options(
         &self,
         global_options: &GlobalOptions,
         mut opts: StandaloneOptions,
@@ -373,7 +378,11 @@ impl StartCommand {
     #[allow(unreachable_code)]
     #[allow(unused_variables)]
     #[allow(clippy::diverging_sub_expression)]
-    async fn build(self, opts: StandaloneOptions) -> Result<Instance> {
+    async fn build(&self, opts: StandaloneOptions) -> Result<Instance> {
+        let guard =
+            common_telemetry::init_global_logging(APP_NAME, &opts.logging, &opts.tracing, None);
+        log_versions(version!(), short_version!());
+
         info!("Standalone start command: {:#?}", self);
         info!("Building standalone instance with {opts:#?}");
 
@@ -516,6 +525,7 @@ impl StartCommand {
             frontend,
             procedure_manager,
             wal_options_allocator,
+            _guard: guard,
         })
     }
 
@@ -665,10 +675,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Standalone(options) = cmd.load_options(&GlobalOptions::default()).unwrap()
-        else {
-            unreachable!()
-        };
+        let options = cmd.load_options(&GlobalOptions::default()).unwrap();
         let fe_opts = options.frontend_options();
         let dn_opts = options.datanode_options();
         let logging_opts = options.logging;
@@ -721,7 +728,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Standalone(opts) = cmd
+        let opts = cmd
             .load_options(&GlobalOptions {
                 log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
                 log_level: Some("debug".to_string()),
@@ -729,10 +736,7 @@ mod tests {
                 #[cfg(feature = "tokio-console")]
                 tokio_console_addr: None,
             })
-            .unwrap()
-        else {
-            unreachable!()
-        };
+            .unwrap();
 
         assert_eq!("/tmp/greptimedb/test/logs", opts.logging.dir);
         assert_eq!("debug", opts.logging.level.unwrap());
@@ -794,11 +798,7 @@ mod tests {
                     ..Default::default()
                 };
 
-                let Options::Standalone(opts) =
-                    command.load_options(&GlobalOptions::default()).unwrap()
-                else {
-                    unreachable!()
-                };
+                let opts = command.load_options(&GlobalOptions::default()).unwrap();
 
                 // Should be read from env, env > default values.
                 assert_eq!(opts.logging.dir, "/other/log/dir");

src/common/function/src/table/flush_compact_table.rs

@@ -13,7 +13,9 @@
 // limitations under the License.
 
 use std::fmt;
+use std::str::FromStr;
 
+use api::v1::region::{compact_request, StrictWindow};
 use common_error::ext::BoxedError;
 use common_macro::admin_fn;
 use common_query::error::Error::ThreadJoin;
@@ -22,7 +24,7 @@ use common_query::error::{
     UnsupportedInputDataTypeSnafu,
 };
 use common_query::prelude::{Signature, Volatility};
-use common_telemetry::error;
+use common_telemetry::{error, info};
 use datatypes::prelude::*;
 use datatypes::vectors::VectorRef;
 use session::context::QueryContextRef;
@@ -34,71 +36,78 @@ use crate::ensure_greptime;
 use crate::function::{Function, FunctionContext};
 use crate::handlers::TableMutationHandlerRef;
 
-macro_rules! define_table_function {
-    ($name: expr, $display_name_str: expr, $display_name: ident, $func: ident, $request: ident) => {
-        /// A function to $func table, such as `$display_name(table_name)`.
-        #[admin_fn(name = $name, display_name = $display_name_str, sig_fn = "signature", ret = "uint64")]
-        pub(crate) async fn $display_name(
-            table_mutation_handler: &TableMutationHandlerRef,
-            query_ctx: &QueryContextRef,
-            params: &[ValueRef<'_>],
-        ) -> Result<Value> {
-            ensure!(
-                params.len() == 1,
-                InvalidFuncArgsSnafu {
-                    err_msg: format!(
-                        "The length of the args is not correct, expect 1, have: {}",
-                        params.len()
-                    ),
-                }
-            );
+/// Compact type: strict window.
+const COMPACT_TYPE_STRICT_WINDOW: &str = "strict_window";
 
-            let ValueRef::String(table_name) = params[0] else {
-                return UnsupportedInputDataTypeSnafu {
-                    function: $display_name_str,
-                    datatypes: params.iter().map(|v| v.data_type()).collect::<Vec<_>>(),
-                }
-                .fail();
-            };
-
-            let (catalog_name, schema_name, table_name) =
-                table_name_to_full_name(table_name, &query_ctx)
-                    .map_err(BoxedError::new)
-                    .context(TableMutationSnafu)?;
-
-            let affected_rows = table_mutation_handler
-                .$func(
-                    $request {
-                        catalog_name,
-                        schema_name,
-                        table_name,
-                    },
-                    query_ctx.clone(),
-                )
-                .await?;
-
-            Ok(Value::from(affected_rows as u64))
+#[admin_fn(
+    name = "FlushTableFunction",
+    display_name = "flush_table",
+    sig_fn = "flush_signature",
+    ret = "uint64"
+)]
+pub(crate) async fn flush_table(
+    table_mutation_handler: &TableMutationHandlerRef,
+    query_ctx: &QueryContextRef,
+    params: &[ValueRef<'_>],
+) -> Result<Value> {
+    ensure!(
+        params.len() == 1,
+        InvalidFuncArgsSnafu {
+            err_msg: format!(
+                "The length of the args is not correct, expect 1, have: {}",
+                params.len()
+            ),
         }
+    );
+
+    let ValueRef::String(table_name) = params[0] else {
+        return UnsupportedInputDataTypeSnafu {
+            function: "flush_table",
+            datatypes: params.iter().map(|v| v.data_type()).collect::<Vec<_>>(),
+        }
+        .fail();
     };
+
+    let (catalog_name, schema_name, table_name) = table_name_to_full_name(table_name, query_ctx)
+        .map_err(BoxedError::new)
+        .context(TableMutationSnafu)?;
+
+    let affected_rows = table_mutation_handler
+        .flush(
+            FlushTableRequest {
+                catalog_name,
+                schema_name,
+                table_name,
+            },
+            query_ctx.clone(),
+        )
+        .await?;
+
+    Ok(Value::from(affected_rows as u64))
 }
 
-define_table_function!(
-    "FlushTableFunction",
-    "flush_table",
-    flush_table,
-    flush,
-    FlushTableRequest
-);
+#[admin_fn(
+    name = "CompactTableFunction",
+    display_name = "compact_table",
+    sig_fn = "compact_signature",
+    ret = "uint64"
+)]
+pub(crate) async fn compact_table(
+    table_mutation_handler: &TableMutationHandlerRef,
+    query_ctx: &QueryContextRef,
+    params: &[ValueRef<'_>],
+) -> Result<Value> {
+    let request = parse_compact_params(params, query_ctx)?;
+    info!("Compact table request: {:?}", request);
 
-define_table_function!(
-    "CompactTableFunction",
-    "compact_table",
-    compact_table,
-    compact,
-    CompactTableRequest
-);
+    let affected_rows = table_mutation_handler
+        .compact(request, query_ctx.clone())
+        .await?;
 
-fn signature() -> Signature {
+    Ok(Value::from(affected_rows as u64))
+}
+
+fn flush_signature() -> Signature {
     Signature::uniform(
         1,
         vec![ConcreteDataType::string_datatype()],
@@ -106,12 +115,98 @@ fn signature() -> Signature {
     )
 }
 
+fn compact_signature() -> Signature {
+    Signature::variadic(
+        vec![ConcreteDataType::string_datatype()],
+        Volatility::Immutable,
+    )
+}
+
+/// Parses `compact_table` UDF parameters. This function accepts following combinations:
+/// - `[<table_name>]`: only tables name provided, using default compaction type: regular
+/// - `[<table_name>, <type>]`: specify table name and compaction type. The compaction options will be default.
+/// - `[<table_name>, <type>, <options>]`: provides both type and type-specific options.
+fn parse_compact_params(
+    params: &[ValueRef<'_>],
+    query_ctx: &QueryContextRef,
+) -> Result<CompactTableRequest> {
+    ensure!(
+        !params.is_empty(),
+        InvalidFuncArgsSnafu {
+            err_msg: "Args cannot be empty",
+        }
+    );
+
+    let (table_name, compact_type) = match params {
+        [ValueRef::String(table_name)] => (
+            table_name,
+            compact_request::Options::Regular(Default::default()),
+        ),
+        [ValueRef::String(table_name), ValueRef::String(compact_ty_str)] => {
+            let compact_type = parse_compact_type(compact_ty_str, None)?;
+            (table_name, compact_type)
+        }
+
+        [ValueRef::String(table_name), ValueRef::String(compact_ty_str), ValueRef::String(options_str)] =>
+        {
+            let compact_type = parse_compact_type(compact_ty_str, Some(options_str))?;
+            (table_name, compact_type)
+        }
+        _ => {
+            return UnsupportedInputDataTypeSnafu {
+                function: "compact_table",
+                datatypes: params.iter().map(|v| v.data_type()).collect::<Vec<_>>(),
+            }
+            .fail()
+        }
+    };
+
+    let (catalog_name, schema_name, table_name) = table_name_to_full_name(table_name, query_ctx)
+        .map_err(BoxedError::new)
+        .context(TableMutationSnafu)?;
+
+    Ok(CompactTableRequest {
+        catalog_name,
+        schema_name,
+        table_name,
+        compact_options: compact_type,
+    })
+}
+
+fn parse_compact_type(type_str: &str, option: Option<&str>) -> Result<compact_request::Options> {
+    if type_str.eq_ignore_ascii_case(COMPACT_TYPE_STRICT_WINDOW) {
+        let window_seconds = option
+            .map(|v| {
+                i64::from_str(v).map_err(|_| {
+                    InvalidFuncArgsSnafu {
+                        err_msg: format!(
+                            "Compact window is expected to be a valid number, provided: {}",
+                            v
+                        ),
+                    }
+                    .build()
+                })
+            })
+            .transpose()?
+            .unwrap_or(0);
+
+        Ok(compact_request::Options::StrictWindow(StrictWindow {
+            window_seconds,
+        }))
+    } else {
+        Ok(compact_request::Options::Regular(Default::default()))
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;
 
+    use api::v1::region::compact_request::Options;
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
     use common_query::prelude::TypeSignature;
     use datatypes::vectors::{StringVector, UInt64Vector};
+    use session::context::QueryContext;
 
     use super::*;
 
@@ -174,5 +269,109 @@ mod tests {
 
     define_table_function_test!(flush_table, FlushTableFunction);
 
-    define_table_function_test!(compact_table, CompactTableFunction);
+    fn check_parse_compact_params(cases: &[(&[&str], CompactTableRequest)]) {
+        for (params, expected) in cases {
+            let params = params
+                .iter()
+                .map(|s| ValueRef::String(s))
+                .collect::<Vec<_>>();
+
+            assert_eq!(
+                expected,
+                &parse_compact_params(&params, &QueryContext::arc()).unwrap()
+            );
+        }
+    }
+
+    #[test]
+    fn test_parse_compact_params() {
+        check_parse_compact_params(&[
+            (
+                &["table"],
+                CompactTableRequest {
+                    catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+                    schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "table".to_string(),
+                    compact_options: Options::Regular(Default::default()),
+                },
+            ),
+            (
+                &[&format!("{}.table", DEFAULT_SCHEMA_NAME)],
+                CompactTableRequest {
+                    catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+                    schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "table".to_string(),
+                    compact_options: Options::Regular(Default::default()),
+                },
+            ),
+            (
+                &[&format!(
+                    "{}.{}.table",
+                    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME
+                )],
+                CompactTableRequest {
+                    catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+                    schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "table".to_string(),
+                    compact_options: Options::Regular(Default::default()),
+                },
+            ),
+            (
+                &["table", "regular"],
+                CompactTableRequest {
+                    catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+                    schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "table".to_string(),
+                    compact_options: Options::Regular(Default::default()),
+                },
+            ),
+            (
+                &["table", "strict_window"],
+                CompactTableRequest {
+                    catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+                    schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "table".to_string(),
+                    compact_options: Options::StrictWindow(StrictWindow { window_seconds: 0 }),
+                },
+            ),
+            (
+                &["table", "strict_window", "3600"],
+                CompactTableRequest {
+                    catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+                    schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "table".to_string(),
+                    compact_options: Options::StrictWindow(StrictWindow {
+                        window_seconds: 3600,
+                    }),
+                },
+            ),
+            (
+                &["table", "regular", "abcd"],
+                CompactTableRequest {
+                    catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+                    schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "table".to_string(),
+                    compact_options: Options::Regular(Default::default()),
+                },
+            ),
+        ]);
+
+        assert!(parse_compact_params(
+            &["table", "strict_window", "abc"]
+                .into_iter()
+                .map(ValueRef::String)
+                .collect::<Vec<_>>(),
+            &QueryContext::arc(),
+        )
+        .is_err());
+
+        assert!(parse_compact_params(
+            &["a.b.table", "strict_window", "abc"]
+                .into_iter()
+                .map(ValueRef::String)
+                .collect::<Vec<_>>(),
+            &QueryContext::arc(),
+        )
+        .is_err());
+    }
 }

src/common/grpc/tests/mod.rs

@@ -23,9 +23,9 @@ async fn test_mtls_config() {
 
     // test wrong file
     let config = ChannelConfig::new().client_tls_config(ClientTlsOption {
-        server_ca_cert_path: "tests/tls/wrong_server.cert.pem".to_string(),
-        client_cert_path: "tests/tls/wrong_client.cert.pem".to_string(),
-        client_key_path: "tests/tls/wrong_client.key.pem".to_string(),
+        server_ca_cert_path: "tests/tls/wrong_ca.pem".to_string(),
+        client_cert_path: "tests/tls/wrong_client.pem".to_string(),
+        client_key_path: "tests/tls/wrong_client.key".to_string(),
     });
 
     let re = ChannelManager::with_tls_config(config);
@@ -33,8 +33,8 @@ async fn test_mtls_config() {
 
     // test corrupted file content
     let config = ChannelConfig::new().client_tls_config(ClientTlsOption {
-        server_ca_cert_path: "tests/tls/server.cert.pem".to_string(),
-        client_cert_path: "tests/tls/client.cert.pem".to_string(),
+        server_ca_cert_path: "tests/tls/ca.pem".to_string(),
+        client_cert_path: "tests/tls/client.pem".to_string(),
         client_key_path: "tests/tls/corrupted".to_string(),
     });
 
@@ -44,9 +44,9 @@ async fn test_mtls_config() {
 
     // success
     let config = ChannelConfig::new().client_tls_config(ClientTlsOption {
-        server_ca_cert_path: "tests/tls/server.cert.pem".to_string(),
-        client_cert_path: "tests/tls/client.cert.pem".to_string(),
-        client_key_path: "tests/tls/client.key.pem".to_string(),
+        server_ca_cert_path: "tests/tls/ca.pem".to_string(),
+        client_cert_path: "tests/tls/client.pem".to_string(),
+        client_key_path: "tests/tls/client.key".to_string(),
     });
 
     let re = ChannelManager::with_tls_config(config).unwrap();

src/common/grpc/tests/tls/ca.pem

@@ -0,0 +1,28 @@
+-----BEGIN CERTIFICATE-----
+MIIE3DCCA0SgAwIBAgIRAObeYbJFiVQSGR8yk44dsOYwDQYJKoZIhvcNAQELBQAw
+gYUxHjAcBgNVBAoTFW1rY2VydCBkZXZlbG9wbWVudCBDQTEtMCsGA1UECwwkbHVj
+aW9ATHVjaW9zLVdvcmstTUJQIChMdWNpbyBGcmFuY28pMTQwMgYDVQQDDCtta2Nl
+cnQgbHVjaW9ATHVjaW9zLVdvcmstTUJQIChMdWNpbyBGcmFuY28pMB4XDTE5MDky
+OTIzMzUzM1oXDTI5MDkyOTIzMzUzM1owgYUxHjAcBgNVBAoTFW1rY2VydCBkZXZl
+bG9wbWVudCBDQTEtMCsGA1UECwwkbHVjaW9ATHVjaW9zLVdvcmstTUJQIChMdWNp
+byBGcmFuY28pMTQwMgYDVQQDDCtta2NlcnQgbHVjaW9ATHVjaW9zLVdvcmstTUJQ
+IChMdWNpbyBGcmFuY28pMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA
+y/vE61ItbN/1qMYt13LMf+le1svwfkCCOPsygk7nWeRXmomgUpymqn1LnWiuB0+e
+4IdVH2f5E9DknWEpPhKIDMRTCbz4jTwQfHrxCb8EGj3I8oO73pJO5S/xCedM9OrZ
+qWcYWwN0GQ8cO/ogazaoZf1uTrRNHyzRyQsKyb412kDBTNEeldJZ2ljKgXXvh4HO
+2ZIk9K/ZAaAf6VN8K/89rlJ9/KPgRVNsyAapE+Pb8XXKtpzeFiEcUfuXVYWtkoW+
+xyn/Zu8A1L2CXMQ1sARh7P/42BTMKr5pfraYgcBGxKXLrxoySpxCO9KqeVveKy1q
+fPm5FCwFsXDr0koFLrCiR58mcIO/04Q9DKKTV4Z2a+LoqDJRY37KfBSc8sDMPhw5
+k7g3WPoa6QwXRjZTCA5fHWVgLOtcwLsnju5tBE4LDxwF6s+1wPF8NI5yUfufcEjJ
+Z6JBwgoWYosVj27Lx7KBNLU/57PX9ryee691zmtswt0tP0WVBAgalhYWg99RXoa3
+AgMBAAGjRTBDMA4GA1UdDwEB/wQEAwICBDASBgNVHRMBAf8ECDAGAQH/AgEAMB0G
+A1UdDgQWBBQdvlE4Bdcsjc9oaxjDCRu5FiuZkzANBgkqhkiG9w0BAQsFAAOCAYEA
+BP/6o1kPINksMJZSSXgNCPZskDLyGw7auUZBnQ0ocDT3W6gXQvT/27LM1Hxoj9Eh
+qU1TYdEt7ppecLQSGvzQ02MExG7H75art75oLiB+A5agDira937YbK4MCjqW481d
+bDhw6ixJnY1jIvwjEZxyH6g94YyL927aSPch51fys0kSnjkFzC2RmuzDADScc4XH
+5P1+/3dnIm3M5yfpeUzoaOrTXNmhn8p0RDIGrZ5kA5eISIGGD3Mm8FDssUNKndtO
+g4ojHUsxb14icnAYGeye1NOhGiqN6TEFcgr6MPd0XdFNZ5c0HUaBCfN6bc+JxDV5
+MKZVJdNeJsYYwilgJNHAyZgCi30JC20xeYVtTF7CEEsMrFDGJ70Kz7o/FnRiFsA1
+ZSwVVWhhkHG2VkT4vlo0O3fYeZpenYicvy+wZNTbGK83gzHWqxxNC1z3Etg5+HRJ
+F9qeMWPyfA3IHYXygiMcviyLcyNGG/SJ0EhUpYBN/Gg7wI5yFkcsxUDPPzd23O0M
+-----END CERTIFICATE-----

src/common/grpc/tests/tls/server.cert.pem

@@ -1,40 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIG+jCCBOKgAwIBAgIBAjANBgkqhkiG9w0BAQsFADCBhzELMAkGA1UEBhMCSU4x
-EjAQBgNVBAgMCUthcm5hdGFrYTESMBAGA1UEBwwJQkFOR0FMT1JFMRUwEwYDVQQK
-DAxHb0xpbnV4Q2xvdWQxEjAQBgNVBAMMCWNhLXNlcnZlcjElMCMGCSqGSIb3DQEJ
-ARYWYWRtaW5AZ29saW51eGNsb3VkLmNvbTAeFw0yMzAyMTQxMTM5NDBaFw0yNzA4
-MjIxMTM5NDBaMHAxCzAJBgNVBAYTAklOMRIwEAYDVQQIDAlLYXJuYXRha2ExFTAT
-BgNVBAoMDEdvTGludXhDbG91ZDEPMA0GA1UEAwwGc2VydmVyMSUwIwYJKoZIhvcN
-AQkBFhZhZG1pbkBnb2xpbnV4Y2xvdWQuY29tMIICIjANBgkqhkiG9w0BAQEFAAOC
-Ag8AMIICCgKCAgEAvVtxAoRjLRs3Ei4+CgzqJ2+bpc0sBdUm/4LM/D+0KbXxwD7w
-HP6GcKl/9zf9GJg56pVXxXMaerMDLS4Est25+mBgqcePC6utCBYrKA25pKbkFkxZ
-TPh9/R4RHGVJ3KHy9vc4VzqoV7XFMJFFUQ2fQywHZlXh6MNz0WPTIGaH7hvYoHbK
-I3NpPq8TjRuuV61XB0hK+RW0K6/5Yuj74h/mfheX1VIUOjGwKnTPccZQAlrKYjeW
-BZBS4YqahkTIaGLa06SdUSkuhL85rqAxWvhK9GIRlQLNYJOzg+E3jGyqf566xX60
-fxM6alLYf+ZzCwSBuDDj5f+j752gPLYUI82YL4xQ+AEHNR8U1uMvt0EzzFt7mSRe
-fobVr+Y2zpci+mo7kcQGOhenzGclsm+qXwMhYUnJcOYFZWtTJlFaaPreL4M3Dh+2
-pmKj23ZU6zcT3MYtE6phjCLJl0DsFIcOn+tSqMdpwB20EeQjo9bVJuw/HJrlpcnY
-U9aLsnm/4Ls5A0BQutZnxKBIJjpzp8VfK0WU8a4iKok3AS0z1/K+atNrgSUB9DCH
-0MvLqqQmM9TdLcZj7NSEfLyyFVwPRc5dt4CrNDL7JUpMzt36ezU83JU+nfqWDZsL
-+2JOaE4gGLZDcA3cfP83/mYRaAnYW/9W4vEnIpa6subzq1aFOeY/3dKLTx8CAwEA
-AaOCAYUwggGBMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG
-+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYD
-VR0OBBYEFLijeA+RFDQtuVeMUkaXqF7LF50GMIG8BgNVHSMEgbQwgbGAFKVZwpSJ
-CPkNwGXyJX1sl2Pbby4FoYGNpIGKMIGHMQswCQYDVQQGEwJJTjESMBAGA1UECAwJ
-S2FybmF0YWthMRIwEAYDVQQHDAlCQU5HQUxPUkUxFTATBgNVBAoMDEdvTGludXhD
-bG91ZDESMBAGA1UEAwwJY2Etc2VydmVyMSUwIwYJKoZIhvcNAQkBFhZhZG1pbkBn
-b2xpbnV4Y2xvdWQuY29tggkA7NvbvF8jodEwDgYDVR0PAQH/BAQDAgWgMBMGA1Ud
-JQQMMAoGCCsGAQUFBwMBMCkGA1UdEQQiMCCHBMCoAHKHBAoAAg+CEnNlcnZlci5l
-eGFtcGxlLmNvbTANBgkqhkiG9w0BAQsFAAOCAgEAXvaS9+y5g2Kw/4EPsnhjpN1v
-CxXW0+UYSWOaxVJdEAjGQI/1m9LOiF9IHImmiwluJ/Bex1TzuaTCKmpluPwGvd9D
-Zgf0A5SmVqW4WTT4d2nSecxw4OICJ3j6ubKkvMVf9s+ZJwb+fMMUaSt80bWqp1TY
-XbZguv67PkBECPqVe6rgzXnTLwM3lE8EgG8VtM3IOy9a5SIEjm5L8SQ2I2hiytmE
-e4jR1fbZsB5NbBdfA3GFMKQEE2dIymkG3Bz71M3tZi1y4RnHtRKdrFtrIlgclrwd
-nVnQn/NiXUOOzsL2+vwSF32SSbiLvOxu63qO1YDBkKVChog3P/2f6xcJ23wkbHlL
-qaL2jvLo6ylvMPUYHf5ZWat5zayaGUMHYDKcbD4Dw7aY3M0tNgEHdqUqNePmKvmn
-luyXof3KmmLgWlcfBoX96a7hXDtxFyB2N4nzfQBXh+0VAlgqa+ZZhpdEqRQaWkkR
-MDBdsVJ9O3812IaNfMzpS1vb701GFDCM5Hcyw6a/v6Ln08NMhYut4saLi13kHilS
-Wq7wOAfW3rzxuhjOJJxsi0jJNI775q+a/BbbG/CPl826bXPGH43BdPV8mKwsX5HM
-wwDKf3otP/v7bxwJabfhv2EKUy+W1kkFW9FEZ919yTtfhSDrTNcrXtE7RkiAepfm
-95I025URIlhJGLGBUlA=
------END CERTIFICATE-----

src/common/grpc/tests/tls/server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDyptbMyYWztgta
+t1MXLMzIkaQdeeVbs1Y/qCpAdwZe/Y5ZpbzjGIjCxbB6vNRSnEbYKpytKHPzYfM7
+8d8K8bPvpnqXIiTXFT0JQlw1OHLC1fr4e598GJumAmpMYFrtqv0fbmUFTuQGbHxe
+OH2vji0bvr3NKZubMfkEZP3X4sNXXoXIuW2LaS8OMGKoJaeCBvdbszEiSGj/v9Bj
+pM0yLTH89NNMX1T+FtTKnuXag5g7pr6lzJj83+MzAGy4nOjseSuUimuiyG90/C5t
+A5wC0Qh5RbDnkFYhC44Kxof/i6+jnfateIPNiIIwQV+2f6G/aK1hgjekT10m/eoR
+YDTf+e5ZAgMBAAECggEACODt7yRYjhDVLYaTtb9f5t7dYG67Y7WWLFIc6arxQryI
+XuNfm/ej2WyeXn9WTYeGWBaHERbv1zH4UnMxNBdP/C7dQXZwXqZaS2JwOUpNeK+X
+tUvgtAu6dkKUXSMRcKzXAjVp4N3YHhwOGOx8PNY49FDwZPdmyDD16aFAYIvdle6/
+PSMrj38rB1sbQQdmRob2FjJBSDZ44nsr+/nilrcOFNfNnWv7tQIWYVXNcLfdK/WJ
+ZCDFhA8lr/Yon6MEq6ApTj2ZYRRGXPd6UeASJkmTZEUIUbeDcje/MO8cHkREpuRH
+wm3pCjR7OdO4vc+/d/QmEvu5ns6wbTauelYnL616YQKBgQD414gJtpCHauNEUlFB
+v/R3DzPI5NGp9PAqovOD8nCbI49Mw61gP/ExTIPKiR5uUX/5EL04uspaNkuohXk+
+ys0G5At0NfV7W39lzhvALEaSfleybvYxppbBrc20/q8Gvi/i30NY+1LM3RdtMiEw
+hKHjU0SnFhJq0InFg3AO/iCeTQKBgQD5obkbzpOidSsa55aNsUlO2qjiUY9leq9b
+irAohIZ8YnuuixYvkOeSeSz1eIrA4tECeAFSgTZxYe1Iz+USru2Xg/0xNte11dJD
+rBoH/yMn2gDvBK7xQ6uFMPTeYtKG0vfvpXZYSWZzGntyrHTwFk6UV+xdrt9MBdd1
+XdSn7bwOPQKBgC9VQAko8uDvUf+C8PXiv2uONrl13PPJJY3WpR9qFEVOREnDxszS
+HNzVwxPZdTJiykbkCjoqPadfQJDzopZxGQLAifU29lTamKcSx3CMe3gOFDxaovXa
+zD5XAxP0hfJwZsdu1G6uj5dsTrJ0oJ+L+wc0pZBqwGIU/L/XOo9/g1DZAoGAUebL
+kuH98ik7EUK2VJq8EJERI9/ailLsQb6I+WIxtZGiPqwHhWencpkrNQZtj8dbB9JT
+rLwUHrMgZOlAoRafgTyez4zMzS3wJJ/Mkp8U67hM4h7JPwMSvUpIrMYDiJSjIA9L
+er/qSw1/Pypx22uWMHmAZWRAgvLPtAQrB0Wqk4kCgYEAr2H1PvfbwZwkSvlMt5o8
+WLnBbxcM3AKglLRbkShxxgiZYdEP71/uOtRMiL26du5XX8evItITN0DsvmXL/kcd
+h29LK7LM5uLw7efz0Qxs03G6kEyIHVkacowHi5I5Ul1qI61SoV3yMB1TjIU+bXZt
+0ZjC07totO0fqPOLQxonjQg=
+-----END PRIVATE KEY-----

src/common/grpc/tests/tls/server.pem

@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIEmDCCAwCgAwIBAgIQVEJFCgU/CZk9JEwTucWPpzANBgkqhkiG9w0BAQsFADCB
+hTEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMS0wKwYDVQQLDCRsdWNp
+b0BMdWNpb3MtV29yay1NQlAgKEx1Y2lvIEZyYW5jbykxNDAyBgNVBAMMK21rY2Vy
+dCBsdWNpb0BMdWNpb3MtV29yay1NQlAgKEx1Y2lvIEZyYW5jbykwHhcNMTkwNjAx
+MDAwMDAwWhcNMjkwOTI5MjMzNTM0WjBYMScwJQYDVQQKEx5ta2NlcnQgZGV2ZWxv
+cG1lbnQgY2VydGlmaWNhdGUxLTArBgNVBAsMJGx1Y2lvQEx1Y2lvcy1Xb3JrLU1C
+UCAoTHVjaW8gRnJhbmNvKTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+APKm1szJhbO2C1q3UxcszMiRpB155VuzVj+oKkB3Bl79jlmlvOMYiMLFsHq81FKc
+RtgqnK0oc/Nh8zvx3wrxs++mepciJNcVPQlCXDU4csLV+vh7n3wYm6YCakxgWu2q
+/R9uZQVO5AZsfF44fa+OLRu+vc0pm5sx+QRk/dfiw1dehci5bYtpLw4wYqglp4IG
+91uzMSJIaP+/0GOkzTItMfz000xfVP4W1Mqe5dqDmDumvqXMmPzf4zMAbLic6Ox5
+K5SKa6LIb3T8Lm0DnALRCHlFsOeQViELjgrGh/+Lr6Od9q14g82IgjBBX7Z/ob9o
+rWGCN6RPXSb96hFgNN/57lkCAwEAAaOBrzCBrDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBQdvlE4
+Bdcsjc9oaxjDCRu5FiuZkzBWBgNVHREETzBNggtleGFtcGxlLmNvbYINKi5leGFt
+cGxlLmNvbYIMZXhhbXBsZS50ZXN0gglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAA
+AAAAAAAAAAEwDQYJKoZIhvcNAQELBQADggGBAKb2TJ8l+e1eraNwZWizLw5fccAf
+y59J1JAWdLxZyAI/bkiTlVO3DQoPZpw7XwLhefCvILkwKAL4TtIGGVC9yTb5Q5eg
+rqGO3FC0yg1fn65Kf1VpVxxUVyoiM5PQ4pFJb4AicAv88rCOLD9FFuE0PKOKU/dm
+Tw0WgPStoh9wsJ1RXUuTJYZs1nd1kMBlfv9NbLilnL+cR2sLktS54X5XagsBYVlf
+oapRb0JtABOoQhX3U8QMq8UF8yzceRHNTN9yfLOUrW26s9nKtlWVniNhw1uPxZw9
+RHM7w9/4+a9LXtEDYg4IP/1mm0ywBoUqy1O6hA73uId+Yi/kFBks/GyYaGjKgYcO
+23B75tkPGYEdGuGZYLzZNHbXg4V0UxFQG3KA1pUiSnD3bN2Rxs+CMpzORnOeK3xi
+EooKgAPYsehItoQOMPpccI2xHdSAMWtwUgOKrefUQujkx2Op+KFlspF0+WJ6AZEe
+2D4hyWaEZsvvILXapwqHDCuN3/jSUlTIqUoE1w==
+-----END CERTIFICATE-----

src/common/meta/src/cache/flow/table_flownode.rs

@@ -180,7 +180,7 @@ mod tests {
                     catalog_name: DEFAULT_CATALOG_NAME.to_string(),
                     flow_name: "my_flow".to_string(),
                     raw_sql: "sql".to_string(),
-                    expire_when: "expire".to_string(),
+                    expire_after: Some(300),
                     comment: "comment".to_string(),
                     options: Default::default(),
                 },

src/common/meta/src/ddl/create_flow.rs

@@ -18,6 +18,7 @@ use std::collections::BTreeMap;
 
 use api::v1::flow::flow_request::Body as PbFlowRequest;
 use api::v1::flow::{CreateRequest, FlowRequest, FlowRequestHeader};
+use api::v1::ExpireAfter;
 use async_trait::async_trait;
 use common_catalog::format_full_flow_name;
 use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
@@ -119,12 +120,11 @@ impl CreateFlowProcedure {
                 &sink_table_name.table_name,
             ))
             .await?;
-        ensure!(
-            !exists,
-            error::TableAlreadyExistsSnafu {
-                table_name: sink_table_name.to_string(),
-            }
-        );
+        // TODO(discord9): due to undefined behavior in flow's plan in how to transform types in mfp, sometime flow can't deduce correct schema
+        // and require manually create sink table
+        if exists {
+            common_telemetry::warn!("Table already exists, table: {}", sink_table_name);
+        }
 
         self.collect_source_tables().await?;
         self.allocate_flow_id().await?;
@@ -284,7 +284,7 @@ impl From<&CreateFlowData> for CreateRequest {
             sink_table_name: Some(value.task.sink_table_name.clone().into()),
             // Always be true
             create_if_not_exists: true,
-            expire_when: value.task.expire_when.clone(),
+            expire_after: value.task.expire_after.map(|value| ExpireAfter { value }),
             comment: value.task.comment.clone(),
             sql: value.task.sql.clone(),
             flow_options: value.task.flow_options.clone(),
@@ -298,7 +298,7 @@ impl From<&CreateFlowData> for FlowInfoValue {
             catalog_name,
             flow_name,
             sink_table_name,
-            expire_when,
+            expire_after,
             comment,
             sql,
             flow_options: options,
@@ -319,7 +319,7 @@ impl From<&CreateFlowData> for FlowInfoValue {
             catalog_name,
             flow_name,
             raw_sql: sql,
-            expire_when,
+            expire_after,
             comment,
             options,
         }

src/common/meta/src/ddl/create_view.rs

@@ -30,10 +30,10 @@ use crate::lock_key::{CatalogLock, SchemaLock, TableNameLock};
 use crate::rpc::ddl::CreateViewTask;
 use crate::{metrics, ClusterId};
 
-// The proceudure to execute `[CreateViewTask]`.
+// The procedure to execute `[CreateViewTask]`.
 pub struct CreateViewProcedure {
     pub context: DdlContext,
-    pub creator: ViewCreator,
+    pub data: CreateViewData,
 }
 
 impl CreateViewProcedure {
@@ -42,24 +42,27 @@ impl CreateViewProcedure {
     pub fn new(cluster_id: ClusterId, task: CreateViewTask, context: DdlContext) -> Self {
         Self {
             context,
-            creator: ViewCreator::new(cluster_id, task),
+            data: CreateViewData {
+                state: CreateViewState::Prepare,
+                cluster_id,
+                task,
+                need_update: false,
+            },
         }
     }
 
     pub fn from_json(json: &str, context: DdlContext) -> ProcedureResult<Self> {
         let data = serde_json::from_str(json).context(FromJsonSnafu)?;
 
-        let creator = ViewCreator { data };
-
-        Ok(CreateViewProcedure { context, creator })
+        Ok(CreateViewProcedure { context, data })
     }
 
     fn view_info(&self) -> &RawTableInfo {
-        &self.creator.data.task.view_info
+        &self.data.task.view_info
     }
 
     fn need_update(&self) -> bool {
-        self.creator.data.need_update
+        self.data.need_update
     }
 
     pub(crate) fn view_id(&self) -> TableId {
@@ -68,7 +71,7 @@ impl CreateViewProcedure {
 
     #[cfg(any(test, feature = "testing"))]
     pub fn set_allocated_metadata(&mut self, view_id: TableId) {
-        self.creator.set_allocated_metadata(view_id, false)
+        self.data.set_allocated_metadata(view_id, false)
     }
 
     /// On the prepare step, it performs:
@@ -79,7 +82,7 @@ impl CreateViewProcedure {
     /// - ViewName exists and `create_if_not_exists` is false.
     /// - Failed to allocate [ViewMetadata].
     pub(crate) async fn on_prepare(&mut self) -> Result<Status> {
-        let expr = &self.creator.data.task.create_view;
+        let expr = &self.data.task.create_view;
         let view_name_value = self
             .context
             .table_metadata_manager
@@ -102,7 +105,7 @@ impl CreateViewProcedure {
             ensure!(
                 expr.create_if_not_exists || expr.or_replace,
                 error::ViewAlreadyExistsSnafu {
-                    view_name: self.creator.data.table_ref().to_string(),
+                    view_name: self.data.table_ref().to_string(),
                 }
             );
 
@@ -122,18 +125,18 @@ impl CreateViewProcedure {
                 .get(view_id)
                 .await?
                 .with_context(|| error::TableInfoNotFoundSnafu {
-                    table: self.creator.data.table_ref().to_string(),
+                    table: self.data.table_ref().to_string(),
                 })?;
 
             // Ensure the exists one is view, we can't replace a table.
             ensure!(
                 view_info_value.table_info.table_type == TableType::View,
                 error::TableAlreadyExistsSnafu {
-                    table_name: self.creator.data.table_ref().to_string(),
+                    table_name: self.data.table_ref().to_string(),
                 }
             );
 
-            self.creator.set_allocated_metadata(view_id, true);
+            self.data.set_allocated_metadata(view_id, true);
         } else {
             // Allocate the new `view_id`.
             let TableMetadata { table_id, .. } = self
@@ -141,15 +144,15 @@ impl CreateViewProcedure {
                 .table_metadata_allocator
                 .create_view(
                     &TableMetadataAllocatorContext {
-                        cluster_id: self.creator.data.cluster_id,
+                        cluster_id: self.data.cluster_id,
                     },
                     &None,
                 )
                 .await?;
-            self.creator.set_allocated_metadata(table_id, false);
+            self.data.set_allocated_metadata(table_id, false);
         }
 
-        self.creator.data.state = CreateViewState::CreateMetadata;
+        self.data.state = CreateViewState::CreateMetadata;
 
         Ok(Status::executing(true))
     }
@@ -169,9 +172,9 @@ impl CreateViewProcedure {
                 .get(view_id)
                 .await?
                 .with_context(|| error::ViewNotFoundSnafu {
-                    view_name: self.creator.data.table_ref().to_string(),
+                    view_name: self.data.table_ref().to_string(),
                 })?;
-            let new_logical_plan = self.creator.data.task.raw_logical_plan().clone();
+            let new_logical_plan = self.data.task.raw_logical_plan().clone();
             manager
                 .update_view_info(view_id, &current_view_info, new_logical_plan)
                 .await?;
@@ -180,7 +183,7 @@ impl CreateViewProcedure {
         } else {
             let raw_view_info = self.view_info().clone();
             manager
-                .create_view_metadata(raw_view_info, self.creator.data.task.raw_logical_plan())
+                .create_view_metadata(raw_view_info, self.data.task.raw_logical_plan())
                 .await?;
 
             info!(
@@ -200,7 +203,7 @@ impl Procedure for CreateViewProcedure {
     }
 
     async fn execute(&mut self, ctx: &ProcedureContext) -> ProcedureResult<Status> {
-        let state = &self.creator.data.state;
+        let state = &self.data.state;
 
         let _timer = metrics::METRIC_META_PROCEDURE_CREATE_VIEW
             .with_label_values(&[state.as_ref()])
@@ -214,11 +217,11 @@ impl Procedure for CreateViewProcedure {
     }
 
     fn dump(&self) -> ProcedureResult<String> {
-        serde_json::to_string(&self.creator.data).context(ToJsonSnafu)
+        serde_json::to_string(&self.data).context(ToJsonSnafu)
     }
 
     fn lock_key(&self) -> LockKey {
-        let table_ref = &self.creator.data.table_ref();
+        let table_ref = &self.data.table_ref();
 
         LockKey::new(vec![
             CatalogLock::Read(table_ref.catalog).into(),
@@ -228,30 +231,6 @@ impl Procedure for CreateViewProcedure {
     }
 }
 
-/// The VIEW creator
-pub struct ViewCreator {
-    /// The serializable data.
-    pub data: CreateViewData,
-}
-
-impl ViewCreator {
-    pub fn new(cluster_id: u64, task: CreateViewTask) -> Self {
-        Self {
-            data: CreateViewData {
-                state: CreateViewState::Prepare,
-                cluster_id,
-                task,
-                need_update: false,
-            },
-        }
-    }
-
-    fn set_allocated_metadata(&mut self, view_id: TableId, need_update: bool) {
-        self.data.task.view_info.ident.table_id = view_id;
-        self.data.need_update = need_update;
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize, AsRefStr, PartialEq)]
 pub enum CreateViewState {
     /// Prepares to create the table
@@ -270,6 +249,11 @@ pub struct CreateViewData {
 }
 
 impl CreateViewData {
+    fn set_allocated_metadata(&mut self, view_id: TableId, need_update: bool) {
+        self.task.view_info.ident.table_id = view_id;
+        self.need_update = need_update;
+    }
+
     fn table_ref(&self) -> TableReference<'_> {
         self.task.table_ref()
     }

src/common/meta/src/ddl/table_meta.rs

@@ -184,10 +184,10 @@ impl TableMetadataAllocator {
 
 pub type PeerAllocatorRef = Arc<dyn PeerAllocator>;
 
-/// [PeerAllocator] allocates [Peer]s for creating regions.
+/// [`PeerAllocator`] allocates [`Peer`]s for creating regions.
 #[async_trait]
 pub trait PeerAllocator: Send + Sync {
-    /// Allocates `regions` size [Peer]s.
+    /// Allocates `regions` size [`Peer`]s.
     async fn alloc(&self, ctx: &TableMetadataAllocatorContext, regions: usize)
         -> Result<Vec<Peer>>;
 }

src/common/meta/src/ddl/tests/create_flow.rs

@@ -44,7 +44,7 @@ pub(crate) fn test_create_flow_task(
         sink_table_name,
         or_replace: false,
         create_if_not_exists,
-        expire_when: "".to_string(),
+        expire_after: Some(300),
         comment: "".to_string(),
         sql: "raw_sql".to_string(),
         flow_options: Default::default(),

src/common/meta/src/key/flow.rs

@@ -328,7 +328,7 @@ mod tests {
             sink_table_name,
             flownode_ids,
             raw_sql: "raw".to_string(),
-            expire_when: "expr".to_string(),
+            expire_after: Some(300),
             comment: "hi".to_string(),
             options: Default::default(),
         }
@@ -420,7 +420,7 @@ mod tests {
             sink_table_name: another_sink_table_name,
             flownode_ids: [(0, 1u64)].into(),
             raw_sql: "raw".to_string(),
-            expire_when: "expr".to_string(),
+            expire_after: Some(300),
             comment: "hi".to_string(),
             options: Default::default(),
         };

src/common/meta/src/key/flow/flow_info.rs

@@ -123,7 +123,8 @@ pub struct FlowInfoValue {
     /// The raw sql.
     pub(crate) raw_sql: String,
     /// The expr of expire.
-    pub(crate) expire_when: String,
+    /// Duration in seconds as `i64`.
+    pub(crate) expire_after: Option<i64>,
     /// The comment.
     pub(crate) comment: String,
     /// The options.

src/common/meta/src/rpc/ddl.rs

@@ -28,8 +28,8 @@ use api::v1::meta::{
 };
 use api::v1::{
     AlterExpr, CreateDatabaseExpr, CreateFlowExpr, CreateTableExpr, CreateViewExpr,
-    DropDatabaseExpr, DropFlowExpr, DropTableExpr, DropViewExpr, QueryContext as PbQueryContext,
-    TruncateTableExpr,
+    DropDatabaseExpr, DropFlowExpr, DropTableExpr, DropViewExpr, ExpireAfter,
+    QueryContext as PbQueryContext, TruncateTableExpr,
 };
 use base64::engine::general_purpose;
 use base64::Engine as _;
@@ -898,7 +898,8 @@ pub struct CreateFlowTask {
     pub sink_table_name: TableName,
     pub or_replace: bool,
     pub create_if_not_exists: bool,
-    pub expire_when: String,
+    /// Duration in seconds. Data older than this duration will not be used.
+    pub expire_after: Option<i64>,
     pub comment: String,
     pub sql: String,
     pub flow_options: HashMap<String, String>,
@@ -915,7 +916,7 @@ impl TryFrom<PbCreateFlowTask> for CreateFlowTask {
             sink_table_name,
             or_replace,
             create_if_not_exists,
-            expire_when,
+            expire_after,
             comment,
             sql,
             flow_options,
@@ -934,7 +935,7 @@ impl TryFrom<PbCreateFlowTask> for CreateFlowTask {
                 .into(),
             or_replace,
             create_if_not_exists,
-            expire_when,
+            expire_after: expire_after.map(|e| e.value),
             comment,
             sql,
             flow_options,
@@ -951,7 +952,7 @@ impl From<CreateFlowTask> for PbCreateFlowTask {
             sink_table_name,
             or_replace,
             create_if_not_exists,
-            expire_when,
+            expire_after,
             comment,
             sql,
             flow_options,
@@ -965,7 +966,7 @@ impl From<CreateFlowTask> for PbCreateFlowTask {
                 sink_table_name: Some(sink_table_name.into()),
                 or_replace,
                 create_if_not_exists,
-                expire_when,
+                expire_after: expire_after.map(|value| ExpireAfter { value }),
                 comment,
                 sql,
                 flow_options,

src/common/procedure/src/local.rs

@@ -36,7 +36,7 @@ use crate::error::{
 };
 use crate::local::runner::Runner;
 use crate::procedure::{BoxedProcedureLoader, InitProcedureState};
-use crate::store::{ProcedureMessage, ProcedureStore, StateStoreRef};
+use crate::store::{ProcedureMessage, ProcedureMessages, ProcedureStore, StateStoreRef};
 use crate::{
     BoxedProcedure, ContextProvider, LockKey, ProcedureId, ProcedureManager, ProcedureState,
     ProcedureWithId, Watcher,
@@ -534,8 +534,11 @@ impl LocalManager {
         info!("LocalManager start to recover");
         let recover_start = Instant::now();
 
-        let (messages, rollback_messages, finished_ids) =
-            self.procedure_store.load_messages().await?;
+        let ProcedureMessages {
+            messages,
+            rollback_messages,
+            finished_ids,
+        } = self.procedure_store.load_messages().await?;
         // Submits recovered messages first.
         self.submit_recovered_messages(rollback_messages, InitProcedureState::RollingBack);
         self.submit_recovered_messages(messages, InitProcedureState::Running);

src/common/procedure/src/store.rs

@@ -55,6 +55,17 @@ pub struct ProcedureMessage {
     pub error: Option<String>,
 }
 
+/// A collection of all procedures' messages.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcedureMessages {
+    /// A map of uncommitted procedures
+    pub messages: HashMap<ProcedureId, ProcedureMessage>,
+    /// A map of rolling back procedures
+    pub rollback_messages: HashMap<ProcedureId, ProcedureMessage>,
+    /// A list of finished procedures' ids
+    pub finished_ids: Vec<ProcedureId>,
+}
+
 /// Procedure storage layer.
 pub(crate) struct ProcedureStore {
     proc_path: String,
@@ -182,17 +193,7 @@ impl ProcedureStore {
     }
 
     /// Load procedures from the storage.
-    /// Returns:
-    /// - a map of uncommitted procedures
-    /// - a map of rolling back procedures
-    /// - a list of finished procedures' ids
-    pub(crate) async fn load_messages(
-        &self,
-    ) -> Result<(
-        HashMap<ProcedureId, ProcedureMessage>,
-        HashMap<ProcedureId, ProcedureMessage>,
-        Vec<ProcedureId>,
-    )> {
+    pub(crate) async fn load_messages(&self) -> Result<ProcedureMessages> {
         // Track the key-value pair by procedure id.
         let mut procedure_key_values: HashMap<_, (ParsedKey, Vec<u8>)> = HashMap::new();
 
@@ -242,7 +243,11 @@ impl ProcedureStore {
             }
         }
 
-        Ok((messages, rollback_messages, finished_ids))
+        Ok(ProcedureMessages {
+            messages,
+            rollback_messages,
+            finished_ids,
+        })
     }
 
     fn load_one_message(&self, key: &ParsedKey, value: &[u8]) -> Option<ProcedureMessage> {
@@ -515,10 +520,14 @@ mod tests {
             .await
             .unwrap();
 
-        let (messages, rollback_messages, finished) = store.load_messages().await.unwrap();
+        let ProcedureMessages {
+            messages,
+            rollback_messages,
+            finished_ids,
+        } = store.load_messages().await.unwrap();
         assert_eq!(1, messages.len());
         assert!(rollback_messages.is_empty());
-        assert!(finished.is_empty());
+        assert!(finished_ids.is_empty());
         let msg = messages.get(&procedure_id).unwrap();
         let expect = ProcedureMessage {
             type_name: "MockProcedure".to_string(),
@@ -545,10 +554,14 @@ mod tests {
             .unwrap();
         store.commit_procedure(procedure_id, 1).await.unwrap();
 
-        let (messages, rollback_messages, finished) = store.load_messages().await.unwrap();
+        let ProcedureMessages {
+            messages,
+            rollback_messages,
+            finished_ids,
+        } = store.load_messages().await.unwrap();
         assert!(messages.is_empty());
         assert!(rollback_messages.is_empty());
-        assert_eq!(&[procedure_id], &finished[..]);
+        assert_eq!(&[procedure_id], &finished_ids[..]);
     }
 
     #[tokio::test]
@@ -582,10 +595,14 @@ mod tests {
             .await
             .unwrap();
 
-        let (messages, rollback_messages, finished) = store.load_messages().await.unwrap();
+        let ProcedureMessages {
+            messages,
+            rollback_messages,
+            finished_ids,
+        } = store.load_messages().await.unwrap();
         assert!(messages.is_empty());
         assert_eq!(1, rollback_messages.len());
-        assert!(finished.is_empty());
+        assert!(finished_ids.is_empty());
         assert!(rollback_messages.contains_key(&procedure_id));
     }
 
@@ -611,10 +628,14 @@ mod tests {
 
         store.delete_procedure(procedure_id).await.unwrap();
 
-        let (messages, rollback_messages, finished) = store.load_messages().await.unwrap();
+        let ProcedureMessages {
+            messages,
+            rollback_messages,
+            finished_ids,
+        } = store.load_messages().await.unwrap();
         assert!(messages.is_empty());
         assert!(rollback_messages.is_empty());
-        assert!(finished.is_empty());
+        assert!(finished_ids.is_empty());
     }
 
     #[tokio::test]
@@ -642,10 +663,14 @@ mod tests {
 
         store.delete_procedure(procedure_id).await.unwrap();
 
-        let (messages, rollback_messages, finished) = store.load_messages().await.unwrap();
+        let ProcedureMessages {
+            messages,
+            rollback_messages,
+            finished_ids,
+        } = store.load_messages().await.unwrap();
         assert!(messages.is_empty());
         assert!(rollback_messages.is_empty());
-        assert!(finished.is_empty());
+        assert!(finished_ids.is_empty());
     }
 
     #[tokio::test]
@@ -705,10 +730,14 @@ mod tests {
             .await
             .unwrap();
 
-        let (messages, rollback_messages, finished) = store.load_messages().await.unwrap();
+        let ProcedureMessages {
+            messages,
+            rollback_messages,
+            finished_ids,
+        } = store.load_messages().await.unwrap();
         assert_eq!(2, messages.len());
         assert!(rollback_messages.is_empty());
-        assert_eq!(1, finished.len());
+        assert_eq!(1, finished_ids.len());
 
         let msg = messages.get(&id0).unwrap();
         assert_eq!("id0-2", msg.data);

src/common/query/src/logical_plan.rs

@@ -23,7 +23,6 @@ use datatypes::prelude::ConcreteDataType;
 pub use expr::build_filter_from_timestamp;
 
 pub use self::accumulator::{Accumulator, AggregateFunctionCreator, AggregateFunctionCreatorRef};
-pub use self::expr::{DfExpr, Expr};
 pub use self::udaf::AggregateFunction;
 pub use self::udf::ScalarUdf;
 use crate::function::{ReturnTypeFunction, ScalarFunctionImplementation};

src/common/query/src/logical_plan/expr.rs

@@ -16,28 +16,9 @@ use common_time::range::TimestampRange;
 use common_time::timestamp::TimeUnit;
 use common_time::Timestamp;
 use datafusion_common::{Column, ScalarValue};
-pub use datafusion_expr::expr::Expr as DfExpr;
+use datafusion_expr::expr::Expr;
 use datafusion_expr::{and, binary_expr, Operator};
 
-/// Central struct of query API.
-/// Represent logical expressions such as `A + 1`, or `CAST(c1 AS int)`.
-#[derive(Clone, PartialEq, Eq, Hash, Debug)]
-pub struct Expr {
-    df_expr: DfExpr,
-}
-
-impl Expr {
-    pub fn df_expr(&self) -> &DfExpr {
-        &self.df_expr
-    }
-}
-
-impl From<DfExpr> for Expr {
-    fn from(df_expr: DfExpr) -> Self {
-        Self { df_expr }
-    }
-}
-
 /// Builds an `Expr` that filters timestamp column from given timestamp range.
 /// Returns [None] if time range is [None] or full time range.
 pub fn build_filter_from_timestamp(
@@ -45,12 +26,12 @@ pub fn build_filter_from_timestamp(
     time_range: Option<&TimestampRange>,
 ) -> Option<Expr> {
     let time_range = time_range?;
-    let ts_col_expr = DfExpr::Column(Column {
+    let ts_col_expr = Expr::Column(Column {
         relation: None,
         name: ts_col_name.to_string(),
     });
 
-    let df_expr = match (time_range.start(), time_range.end()) {
+    match (time_range.start(), time_range.end()) {
         (None, None) => None,
         (Some(start), None) => Some(binary_expr(
             ts_col_expr,
@@ -70,20 +51,18 @@ pub fn build_filter_from_timestamp(
             ),
             binary_expr(ts_col_expr, Operator::Lt, timestamp_to_literal(end)),
         )),
-    };
-
-    df_expr.map(Expr::from)
+    }
 }
 
 /// Converts a [Timestamp] to datafusion literal value.
-fn timestamp_to_literal(timestamp: &Timestamp) -> DfExpr {
+fn timestamp_to_literal(timestamp: &Timestamp) -> Expr {
     let scalar_value = match timestamp.unit() {
         TimeUnit::Second => ScalarValue::TimestampSecond(Some(timestamp.value()), None),
         TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(Some(timestamp.value()), None),
         TimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(Some(timestamp.value()), None),
         TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(Some(timestamp.value()), None),
     };
-    DfExpr::Literal(scalar_value)
+    Expr::Literal(scalar_value)
 }
 
 #[cfg(test)]
@@ -91,11 +70,21 @@ mod tests {
     use super::*;
 
     #[test]
-    fn test_from_df_expr() {
-        let df_expr = DfExpr::Wildcard { qualifier: None };
+    fn test_timestamp_to_literal() {
+        let timestamp = Timestamp::new(123456789, TimeUnit::Second);
+        let expected = Expr::Literal(ScalarValue::TimestampSecond(Some(123456789), None));
+        assert_eq!(timestamp_to_literal(&timestamp), expected);
 
-        let expr: Expr = df_expr.into();
+        let timestamp = Timestamp::new(123456789, TimeUnit::Millisecond);
+        let expected = Expr::Literal(ScalarValue::TimestampMillisecond(Some(123456789), None));
+        assert_eq!(timestamp_to_literal(&timestamp), expected);
 
-        assert_eq!(DfExpr::Wildcard { qualifier: None }, *expr.df_expr());
+        let timestamp = Timestamp::new(123456789, TimeUnit::Microsecond);
+        let expected = Expr::Literal(ScalarValue::TimestampMicrosecond(Some(123456789), None));
+        assert_eq!(timestamp_to_literal(&timestamp), expected);
+
+        let timestamp = Timestamp::new(123456789, TimeUnit::Nanosecond);
+        let expected = Expr::Literal(ScalarValue::TimestampNanosecond(Some(123456789), None));
+        assert_eq!(timestamp_to_literal(&timestamp), expected);
     }
 }

src/common/query/src/prelude.rs

@@ -16,7 +16,7 @@ pub use datafusion_common::ScalarValue;
 
 pub use crate::columnar_value::ColumnarValue;
 pub use crate::function::*;
-pub use crate::logical_plan::{create_udf, AggregateFunction, Expr, ScalarUdf};
+pub use crate::logical_plan::{create_udf, AggregateFunction, ScalarUdf};
 pub use crate::signature::{Signature, TypeSignature, Volatility};
 
 /// Default timestamp column name for Prometheus metrics.

src/common/recordbatch/src/filter.rs

@@ -14,7 +14,7 @@
 
 //! Util record batch stream wrapper that can perform precise filter.
 
-use datafusion::logical_expr::{Expr, Operator};
+use datafusion::logical_expr::{Expr, Literal, Operator};
 use datafusion_common::arrow::array::{ArrayRef, Datum, Scalar};
 use datafusion_common::arrow::buffer::BooleanBuffer;
 use datafusion_common::arrow::compute::kernels::cmp;
@@ -43,6 +43,28 @@ pub struct SimpleFilterEvaluator {
 }
 
 impl SimpleFilterEvaluator {
+    pub fn new<T: Literal>(column_name: String, lit: T, op: Operator) -> Option<Self> {
+        match op {
+            Operator::Eq
+            | Operator::NotEq
+            | Operator::Lt
+            | Operator::LtEq
+            | Operator::Gt
+            | Operator::GtEq => {}
+            _ => return None,
+        }
+
+        let Expr::Literal(val) = lit.lit() else {
+            return None;
+        };
+
+        Some(Self {
+            column_name,
+            literal: val.to_scalar().ok()?,
+            op,
+        })
+    }
+
     pub fn try_new(predicate: &Expr) -> Option<Self> {
         match predicate {
             Expr::BinaryExpr(binary) => {

src/common/telemetry/src/logging.rs

@@ -124,139 +124,144 @@ pub fn init_global_logging(
     tracing_opts: &TracingOptions,
     node_id: Option<String>,
 ) -> Vec<WorkerGuard> {
+    static START: Once = Once::new();
     let mut guards = vec![];
-    let dir = &opts.dir;
-    let level = &opts.level;
-    let enable_otlp_tracing = opts.enable_otlp_tracing;
 
-    // Enable log compatible layer to convert log record to tracing span.
-    LogTracer::init().expect("log tracer must be valid");
+    START.call_once(|| {
+        let dir = &opts.dir;
+        let level = &opts.level;
+        let enable_otlp_tracing = opts.enable_otlp_tracing;
 
-    // stdout log layer.
-    let stdout_logging_layer = if opts.append_stdout {
-        let (stdout_writer, stdout_guard) = tracing_appender::non_blocking(std::io::stdout());
-        guards.push(stdout_guard);
+        // Enable log compatible layer to convert log record to tracing span.
+        LogTracer::init().expect("log tracer must be valid");
 
-        Some(
-            Layer::new()
-                .with_writer(stdout_writer)
-                .with_ansi(atty::is(atty::Stream::Stdout)),
-        )
-    } else {
-        None
-    };
-
-    // file log layer.
-    let rolling_appender = RollingFileAppender::new(Rotation::HOURLY, dir, app_name);
-    let (rolling_writer, rolling_writer_guard) = tracing_appender::non_blocking(rolling_appender);
-    let file_logging_layer = Layer::new().with_writer(rolling_writer).with_ansi(false);
-    guards.push(rolling_writer_guard);
-
-    // error file log layer.
-    let err_rolling_appender =
-        RollingFileAppender::new(Rotation::HOURLY, dir, format!("{}-{}", app_name, "err"));
-    let (err_rolling_writer, err_rolling_writer_guard) =
-        tracing_appender::non_blocking(err_rolling_appender);
-    let err_file_logging_layer = Layer::new()
-        .with_writer(err_rolling_writer)
-        .with_ansi(false);
-    guards.push(err_rolling_writer_guard);
-
-    // resolve log level settings from:
-    // - options from command line or config files
-    // - environment variable: RUST_LOG
-    // - default settings
-    let rust_log_env = std::env::var(EnvFilter::DEFAULT_ENV).ok();
-    let targets_string = level
-        .as_deref()
-        .or(rust_log_env.as_deref())
-        .unwrap_or(DEFAULT_LOG_TARGETS);
-    let filter = targets_string
-        .parse::<filter::Targets>()
-        .expect("error parsing log level string");
-    let sampler = opts
-        .tracing_sample_ratio
-        .as_ref()
-        .map(create_sampler)
-        .map(Sampler::ParentBased)
-        .unwrap_or(Sampler::ParentBased(Box::new(Sampler::AlwaysOn)));
-    // Must enable 'tokio_unstable' cfg to use this feature.
-    // For example: `RUSTFLAGS="--cfg tokio_unstable" cargo run -F common-telemetry/console -- standalone start`
-    #[cfg(feature = "tokio-console")]
-    let subscriber = {
-        let tokio_console_layer = if let Some(tokio_console_addr) = &tracing_opts.tokio_console_addr
-        {
-            let addr: std::net::SocketAddr = tokio_console_addr.parse().unwrap_or_else(|e| {
-                panic!("Invalid binding address '{tokio_console_addr}' for tokio-console: {e}");
-            });
-            println!("tokio-console listening on {addr}");
+        // stdout log layer.
+        let stdout_logging_layer = if opts.append_stdout {
+            let (stdout_writer, stdout_guard) = tracing_appender::non_blocking(std::io::stdout());
+            guards.push(stdout_guard);
 
             Some(
-                console_subscriber::ConsoleLayer::builder()
-                    .server_addr(addr)
-                    .spawn(),
+                Layer::new()
+                    .with_writer(stdout_writer)
+                    .with_ansi(atty::is(atty::Stream::Stdout)),
             )
         } else {
             None
         };
 
-        let stdout_logging_layer = stdout_logging_layer.map(|x| x.with_filter(filter.clone()));
+        // file log layer.
+        let rolling_appender = RollingFileAppender::new(Rotation::HOURLY, dir, app_name);
+        let (rolling_writer, rolling_writer_guard) =
+            tracing_appender::non_blocking(rolling_appender);
+        let file_logging_layer = Layer::new().with_writer(rolling_writer).with_ansi(false);
+        guards.push(rolling_writer_guard);
 
-        let file_logging_layer = file_logging_layer.with_filter(filter);
+        // error file log layer.
+        let err_rolling_appender =
+            RollingFileAppender::new(Rotation::HOURLY, dir, format!("{}-{}", app_name, "err"));
+        let (err_rolling_writer, err_rolling_writer_guard) =
+            tracing_appender::non_blocking(err_rolling_appender);
+        let err_file_logging_layer = Layer::new()
+            .with_writer(err_rolling_writer)
+            .with_ansi(false);
+        guards.push(err_rolling_writer_guard);
 
-        Registry::default()
-            .with(tokio_console_layer)
+        // resolve log level settings from:
+        // - options from command line or config files
+        // - environment variable: RUST_LOG
+        // - default settings
+        let rust_log_env = std::env::var(EnvFilter::DEFAULT_ENV).ok();
+        let targets_string = level
+            .as_deref()
+            .or(rust_log_env.as_deref())
+            .unwrap_or(DEFAULT_LOG_TARGETS);
+        let filter = targets_string
+            .parse::<filter::Targets>()
+            .expect("error parsing log level string");
+        let sampler = opts
+            .tracing_sample_ratio
+            .as_ref()
+            .map(create_sampler)
+            .map(Sampler::ParentBased)
+            .unwrap_or(Sampler::ParentBased(Box::new(Sampler::AlwaysOn)));
+        // Must enable 'tokio_unstable' cfg to use this feature.
+        // For example: `RUSTFLAGS="--cfg tokio_unstable" cargo run -F common-telemetry/console -- standalone start`
+        #[cfg(feature = "tokio-console")]
+        let subscriber = {
+            let tokio_console_layer =
+                if let Some(tokio_console_addr) = &tracing_opts.tokio_console_addr {
+                    let addr: std::net::SocketAddr = tokio_console_addr.parse().unwrap_or_else(|e| {
+                    panic!("Invalid binding address '{tokio_console_addr}' for tokio-console: {e}");
+                });
+                    println!("tokio-console listening on {addr}");
+
+                    Some(
+                        console_subscriber::ConsoleLayer::builder()
+                            .server_addr(addr)
+                            .spawn(),
+                    )
+                } else {
+                    None
+                };
+
+            let stdout_logging_layer = stdout_logging_layer.map(|x| x.with_filter(filter.clone()));
+
+            let file_logging_layer = file_logging_layer.with_filter(filter);
+
+            Registry::default()
+                .with(tokio_console_layer)
+                .with(stdout_logging_layer)
+                .with(file_logging_layer)
+                .with(err_file_logging_layer.with_filter(filter::LevelFilter::ERROR))
+        };
+
+        // consume the `tracing_opts`, to avoid "unused" warnings
+        let _ = tracing_opts;
+
+        #[cfg(not(feature = "tokio-console"))]
+        let subscriber = Registry::default()
+            .with(filter)
             .with(stdout_logging_layer)
             .with(file_logging_layer)
-            .with(err_file_logging_layer.with_filter(filter::LevelFilter::ERROR))
-    };
+            .with(err_file_logging_layer.with_filter(filter::LevelFilter::ERROR));
 
-    // consume the `tracing_opts`, to avoid "unused" warnings
-    let _ = tracing_opts;
-
-    #[cfg(not(feature = "tokio-console"))]
-    let subscriber = Registry::default()
-        .with(filter)
-        .with(stdout_logging_layer)
-        .with(file_logging_layer)
-        .with(err_file_logging_layer.with_filter(filter::LevelFilter::ERROR));
-
-    if enable_otlp_tracing {
-        global::set_text_map_propagator(TraceContextPropagator::new());
-        // otlp exporter
-        let tracer = opentelemetry_otlp::new_pipeline()
-            .tracing()
-            .with_exporter(
-                opentelemetry_otlp::new_exporter().tonic().with_endpoint(
-                    opts.otlp_endpoint
-                        .as_ref()
-                        .map(|e| format!("http://{}", e))
-                        .unwrap_or(DEFAULT_OTLP_ENDPOINT.to_string()),
-                ),
-            )
-            .with_trace_config(
-                opentelemetry_sdk::trace::config()
-                    .with_sampler(sampler)
-                    .with_resource(opentelemetry_sdk::Resource::new(vec![
-                        KeyValue::new(resource::SERVICE_NAME, app_name.to_string()),
-                        KeyValue::new(
-                            resource::SERVICE_INSTANCE_ID,
-                            node_id.unwrap_or("none".to_string()),
-                        ),
-                        KeyValue::new(resource::SERVICE_VERSION, env!("CARGO_PKG_VERSION")),
-                        KeyValue::new(resource::PROCESS_PID, std::process::id().to_string()),
-                    ])),
-            )
-            .install_batch(opentelemetry_sdk::runtime::Tokio)
-            .expect("otlp tracer install failed");
-        let tracing_layer = Some(tracing_opentelemetry::layer().with_tracer(tracer));
-        let subscriber = subscriber.with(tracing_layer);
-        tracing::subscriber::set_global_default(subscriber)
-            .expect("error setting global tracing subscriber");
-    } else {
-        tracing::subscriber::set_global_default(subscriber)
-            .expect("error setting global tracing subscriber");
-    }
+        if enable_otlp_tracing {
+            global::set_text_map_propagator(TraceContextPropagator::new());
+            // otlp exporter
+            let tracer = opentelemetry_otlp::new_pipeline()
+                .tracing()
+                .with_exporter(
+                    opentelemetry_otlp::new_exporter().tonic().with_endpoint(
+                        opts.otlp_endpoint
+                            .as_ref()
+                            .map(|e| format!("http://{}", e))
+                            .unwrap_or(DEFAULT_OTLP_ENDPOINT.to_string()),
+                    ),
+                )
+                .with_trace_config(
+                    opentelemetry_sdk::trace::config()
+                        .with_sampler(sampler)
+                        .with_resource(opentelemetry_sdk::Resource::new(vec![
+                            KeyValue::new(resource::SERVICE_NAME, app_name.to_string()),
+                            KeyValue::new(
+                                resource::SERVICE_INSTANCE_ID,
+                                node_id.unwrap_or("none".to_string()),
+                            ),
+                            KeyValue::new(resource::SERVICE_VERSION, env!("CARGO_PKG_VERSION")),
+                            KeyValue::new(resource::PROCESS_PID, std::process::id().to_string()),
+                        ])),
+                )
+                .install_batch(opentelemetry_sdk::runtime::Tokio)
+                .expect("otlp tracer install failed");
+            let tracing_layer = Some(tracing_opentelemetry::layer().with_tracer(tracer));
+            let subscriber = subscriber.with(tracing_layer);
+            tracing::subscriber::set_global_default(subscriber)
+                .expect("error setting global tracing subscriber");
+        } else {
+            tracing::subscriber::set_global_default(subscriber)
+                .expect("error setting global tracing subscriber");
+        }
+    });
 
     guards
 }

src/datanode/src/datanode.rs

@@ -516,6 +516,7 @@ mod tests {
     use common_meta::key::datanode_table::DatanodeTableManager;
     use common_meta::kv_backend::memory::MemoryKvBackend;
     use common_meta::kv_backend::KvBackendRef;
+    use mito2::engine::MITO_ENGINE_NAME;
     use store_api::region_request::RegionRequest;
     use store_api::storage::RegionId;
 
@@ -528,7 +529,7 @@ mod tests {
         let txn = mgr
             .build_create_txn(
                 1028,
-                "mock",
+                MITO_ENGINE_NAME,
                 "foo/bar/weny",
                 HashMap::from([("foo".to_string(), "bar".to_string())]),
                 HashMap::default(),
@@ -542,8 +543,9 @@ mod tests {
 
     #[tokio::test]
     async fn test_initialize_region_server() {
+        common_telemetry::init_default_ut_logging();
         let mut mock_region_server = mock_region_server();
-        let (mock_region, mut mock_region_handler) = MockRegionEngine::new();
+        let (mock_region, mut mock_region_handler) = MockRegionEngine::new(MITO_ENGINE_NAME);
 
         mock_region_server.register_engine(mock_region.clone());
 

src/datanode/src/error.rs

@@ -367,6 +367,19 @@ pub enum Error {
         #[snafu(source(from(common_config::error::Error, Box::new)))]
         source: Box<common_config::error::Error>,
     },
+
+    #[snafu(display(
+        "Failed to get region metadata from engine {} for region_id {}",
+        engine,
+        region_id,
+    ))]
+    GetRegionMetadata {
+        engine: String,
+        region_id: RegionId,
+        #[snafu(implicit)]
+        location: Location,
+        source: BoxedError,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -433,7 +446,9 @@ impl ErrorExt for Error {
             TableIdProviderNotFound { .. } | UnsupportedGrpcRequest { .. } => {
                 StatusCode::Unsupported
             }
-            HandleRegionRequest { source, .. } => source.status_code(),
+            HandleRegionRequest { source, .. } | GetRegionMetadata { source, .. } => {
+                source.status_code()
+            }
             StopRegionEngine { source, .. } => source.status_code(),
 
             FindLogicalRegions { source, .. } => source.status_code(),

src/datanode/src/heartbeat.rs

@@ -109,13 +109,10 @@ impl HeartbeatTask {
         let mut last_received_lease = Instant::now();
 
         let _handle = common_runtime::spawn_bg(async move {
-            while let Some(res) = match rx.message().await {
-                Ok(m) => m,
-                Err(e) => {
-                    error!(e; "Error while reading heartbeat response");
-                    None
-                }
-            } {
+            while let Some(res) = rx.message().await.unwrap_or_else(|e| {
+                error!(e; "Error while reading heartbeat response");
+                None
+            }) {
                 if let Some(msg) = res.mailbox_message.as_ref() {
                     info!("Received mailbox message: {msg:?}, meta_client id: {client_id:?}");
                 }
@@ -238,7 +235,7 @@ impl HeartbeatTask {
                                     Some(req)
                                 }
                                 Err(e) => {
-                                    error!(e;"Failed to encode mailbox messages!");
+                                    error!(e; "Failed to encode mailbox messages!");
                                     None
                                 }
                             }
@@ -276,7 +273,7 @@ impl HeartbeatTask {
                 if let Some(req) = req {
                     debug!("Sending heartbeat request: {:?}", req);
                     if let Err(e) = tx.send(req).await {
-                        error!("Failed to send heartbeat to metasrv, error: {:?}", e);
+                        error!(e; "Failed to send heartbeat to metasrv");
                         match Self::create_streams(
                             &meta_client,
                             running.clone(),
@@ -301,7 +298,7 @@ impl HeartbeatTask {
                                     Instant::now()
                                         + Duration::from_secs(META_KEEP_ALIVE_INTERVAL_SECS),
                                 );
-                                error!(e;"Failed to reconnect to metasrv!");
+                                error!(e; "Failed to reconnect to metasrv!");
                             }
                         }
                     }

src/datanode/src/heartbeat/handler/upgrade_region.rs

@@ -121,6 +121,7 @@ mod tests {
     use std::time::Duration;
 
     use common_meta::instruction::{InstructionReply, UpgradeRegion};
+    use mito2::engine::MITO_ENGINE_NAME;
     use store_api::region_engine::RegionRole;
     use store_api::storage::RegionId;
     use tokio::time::Instant;
@@ -133,7 +134,7 @@ mod tests {
     #[tokio::test]
     async fn test_region_not_exist() {
         let mut mock_region_server = mock_region_server();
-        let (mock_engine, _) = MockRegionEngine::new();
+        let (mock_engine, _) = MockRegionEngine::new(MITO_ENGINE_NAME);
         mock_region_server.register_engine(mock_engine);
 
         let handler_context = HandlerContext {
@@ -167,13 +168,14 @@ mod tests {
         let mock_region_server = mock_region_server();
         let region_id = RegionId::new(1024, 1);
 
-        let (mock_engine, _) = MockRegionEngine::with_custom_apply_fn(|region_engine| {
-            region_engine.mock_role = Some(Some(RegionRole::Leader));
-            region_engine.handle_request_mock_fn = Some(Box::new(|_, _| {
-                // Should be unreachable.
-                unreachable!();
-            }));
-        });
+        let (mock_engine, _) =
+            MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
+                region_engine.mock_role = Some(Some(RegionRole::Leader));
+                region_engine.handle_request_mock_fn = Some(Box::new(|_, _| {
+                    // Should be unreachable.
+                    unreachable!();
+                }));
+            });
         mock_region_server.register_test_region(region_id, mock_engine);
 
         let handler_context = HandlerContext {
@@ -207,13 +209,14 @@ mod tests {
         let mock_region_server = mock_region_server();
         let region_id = RegionId::new(1024, 1);
 
-        let (mock_engine, _) = MockRegionEngine::with_custom_apply_fn(|region_engine| {
-            // Region is not ready.
-            region_engine.mock_role = Some(Some(RegionRole::Follower));
-            region_engine.handle_request_mock_fn = Some(Box::new(|_, _| Ok(0)));
-            // Note: Don't change.
-            region_engine.handle_request_delay = Some(Duration::from_secs(100));
-        });
+        let (mock_engine, _) =
+            MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
+                // Region is not ready.
+                region_engine.mock_role = Some(Some(RegionRole::Follower));
+                region_engine.handle_request_mock_fn = Some(Box::new(|_, _| Ok(0)));
+                // Note: Don't change.
+                region_engine.handle_request_delay = Some(Duration::from_secs(100));
+            });
         mock_region_server.register_test_region(region_id, mock_engine);
 
         let handler_context = HandlerContext {
@@ -247,13 +250,14 @@ mod tests {
         let mock_region_server = mock_region_server();
         let region_id = RegionId::new(1024, 1);
 
-        let (mock_engine, _) = MockRegionEngine::with_custom_apply_fn(|region_engine| {
-            // Region is not ready.
-            region_engine.mock_role = Some(Some(RegionRole::Follower));
-            region_engine.handle_request_mock_fn = Some(Box::new(|_, _| Ok(0)));
-            // Note: Don't change.
-            region_engine.handle_request_delay = Some(Duration::from_millis(300));
-        });
+        let (mock_engine, _) =
+            MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
+                // Region is not ready.
+                region_engine.mock_role = Some(Some(RegionRole::Follower));
+                region_engine.handle_request_mock_fn = Some(Box::new(|_, _| Ok(0)));
+                // Note: Don't change.
+                region_engine.handle_request_delay = Some(Duration::from_millis(300));
+            });
         mock_region_server.register_test_region(region_id, mock_engine);
 
         let waits = vec![
@@ -308,18 +312,19 @@ mod tests {
         let mock_region_server = mock_region_server();
         let region_id = RegionId::new(1024, 1);
 
-        let (mock_engine, _) = MockRegionEngine::with_custom_apply_fn(|region_engine| {
-            // Region is not ready.
-            region_engine.mock_role = Some(Some(RegionRole::Follower));
-            region_engine.handle_request_mock_fn = Some(Box::new(|_, _| {
-                error::UnexpectedSnafu {
-                    violated: "mock_error".to_string(),
-                }
-                .fail()
-            }));
-            // Note: Don't change.
-            region_engine.handle_request_delay = Some(Duration::from_millis(100));
-        });
+        let (mock_engine, _) =
+            MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
+                // Region is not ready.
+                region_engine.mock_role = Some(Some(RegionRole::Follower));
+                region_engine.handle_request_mock_fn = Some(Box::new(|_, _| {
+                    error::UnexpectedSnafu {
+                        violated: "mock_error".to_string(),
+                    }
+                    .fail()
+                }));
+                // Note: Don't change.
+                region_engine.handle_request_delay = Some(Duration::from_millis(100));
+            });
         mock_region_server.register_test_region(region_id, mock_engine);
 
         let handler_context = HandlerContext {

src/datanode/src/region_server.rs

@@ -34,6 +34,7 @@ use common_telemetry::{info, warn};
 use dashmap::DashMap;
 use futures_util::future::try_join_all;
 use metric_engine::engine::MetricEngine;
+use mito2::engine::MITO_ENGINE_NAME;
 use prost::Message;
 pub use query::dummy_catalog::{
     DummyCatalogList, DummyTableProviderFactory, TableProviderFactoryRef,
@@ -44,7 +45,9 @@ use servers::grpc::flight::{FlightCraft, FlightRecordBatchStream, TonicStream};
 use servers::grpc::region_server::RegionServerHandler;
 use session::context::{QueryContextBuilder, QueryContextRef};
 use snafu::{OptionExt, ResultExt};
-use store_api::metric_engine_consts::{METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY};
+use store_api::metric_engine_consts::{
+    FILE_ENGINE_NAME, LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME,
+};
 use store_api::region_engine::{RegionEngineRef, RegionRole, SetReadonlyResponse};
 use store_api::region_request::{AffectedRows, RegionCloseRequest, RegionRequest};
 use store_api::storage::RegionId;
@@ -216,6 +219,7 @@ impl RegionServerHandler for RegionServer {
             .context(BuildRegionRequestsSnafu)
             .map_err(BoxedError::new)
             .context(ExecuteGrpcRequestSnafu)?;
+
         let tracing_context = TracingContext::from_current_span();
 
         let results = if is_parallel {
@@ -403,7 +407,7 @@ impl RegionServerInner {
         let current_region_status = self.region_map.get(&region_id);
 
         let engine = match region_change {
-            RegionChange::Register(ref engine_type, _) => match current_region_status {
+            RegionChange::Register(attribute) => match current_region_status {
                 Some(status) => match status.clone() {
                     RegionEngineWithStatus::Registering(_) => {
                         return Ok(CurrentEngine::EarlyReturn(0))
@@ -417,8 +421,10 @@ impl RegionServerInner {
                     .engines
                     .read()
                     .unwrap()
-                    .get(engine_type)
-                    .with_context(|| RegionEngineNotFoundSnafu { name: engine_type })?
+                    .get(attribute.engine())
+                    .with_context(|| RegionEngineNotFoundSnafu {
+                        name: attribute.engine(),
+                    })?
                     .clone(),
             },
             RegionChange::Deregisters => match current_region_status {
@@ -461,11 +467,13 @@ impl RegionServerInner {
             .start_timer();
 
         let region_change = match &request {
-            RegionRequest::Create(create) => RegionChange::Register(create.engine.clone(), false),
+            RegionRequest::Create(create) => {
+                let attribute = parse_region_attribute(&create.engine, &create.options)?;
+                RegionChange::Register(attribute)
+            }
             RegionRequest::Open(open) => {
-                let is_opening_physical_region =
-                    open.options.contains_key(PHYSICAL_TABLE_METADATA_KEY);
-                RegionChange::Register(open.engine.clone(), is_opening_physical_region)
+                let attribute = parse_region_attribute(&open.engine, &open.options)?;
+                RegionChange::Register(attribute)
             }
             RegionRequest::Close(_) | RegionRequest::Drop(_) => RegionChange::Deregisters,
             RegionRequest::Put(_)
@@ -514,7 +522,7 @@ impl RegionServerInner {
         region_change: &RegionChange,
     ) {
         match region_change {
-            RegionChange::Register(_, _) => {
+            RegionChange::Register(_) => {
                 self.region_map.insert(
                     region_id,
                     RegionEngineWithStatus::Registering(engine.clone()),
@@ -533,7 +541,7 @@ impl RegionServerInner {
     fn unset_region_status(&self, region_id: RegionId, region_change: RegionChange) {
         match region_change {
             RegionChange::None => {}
-            RegionChange::Register(_, _) | RegionChange::Deregisters => {
+            RegionChange::Register(_) | RegionChange::Deregisters => {
                 self.region_map.remove(&region_id);
             }
         }
@@ -548,15 +556,28 @@ impl RegionServerInner {
         let engine_type = engine.name();
         match region_change {
             RegionChange::None => {}
-            RegionChange::Register(_, is_opening_physical_region) => {
-                if is_opening_physical_region {
-                    self.register_logical_regions(&engine, region_id).await?;
-                }
-
-                info!("Region {region_id} is registered to engine {engine_type}");
+            RegionChange::Register(attribute) => {
+                info!(
+                    "Region {region_id} is registered to engine {}",
+                    attribute.engine()
+                );
                 self.region_map
-                    .insert(region_id, RegionEngineWithStatus::Ready(engine));
-                self.event_listener.on_region_registered(region_id);
+                    .insert(region_id, RegionEngineWithStatus::Ready(engine.clone()));
+
+                match attribute {
+                    RegionAttribute::Metric { physical } => {
+                        if physical {
+                            // Registers the logical regions belong to the physical region (`region_id`).
+                            self.register_logical_regions(&engine, region_id).await?;
+                            // We only send the `on_region_registered` event of the physical region.
+                            self.event_listener.on_region_registered(region_id);
+                        }
+                    }
+                    RegionAttribute::Mito => self.event_listener.on_region_registered(region_id),
+                    RegionAttribute::File => {
+                        // do nothing
+                    }
+                }
             }
             RegionChange::Deregisters => {
                 info!("Region {region_id} is deregistered from engine {engine_type}");
@@ -699,10 +720,45 @@ impl RegionServerInner {
 
 enum RegionChange {
     None,
-    Register(String, bool),
+    Register(RegionAttribute),
     Deregisters,
 }
 
+fn parse_region_attribute(
+    engine: &str,
+    options: &HashMap<String, String>,
+) -> Result<RegionAttribute> {
+    match engine {
+        MITO_ENGINE_NAME => Ok(RegionAttribute::Mito),
+        METRIC_ENGINE_NAME => {
+            let physical = !options.contains_key(LOGICAL_TABLE_METADATA_KEY);
+
+            Ok(RegionAttribute::Metric { physical })
+        }
+        FILE_ENGINE_NAME => Ok(RegionAttribute::File),
+        _ => error::UnexpectedSnafu {
+            violated: format!("Unknown engine: {}", engine),
+        }
+        .fail(),
+    }
+}
+
+enum RegionAttribute {
+    Mito,
+    Metric { physical: bool },
+    File,
+}
+
+impl RegionAttribute {
+    fn engine(&self) -> &'static str {
+        match self {
+            RegionAttribute::Mito => MITO_ENGINE_NAME,
+            RegionAttribute::Metric { .. } => METRIC_ENGINE_NAME,
+            RegionAttribute::File => FILE_ENGINE_NAME,
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
 
@@ -723,7 +779,7 @@ mod tests {
         common_telemetry::init_default_ut_logging();
 
         let mut mock_region_server = mock_region_server();
-        let (engine, _receiver) = MockRegionEngine::new();
+        let (engine, _receiver) = MockRegionEngine::new(MITO_ENGINE_NAME);
         let engine_name = engine.name();
 
         mock_region_server.register_engine(engine.clone());
@@ -781,7 +837,7 @@ mod tests {
         common_telemetry::init_default_ut_logging();
 
         let mut mock_region_server = mock_region_server();
-        let (engine, _receiver) = MockRegionEngine::new();
+        let (engine, _receiver) = MockRegionEngine::new(MITO_ENGINE_NAME);
 
         mock_region_server.register_engine(engine.clone());
 
@@ -832,7 +888,7 @@ mod tests {
         common_telemetry::init_default_ut_logging();
 
         let mut mock_region_server = mock_region_server();
-        let (engine, _receiver) = MockRegionEngine::new();
+        let (engine, _receiver) = MockRegionEngine::new(MITO_ENGINE_NAME);
 
         mock_region_server.register_engine(engine.clone());
 
@@ -857,13 +913,15 @@ mod tests {
         common_telemetry::init_default_ut_logging();
 
         let mut mock_region_server = mock_region_server();
-        let (engine, _receiver) =
-            MockRegionEngine::with_mock_fn(Box::new(|_region_id, _request| {
+        let (engine, _receiver) = MockRegionEngine::with_mock_fn(
+            MITO_ENGINE_NAME,
+            Box::new(|_region_id, _request| {
                 error::UnexpectedSnafu {
                     violated: "test".to_string(),
                 }
                 .fail()
-            }));
+            }),
+        );
 
         mock_region_server.register_engine(engine.clone());
 
@@ -904,7 +962,7 @@ mod tests {
         common_telemetry::init_default_ut_logging();
 
         let mut mock_region_server = mock_region_server();
-        let (engine, _) = MockRegionEngine::new();
+        let (engine, _) = MockRegionEngine::new(MITO_ENGINE_NAME);
         mock_region_server.register_engine(engine.clone());
 
         let region_id = RegionId::new(1024, 1);
@@ -950,7 +1008,7 @@ mod tests {
             CurrentEngineTest {
                 region_id,
                 current_region_status: None,
-                region_change: RegionChange::Register(engine.name().to_string(), false),
+                region_change: RegionChange::Register(RegionAttribute::Mito),
                 assert: Box::new(|result| {
                     let current_engine = result.unwrap();
                     assert_matches!(current_engine, CurrentEngine::Engine(_));
@@ -959,7 +1017,7 @@ mod tests {
             CurrentEngineTest {
                 region_id,
                 current_region_status: Some(RegionEngineWithStatus::Registering(engine.clone())),
-                region_change: RegionChange::Register(engine.name().to_string(), false),
+                region_change: RegionChange::Register(RegionAttribute::Mito),
                 assert: Box::new(|result| {
                     let current_engine = result.unwrap();
                     assert_matches!(current_engine, CurrentEngine::EarlyReturn(_));
@@ -968,7 +1026,7 @@ mod tests {
             CurrentEngineTest {
                 region_id,
                 current_region_status: Some(RegionEngineWithStatus::Deregistering(engine.clone())),
-                region_change: RegionChange::Register(engine.name().to_string(), false),
+                region_change: RegionChange::Register(RegionAttribute::Mito),
                 assert: Box::new(|result| {
                     let err = result.unwrap_err();
                     assert_eq!(err.status_code(), StatusCode::RegionBusy);
@@ -977,7 +1035,7 @@ mod tests {
             CurrentEngineTest {
                 region_id,
                 current_region_status: Some(RegionEngineWithStatus::Ready(engine.clone())),
-                region_change: RegionChange::Register(engine.name().to_string(), false),
+                region_change: RegionChange::Register(RegionAttribute::Mito),
                 assert: Box::new(|result| {
                     let current_engine = result.unwrap();
                     assert_matches!(current_engine, CurrentEngine::Engine(_));

src/datanode/src/service.rs

@@ -21,6 +21,7 @@ use servers::grpc::{GrpcServer, GrpcServerConfig};
 use servers::http::HttpServerBuilder;
 use servers::metrics_handler::MetricsHandler;
 use servers::server::{ServerHandler, ServerHandlers};
+use servers::tls::TlsOption;
 use snafu::ResultExt;
 
 use crate::config::DatanodeOptions;
@@ -95,6 +96,7 @@ impl<'a> DatanodeServiceBuilder<'a> {
         let config = GrpcServerConfig {
             max_recv_message_size: opts.rpc_max_recv_message_size.as_bytes() as usize,
             max_send_message_size: opts.rpc_max_send_message_size.as_bytes() as usize,
+            tls: TlsOption::default(),
         };
 
         GrpcServerBuilder::new(config, region_server.runtime())

src/datanode/src/tests.rs

@@ -23,7 +23,6 @@ use common_function::function::FunctionRef;
 use common_function::scalars::aggregate::AggregateFunctionMetaRef;
 use common_query::prelude::ScalarUdf;
 use common_query::Output;
-use common_recordbatch::SendableRecordBatchStream;
 use common_runtime::Runtime;
 use query::dataframe::DataFrame;
 use query::plan::LogicalPlan;
@@ -32,7 +31,7 @@ use query::query_engine::DescribeResult;
 use query::{QueryEngine, QueryEngineContext};
 use session::context::QueryContextRef;
 use store_api::metadata::RegionMetadataRef;
-use store_api::region_engine::{RegionEngine, RegionRole, SetReadonlyResponse};
+use store_api::region_engine::{RegionEngine, RegionRole, RegionScannerRef, SetReadonlyResponse};
 use store_api::region_request::{AffectedRows, RegionRequest};
 use store_api::storage::{RegionId, ScanRequest};
 use table::TableRef;
@@ -106,10 +105,11 @@ pub struct MockRegionEngine {
     pub(crate) handle_request_delay: Option<Duration>,
     pub(crate) handle_request_mock_fn: Option<MockRequestHandler>,
     pub(crate) mock_role: Option<Option<RegionRole>>,
+    engine: String,
 }
 
 impl MockRegionEngine {
-    pub fn new() -> (Arc<Self>, Receiver<(RegionId, RegionRequest)>) {
+    pub fn new(engine: &str) -> (Arc<Self>, Receiver<(RegionId, RegionRequest)>) {
         let (tx, rx) = tokio::sync::mpsc::channel(8);
 
         (
@@ -118,12 +118,14 @@ impl MockRegionEngine {
                 sender: tx,
                 handle_request_mock_fn: None,
                 mock_role: None,
+                engine: engine.to_string(),
             }),
             rx,
         )
     }
 
     pub fn with_mock_fn(
+        engine: &str,
         mock_fn: MockRequestHandler,
     ) -> (Arc<Self>, Receiver<(RegionId, RegionRequest)>) {
         let (tx, rx) = tokio::sync::mpsc::channel(8);
@@ -134,12 +136,16 @@ impl MockRegionEngine {
                 sender: tx,
                 handle_request_mock_fn: Some(mock_fn),
                 mock_role: None,
+                engine: engine.to_string(),
             }),
             rx,
         )
     }
 
-    pub fn with_custom_apply_fn<F>(apply: F) -> (Arc<Self>, Receiver<(RegionId, RegionRequest)>)
+    pub fn with_custom_apply_fn<F>(
+        engine: &str,
+        apply: F,
+    ) -> (Arc<Self>, Receiver<(RegionId, RegionRequest)>)
     where
         F: FnOnce(&mut MockRegionEngine),
     {
@@ -149,6 +155,7 @@ impl MockRegionEngine {
             sender: tx,
             handle_request_mock_fn: None,
             mock_role: None,
+            engine: engine.to_string(),
         };
 
         apply(&mut region_engine);
@@ -160,7 +167,7 @@ impl MockRegionEngine {
 #[async_trait::async_trait]
 impl RegionEngine for MockRegionEngine {
     fn name(&self) -> &str {
-        "mock"
+        &self.engine
     }
 
     async fn handle_request(
@@ -185,7 +192,7 @@ impl RegionEngine for MockRegionEngine {
         &self,
         _region_id: RegionId,
         _request: ScanRequest,
-    ) -> Result<SendableRecordBatchStream, BoxedError> {
+    ) -> Result<RegionScannerRef, BoxedError> {
         unimplemented!()
     }
 

src/datatypes/src/arrow_array.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub type BinaryArray = arrow::array::LargeBinaryArray;
-pub type MutableBinaryArray = arrow::array::LargeBinaryBuilder;
+pub type BinaryArray = arrow::array::BinaryArray;
+pub type MutableBinaryArray = arrow::array::BinaryBuilder;
 pub type StringArray = arrow::array::StringArray;
 pub type MutableStringArray = arrow::array::StringBuilder;

src/datatypes/src/types/binary_type.rs

@@ -47,7 +47,7 @@ impl DataType for BinaryType {
     }
 
     fn as_arrow_type(&self) -> ArrowDataType {
-        ArrowDataType::LargeBinary
+        ArrowDataType::Binary
     }
 
     fn create_mutable_vector(&self, capacity: usize) -> Box<dyn MutableVector> {

src/datatypes/src/value.rs

@@ -342,7 +342,7 @@ impl Value {
             Value::Float32(v) => ScalarValue::Float32(Some(v.0)),
             Value::Float64(v) => ScalarValue::Float64(Some(v.0)),
             Value::String(v) => ScalarValue::Utf8(Some(v.as_utf8().to_string())),
-            Value::Binary(v) => ScalarValue::LargeBinary(Some(v.to_vec())),
+            Value::Binary(v) => ScalarValue::Binary(Some(v.to_vec())),
             Value::Date(v) => ScalarValue::Date32(Some(v.val())),
             Value::DateTime(v) => ScalarValue::Date64(Some(v.val())),
             Value::Null => to_null_scalar_value(output_type)?,
@@ -413,7 +413,7 @@ pub fn to_null_scalar_value(output_type: &ConcreteDataType) -> Result<ScalarValu
         ConcreteDataType::UInt64(_) => ScalarValue::UInt64(None),
         ConcreteDataType::Float32(_) => ScalarValue::Float32(None),
         ConcreteDataType::Float64(_) => ScalarValue::Float64(None),
-        ConcreteDataType::Binary(_) => ScalarValue::LargeBinary(None),
+        ConcreteDataType::Binary(_) => ScalarValue::Binary(None),
         ConcreteDataType::String(_) => ScalarValue::Utf8(None),
         ConcreteDataType::Date(_) => ScalarValue::Date32(None),
         ConcreteDataType::DateTime(_) => ScalarValue::Date64(None),
@@ -2105,7 +2105,7 @@ mod tests {
                 .unwrap()
         );
         assert_eq!(
-            ScalarValue::LargeBinary(Some("world".as_bytes().to_vec())),
+            ScalarValue::Binary(Some("world".as_bytes().to_vec())),
             Value::Binary(Bytes::from("world".as_bytes()))
                 .try_to_scalar_value(&ConcreteDataType::binary_datatype())
                 .unwrap()
@@ -2187,7 +2187,7 @@ mod tests {
                 .unwrap()
         );
         assert_eq!(
-            ScalarValue::LargeBinary(None),
+            ScalarValue::Binary(None),
             Value::Null
                 .try_to_scalar_value(&ConcreteDataType::binary_datatype())
                 .unwrap()

src/datatypes/src/vectors/binary.rs

@@ -52,6 +52,14 @@ impl From<Vec<Option<Vec<u8>>>> for BinaryVector {
     }
 }
 
+impl From<Vec<&[u8]>> for BinaryVector {
+    fn from(data: Vec<&[u8]>) -> Self {
+        Self {
+            array: BinaryArray::from_iter_values(data),
+        }
+    }
+}
+
 impl Vector for BinaryVector {
     fn data_type(&self) -> ConcreteDataType {
         ConcreteDataType::binary_datatype()
@@ -257,7 +265,7 @@ mod tests {
 
         let arrow_arr = v.to_arrow_array();
         assert_eq!(2, arrow_arr.len());
-        assert_eq!(&ArrowDataType::LargeBinary, arrow_arr.data_type());
+        assert_eq!(&ArrowDataType::Binary, arrow_arr.data_type());
     }
 
     #[test]

src/datatypes/src/vectors/helper.rs

@@ -258,9 +258,9 @@ impl Helper {
         Ok(match array.as_ref().data_type() {
             ArrowDataType::Null => Arc::new(NullVector::try_from_arrow_array(array)?),
             ArrowDataType::Boolean => Arc::new(BooleanVector::try_from_arrow_array(array)?),
-            ArrowDataType::LargeBinary => Arc::new(BinaryVector::try_from_arrow_array(array)?),
-            ArrowDataType::FixedSizeBinary(_) | ArrowDataType::Binary => {
-                let array = arrow::compute::cast(array.as_ref(), &ArrowDataType::LargeBinary)
+            ArrowDataType::Binary => Arc::new(BinaryVector::try_from_arrow_array(array)?),
+            ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => {
+                let array = arrow::compute::cast(array.as_ref(), &ArrowDataType::Binary)
                     .context(crate::error::ArrowComputeSnafu)?;
                 Arc::new(BinaryVector::try_from_arrow_array(array)?)
             }
@@ -278,7 +278,7 @@ impl Helper {
             ArrowDataType::LargeUtf8 => {
                 let array = arrow::compute::cast(array.as_ref(), &ArrowDataType::Utf8)
                     .context(crate::error::ArrowComputeSnafu)?;
-                Arc::new(BinaryVector::try_from_arrow_array(array)?)
+                Arc::new(StringVector::try_from_arrow_array(array)?)
             }
             ArrowDataType::Date32 => Arc::new(DateVector::try_from_arrow_array(array)?),
             ArrowDataType::Date64 => Arc::new(DateTimeVector::try_from_arrow_array(array)?),
@@ -402,8 +402,10 @@ mod tests {
         TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
         TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
     };
+    use arrow::buffer::Buffer;
     use arrow::datatypes::Int32Type;
-    use arrow_array::DictionaryArray;
+    use arrow_array::{BinaryArray, DictionaryArray, FixedSizeBinaryArray, LargeStringArray};
+    use arrow_schema::DataType;
     use common_decimal::Decimal128;
     use common_time::time::Time;
     use common_time::timestamp::TimeUnit;
@@ -576,10 +578,6 @@ mod tests {
     fn test_try_into_vector() {
         check_try_into_vector(NullArray::new(2));
         check_try_into_vector(BooleanArray::from(vec![true, false]));
-        check_try_into_vector(LargeBinaryArray::from(vec![
-            "hello".as_bytes(),
-            "world".as_bytes(),
-        ]));
         check_try_into_vector(Int8Array::from(vec![1, 2, 3]));
         check_try_into_vector(Int16Array::from(vec![1, 2, 3]));
         check_try_into_vector(Int32Array::from(vec![1, 2, 3]));
@@ -611,6 +609,52 @@ mod tests {
         Helper::try_into_vector(array).unwrap_err();
     }
 
+    #[test]
+    fn test_try_binary_array_into_vector() {
+        let input_vec: Vec<&[u8]> = vec!["hello".as_bytes(), "world".as_bytes()];
+        let assertion_vector = BinaryVector::from(input_vec.clone());
+
+        let input_arrays: Vec<ArrayRef> = vec![
+            Arc::new(LargeBinaryArray::from(input_vec.clone())) as ArrayRef,
+            Arc::new(BinaryArray::from(input_vec.clone())) as ArrayRef,
+            Arc::new(FixedSizeBinaryArray::new(
+                5,
+                Buffer::from_vec("helloworld".as_bytes().to_vec()),
+                None,
+            )) as ArrayRef,
+        ];
+
+        for input_array in input_arrays {
+            let vector = Helper::try_into_vector(input_array).unwrap();
+
+            assert_eq!(2, vector.len());
+            assert_eq!(0, vector.null_count());
+
+            let output_arrow_array: ArrayRef = vector.to_arrow_array();
+            assert_eq!(&DataType::Binary, output_arrow_array.data_type());
+            assert_eq!(&assertion_vector.to_arrow_array(), &output_arrow_array);
+        }
+    }
+
+    #[test]
+    fn test_large_string_array_into_vector() {
+        let input_vec = vec!["a", "b"];
+        let assertion_array = StringArray::from(input_vec.clone());
+
+        let large_string_array: ArrayRef = Arc::new(LargeStringArray::from(input_vec));
+        let vector = Helper::try_into_vector(large_string_array).unwrap();
+        assert_eq!(2, vector.len());
+        assert_eq!(0, vector.null_count());
+
+        let output_arrow_array: StringArray = vector
+            .to_arrow_array()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .clone();
+        assert_eq!(&assertion_array, &output_arrow_array);
+    }
+
     #[test]
     fn test_try_from_scalar_time_value() {
         let vector = Helper::try_from_scalar_value(ScalarValue::Time32Second(Some(42)), 3).unwrap();

src/file-engine/src/engine.rs

@@ -25,7 +25,9 @@ use common_telemetry::{error, info};
 use object_store::ObjectStore;
 use snafu::{ensure, OptionExt};
 use store_api::metadata::RegionMetadataRef;
-use store_api::region_engine::{RegionEngine, RegionRole, SetReadonlyResponse};
+use store_api::region_engine::{
+    RegionEngine, RegionRole, RegionScannerRef, SetReadonlyResponse, SinglePartitionScanner,
+};
 use store_api::region_request::{
     AffectedRows, RegionCloseRequest, RegionCreateRequest, RegionDropRequest, RegionOpenRequest,
     RegionRequest,
@@ -49,6 +51,20 @@ impl FileRegionEngine {
             inner: Arc::new(EngineInner::new(object_store)),
         }
     }
+
+    async fn handle_query(
+        &self,
+        region_id: RegionId,
+        request: ScanRequest,
+    ) -> Result<SendableRecordBatchStream, BoxedError> {
+        self.inner
+            .get_region(region_id)
+            .await
+            .context(RegionNotFoundSnafu { region_id })
+            .map_err(BoxedError::new)?
+            .query(request)
+            .map_err(BoxedError::new)
+    }
 }
 
 #[async_trait]
@@ -72,14 +88,10 @@ impl RegionEngine for FileRegionEngine {
         &self,
         region_id: RegionId,
         request: ScanRequest,
-    ) -> Result<SendableRecordBatchStream, BoxedError> {
-        self.inner
-            .get_region(region_id)
-            .await
-            .context(RegionNotFoundSnafu { region_id })
-            .map_err(BoxedError::new)?
-            .query(request)
-            .map_err(BoxedError::new)
+    ) -> Result<RegionScannerRef, BoxedError> {
+        let stream = self.handle_query(region_id, request).await?;
+        let scanner = Arc::new(SinglePartitionScanner::new(stream));
+        Ok(scanner)
     }
 
     async fn get_metadata(&self, region_id: RegionId) -> Result<RegionMetadataRef, BoxedError> {

src/file-engine/src/query.rs

@@ -21,11 +21,11 @@ use std::task::{Context, Poll};
 
 use common_datasource::object_store::build_backend;
 use common_error::ext::BoxedError;
-use common_query::prelude::Expr;
 use common_recordbatch::adapter::RecordBatchMetrics;
 use common_recordbatch::error::{CastVectorSnafu, ExternalSnafu, Result as RecordBatchResult};
 use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream};
 use datafusion::logical_expr::utils as df_logical_expr_utils;
+use datafusion_expr::expr::Expr;
 use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 use datatypes::vectors::VectorRef;
@@ -113,7 +113,7 @@ impl FileRegion {
 
         let mut aux_column_set = HashSet::new();
         for scan_filter in scan_filters {
-            df_logical_expr_utils::expr_to_columns(scan_filter.df_expr(), &mut aux_column_set)
+            df_logical_expr_utils::expr_to_columns(scan_filter, &mut aux_column_set)
                 .context(ExtractColumnFromFilterSnafu)?;
 
             let all_file_columns = aux_column_set

src/file-engine/src/query/file_stream.rs

@@ -19,7 +19,6 @@ use common_datasource::file_format::json::{JsonFormat, JsonOpener};
 use common_datasource::file_format::orc::{OrcFormat, OrcOpener};
 use common_datasource::file_format::parquet::{DefaultParquetFileReaderFactory, ParquetFormat};
 use common_datasource::file_format::Format;
-use common_query::prelude::Expr;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
 use common_recordbatch::SendableRecordBatchStream;
 use datafusion::common::{Statistics, ToDFSchema};
@@ -32,6 +31,7 @@ use datafusion::physical_expr::execution_props::ExecutionProps;
 use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::SessionContext;
+use datafusion_expr::expr::Expr;
 use datafusion_expr::utils::conjunction;
 use datatypes::arrow::datatypes::Schema as ArrowSchema;
 use datatypes::schema::SchemaRef;
@@ -182,10 +182,7 @@ fn new_parquet_stream_with_exec_plan(
     };
 
     // build predicate filter
-    let filters = filters
-        .iter()
-        .map(|f| f.df_expr().clone())
-        .collect::<Vec<_>>();
+    let filters = filters.to_vec();
     let filters = if let Some(expr) = conjunction(filters) {
         let df_schema = file_schema
             .clone()

src/flow/Cargo.toml

@@ -26,7 +26,10 @@ futures = "0.3"
 # This fork is simply for keeping our dependency in our org, and pin the version
 # it is the same with upstream repo
 async-trait.workspace = true
+common-function.workspace = true
 common-meta.workspace = true
+common-query.workspace = true
+common-recordbatch.workspace = true
 enum-as-inner = "0.6.0"
 greptime-proto.workspace = true
 hydroflow = { git = "https://github.com/GreptimeTeam/hydroflow.git", branch = "main" }

src/flow/src/adapter.rs

@@ -18,7 +18,7 @@
 
 use std::collections::{BTreeMap, HashMap};
 use std::sync::Arc;
-use std::time::Instant;
+use std::time::{Instant, SystemTime};
 
 use api::v1::{RowDeleteRequest, RowDeleteRequests, RowInsertRequest, RowInsertRequests};
 use catalog::CatalogManagerRef;
@@ -26,6 +26,7 @@ use common_base::Plugins;
 use common_error::ext::BoxedError;
 use common_frontend::handler::FrontendInvoker;
 use common_meta::key::TableMetadataManagerRef;
+use common_query::prelude::GREPTIME_TIMESTAMP;
 use common_runtime::JoinHandle;
 use common_telemetry::{debug, info};
 use datatypes::schema::ColumnSchema;
@@ -42,14 +43,13 @@ use tokio::sync::{oneshot, watch, Mutex, RwLock};
 
 use crate::adapter::error::{ExternalSnafu, TableNotFoundSnafu, UnexpectedSnafu};
 pub(crate) use crate::adapter::node_context::FlownodeContext;
-use crate::adapter::parse_expr::parse_fixed;
 use crate::adapter::table_source::TableSource;
 use crate::adapter::util::column_schemas_to_proto;
 use crate::adapter::worker::{create_worker, Worker, WorkerHandle};
 use crate::compute::ErrCollector;
 use crate::expr::GlobalId;
 use crate::repr::{self, DiffRow, Row};
-use crate::transform::sql_to_flow_plan;
+use crate::transform::{register_function_to_query_engine, sql_to_flow_plan};
 
 pub(crate) mod error;
 mod flownode_impl;
@@ -120,6 +120,8 @@ impl FlownodeBuilder {
         );
         let query_engine = query_engine_factory.query_engine();
 
+        register_function_to_query_engine(&query_engine);
+
         let (tx, rx) = oneshot::channel();
 
         let node_id = Some(self.flow_node_id);
@@ -261,7 +263,7 @@ impl FlownodeManager {
             let ctx = Arc::new(QueryContext::with(&catalog, &schema));
             // TODO(discord9): instead of auto build table from request schema, actually build table
             // before `create flow` to be able to assign pk and ts etc.
-            let (primary_keys, schema) = if let Some(table_id) = self
+            let (primary_keys, schema, is_auto_create) = if let Some(table_id) = self
                 .table_info_source
                 .get_table_id_from_name(&table_name)
                 .await?
@@ -278,54 +280,73 @@ impl FlownodeManager {
                     .map(|i| meta.schema.column_schemas[i].name.clone())
                     .collect_vec();
                 let schema = meta.schema.column_schemas;
-                (primary_keys, schema)
+                let is_auto_create = schema
+                    .last()
+                    .map(|s| s.name == GREPTIME_TIMESTAMP)
+                    .unwrap_or(false);
+                (primary_keys, schema, is_auto_create)
             } else {
-                // TODO(discord9): get ts column from `RelationType` once we are done rewriting flow plan to attach ts
-                let (primary_keys, schema) = {
-                    let node_ctx = self.node_context.lock().await;
-                    let gid: GlobalId = node_ctx
-                        .table_repr
-                        .get_by_name(&table_name)
-                        .map(|x| x.1)
-                        .unwrap();
-                    let schema = node_ctx
-                        .schema
-                        .get(&gid)
-                        .with_context(|| TableNotFoundSnafu {
-                            name: format!("Table name = {:?}", table_name),
-                        })?
-                        .clone();
-                    // TODO(discord9): use default key from schema
-                    let primary_keys = schema
-                        .keys
-                        .first()
-                        .map(|v| {
-                            v.column_indices
-                                .iter()
-                                .map(|i| format!("Col_{i}"))
-                                .collect_vec()
-                        })
-                        .unwrap_or_default();
-                    let ts_col = ColumnSchema::new(
-                        "ts",
-                        ConcreteDataType::timestamp_millisecond_datatype(),
-                        true,
-                    )
-                    .with_time_index(true);
+                // TODO(discord9): condiser remove buggy auto create by schema
 
-                    let wout_ts = schema
-                        .column_types
-                        .into_iter()
-                        .enumerate()
-                        .map(|(idx, typ)| {
-                            ColumnSchema::new(format!("Col_{idx}"), typ.scalar_type, typ.nullable)
-                        })
-                        .collect_vec();
-                    let mut with_ts = wout_ts.clone();
-                    with_ts.push(ts_col);
-                    (primary_keys, with_ts)
-                };
-                (primary_keys, schema)
+                let node_ctx = self.node_context.lock().await;
+                let gid: GlobalId = node_ctx
+                    .table_repr
+                    .get_by_name(&table_name)
+                    .map(|x| x.1)
+                    .unwrap();
+                let schema = node_ctx
+                    .schema
+                    .get(&gid)
+                    .with_context(|| TableNotFoundSnafu {
+                        name: format!("Table name = {:?}", table_name),
+                    })?
+                    .clone();
+                // TODO(discord9): use default key from schema
+                let primary_keys = schema
+                    .typ()
+                    .keys
+                    .first()
+                    .map(|v| {
+                        v.column_indices
+                            .iter()
+                            .map(|i| format!("Col_{i}"))
+                            .collect_vec()
+                    })
+                    .unwrap_or_default();
+                let update_at = ColumnSchema::new(
+                    "update_at",
+                    ConcreteDataType::timestamp_millisecond_datatype(),
+                    true,
+                );
+                // TODO(discord9): bugged so we can't infer time index from flow plan, so we have to manually set one
+                let ts_col = ColumnSchema::new(
+                    GREPTIME_TIMESTAMP,
+                    ConcreteDataType::timestamp_millisecond_datatype(),
+                    true,
+                )
+                .with_time_index(true);
+
+                let wout_ts = schema
+                    .typ()
+                    .column_types
+                    .clone()
+                    .into_iter()
+                    .enumerate()
+                    .map(|(idx, typ)| {
+                        let name = schema
+                            .names
+                            .get(idx)
+                            .cloned()
+                            .unwrap_or(format!("Col_{}", idx));
+                        ColumnSchema::new(name, typ.scalar_type, typ.nullable)
+                    })
+                    .collect_vec();
+
+                let mut with_ts = wout_ts.clone();
+                with_ts.push(update_at);
+                with_ts.push(ts_col);
+
+                (primary_keys, with_ts, true)
             };
 
             let proto_schema = column_schemas_to_proto(schema, &primary_keys)?;
@@ -336,16 +357,32 @@ impl FlownodeManager {
                 table_name.join("."),
                 reqs
             );
-
+            let now = SystemTime::now();
+            let now = now
+                .duration_since(SystemTime::UNIX_EPOCH)
+                .map(|s| s.as_millis() as repr::Timestamp)
+                .unwrap_or_else(|_| {
+                    -(SystemTime::UNIX_EPOCH
+                        .duration_since(now)
+                        .unwrap()
+                        .as_millis() as repr::Timestamp)
+                });
             for req in reqs {
                 match req {
                     DiffRequest::Insert(insert) => {
                         let rows_proto: Vec<v1::Row> = insert
                             .into_iter()
                             .map(|(mut row, _ts)| {
-                                row.extend(Some(Value::from(
-                                    common_time::Timestamp::new_millisecond(0),
-                                )));
+                                // `update_at` col
+                                row.extend([Value::from(common_time::Timestamp::new_millisecond(
+                                    now,
+                                ))]);
+                                // ts col, if auto create
+                                if is_auto_create {
+                                    row.extend([Value::from(
+                                        common_time::Timestamp::new_millisecond(0),
+                                    )]);
+                                }
                                 row.into()
                             })
                             .collect::<Vec<_>>();
@@ -476,26 +513,26 @@ impl FlownodeManager {
     /// However this is not blocking and can sometimes return while actual computation is still running in worker thread
     /// TODO(discord9): add flag for subgraph that have input since last run
     pub async fn run_available(&self) -> Result<(), Error> {
-        let now = self.tick_manager.tick();
-
         loop {
+            let now = self.tick_manager.tick();
             for worker in self.worker_handles.iter() {
                 // TODO(discord9): consider how to handle error in individual worker
                 worker.lock().await.run_available(now).await.unwrap();
             }
             // first check how many inputs were sent
-            let send_cnt = match self.node_context.lock().await.flush_all_sender() {
-                Ok(cnt) => cnt,
+            match self.node_context.lock().await.flush_all_sender() {
+                Ok(_) => (),
                 Err(err) => {
                     common_telemetry::error!("Flush send buf errors: {:?}", err);
                     break;
                 }
             };
-            // if no inputs
-            if send_cnt == 0 {
+            // if no thing in send buf then break
+            let buf_len = self.node_context.lock().await.get_send_buf_size();
+            if buf_len == 0 {
                 break;
             } else {
-                debug!("FlownodeManager::run_available: send_cnt={}", send_cnt);
+                debug!("Send buf len = {}", buf_len);
             }
         }
 
@@ -515,6 +552,8 @@ impl FlownodeManager {
         );
         let table_id = region_id.table_id();
         self.node_context.lock().await.send(table_id, rows)?;
+        // TODO(discord9): put it in a background task?
+        self.run_available().await?;
         Ok(())
     }
 }
@@ -536,7 +575,7 @@ impl FlownodeManager {
     /// Return task id if a new task is created, otherwise return None
     ///
     /// steps to create task:
-    /// 1. parse query into typed plan(and optional parse expire_when expr)
+    /// 1. parse query into typed plan(and optional parse expire_after expr)
     /// 2. render source/sink with output table id and used input table id
     #[allow(clippy::too_many_arguments)]
     pub async fn create_flow(
@@ -544,14 +583,14 @@ impl FlownodeManager {
         flow_id: FlowId,
         sink_table_name: TableName,
         source_table_ids: &[TableId],
-        create_if_not_exist: bool,
-        expire_when: Option<String>,
+        create_if_not_exists: bool,
+        expire_after: Option<i64>,
         comment: Option<String>,
         sql: String,
         flow_options: HashMap<String, String>,
         query_ctx: Option<QueryContext>,
     ) -> Result<Option<FlowId>, Error> {
-        if create_if_not_exist {
+        if create_if_not_exists {
             // check if the task already exists
             for handle in self.worker_handles.iter() {
                 if handle.lock().await.contains_flow(flow_id).await? {
@@ -579,22 +618,6 @@ impl FlownodeManager {
         debug!("Flow {:?}'s Plan is {:?}", flow_id, flow_plan);
         node_ctx.assign_table_schema(&sink_table_name, flow_plan.typ.clone())?;
 
-        let expire_when = expire_when
-            .and_then(|s| {
-                if s.is_empty() || s.split_whitespace().join("").is_empty() {
-                    None
-                } else {
-                    Some(s)
-                }
-            })
-            .map(|d| {
-                let d = d.as_ref();
-                parse_fixed(d)
-                    .map(|(_, n)| n)
-                    .map_err(|err| err.to_string())
-            })
-            .transpose()
-            .map_err(|err| UnexpectedSnafu { reason: err }.build())?;
         let _ = comment;
         let _ = flow_options;
 
@@ -627,8 +650,8 @@ impl FlownodeManager {
             sink_sender,
             source_ids,
             src_recvs: source_receivers,
-            expire_when,
-            create_if_not_exist,
+            expire_after,
+            create_if_not_exists,
             err_collector,
         };
         handle.create_flow(create_request).await?;
@@ -641,21 +664,20 @@ impl FlownodeManager {
 ///
 /// TODO(discord9): better way to do it, and not expose flow tick even to other flow to avoid
 /// TSO coord mess
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct FlowTickManager {
     start: Instant,
-}
-
-impl std::fmt::Debug for FlowTickManager {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("FlowTickManager").finish()
-    }
+    start_timestamp: repr::Timestamp,
 }
 
 impl FlowTickManager {
     pub fn new() -> Self {
         FlowTickManager {
             start: Instant::now(),
+            start_timestamp: SystemTime::now()
+                .duration_since(SystemTime::UNIX_EPOCH)
+                .unwrap()
+                .as_millis() as repr::Timestamp,
         }
     }
 
@@ -665,6 +687,6 @@ impl FlowTickManager {
     pub fn tick(&self) -> repr::Timestamp {
         let current = Instant::now();
         let since_the_epoch = current - self.start;
-        since_the_epoch.as_millis() as repr::Timestamp
+        since_the_epoch.as_millis() as repr::Timestamp + self.start_timestamp
     }
 }

src/flow/src/adapter/flownode_impl.rs

@@ -14,13 +14,17 @@
 
 //! impl `FlowNode` trait for FlowNodeManager so standalone can call them
 
+use std::collections::HashMap;
+
 use api::v1::flow::{flow_request, CreateRequest, DropRequest, FlowRequest, FlowResponse};
 use api::v1::region::InsertRequests;
 use common_error::ext::BoxedError;
 use common_meta::error::{ExternalSnafu, Result, UnexpectedSnafu};
 use common_meta::node_manager::Flownode;
+use common_telemetry::debug;
 use itertools::Itertools;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::RegionId;
 
 use crate::adapter::FlownodeManager;
 use crate::repr::{self, DiffRow};
@@ -45,7 +49,7 @@ impl Flownode for FlownodeManager {
                 source_table_ids,
                 sink_table_name: Some(sink_table_name),
                 create_if_not_exists,
-                expire_when,
+                expire_after,
                 comment,
                 sql,
                 flow_options,
@@ -56,13 +60,14 @@ impl Flownode for FlownodeManager {
                     sink_table_name.schema_name,
                     sink_table_name.table_name,
                 ];
+                let expire_after = expire_after.map(|e| e.value);
                 let ret = self
                     .create_flow(
                         task_id.id as u64,
                         sink_table_name,
                         &source_table_ids,
                         create_if_not_exists,
-                        Some(expire_when),
+                        expire_after,
                         Some(comment),
                         sql,
                         flow_options,
@@ -100,12 +105,57 @@ impl Flownode for FlownodeManager {
     async fn handle_inserts(&self, request: InsertRequests) -> Result<FlowResponse> {
         for write_request in request.requests {
             let region_id = write_request.region_id;
-            let rows_proto = write_request.rows.map(|r| r.rows).unwrap_or(vec![]);
+            let table_id = RegionId::from(region_id).table_id();
+
+            let (insert_schema, rows_proto) = write_request
+                .rows
+                .map(|r| (r.schema, r.rows))
+                .unwrap_or_default();
+
             // TODO(discord9): reconsider time assignment mechanism
             let now = self.tick_manager.tick();
+
+            let fetch_order = {
+                let ctx = self.node_context.lock().await;
+                let table_col_names = ctx
+                    .table_repr
+                    .get_by_table_id(&table_id)
+                    .map(|r| r.1)
+                    .and_then(|id| ctx.schema.get(&id))
+                    .map(|desc| &desc.names)
+                    .context(UnexpectedSnafu {
+                        err_msg: format!("Table not found: {}", table_id),
+                    })?;
+                let name_to_col = HashMap::<_, _>::from_iter(
+                    insert_schema
+                        .iter()
+                        .enumerate()
+                        .map(|(i, name)| (&name.column_name, i)),
+                );
+                let fetch_order: Vec<usize> = table_col_names
+                    .iter()
+                    .map(|names| {
+                        name_to_col.get(names).copied().context(UnexpectedSnafu {
+                            err_msg: format!("Column not found: {}", names),
+                        })
+                    })
+                    .try_collect()?;
+                if !fetch_order.iter().enumerate().all(|(i, &v)| i == v) {
+                    debug!("Reordering columns: {:?}", fetch_order)
+                }
+                fetch_order
+            };
+
             let rows: Vec<DiffRow> = rows_proto
                 .into_iter()
-                .map(repr::Row::from)
+                .map(|r| {
+                    let r = repr::Row::from(r);
+                    let reordered = fetch_order
+                        .iter()
+                        .map(|&i| r.inner[i].clone())
+                        .collect_vec();
+                    repr::Row::new(reordered)
+                })
                 .map(|r| (r, now, 1))
                 .collect_vec();
             self.handle_write_request(region_id.into(), rows)

src/flow/src/adapter/node_context.rs

@@ -27,10 +27,10 @@ use crate::adapter::error::{Error, EvalSnafu, TableNotFoundSnafu};
 use crate::adapter::{FlowId, TableName, TableSource};
 use crate::expr::error::InternalSnafu;
 use crate::expr::GlobalId;
-use crate::repr::{DiffRow, RelationType, BROADCAST_CAP};
+use crate::repr::{DiffRow, RelationDesc, RelationType, BROADCAST_CAP};
 
 /// A context that holds the information of the dataflow
-#[derive(Default)]
+#[derive(Default, Debug)]
 pub struct FlownodeContext {
     /// mapping from source table to tasks, useful for schedule which task to run when a source table is updated
     pub source_to_tasks: BTreeMap<TableId, BTreeSet<FlowId>>,
@@ -51,10 +51,8 @@ pub struct FlownodeContext {
             mpsc::UnboundedReceiver<DiffRow>,
         ),
     >,
-    /// store source in buffer for each source table, in case broadcast channel is full
-    pub send_buffer: BTreeMap<TableId, VecDeque<DiffRow>>,
     /// the schema of the table, query from metasrv or inferred from TypedPlan
-    pub schema: HashMap<GlobalId, RelationType>,
+    pub schema: HashMap<GlobalId, RelationDesc>,
     /// All the tables that have been registered in the worker
     pub table_repr: IdToNameMap,
     pub query_context: Option<Arc<QueryContext>>,
@@ -64,6 +62,7 @@ pub struct FlownodeContext {
 ///
 /// receiver still use tokio broadcast channel, since only sender side need to know
 /// backpressure and adjust dataflow running duration to avoid blocking
+#[derive(Debug)]
 pub struct SourceSender {
     sender: broadcast::Sender<DiffRow>,
     send_buf: VecDeque<DiffRow>,
@@ -108,6 +107,7 @@ impl SourceSender {
         }
         if row_cnt > 0 {
             debug!("Send {} rows", row_cnt);
+            debug!("Send buf len = {}", self.send_buf.len());
         }
 
         Ok(row_cnt)
@@ -139,12 +139,19 @@ impl FlownodeContext {
     }
 
     /// flush all sender's buf
+    ///
+    /// return numbers being sent
     pub fn flush_all_sender(&mut self) -> Result<usize, Error> {
         self.source_sender
             .iter_mut()
             .map(|(_table_id, src_sender)| src_sender.try_send_all())
             .try_fold(0, |acc, x| x.map(|x| x + acc))
     }
+
+    /// Return the sum number of rows in all send buf
+    pub fn get_send_buf_size(&self) -> usize {
+        self.source_sender.values().map(|v| v.send_buf.len()).sum()
+    }
 }
 
 impl FlownodeContext {
@@ -225,7 +232,7 @@ impl FlownodeContext {
     /// Retrieves a GlobalId and table schema representing a table previously registered by calling the [register_table] function.
     ///
     /// Returns an error if no table has been registered with the provided names
-    pub fn table(&self, name: &TableName) -> Result<(GlobalId, RelationType), Error> {
+    pub fn table(&self, name: &TableName) -> Result<(GlobalId, RelationDesc), Error> {
         let id = self
             .table_repr
             .get_by_name(name)
@@ -296,7 +303,7 @@ impl FlownodeContext {
             .get_by_name(table_name)
             .map(|(_, gid)| gid)
             .unwrap();
-        self.schema.insert(gid, schema);
+        self.schema.insert(gid, schema.into_unnamed());
         Ok(())
     }
 

src/flow/src/adapter/table_source.rs

@@ -17,7 +17,6 @@
 use common_error::ext::BoxedError;
 use common_meta::key::table_info::{TableInfoManager, TableInfoValue};
 use common_meta::key::table_name::{TableNameKey, TableNameManager};
-use itertools::Itertools;
 use snafu::{OptionExt, ResultExt};
 use table::metadata::TableId;
 
@@ -25,7 +24,7 @@ use crate::adapter::error::{
     Error, ExternalSnafu, TableNotFoundMetaSnafu, TableNotFoundSnafu, UnexpectedSnafu,
 };
 use crate::adapter::TableName;
-use crate::repr::{self, ColumnType, RelationType};
+use crate::repr::{self, ColumnType, RelationDesc, RelationType};
 
 /// mapping of table name <-> table id should be query from tableinfo manager
 pub struct TableSource {
@@ -107,7 +106,7 @@ impl TableSource {
     pub async fn get_table_name_schema(
         &self,
         table_id: &TableId,
-    ) -> Result<(TableName, RelationType), Error> {
+    ) -> Result<(TableName, RelationDesc), Error> {
         let table_info_value = self
             .get_table_info_value(table_id)
             .await?
@@ -123,14 +122,20 @@ impl TableSource {
         ];
 
         let raw_schema = table_info_value.table_info.meta.schema;
-        let column_types = raw_schema
+        let (column_types, col_names): (Vec<_>, Vec<_>) = raw_schema
             .column_schemas
+            .clone()
             .into_iter()
-            .map(|col| ColumnType {
-                nullable: col.is_nullable(),
-                scalar_type: col.data_type,
+            .map(|col| {
+                (
+                    ColumnType {
+                        nullable: col.is_nullable(),
+                        scalar_type: col.data_type,
+                    },
+                    col.name,
+                )
             })
-            .collect_vec();
+            .unzip();
 
         let key = table_info_value.table_info.meta.primary_key_indices;
         let keys = vec![repr::Key::from(key)];
@@ -138,10 +143,13 @@ impl TableSource {
         let time_index = raw_schema.timestamp_index;
         Ok((
             table_name,
-            RelationType {
-                column_types,
-                keys,
-                time_index,
+            RelationDesc {
+                typ: RelationType {
+                    column_types,
+                    keys,
+                    time_index,
+                },
+                names: col_names,
             },
         ))
     }

src/flow/src/adapter/worker.rs

@@ -232,12 +232,12 @@ impl<'s> Worker<'s> {
         source_ids: &[GlobalId],
         src_recvs: Vec<broadcast::Receiver<DiffRow>>,
         // TODO(discord9): set expire duration for all arrangement and compare to sys timestamp instead
-        expire_when: Option<repr::Duration>,
-        create_if_not_exist: bool,
+        expire_after: Option<repr::Duration>,
+        create_if_not_exists: bool,
         err_collector: ErrCollector,
     ) -> Result<Option<FlowId>, Error> {
-        let already_exist = self.task_states.contains_key(&flow_id);
-        match (already_exist, create_if_not_exist) {
+        let already_exists = self.task_states.contains_key(&flow_id);
+        match (already_exists, create_if_not_exists) {
             (true, true) => return Ok(None),
             (true, false) => FlowAlreadyExistSnafu { id: flow_id }.fail()?,
             (false, _) => (),
@@ -247,7 +247,7 @@ impl<'s> Worker<'s> {
             err_collector,
             ..Default::default()
         };
-        cur_task_state.state.set_expire_after(expire_when);
+        cur_task_state.state.set_expire_after(expire_after);
 
         {
             let mut ctx = cur_task_state.new_ctx(sink_id);
@@ -319,8 +319,8 @@ impl<'s> Worker<'s> {
                 sink_sender,
                 source_ids,
                 src_recvs,
-                expire_when,
-                create_if_not_exist,
+                expire_after,
+                create_if_not_exists,
                 err_collector,
             } => {
                 let task_create_result = self.create_flow(
@@ -330,8 +330,8 @@ impl<'s> Worker<'s> {
                     sink_sender,
                     &source_ids,
                     src_recvs,
-                    expire_when,
-                    create_if_not_exist,
+                    expire_after,
+                    create_if_not_exists,
                     err_collector,
                 );
                 Some((
@@ -368,8 +368,8 @@ pub enum Request {
         sink_sender: mpsc::UnboundedSender<DiffRow>,
         source_ids: Vec<GlobalId>,
         src_recvs: Vec<broadcast::Receiver<DiffRow>>,
-        expire_when: Option<repr::Duration>,
-        create_if_not_exist: bool,
+        expire_after: Option<repr::Duration>,
+        create_if_not_exists: bool,
         err_collector: ErrCollector,
     },
     Remove {
@@ -524,8 +524,8 @@ mod test {
             sink_sender: sink_tx,
             source_ids: src_ids,
             src_recvs: vec![rx],
-            expire_when: None,
-            create_if_not_exist: true,
+            expire_after: None,
+            create_if_not_exists: true,
             err_collector: ErrCollector::default(),
         };
         handle.create_flow(create_reqs).await.unwrap();

src/flow/src/compute/render.rs

@@ -223,11 +223,11 @@ mod test {
     use hydroflow::scheduled::graph::Hydroflow;
     use hydroflow::scheduled::graph_ext::GraphExt;
     use hydroflow::scheduled::handoff::VecHandoff;
+    use pretty_assertions::{assert_eq, assert_ne};
 
     use super::*;
     use crate::expr::BinaryFunc;
     use crate::repr::Row;
-
     pub fn run_and_check(
         state: &mut DataflowState,
         df: &mut Hydroflow,

src/flow/src/compute/render/map.rs

@@ -124,9 +124,13 @@ fn mfp_subgraph(
     // 1. Read all updates that were emitted between the last time this arrangement had updates and the current time.
     // 2. Output the updates.
     // 3. Truncate all updates within that range.
-    let from = arrange.read().last_compaction_time().map(|n| n + 1);
+    let from = arrange.read().last_compaction_time();
     let from = from.unwrap_or(repr::Timestamp::MIN);
-    let output_kv = arrange.read().get_updates_in_range(from..=now);
+    let range = (
+        std::ops::Bound::Excluded(from),
+        std::ops::Bound::Included(now),
+    );
+    let output_kv = arrange.read().get_updates_in_range(range);
     // the output is expected to be key -> empty val
     let output = output_kv
         .into_iter()

src/flow/src/compute/render/reduce.rs

@@ -26,7 +26,7 @@ use crate::adapter::error::{Error, PlanSnafu};
 use crate::compute::render::{Context, SubgraphArg};
 use crate::compute::state::Scheduler;
 use crate::compute::types::{Arranged, Collection, CollectionBundle, ErrCollector, Toff};
-use crate::expr::error::{DataTypeSnafu, InternalSnafu};
+use crate::expr::error::{DataAlreadyExpiredSnafu, DataTypeSnafu, InternalSnafu};
 use crate::expr::{AggregateExpr, EvalError, ScalarExpr};
 use crate::plan::{AccumulablePlan, AggrWithIndex, KeyValPlan, Plan, ReducePlan, TypedPlan};
 use crate::repr::{self, DiffRow, KeyValDiffRow, RelationType, Row};
@@ -301,9 +301,13 @@ fn update_reduce_distinct_arrange(
     // Deal with output:
 
     // 1. Read all updates that were emitted between the last time this arrangement had updates and the current time.
-    let from = arrange.read().last_compaction_time().map(|n| n + 1);
+    let from = arrange.read().last_compaction_time();
     let from = from.unwrap_or(repr::Timestamp::MIN);
-    let output_kv = arrange.read().get_updates_in_range(from..=now);
+    let range = (
+        std::ops::Bound::Excluded(from),
+        std::ops::Bound::Included(now),
+    );
+    let output_kv = arrange.read().get_updates_in_range(range);
 
     // 2. Truncate all updates stored in arrangement within that range.
     let run_compaction = || {
@@ -397,6 +401,24 @@ fn reduce_accum_subgraph(
     // TODO(discord9): consider key-based lock
     let mut arrange = arrange.write();
     for (key, value_diffs) in key_to_vals {
+        if let Some(expire_man) = &arrange.get_expire_state() {
+            let mut is_expired = false;
+            err_collector.run(|| {
+                if let Some(expired) = expire_man.get_expire_duration(now, &key)? {
+                    is_expired = true;
+                    DataAlreadyExpiredSnafu {
+                        expired_by: expired,
+                    }
+                    .fail()
+                } else {
+                    Ok(())
+                }
+            });
+            if is_expired {
+                // errors already collected, we can just continue to next key
+                continue;
+            }
+        }
         let col_diffs = {
             let row_len = value_diffs[0].0.len();
             let res = err_collector.run(|| get_col_diffs(value_diffs, row_len));
@@ -739,6 +761,7 @@ mod test {
     use std::cell::RefCell;
     use std::rc::Rc;
 
+    use common_time::{DateTime, Interval, Timestamp};
     use datatypes::data_type::{ConcreteDataType, ConcreteDataType as CDT};
     use hydroflow::scheduled::graph::Hydroflow;
 
@@ -748,6 +771,165 @@ mod test {
     use crate::expr::{self, AggregateFunc, BinaryFunc, GlobalId, MapFilterProject, UnaryFunc};
     use crate::repr::{ColumnType, RelationType};
 
+    /// SELECT sum(number) FROM numbers_with_ts GROUP BY tumble(ts, '1 second', '2021-07-01 00:00:00')
+    /// input table columns: number, ts
+    /// expected: sum(number), window_start, window_end
+    #[test]
+    fn test_tumble_group_by() {
+        let mut df = Hydroflow::new();
+        let mut state = DataflowState::default();
+        let mut ctx = harness_test_ctx(&mut df, &mut state);
+        const START: i64 = 1625097600000;
+        let rows = vec![
+            (1u32, START + 1000),
+            (2u32, START + 1500),
+            (3u32, START + 2000),
+            (1u32, START + 2500),
+            (2u32, START + 3000),
+            (3u32, START + 3500),
+        ];
+        let rows = rows
+            .into_iter()
+            .map(|(number, ts)| {
+                (
+                    Row::new(vec![number.into(), Timestamp::new_millisecond(ts).into()]),
+                    1,
+                    1,
+                )
+            })
+            .collect_vec();
+
+        let collection = ctx.render_constant(rows.clone());
+        ctx.insert_global(GlobalId::User(1), collection);
+
+        let aggr_expr = AggregateExpr {
+            func: AggregateFunc::SumUInt32,
+            expr: ScalarExpr::Column(0),
+            distinct: false,
+        };
+        let expected = TypedPlan {
+            typ: RelationType::new(vec![
+                ColumnType::new(CDT::uint64_datatype(), true), // sum(number)
+                ColumnType::new(CDT::datetime_datatype(), false), // window start
+                ColumnType::new(CDT::datetime_datatype(), false), // window end
+            ]),
+            // TODO(discord9): mfp indirectly ref to key columns
+            /*
+            .with_key(vec![1])
+            .with_time_index(Some(0)),*/
+            plan: Plan::Mfp {
+                input: Box::new(
+                    Plan::Reduce {
+                        input: Box::new(
+                            Plan::Get {
+                                id: crate::expr::Id::Global(GlobalId::User(1)),
+                            }
+                            .with_types(RelationType::new(vec![
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
+                                ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                            ])),
+                        ),
+                        key_val_plan: KeyValPlan {
+                            key_plan: MapFilterProject::new(2)
+                                .map(vec![
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowFloor {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                1_000_000_000,
+                                            ),
+                                            start_time: Some(DateTime::new(1625097600000)),
+                                        },
+                                    ),
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowCeiling {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                1_000_000_000,
+                                            ),
+                                            start_time: Some(DateTime::new(1625097600000)),
+                                        },
+                                    ),
+                                ])
+                                .unwrap()
+                                .project(vec![2, 3])
+                                .unwrap()
+                                .into_safe(),
+                            val_plan: MapFilterProject::new(2)
+                                .project(vec![0, 1])
+                                .unwrap()
+                                .into_safe(),
+                        },
+                        reduce_plan: ReducePlan::Accumulable(AccumulablePlan {
+                            full_aggrs: vec![aggr_expr.clone()],
+                            simple_aggrs: vec![AggrWithIndex::new(aggr_expr.clone(), 0, 0)],
+                            distinct_aggrs: vec![],
+                        }),
+                    }
+                    .with_types(
+                        RelationType::new(vec![
+                            ColumnType::new(CDT::datetime_datatype(), false), // window start
+                            ColumnType::new(CDT::datetime_datatype(), false), // window end
+                            ColumnType::new(CDT::uint64_datatype(), true),    //sum(number)
+                        ])
+                        .with_key(vec![1])
+                        .with_time_index(Some(0)),
+                    ),
+                ),
+                mfp: MapFilterProject::new(3)
+                    .map(vec![
+                        ScalarExpr::Column(2),
+                        ScalarExpr::Column(3),
+                        ScalarExpr::Column(0),
+                        ScalarExpr::Column(1),
+                    ])
+                    .unwrap()
+                    .project(vec![4, 5, 6])
+                    .unwrap(),
+            },
+        };
+
+        let bundle = ctx.render_plan(expected).unwrap();
+
+        let output = get_output_handle(&mut ctx, bundle);
+        drop(ctx);
+        let expected = BTreeMap::from([(
+            1,
+            vec![
+                (
+                    Row::new(vec![
+                        3u64.into(),
+                        Timestamp::new_millisecond(START + 1000).into(),
+                        Timestamp::new_millisecond(START + 2000).into(),
+                    ]),
+                    1,
+                    1,
+                ),
+                (
+                    Row::new(vec![
+                        4u64.into(),
+                        Timestamp::new_millisecond(START + 2000).into(),
+                        Timestamp::new_millisecond(START + 3000).into(),
+                    ]),
+                    1,
+                    1,
+                ),
+                (
+                    Row::new(vec![
+                        5u64.into(),
+                        Timestamp::new_millisecond(START + 3000).into(),
+                        Timestamp::new_millisecond(START + 4000).into(),
+                    ]),
+                    1,
+                    1,
+                ),
+            ],
+        )]);
+        run_and_check(&mut state, &mut df, 1..2, expected, output);
+    }
+
     /// select avg(number) from number;
     #[test]
     fn test_avg_eval() {

src/flow/src/compute/render/src_sink.rs

@@ -55,9 +55,12 @@ impl<'referred, 'df> Context<'referred, 'df> {
             .df
             .add_subgraph_source("source", send_port, move |_ctx, send| {
                 let now = *now.borrow();
-                let arr = arrange_handler_inner.write().get_updates_in_range(..=now);
-                err_collector.run(|| arrange_handler_inner.write().compact_to(now));
+                // write lock to prevent unexpected mutation
+                let mut arranged = arrange_handler_inner.write();
+                let arr = arranged.get_updates_in_range(..=now);
+                err_collector.run(|| arranged.compact_to(now));
 
+                debug!("Call source");
                 let prev_avail = arr.into_iter().map(|((k, _), t, d)| (k, t, d));
                 let mut to_send = Vec::new();
                 let mut to_arrange = Vec::new();
@@ -77,10 +80,10 @@ impl<'referred, 'df> Context<'referred, 'df> {
                         to_arrange.len()
                     );
                 }
-                err_collector.run(|| arrange_handler_inner.write().apply_updates(now, to_arrange));
+                err_collector.run(|| arranged.apply_updates(now, to_arrange));
                 send.give(all);
-                // always schedule source to run at next tick
-                inner_schd.schedule_at(now + 1);
+                // always schedule source to run at now so we can repeatedly run source if needed
+                inner_schd.schedule_at(now);
             });
         schd.set_cur_subgraph(sub);
         let arranged = Arranged::new(arrange_handler);

src/flow/src/expr/error.rs

@@ -100,4 +100,11 @@ pub enum EvalError {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Incoming data already expired by {} ms", expired_by))]
+    DataAlreadyExpired {
+        expired_by: i64,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }

src/flow/src/expr/func.rs

@@ -17,8 +17,10 @@
 use std::collections::HashMap;
 use std::sync::OnceLock;
 
+use common_error::ext::BoxedError;
 use common_telemetry::debug;
-use common_time::DateTime;
+use common_time::timestamp::TimeUnit;
+use common_time::{DateTime, Timestamp};
 use datafusion_expr::Operator;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::types::cast;
@@ -30,14 +32,14 @@ use snafu::{ensure, OptionExt, ResultExt};
 use strum::{EnumIter, IntoEnumIterator};
 use substrait::df_logical_plan::consumer::name_to_op;
 
-use crate::adapter::error::{Error, InvalidQuerySnafu, PlanSnafu};
+use crate::adapter::error::{Error, ExternalSnafu, InvalidQuerySnafu, PlanSnafu};
 use crate::expr::error::{
-    CastValueSnafu, DivisionByZeroSnafu, EvalError, InternalSnafu, TryFromValueSnafu,
-    TypeMismatchSnafu,
+    CastValueSnafu, DivisionByZeroSnafu, EvalError, InternalSnafu, OverflowSnafu,
+    TryFromValueSnafu, TypeMismatchSnafu,
 };
 use crate::expr::signature::{GenericFn, Signature};
-use crate::expr::{InvalidArgumentSnafu, ScalarExpr};
-use crate::repr::{value_to_internal_ts, Row};
+use crate::expr::{InvalidArgumentSnafu, ScalarExpr, TypedExpr};
+use crate::repr::{self, value_to_internal_ts, Row};
 
 /// UnmaterializableFunc is a function that can't be eval independently,
 /// and require special handling
@@ -45,6 +47,11 @@ use crate::repr::{value_to_internal_ts, Row};
 pub enum UnmaterializableFunc {
     Now,
     CurrentSchema,
+    TumbleWindow {
+        ts: Box<TypedExpr>,
+        window_size: common_time::Interval,
+        start_time: Option<DateTime>,
+    },
 }
 
 impl UnmaterializableFunc {
@@ -61,14 +68,51 @@ impl UnmaterializableFunc {
                 output: ConcreteDataType::string_datatype(),
                 generic_fn: GenericFn::CurrentSchema,
             },
+            Self::TumbleWindow { .. } => Signature {
+                input: smallvec![ConcreteDataType::timestamp_millisecond_datatype()],
+                output: ConcreteDataType::timestamp_millisecond_datatype(),
+                generic_fn: GenericFn::TumbleWindow,
+            },
         }
     }
 
     /// Create a UnmaterializableFunc from a string of the function name
-    pub fn from_str(name: &str) -> Result<Self, Error> {
-        match name {
+    pub fn from_str_args(name: &str, args: Vec<TypedExpr>) -> Result<Self, Error> {
+        match name.to_lowercase().as_str() {
             "now" => Ok(Self::Now),
             "current_schema" => Ok(Self::CurrentSchema),
+            "tumble" => {
+                let ts = args.first().context(InvalidQuerySnafu {
+                    reason: "Tumble window function requires a timestamp argument",
+                })?;
+                let window_size = args
+                    .get(1)
+                    .and_then(|expr| expr.expr.as_literal())
+                    .context(InvalidQuerySnafu {
+                        reason: "Tumble window function requires a window size argument"
+                    })?.as_string() // TODO(discord9): since df to substrait convertor does not support interval type yet, we need to take a string and cast it to interval instead
+                    .map(|s|cast(Value::from(s), &ConcreteDataType::interval_month_day_nano_datatype())).transpose().map_err(BoxedError::new).context(
+                        ExternalSnafu
+                    )?.and_then(|v|v.as_interval())
+                    .with_context(||InvalidQuerySnafu {
+                        reason: format!("Tumble window function requires window size argument to be a string describe a interval, found {:?}", args.get(1))
+                    })?;
+                let start_time = match args.get(2) {
+                    Some(start_time) => start_time.expr.as_literal(),
+                    None => None,
+                }
+                .map(|s| cast(s.clone(), &ConcreteDataType::datetime_datatype())).transpose().map_err(BoxedError::new).context(ExternalSnafu)?.map(|v|v.as_datetime().with_context(
+                    ||InvalidQuerySnafu {
+                        reason: format!("Tumble window function requires start time argument to be a datetime describe in string, found {:?}", args.get(2))
+                    }
+                )).transpose()?;
+
+                Ok(Self::TumbleWindow {
+                    ts: Box::new(ts.clone()),
+                    window_size,
+                    start_time,
+                })
+            }
             _ => InvalidQuerySnafu {
                 reason: format!("Unknown unmaterializable function: {}", name),
             }
@@ -87,6 +131,14 @@ pub enum UnaryFunc {
     IsFalse,
     StepTimestamp,
     Cast(ConcreteDataType),
+    TumbleWindowFloor {
+        window_size: common_time::Interval,
+        start_time: Option<DateTime>,
+    },
+    TumbleWindowCeiling {
+        window_size: common_time::Interval,
+        start_time: Option<DateTime>,
+    },
 }
 
 impl UnaryFunc {
@@ -118,6 +170,16 @@ impl UnaryFunc {
                 output: to.clone(),
                 generic_fn: GenericFn::Cast,
             },
+            Self::TumbleWindowFloor { .. } => Signature {
+                input: smallvec![ConcreteDataType::timestamp_millisecond_datatype()],
+                output: ConcreteDataType::timestamp_millisecond_datatype(),
+                generic_fn: GenericFn::TumbleWindow,
+            },
+            Self::TumbleWindowCeiling { .. } => Signature {
+                input: smallvec![ConcreteDataType::timestamp_millisecond_datatype()],
+                output: ConcreteDataType::timestamp_millisecond_datatype(),
+                generic_fn: GenericFn::TumbleWindow,
+            },
         }
     }
 
@@ -211,10 +273,80 @@ impl UnaryFunc {
                 debug!("Cast to type: {to:?}, result: {:?}", res);
                 res
             }
+            Self::TumbleWindowFloor {
+                window_size,
+                start_time,
+            } => {
+                let ts = get_ts_as_millisecond(arg)?;
+                let start_time = start_time.map(|t| t.val());
+                let window_size = (window_size.to_nanosecond() / 1_000_000) as repr::Duration; // nanosecond to millisecond
+                let window_start = get_window_start(ts, window_size, start_time);
+
+                let ret = Timestamp::new_millisecond(window_start);
+                Ok(Value::from(ret))
+            }
+            Self::TumbleWindowCeiling {
+                window_size,
+                start_time,
+            } => {
+                let ts = get_ts_as_millisecond(arg)?;
+                let start_time = start_time.map(|t| t.val());
+                let window_size = (window_size.to_nanosecond() / 1_000_000) as repr::Duration; // nanosecond to millisecond
+                let window_start = get_window_start(ts, window_size, start_time);
+
+                let window_end = window_start + window_size;
+                let ret = Timestamp::new_millisecond(window_end);
+                Ok(Value::from(ret))
+            }
         }
     }
 }
 
+fn get_window_start(
+    ts: repr::Timestamp,
+    window_size: repr::Duration,
+    start_time: Option<repr::Timestamp>,
+) -> repr::Timestamp {
+    let start_time = start_time.unwrap_or(0);
+    // left close right open
+    if ts >= start_time {
+        start_time + (ts - start_time) / window_size * window_size
+    } else {
+        start_time + (ts - start_time) / window_size * window_size
+            - if ((start_time - ts) % window_size) != 0 {
+                window_size
+            } else {
+                0
+            }
+    }
+}
+
+#[test]
+fn test_get_window_start() {
+    assert_eq!(get_window_start(1, 3, None), 0);
+    assert_eq!(get_window_start(3, 3, None), 3);
+    assert_eq!(get_window_start(0, 3, None), 0);
+
+    assert_eq!(get_window_start(-1, 3, None), -3);
+    assert_eq!(get_window_start(-3, 3, None), -3);
+}
+
+fn get_ts_as_millisecond(arg: Value) -> Result<repr::Timestamp, EvalError> {
+    let ts = if let Some(ts) = arg.as_timestamp() {
+        ts.convert_to(TimeUnit::Millisecond)
+            .context(OverflowSnafu)?
+            .value()
+    } else if let Some(ts) = arg.as_datetime() {
+        ts.val()
+    } else {
+        InvalidArgumentSnafu {
+            reason: "Expect input to be timestamp or datetime type",
+        }
+        .fail()?
+    };
+    Ok(ts)
+}
+
 /// BinaryFunc is a function that takes two arguments.
 /// Also notice this enum doesn't contain function arguments, since the arguments are stored in the expression.
 ///

src/flow/src/expr/scalar.rs

@@ -26,10 +26,10 @@ use crate::adapter::error::{
 };
 use crate::expr::error::{EvalError, InvalidArgumentSnafu, OptimizeSnafu};
 use crate::expr::func::{BinaryFunc, UnaryFunc, UnmaterializableFunc, VariadicFunc};
-use crate::repr::ColumnType;
+use crate::repr::{ColumnType, RelationType};
 
 /// A scalar expression with a known type.
-#[derive(Debug, Clone)]
+#[derive(Ord, PartialOrd, Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Hash)]
 pub struct TypedExpr {
     /// The expression.
     pub expr: ScalarExpr,
@@ -43,7 +43,75 @@ impl TypedExpr {
     }
 }
 
-/// TODO(discord9): add tumble function here
+impl TypedExpr {
+    /// expand multi-value expression to multiple expressions with new indices
+    ///
+    /// Currently it just mean expand `TumbleWindow` to `TumbleWindowFloor` and `TumbleWindowCeiling`
+    pub fn expand_multi_value(
+        input_typ: &RelationType,
+        exprs: &[TypedExpr],
+    ) -> Result<Vec<TypedExpr>, Error> {
+        // old indices in mfp, expanded expr
+        let mut ret = vec![];
+        let input_arity = input_typ.column_types.len();
+        for (old_idx, expr) in exprs.iter().enumerate() {
+            if let ScalarExpr::CallUnmaterializable(UnmaterializableFunc::TumbleWindow {
+                ts,
+                window_size,
+                start_time,
+            }) = &expr.expr
+            {
+                let floor = UnaryFunc::TumbleWindowFloor {
+                    window_size: *window_size,
+                    start_time: *start_time,
+                };
+                let ceil = UnaryFunc::TumbleWindowCeiling {
+                    window_size: *window_size,
+                    start_time: *start_time,
+                };
+                let floor = ScalarExpr::CallUnary {
+                    func: floor,
+                    expr: Box::new(ts.expr.clone()),
+                }
+                .with_type(ts.typ.clone());
+                ret.push((None, floor));
+
+                let ceil = ScalarExpr::CallUnary {
+                    func: ceil,
+                    expr: Box::new(ts.expr.clone()),
+                }
+                .with_type(ts.typ.clone());
+                ret.push((None, ceil));
+            } else {
+                ret.push((Some(input_arity + old_idx), expr.clone()))
+            }
+        }
+
+        // get shuffled index(old_idx -> new_idx)
+        // note index is offset by input_arity because mfp is designed to be first include input columns then intermediate columns
+        let shuffle = ret
+            .iter()
+            .map(|(old_idx, _)| *old_idx) // [Option<opt_idx>]
+            .enumerate()
+            .map(|(new, old)| (old, new + input_arity))
+            .flat_map(|(old, new)| old.map(|o| (o, new)))
+            .chain((0..input_arity).map(|i| (i, i))) // also remember to chain the input columns as not changed
+            .collect::<BTreeMap<_, _>>();
+
+        // shuffle expr's index
+        let exprs = ret
+            .into_iter()
+            .map(|(_, mut expr)| {
+                // invariant: it is expect that no expr will try to refer the column being expanded
+                expr.expr.permute_map(&shuffle)?;
+                Ok(expr)
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+
+        Ok(exprs)
+    }
+}
+
 /// A scalar expression, which can be evaluated to a value.
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum ScalarExpr {
@@ -84,6 +152,10 @@ pub enum ScalarExpr {
 }
 
 impl ScalarExpr {
+    pub fn with_type(self, typ: ColumnType) -> TypedExpr {
+        TypedExpr::new(self, typ)
+    }
+
     /// try to determine the type of the expression
     pub fn typ(&self, context: &[ColumnType]) -> Result<ColumnType, Error> {
         match self {

src/flow/src/expr/signature.rs

@@ -64,4 +64,5 @@ pub enum GenericFn {
     // unmaterized func
     Now,
     CurrentSchema,
+    TumbleWindow,
 }

src/flow/src/repr/relation.rs

@@ -206,6 +206,15 @@ impl RelationType {
         self
     }
 
+    /// will also remove time index from keys if it's in keys
+    pub fn with_time_index(mut self, time_index: Option<usize>) -> Self {
+        self.time_index = time_index;
+        for key in &mut self.keys {
+            key.remove_col(time_index.unwrap_or(usize::MAX));
+        }
+        self
+    }
+
     /// Computes the number of columns in the relation.
     pub fn arity(&self) -> usize {
         self.column_types.len()
@@ -253,6 +262,19 @@ impl RelationType {
 
         true
     }
+
+    /// Return relation describe with column names
+    pub fn into_named(self, names: Vec<ColumnName>) -> RelationDesc {
+        RelationDesc { typ: self, names }
+    }
+
+    /// Return relation describe without column names
+    pub fn into_unnamed(self) -> RelationDesc {
+        RelationDesc {
+            typ: self,
+            names: vec![],
+        }
+    }
 }
 
 /// The type of a `Value`
@@ -316,8 +338,8 @@ fn return_true() -> bool {
 /// Individual column names are optional.
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Hash)]
 pub struct RelationDesc {
-    typ: RelationType,
-    names: Vec<ColumnName>,
+    pub typ: RelationType,
+    pub names: Vec<ColumnName>,
 }
 
 impl RelationDesc {

src/flow/src/transform.rs

@@ -130,12 +130,60 @@ pub async fn sql_to_flow_plan(
     Ok(flow_plan)
 }
 
+/// register flow-specific functions to the query engine
+pub fn register_function_to_query_engine(engine: &Arc<dyn QueryEngine>) {
+    engine.register_function(Arc::new(TumbleFunction {}));
+}
+
+#[derive(Debug)]
+pub struct TumbleFunction {}
+
+const TUMBLE_NAME: &str = "tumble";
+
+impl std::fmt::Display for TumbleFunction {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "{}", TUMBLE_NAME.to_ascii_uppercase())
+    }
+}
+
+impl common_function::function::Function for TumbleFunction {
+    fn name(&self) -> &str {
+        TUMBLE_NAME
+    }
+
+    fn return_type(&self, _input_types: &[CDT]) -> common_query::error::Result<CDT> {
+        Ok(CDT::datetime_datatype())
+    }
+
+    fn signature(&self) -> common_query::prelude::Signature {
+        common_query::prelude::Signature::variadic_any(common_query::prelude::Volatility::Immutable)
+    }
+
+    fn eval(
+        &self,
+        _func_ctx: common_function::function::FunctionContext,
+        _columns: &[datatypes::prelude::VectorRef],
+    ) -> common_query::error::Result<datatypes::prelude::VectorRef> {
+        UnexpectedSnafu {
+            reason: "Tumbler function is not implemented for datafusion executor",
+        }
+        .fail()
+        .map_err(BoxedError::new)
+        .context(common_query::error::ExecuteSnafu)
+    }
+}
+
 #[cfg(test)]
 mod test {
     use std::sync::Arc;
 
     use catalog::RegisterTableRequest;
     use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, NUMBERS_TABLE_ID};
+    use common_time::{Date, DateTime};
+    use datatypes::prelude::*;
+    use datatypes::schema::Schema;
+    use datatypes::vectors::VectorRef;
+    use itertools::Itertools;
     use prost::Message;
     use query::parser::QueryLanguageParser;
     use query::plan::LogicalPlan;
@@ -144,23 +192,45 @@ mod test {
     use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
     use substrait_proto::proto;
     use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
+    use table::test_util::MemTable;
 
     use super::*;
     use crate::adapter::node_context::IdToNameMap;
     use crate::repr::ColumnType;
 
     pub fn create_test_ctx() -> FlownodeContext {
-        let gid = GlobalId::User(0);
-        let name = [
-            "greptime".to_string(),
-            "public".to_string(),
-            "numbers".to_string(),
-        ];
-        let schema = RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)]);
+        let mut schemas = HashMap::new();
         let mut tri_map = IdToNameMap::new();
-        tri_map.insert(Some(name.clone()), Some(0), gid);
+        {
+            let gid = GlobalId::User(0);
+            let name = [
+                "greptime".to_string(),
+                "public".to_string(),
+                "numbers".to_string(),
+            ];
+            let schema = RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)]);
+
+            tri_map.insert(Some(name.clone()), Some(1024), gid);
+            schemas.insert(gid, schema.into_unnamed());
+        }
+
+        {
+            let gid = GlobalId::User(1);
+            let name = [
+                "greptime".to_string(),
+                "public".to_string(),
+                "numbers_with_ts".to_string(),
+            ];
+            let schema = RelationType::new(vec![
+                ColumnType::new(CDT::uint32_datatype(), false),
+                ColumnType::new(CDT::datetime_datatype(), false),
+            ]);
+            schemas.insert(gid, schema.into_unnamed());
+            tri_map.insert(Some(name.clone()), Some(1025), gid);
+        }
+
         FlownodeContext {
-            schema: HashMap::from([(gid, schema)]),
+            schema: schemas,
             table_repr: tri_map,
             query_context: Some(Arc::new(QueryContext::with("greptime", "public"))),
             ..Default::default()
@@ -177,9 +247,37 @@ mod test {
             table: NumbersTable::table(NUMBERS_TABLE_ID),
         };
         catalog_list.register_table_sync(req).unwrap();
+
+        let schema = vec![
+            datatypes::schema::ColumnSchema::new("number", CDT::uint32_datatype(), false),
+            datatypes::schema::ColumnSchema::new("ts", CDT::datetime_datatype(), false),
+        ];
+        let mut columns = vec![];
+        let numbers = (1..=10).collect_vec();
+        let column: VectorRef = Arc::new(<u32 as Scalar>::VectorType::from_vec(numbers));
+        columns.push(column);
+
+        let ts = (1..=10).collect_vec();
+        let column: VectorRef = Arc::new(<DateTime as Scalar>::VectorType::from_vec(ts));
+        columns.push(column);
+
+        let schema = Arc::new(Schema::new(schema));
+        let recordbatch = common_recordbatch::RecordBatch::new(schema, columns).unwrap();
+        let table = MemTable::table("numbers_with_ts", recordbatch);
+
+        let req_with_ts = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "numbers_with_ts".to_string(),
+            table_id: 1024,
+            table,
+        };
+        catalog_list.register_table_sync(req_with_ts).unwrap();
+
         let factory = query::QueryEngineFactory::new(catalog_list, None, None, None, false);
 
         let engine = factory.query_engine();
+        engine.register_function(Arc::new(TumbleFunction {}));
 
         assert_eq!("datafusion", engine.name());
         engine

src/flow/src/transform/aggr.rs

@@ -302,8 +302,26 @@ impl TypedPlan {
             return not_impl_err!("Aggregate without an input is not supported");
         };
 
-        let group_exprs =
-            TypedExpr::from_substrait_agg_grouping(ctx, &agg.groupings, &input.typ, extensions)?;
+        let group_exprs = {
+            let group_exprs = TypedExpr::from_substrait_agg_grouping(
+                ctx,
+                &agg.groupings,
+                &input.typ,
+                extensions,
+            )?;
+
+            TypedExpr::expand_multi_value(&input.typ, &group_exprs)?
+        };
+
+        let time_index = group_exprs.iter().position(|expr| {
+            matches!(
+                &expr.expr,
+                ScalarExpr::CallUnary {
+                    func: UnaryFunc::TumbleWindowFloor { .. },
+                    expr: _
+                }
+            )
+        });
 
         let (mut aggr_exprs, post_mfp) =
             AggregateExpr::from_substrait_agg_measures(ctx, &agg.measures, &input.typ, extensions)?;
@@ -314,6 +332,7 @@ impl TypedPlan {
             input.typ.column_types.len(),
         )?;
 
+        // output type is group_exprs + aggr_exprs
         let output_type = {
             let mut output_types = Vec::new();
             // first append group_expr as key, then aggr_expr as value
@@ -332,7 +351,8 @@ impl TypedPlan {
             } else {
                 RelationType::new(output_types).with_key((0..group_exprs.len()).collect_vec())
             }
-        };
+        }
+        .with_time_index(time_index);
 
         // copy aggr_exprs to full_aggrs, and split them into simple_aggrs and distinct_aggrs
         // also set them input/output column
@@ -406,6 +426,7 @@ impl TypedPlan {
 
 #[cfg(test)]
 mod test {
+    use common_time::{DateTime, Interval};
     use datatypes::prelude::ConcreteDataType;
     use pretty_assertions::{assert_eq, assert_ne};
 
@@ -414,6 +435,336 @@ mod test {
     use crate::repr::{self, ColumnType, RelationType};
     use crate::transform::test::{create_test_ctx, create_test_query_engine, sql_to_substrait};
 
+    /// TODO(discord9): add more illegal sql tests
+    #[tokio::test]
+    async fn test_tumble_composite() {
+        let engine = create_test_query_engine();
+        let sql =
+            "SELECT number, avg(number) FROM numbers_with_ts GROUP BY tumble(ts, '1 hour'), number";
+        let plan = sql_to_substrait(engine.clone(), sql).await;
+
+        let mut ctx = create_test_ctx();
+        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).unwrap();
+
+        let aggr_exprs = vec![
+            AggregateExpr {
+                func: AggregateFunc::SumUInt32,
+                expr: ScalarExpr::Column(0),
+                distinct: false,
+            },
+            AggregateExpr {
+                func: AggregateFunc::Count,
+                expr: ScalarExpr::Column(0),
+                distinct: false,
+            },
+        ];
+        let avg_expr = ScalarExpr::If {
+            cond: Box::new(ScalarExpr::Column(4).call_binary(
+                ScalarExpr::Literal(Value::from(0i64), CDT::int64_datatype()),
+                BinaryFunc::NotEq,
+            )),
+            then: Box::new(ScalarExpr::Column(3).call_binary(
+                ScalarExpr::Column(4).call_unary(UnaryFunc::Cast(CDT::uint64_datatype())),
+                BinaryFunc::DivUInt64,
+            )),
+            els: Box::new(ScalarExpr::Literal(Value::Null, CDT::uint64_datatype())),
+        };
+        let expected = TypedPlan {
+            // TODO(discord9): mfp indirectly ref to key columns
+            /*
+            .with_key(vec![1])
+            .with_time_index(Some(0)),*/
+            plan: Plan::Mfp {
+                input: Box::new(
+                    Plan::Reduce {
+                        input: Box::new(
+                            Plan::Get {
+                                id: crate::expr::Id::Global(GlobalId::User(1)),
+                            }
+                            .with_types(RelationType::new(vec![
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
+                                ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                            ])),
+                        ),
+                        key_val_plan: KeyValPlan {
+                            key_plan: MapFilterProject::new(2)
+                                .map(vec![
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowFloor {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                3_600_000_000_000,
+                                            ),
+                                            start_time: None,
+                                        },
+                                    ),
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowCeiling {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                3_600_000_000_000,
+                                            ),
+                                            start_time: None,
+                                        },
+                                    ),
+                                    ScalarExpr::Column(0),
+                                ])
+                                .unwrap()
+                                .project(vec![2, 3, 4])
+                                .unwrap()
+                                .into_safe(),
+                            val_plan: MapFilterProject::new(2)
+                                .project(vec![0, 1])
+                                .unwrap()
+                                .into_safe(),
+                        },
+                        reduce_plan: ReducePlan::Accumulable(AccumulablePlan {
+                            full_aggrs: aggr_exprs.clone(),
+                            simple_aggrs: vec![
+                                AggrWithIndex::new(aggr_exprs[0].clone(), 0, 0),
+                                AggrWithIndex::new(aggr_exprs[1].clone(), 0, 1),
+                            ],
+                            distinct_aggrs: vec![],
+                        }),
+                    }
+                    .with_types(
+                        RelationType::new(vec![
+                            // keys
+                            ColumnType::new(CDT::datetime_datatype(), false), // window start(time index)
+                            ColumnType::new(CDT::datetime_datatype(), false), // window end(pk)
+                            ColumnType::new(CDT::uint32_datatype(), false),   // number(pk)
+                            // values
+                            ColumnType::new(CDT::uint64_datatype(), true), // avg.sum(number)
+                            ColumnType::new(CDT::int64_datatype(), true),  // avg.count(number)
+                        ])
+                        .with_key(vec![1, 2])
+                        .with_time_index(Some(0)),
+                    ),
+                ),
+                mfp: MapFilterProject::new(5)
+                    .map(vec![
+                        avg_expr,
+                        ScalarExpr::Column(2), // number(pk)
+                        ScalarExpr::Column(5), // avg.sum(number)
+                        ScalarExpr::Column(0), // window start
+                        ScalarExpr::Column(1), // window end
+                    ])
+                    .unwrap()
+                    .project(vec![6, 7, 8, 9])
+                    .unwrap(),
+            },
+            typ: RelationType::new(vec![
+                ColumnType::new(CDT::uint32_datatype(), false), // number
+                ColumnType::new(CDT::uint64_datatype(), true),  // avg(number)
+                ColumnType::new(CDT::datetime_datatype(), false), // window start
+                ColumnType::new(CDT::datetime_datatype(), false), // window end
+            ]),
+        };
+        assert_eq!(flow_plan, expected);
+    }
+
+    #[tokio::test]
+    async fn test_tumble_parse_optional() {
+        let engine = create_test_query_engine();
+        let sql = "SELECT sum(number) FROM numbers_with_ts GROUP BY tumble(ts, '1 hour')";
+        let plan = sql_to_substrait(engine.clone(), sql).await;
+
+        let mut ctx = create_test_ctx();
+        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).unwrap();
+
+        let aggr_expr = AggregateExpr {
+            func: AggregateFunc::SumUInt32,
+            expr: ScalarExpr::Column(0),
+            distinct: false,
+        };
+        let expected = TypedPlan {
+            typ: RelationType::new(vec![
+                ColumnType::new(CDT::uint64_datatype(), true), // sum(number)
+                ColumnType::new(CDT::datetime_datatype(), false), // window start
+                ColumnType::new(CDT::datetime_datatype(), false), // window end
+            ]),
+            // TODO(discord9): mfp indirectly ref to key columns
+            /*
+            .with_key(vec![1])
+            .with_time_index(Some(0)),*/
+            plan: Plan::Mfp {
+                input: Box::new(
+                    Plan::Reduce {
+                        input: Box::new(
+                            Plan::Get {
+                                id: crate::expr::Id::Global(GlobalId::User(1)),
+                            }
+                            .with_types(RelationType::new(vec![
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
+                                ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                            ])),
+                        ),
+                        key_val_plan: KeyValPlan {
+                            key_plan: MapFilterProject::new(2)
+                                .map(vec![
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowFloor {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                3_600_000_000_000,
+                                            ),
+                                            start_time: None,
+                                        },
+                                    ),
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowCeiling {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                3_600_000_000_000,
+                                            ),
+                                            start_time: None,
+                                        },
+                                    ),
+                                ])
+                                .unwrap()
+                                .project(vec![2, 3])
+                                .unwrap()
+                                .into_safe(),
+                            val_plan: MapFilterProject::new(2)
+                                .project(vec![0, 1])
+                                .unwrap()
+                                .into_safe(),
+                        },
+                        reduce_plan: ReducePlan::Accumulable(AccumulablePlan {
+                            full_aggrs: vec![aggr_expr.clone()],
+                            simple_aggrs: vec![AggrWithIndex::new(aggr_expr.clone(), 0, 0)],
+                            distinct_aggrs: vec![],
+                        }),
+                    }
+                    .with_types(
+                        RelationType::new(vec![
+                            ColumnType::new(CDT::datetime_datatype(), false), // window start
+                            ColumnType::new(CDT::datetime_datatype(), false), // window end
+                            ColumnType::new(CDT::uint64_datatype(), true),    //sum(number)
+                        ])
+                        .with_key(vec![1])
+                        .with_time_index(Some(0)),
+                    ),
+                ),
+                mfp: MapFilterProject::new(3)
+                    .map(vec![
+                        ScalarExpr::Column(2),
+                        ScalarExpr::Column(3),
+                        ScalarExpr::Column(0),
+                        ScalarExpr::Column(1),
+                    ])
+                    .unwrap()
+                    .project(vec![4, 5, 6])
+                    .unwrap(),
+            },
+        };
+        assert_eq!(flow_plan, expected);
+    }
+
+    #[tokio::test]
+    async fn test_tumble_parse() {
+        let engine = create_test_query_engine();
+        let sql = "SELECT sum(number) FROM numbers_with_ts GROUP BY tumble(ts, '1 hour', '2021-07-01 00:00:00')";
+        let plan = sql_to_substrait(engine.clone(), sql).await;
+
+        let mut ctx = create_test_ctx();
+        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).unwrap();
+
+        let aggr_expr = AggregateExpr {
+            func: AggregateFunc::SumUInt32,
+            expr: ScalarExpr::Column(0),
+            distinct: false,
+        };
+        let expected = TypedPlan {
+            typ: RelationType::new(vec![
+                ColumnType::new(CDT::uint64_datatype(), true), // sum(number)
+                ColumnType::new(CDT::datetime_datatype(), false), // window start
+                ColumnType::new(CDT::datetime_datatype(), false), // window end
+            ]),
+            // TODO(discord9): mfp indirectly ref to key columns
+            /*
+            .with_key(vec![1])
+            .with_time_index(Some(0)),*/
+            plan: Plan::Mfp {
+                input: Box::new(
+                    Plan::Reduce {
+                        input: Box::new(
+                            Plan::Get {
+                                id: crate::expr::Id::Global(GlobalId::User(1)),
+                            }
+                            .with_types(RelationType::new(vec![
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
+                                ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                            ])),
+                        ),
+                        key_val_plan: KeyValPlan {
+                            key_plan: MapFilterProject::new(2)
+                                .map(vec![
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowFloor {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                3_600_000_000_000,
+                                            ),
+                                            start_time: Some(DateTime::new(1625097600000)),
+                                        },
+                                    ),
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowCeiling {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                3_600_000_000_000,
+                                            ),
+                                            start_time: Some(DateTime::new(1625097600000)),
+                                        },
+                                    ),
+                                ])
+                                .unwrap()
+                                .project(vec![2, 3])
+                                .unwrap()
+                                .into_safe(),
+                            val_plan: MapFilterProject::new(2)
+                                .project(vec![0, 1])
+                                .unwrap()
+                                .into_safe(),
+                        },
+                        reduce_plan: ReducePlan::Accumulable(AccumulablePlan {
+                            full_aggrs: vec![aggr_expr.clone()],
+                            simple_aggrs: vec![AggrWithIndex::new(aggr_expr.clone(), 0, 0)],
+                            distinct_aggrs: vec![],
+                        }),
+                    }
+                    .with_types(
+                        RelationType::new(vec![
+                            ColumnType::new(CDT::datetime_datatype(), false), // window start
+                            ColumnType::new(CDT::datetime_datatype(), false), // window end
+                            ColumnType::new(CDT::uint64_datatype(), true),    //sum(number)
+                        ])
+                        .with_key(vec![1])
+                        .with_time_index(Some(0)),
+                    ),
+                ),
+                mfp: MapFilterProject::new(3)
+                    .map(vec![
+                        ScalarExpr::Column(2),
+                        ScalarExpr::Column(3),
+                        ScalarExpr::Column(0),
+                        ScalarExpr::Column(1),
+                    ])
+                    .unwrap()
+                    .project(vec![4, 5, 6])
+                    .unwrap(),
+            },
+        };
+        assert_eq!(flow_plan, expected);
+    }
+
     #[tokio::test]
     async fn test_avg_group_by() {
         let engine = create_test_query_engine();
@@ -514,7 +865,8 @@ mod test {
         let plan = sql_to_substrait(engine.clone(), sql).await;
 
         let mut ctx = create_test_ctx();
-        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan);
+
+        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).unwrap();
 
         let aggr_exprs = vec![
             AggregateExpr {
@@ -587,7 +939,7 @@ mod test {
                     .unwrap(),
             },
         };
-        assert_eq!(flow_plan.unwrap(), expected);
+        assert_eq!(flow_plan, expected);
     }
 
     #[tokio::test]

src/flow/src/transform/expr.rs

@@ -71,7 +71,7 @@ impl TypedExpr {
                     ),
                 })?;
         let arg_len = f.arguments.len();
-        let arg_exprs: Vec<TypedExpr> = f
+        let arg_typed_exprs: Vec<TypedExpr> = f
             .arguments
             .iter()
             .map(|arg| match &arg.arg_type {
@@ -83,7 +83,8 @@ impl TypedExpr {
             .try_collect()?;
 
         // literal's type is determined by the function and type of other args
-        let (arg_exprs, arg_types): (Vec<_>, Vec<_>) = arg_exprs
+        let (arg_exprs, arg_types): (Vec<_>, Vec<_>) = arg_typed_exprs
+            .clone()
             .into_iter()
             .map(
                 |TypedExpr {
@@ -100,8 +101,7 @@ impl TypedExpr {
             .unzip();
 
         match arg_len {
-            // because variadic function can also have 1 arguments, we need to check if it's a variadic function first
-            1 if VariadicFunc::from_str_and_types(fn_name, &arg_types).is_err() => {
+            1 if UnaryFunc::from_str_and_type(fn_name, None).is_ok() => {
                 let func = UnaryFunc::from_str_and_type(fn_name, None)?;
                 let arg = arg_exprs[0].clone();
                 let ret_type = ColumnType::new_nullable(func.signature().output.clone());
@@ -123,8 +123,13 @@ impl TypedExpr {
 
                 Ok(TypedExpr::new(arg.call_unary(func), ret_type))
             }
-            // because variadic function can also have 2 arguments, we need to check if it's a variadic function first
-            2 if VariadicFunc::from_str_and_types(fn_name, &arg_types).is_err() => {
+            2 if BinaryFunc::from_str_expr_and_type(
+                fn_name,
+                &arg_exprs,
+                arg_types.get(0..2).expect("arg have 2 elements"),
+            )
+            .is_ok() =>
+            {
                 let (func, signature) =
                     BinaryFunc::from_str_expr_and_type(fn_name, &arg_exprs, &arg_types[0..2])?;
 
@@ -174,7 +179,9 @@ impl TypedExpr {
                     };
                     expr.optimize();
                     Ok(TypedExpr::new(expr, ret_type))
-                } else if let Ok(func) = UnmaterializableFunc::from_str(fn_name) {
+                } else if let Ok(func) =
+                    UnmaterializableFunc::from_str_args(fn_name, arg_typed_exprs)
+                {
                     let ret_type = ColumnType::new_nullable(func.signature().output.clone());
                     Ok(TypedExpr::new(
                         ScalarExpr::CallUnmaterializable(func),

src/flow/src/transform/plan.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::BTreeMap;
+
 use itertools::Itertools;
 use snafu::OptionExt;
 use substrait_proto::proto::expression::MaskExpression;
@@ -22,8 +24,8 @@ use substrait_proto::proto::{plan_rel, Plan as SubPlan, Rel};
 use crate::adapter::error::{
     Error, InvalidQuerySnafu, NotImplementedSnafu, PlanSnafu, UnexpectedSnafu,
 };
-use crate::expr::{MapFilterProject, TypedExpr};
-use crate::plan::{Plan, TypedPlan};
+use crate::expr::{MapFilterProject, ScalarExpr, TypedExpr, UnaryFunc};
+use crate::plan::{KeyValPlan, Plan, ReducePlan, TypedPlan};
 use crate::repr::{self, RelationType};
 use crate::transform::{substrait_proto, FlownodeContext, FunctionExtensions};
 
@@ -75,6 +77,7 @@ impl TypedPlan {
                 } else {
                     return not_impl_err!("Projection without an input is not supported");
                 };
+
                 let mut exprs: Vec<TypedExpr> = vec![];
                 for e in &p.expressions {
                     let expr = TypedExpr::from_substrait_rex(e, &input.typ, extensions)?;
@@ -97,6 +100,127 @@ impl TypedPlan {
                     };
                     Ok(TypedPlan { typ, plan })
                 } else {
+                    /// if reduce_plan contains the special function like tumble floor/ceiling, add them to the proj_exprs
+                    fn rewrite_projection_after_reduce(
+                        key_val_plan: KeyValPlan,
+                        _reduce_plan: ReducePlan,
+                        reduce_output_type: &RelationType,
+                        proj_exprs: &mut Vec<TypedExpr>,
+                    ) -> Result<(), Error> {
+                        // TODO: get keys correctly
+                        let key_exprs = key_val_plan
+                            .key_plan
+                            .projection
+                            .clone()
+                            .into_iter()
+                            .map(|i| {
+                                if i < key_val_plan.key_plan.input_arity {
+                                    ScalarExpr::Column(i)
+                                } else {
+                                    key_val_plan.key_plan.expressions
+                                        [i - key_val_plan.key_plan.input_arity]
+                                        .clone()
+                                }
+                            })
+                            .collect_vec();
+                        let mut shift_offset = 0;
+                        let special_keys = key_exprs
+                            .into_iter()
+                            .enumerate()
+                            .filter(|(_idx, p)| {
+                                if matches!(
+                                    p,
+                                    ScalarExpr::CallUnary {
+                                        func: UnaryFunc::TumbleWindowFloor { .. },
+                                        ..
+                                    } | ScalarExpr::CallUnary {
+                                        func: UnaryFunc::TumbleWindowCeiling { .. },
+                                        ..
+                                    }
+                                ) {
+                                    if matches!(
+                                        p,
+                                        ScalarExpr::CallUnary {
+                                            func: UnaryFunc::TumbleWindowFloor { .. },
+                                            ..
+                                        }
+                                    ) {
+                                        shift_offset += 1;
+                                    }
+                                    true
+                                } else {
+                                    false
+                                }
+                            })
+                            .collect_vec();
+                        let spec_key_arity = special_keys.len();
+                        if spec_key_arity == 0 {
+                            return Ok(());
+                        }
+
+                        {
+                            // shift proj_exprs to the right by spec_key_arity
+                            let max_used_col_in_proj = proj_exprs
+                                .iter()
+                                .map(|expr| {
+                                    expr.expr
+                                        .get_all_ref_columns()
+                                        .into_iter()
+                                        .max()
+                                        .unwrap_or_default()
+                                })
+                                .max()
+                                .unwrap_or_default();
+
+                            let shuffle = (0..=max_used_col_in_proj)
+                                .map(|col| (col, col + shift_offset))
+                                .collect::<BTreeMap<_, _>>();
+                            for proj_expr in proj_exprs.iter_mut() {
+                                proj_expr.expr.permute_map(&shuffle)?;
+                            } // add key to the end
+                            for (key_idx, _key_expr) in special_keys {
+                                // here we assume the output type of reduce operator is just first keys columns, then append value columns
+                                proj_exprs.push(
+                                    ScalarExpr::Column(key_idx).with_type(
+                                        reduce_output_type.column_types[key_idx].clone(),
+                                    ),
+                                );
+                            }
+                        }
+
+                        Ok(())
+                    }
+
+                    match input.plan.clone() {
+                        Plan::Reduce {
+                            key_val_plan,
+                            reduce_plan,
+                            ..
+                        } => {
+                            rewrite_projection_after_reduce(
+                                key_val_plan,
+                                reduce_plan,
+                                &input.typ,
+                                &mut exprs,
+                            )?;
+                        }
+                        Plan::Mfp { input, mfp: _ } => {
+                            if let Plan::Reduce {
+                                key_val_plan,
+                                reduce_plan,
+                                ..
+                            } = input.plan
+                            {
+                                rewrite_projection_after_reduce(
+                                    key_val_plan,
+                                    reduce_plan,
+                                    &input.typ,
+                                    &mut exprs,
+                                )?;
+                            }
+                        }
+                        _ => (),
+                    }
                     input.projection(exprs)
                 }
             }
@@ -145,7 +269,7 @@ impl TypedPlan {
                         id: crate::expr::Id::Global(table.0),
                     };
                     let get_table = TypedPlan {
-                        typ: table.1,
+                        typ: table.1.typ().clone(),
                         plan: get_table,
                     };
 

src/flow/src/utils.rs

@@ -18,6 +18,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::ops::Bound;
 use std::sync::Arc;
 
+use common_telemetry::debug;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 use smallvec::{smallvec, SmallVec};
@@ -86,7 +87,7 @@ impl KeyExpiryManager {
     ///
     /// - If given key is expired by now (that is less than `now - expiry_duration`), return the amount of time it's expired.
     /// - If it's not expired, return None
-    pub fn update_event_ts(
+    pub fn get_expire_duration_and_update_event_ts(
         &mut self,
         now: Timestamp,
         row: &Row,
@@ -95,6 +96,33 @@ impl KeyExpiryManager {
             return Ok(None);
         };
 
+        self.event_ts_to_key
+            .entry(event_ts)
+            .or_default()
+            .insert(row.clone());
+
+        if let Some(expire_time) = self.compute_expiration_timestamp(now) {
+            if expire_time > event_ts {
+                // return how much time it's expired
+                return Ok(Some(expire_time - event_ts));
+            }
+        }
+
+        Ok(None)
+    }
+
+    /// Get the expire duration of a key, if it's expired by now.
+    ///
+    /// Return None if the key is not expired
+    pub fn get_expire_duration(
+        &self,
+        now: Timestamp,
+        row: &Row,
+    ) -> Result<Option<Duration>, EvalError> {
+        let Some(event_ts) = self.extract_event_ts(row)? else {
+            return Ok(None);
+        };
+
         if let Some(expire_time) = self.compute_expiration_timestamp(now) {
             if expire_time > event_ts {
                 // return how much time it's expired
@@ -102,10 +130,6 @@ impl KeyExpiryManager {
             }
         }
 
-        self.event_ts_to_key
-            .entry(event_ts)
-            .or_default()
-            .insert(row.clone());
         Ok(None)
     }
 
@@ -189,6 +213,10 @@ impl Arrangement {
         }
     }
 
+    pub fn get_expire_state(&self) -> Option<&KeyExpiryManager> {
+        self.expire_state.as_ref()
+    }
+
     pub fn set_expire_state(&mut self, expire_state: KeyExpiryManager) {
         self.expire_state = Some(expire_state);
     }
@@ -208,8 +236,12 @@ impl Arrangement {
         for ((key, val), update_ts, diff) in updates {
             // check if the key is expired
             if let Some(s) = &mut self.expire_state {
-                if let Some(expired_by) = s.update_event_ts(now, &key)? {
+                if let Some(expired_by) = s.get_expire_duration_and_update_event_ts(now, &key)? {
                     max_expired_by = max_expired_by.max(Some(expired_by));
+                    debug!(
+                        "Expired key: {:?}, expired by: {:?} with time being now={}",
+                        key, expired_by, now
+                    );
                     continue;
                 }
             }
@@ -335,7 +367,9 @@ impl Arrangement {
             for (key, updates) in batch {
                 // check if the key is expired
                 if let Some(s) = &mut self.expire_state {
-                    if let Some(expired_by) = s.update_event_ts(now, &key)? {
+                    if let Some(expired_by) =
+                        s.get_expire_duration_and_update_event_ts(now, &key)?
+                    {
                         max_expired_by = max_expired_by.max(Some(expired_by));
                         continue;
                     }
@@ -540,6 +574,10 @@ impl ArrangeHandler {
     pub fn set_full_arrangement(&self, full: bool) {
         self.write().full_arrangement = full;
     }
+
+    pub fn is_full_arrangement(&self) -> bool {
+        self.read().full_arrangement
+    }
 }
 
 #[cfg(test)]

src/frontend/src/error.rs

@@ -355,6 +355,14 @@ pub enum Error {
         location: Location,
         name: String,
     },
+
+    #[snafu(display("Invalid tls config"))]
+    InvalidTlsConfig {
+        #[snafu(source)]
+        error: common_grpc::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -374,7 +382,8 @@ impl ErrorExt for Error {
             | Error::IllegalAuthConfig { .. }
             | Error::EmptyData { .. }
             | Error::ColumnNoneDefaultValue { .. }
-            | Error::IncompleteGrpcRequest { .. } => StatusCode::InvalidArguments,
+            | Error::IncompleteGrpcRequest { .. }
+            | Error::InvalidTlsConfig { .. } => StatusCode::InvalidArguments,
 
             Error::NotSupported { .. } => StatusCode::Unsupported,
 

src/frontend/src/heartbeat.rs

@@ -151,7 +151,7 @@ impl HeartbeatTask {
                                     Some(req)
                                 }
                                 Err(e) => {
-                                    error!(e;"Failed to encode mailbox messages!");
+                                    error!(e; "Failed to encode mailbox messages");
                                     None
                                 }
                             }

src/frontend/src/server.rs

@@ -76,9 +76,12 @@ where
         let grpc_config = GrpcServerConfig {
             max_recv_message_size: opts.max_recv_message_size.as_bytes() as usize,
             max_send_message_size: opts.max_send_message_size.as_bytes() as usize,
+            tls: opts.tls.clone(),
         };
-
-        Ok(GrpcServerBuilder::new(grpc_config, grpc_runtime))
+        let builder = GrpcServerBuilder::new(grpc_config, grpc_runtime)
+            .with_tls_config(opts.tls.clone())
+            .context(error::InvalidTlsConfigSnafu)?;
+        Ok(builder)
     }
 
     pub fn http_server_builder(&self, opts: &FrontendOptions) -> HttpServerBuilder {