feat: add sqlness test for bloom filter index (#5240 )

* feat: add sqlness test for bloom filter index Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * drop table after finished Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * redact more variables Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
fix: disable path label in opendal for now (#5247 )
2026-01-03 20:02:54 +00:00 · 2024-12-27 06:40:18 +00:00 · 2024-12-27 04:34:19 +00:00 · 2024-12-26 15:07:13 +00:00 · 2024-12-26 12:39:32 +00:00 · 2024-12-26 04:44:03 +00:00
213 changed files with 10450 additions and 2276 deletions
--- a/.github/actions/build-greptime-binary/action.yml
+++ b/.github/actions/build-greptime-binary/action.yml
@@ -54,7 +54,7 @@ runs:
        PROFILE_TARGET: ${{ inputs.cargo-profile == 'dev' && 'debug' || inputs.cargo-profile }}
      with:
        artifacts-dir: ${{ inputs.artifacts-dir }}
-        target-file: ./target/$PROFILE_TARGET/greptime
+        target-files: ./target/$PROFILE_TARGET/greptime
        version: ${{ inputs.version }}
        working-dir: ${{ inputs.working-dir }}

@@ -72,6 +72,6 @@ runs:
      if: ${{ inputs.build-android-artifacts == 'true' }}
      with:
        artifacts-dir: ${{ inputs.artifacts-dir }}
-        target-file: ./target/aarch64-linux-android/release/greptime
+        target-files: ./target/aarch64-linux-android/release/greptime
        version: ${{ inputs.version }}
        working-dir: ${{ inputs.working-dir }}
--- a/.github/actions/build-macos-artifacts/action.yml
+++ b/.github/actions/build-macos-artifacts/action.yml
@@ -90,5 +90,5 @@ runs:
      uses: ./.github/actions/upload-artifacts
      with:
        artifacts-dir: ${{ inputs.artifacts-dir }}
-        target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
+        target-files: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
        version: ${{ inputs.version }}
--- a/.github/actions/build-windows-artifacts/action.yml
+++ b/.github/actions/build-windows-artifacts/action.yml
@@ -76,5 +76,5 @@ runs:
      uses: ./.github/actions/upload-artifacts
      with:
        artifacts-dir: ${{ inputs.artifacts-dir }}
-        target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
+        target-files: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime,target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime.pdb
        version: ${{ inputs.version }}
--- a/.github/actions/setup-greptimedb-cluster/with-minio-and-cache.yaml
+++ b/.github/actions/setup-greptimedb-cluster/with-minio-and-cache.yaml
@@ -5,7 +5,7 @@ meta:

    [datanode]
    [datanode.client]
-    timeout = "60s"
+    timeout = "120s"
 datanode:
  configData: |-
    [runtime]
@@ -21,7 +21,7 @@ frontend:
    global_rt_size = 4

    [meta_client]
-    ddl_timeout = "60s"
+    ddl_timeout = "120s"
 objectStorage:
  s3:
    bucket: default
--- a/.github/actions/setup-greptimedb-cluster/with-minio.yaml
+++ b/.github/actions/setup-greptimedb-cluster/with-minio.yaml
@@ -5,7 +5,7 @@ meta:
    
    [datanode]
    [datanode.client]
-    timeout = "60s"
+    timeout = "120s"
 datanode:
  configData: |-
    [runtime]
@@ -17,7 +17,7 @@ frontend:
    global_rt_size = 4

    [meta_client]
-    ddl_timeout = "60s"
+    ddl_timeout = "120s"
 objectStorage:
  s3:
    bucket: default
--- a/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml
+++ b/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml
@@ -11,7 +11,7 @@ meta:
        
    [datanode]
    [datanode.client]
-    timeout = "60s"
+    timeout = "120s"
 datanode:
  configData: |-
    [runtime]
@@ -28,7 +28,7 @@ frontend:
    global_rt_size = 4

    [meta_client]
-    ddl_timeout = "60s"
+    ddl_timeout = "120s"
 objectStorage:
  s3:
    bucket: default
--- a/.github/actions/upload-artifacts/action.yml
+++ b/.github/actions/upload-artifacts/action.yml
@@ -4,8 +4,8 @@ inputs:
  artifacts-dir:
    description: Directory to store artifacts
    required: true
-  target-file:
-    description: The path of the target artifact
+  target-files:
+    description: The multiple target files to upload, separated by comma
    required: false
  version:
    description: Version of the artifact
@@ -18,12 +18,16 @@ runs:
  using: composite
  steps:
    - name: Create artifacts directory
-      if: ${{ inputs.target-file != '' }}
+      if: ${{ inputs.target-files != '' }}
      working-directory: ${{ inputs.working-dir }}
      shell: bash
      run: |
-        mkdir -p ${{ inputs.artifacts-dir }} && \
-        cp ${{ inputs.target-file }} ${{ inputs.artifacts-dir }}
+        set -e
+        mkdir -p ${{ inputs.artifacts-dir }}
+        IFS=',' read -ra FILES <<< "${{ inputs.target-files }}"
+        for file in "${FILES[@]}"; do
+          cp "$file" ${{ inputs.artifacts-dir }}/
+        done

    # The compressed artifacts will use the following layout:
    # greptime-linux-amd64-pyo3-v0.3.0sha256sum
--- a/.github/workflows/dev-build.yml
+++ b/.github/workflows/dev-build.yml
@@ -29,7 +29,7 @@ on:
      linux_arm64_runner:
        type: choice
        description: The runner uses to build linux-arm64 artifacts
-        default: ec2-c6g.8xlarge-arm64
+        default: ec2-c6g.4xlarge-arm64
        options:
          - ec2-c6g.xlarge-arm64 # 4C8G
          - ec2-c6g.2xlarge-arm64 # 8C16G
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -697,7 +697,7 @@ jobs:
        working-directory: tests-integration/fixtures/postgres
        run: docker compose -f docker-compose-standalone.yml up -d --wait
      - name: Run nextest cases
-        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
+        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard -F pg_kvbackend
        env:
          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
          RUST_BACKTRACE: 1
--- a/.github/workflows/nightly-build.yml
+++ b/.github/workflows/nightly-build.yml
@@ -27,7 +27,7 @@ on:
      linux_arm64_runner:
        type: choice
        description: The runner uses to build linux-arm64 artifacts
-        default: ec2-c6g.8xlarge-arm64
+        default: ec2-c6g.4xlarge-arm64
        options:
          - ec2-c6g.xlarge-arm64 # 4C8G
          - ec2-c6g.2xlarge-arm64 # 8C16G
--- a/.github/workflows/nightly-ci.yml
+++ b/.github/workflows/nightly-ci.yml
@@ -117,7 +117,6 @@ jobs:
  cleanbuild-linux-nix:
    runs-on: ubuntu-latest-8-cores
    timeout-minutes: 60
-    needs: [coverage, fmt, clippy, check]
    steps:
      - uses: actions/checkout@v4
      - uses: cachix/install-nix-action@v27
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -91,7 +91,7 @@ env:
  # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
  NIGHTLY_RELEASE_PREFIX: nightly
  # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
-  NEXT_RELEASE_VERSION: v0.11.0
+  NEXT_RELEASE_VERSION: v0.12.0

 # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
 permissions:
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -188,7 +188,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c"

 [[package]]
 name = "api"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "common-base",
 "common-decimal",
@@ -773,7 +773,7 @@ dependencies = [

 [[package]]
 name = "auth"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "async-trait",
@@ -896,18 +896,6 @@ dependencies = [
 "rand",
 ]

-[[package]]
-name = "backon"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0"
-dependencies = [
- "fastrand",
- "futures-core",
- "pin-project",
- "tokio",
-]
-
 [[package]]
 name = "backon"
 version = "1.2.0"
@@ -1326,7 +1314,7 @@ dependencies = [

 [[package]]
 name = "cache"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "catalog",
 "common-error",
@@ -1360,7 +1348,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"

 [[package]]
 name = "catalog"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "arrow",
@@ -1696,7 +1684,7 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"

 [[package]]
 name = "cli"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-trait",
 "auth",
@@ -1739,7 +1727,7 @@ dependencies = [
 "session",
 "snafu 0.8.5",
 "store-api",
- "substrait 0.11.1",
+ "substrait 0.12.0",
 "table",
 "tempfile",
 "tokio",
@@ -1748,7 +1736,7 @@ dependencies = [

 [[package]]
 name = "client"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "arc-swap",
@@ -1775,7 +1763,7 @@ dependencies = [
 "rand",
 "serde_json",
 "snafu 0.8.5",
- "substrait 0.11.1",
+ "substrait 0.12.0",
 "substrait 0.37.3",
 "tokio",
 "tokio-stream",
@@ -1816,7 +1804,7 @@ dependencies = [

 [[package]]
 name = "cmd"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-trait",
 "auth",
@@ -1876,7 +1864,7 @@ dependencies = [
 "similar-asserts",
 "snafu 0.8.5",
 "store-api",
- "substrait 0.11.1",
+ "substrait 0.12.0",
 "table",
 "temp-env",
 "tempfile",
@@ -1928,7 +1916,7 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335"

 [[package]]
 name = "common-base"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "anymap2",
 "async-trait",
@@ -1950,11 +1938,11 @@ dependencies = [

 [[package]]
 name = "common-catalog"
-version = "0.11.1"
+version = "0.12.0"

 [[package]]
 name = "common-config"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "common-base",
 "common-error",
@@ -1977,7 +1965,7 @@ dependencies = [

 [[package]]
 name = "common-datasource"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "arrow",
 "arrow-schema",
@@ -2013,7 +2001,7 @@ dependencies = [

 [[package]]
 name = "common-decimal"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "bigdecimal 0.4.5",
 "common-error",
@@ -2026,8 +2014,9 @@ dependencies = [

 [[package]]
 name = "common-error"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
+ "http 0.2.12",
 "snafu 0.8.5",
 "strum 0.25.0",
 "tonic 0.11.0",
@@ -2035,7 +2024,7 @@ dependencies = [

 [[package]]
 name = "common-frontend"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-trait",
 "common-error",
@@ -2045,7 +2034,7 @@ dependencies = [

 [[package]]
 name = "common-function"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "approx 0.5.1",
@@ -2089,7 +2078,7 @@ dependencies = [

 [[package]]
 name = "common-greptimedb-telemetry"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-trait",
 "common-runtime",
@@ -2106,7 +2095,7 @@ dependencies = [

 [[package]]
 name = "common-grpc"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "arrow-flight",
@@ -2132,7 +2121,7 @@ dependencies = [

 [[package]]
 name = "common-grpc-expr"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "common-base",
@@ -2151,7 +2140,7 @@ dependencies = [

 [[package]]
 name = "common-macro"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "arc-swap",
 "common-query",
@@ -2165,7 +2154,7 @@ dependencies = [

 [[package]]
 name = "common-mem-prof"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "common-error",
 "common-macro",
@@ -2178,7 +2167,7 @@ dependencies = [

 [[package]]
 name = "common-meta"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "anymap2",
 "api",
@@ -2235,7 +2224,7 @@ dependencies = [

 [[package]]
 name = "common-options"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "common-grpc",
 "humantime-serde",
@@ -2244,11 +2233,11 @@ dependencies = [

 [[package]]
 name = "common-plugins"
-version = "0.11.1"
+version = "0.12.0"

 [[package]]
 name = "common-pprof"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "common-error",
 "common-macro",
@@ -2260,11 +2249,11 @@ dependencies = [

 [[package]]
 name = "common-procedure"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-stream",
 "async-trait",
- "backon 1.2.0",
+ "backon",
 "common-base",
 "common-error",
 "common-macro",
@@ -2287,7 +2276,7 @@ dependencies = [

 [[package]]
 name = "common-procedure-test"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-trait",
 "common-procedure",
@@ -2295,7 +2284,7 @@ dependencies = [

 [[package]]
 name = "common-query"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "async-trait",
@@ -2321,7 +2310,7 @@ dependencies = [

 [[package]]
 name = "common-recordbatch"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "arc-swap",
 "common-error",
@@ -2340,7 +2329,7 @@ dependencies = [

 [[package]]
 name = "common-runtime"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-trait",
 "clap 4.5.19",
@@ -2368,7 +2357,7 @@ dependencies = [

 [[package]]
 name = "common-telemetry"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "atty",
 "backtrace",
@@ -2396,7 +2385,7 @@ dependencies = [

 [[package]]
 name = "common-test-util"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "client",
 "common-query",
@@ -2408,7 +2397,7 @@ dependencies = [

 [[package]]
 name = "common-time"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "arrow",
 "chrono",
@@ -2426,7 +2415,7 @@ dependencies = [

 [[package]]
 name = "common-version"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "build-data",
 "const_format",
@@ -2436,7 +2425,7 @@ dependencies = [

 [[package]]
 name = "common-wal"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "common-base",
 "common-error",
@@ -3235,7 +3224,7 @@ dependencies = [

 [[package]]
 name = "datanode"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "arrow-flight",
@@ -3286,7 +3275,7 @@ dependencies = [
 "session",
 "snafu 0.8.5",
 "store-api",
- "substrait 0.11.1",
+ "substrait 0.12.0",
 "table",
 "tokio",
 "toml 0.8.19",
@@ -3295,7 +3284,7 @@ dependencies = [

 [[package]]
 name = "datatypes"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -3919,7 +3908,7 @@ dependencies = [

 [[package]]
 name = "file-engine"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "async-trait",
@@ -4035,7 +4024,7 @@ checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8"

 [[package]]
 name = "flow"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "arrow",
@@ -4073,6 +4062,7 @@ dependencies = [
 "get-size-derive2",
 "get-size2",
 "greptime-proto",
+ "http 0.2.12",
 "hydroflow",
 "itertools 0.10.5",
 "lazy_static",
@@ -4093,7 +4083,7 @@ dependencies = [
 "snafu 0.8.5",
 "store-api",
 "strum 0.25.0",
- "substrait 0.11.1",
+ "substrait 0.12.0",
 "table",
 "tokio",
 "tonic 0.11.0",
@@ -4131,7 +4121,7 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"

 [[package]]
 name = "frontend"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "arc-swap",
@@ -4164,6 +4154,7 @@ dependencies = [
 "futures",
 "humantime-serde",
 "lazy_static",
+ "log-query",
 "log-store",
 "meta-client",
 "opentelemetry-proto 0.5.0",
@@ -5280,7 +5271,7 @@ dependencies = [

 [[package]]
 name = "index"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-trait",
 "asynchronous-codec",
@@ -5297,6 +5288,7 @@ dependencies = [
 "futures",
 "greptime-proto",
 "mockall",
+ "parquet",
 "pin-project",
 "prost 0.12.6",
 "rand",
@@ -6129,18 +6121,19 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"

 [[package]]
 name = "log-query"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "chrono",
 "common-error",
 "common-macro",
+ "serde",
 "snafu 0.8.5",
 "table",
 ]

 [[package]]
 name = "log-store"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-stream",
 "async-trait",
@@ -6484,7 +6477,7 @@ dependencies = [

 [[package]]
 name = "meta-client"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "async-trait",
@@ -6511,7 +6504,7 @@ dependencies = [

 [[package]]
 name = "meta-srv"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "async-trait",
@@ -6590,7 +6583,7 @@ dependencies = [

 [[package]]
 name = "metric-engine"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "aquamarine",
@@ -6684,7 +6677,7 @@ dependencies = [

 [[package]]
 name = "mito2"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "aquamarine",
@@ -7421,7 +7414,7 @@ dependencies = [

 [[package]]
 name = "object-store"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "anyhow",
 "bytes",
@@ -7481,13 +7474,12 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"

 [[package]]
 name = "opendal"
-version = "0.49.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b04d09b9822c2f75a1d2fc513a2c1279c70e91e7407936fffdf6a6976ec530a"
+version = "0.50.2"
+source = "git+https://github.com/GreptimeTeam/opendal.git?rev=c82605177f2feec83e49dcaa537c505639d94024#c82605177f2feec83e49dcaa537c505639d94024"
 dependencies = [
 "anyhow",
 "async-trait",
- "backon 0.4.4",
+ "backon",
 "base64 0.22.1",
 "bytes",
 "chrono",
@@ -7500,6 +7492,7 @@ dependencies = [
 "md-5",
 "once_cell",
 "percent-encoding",
+ "prometheus",
 "quick-xml 0.36.2",
 "reqsign",
 "reqwest",
@@ -7674,7 +7667,7 @@ dependencies = [

 [[package]]
 name = "operator"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "ahash 0.8.11",
 "api",
@@ -7722,7 +7715,7 @@ dependencies = [
 "sql",
 "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
 "store-api",
- "substrait 0.11.1",
+ "substrait 0.12.0",
 "table",
 "tokio",
 "tokio-util",
@@ -7972,7 +7965,7 @@ dependencies = [

 [[package]]
 name = "partition"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "async-trait",
@@ -8171,7 +8164,7 @@ dependencies = [
 "rand",
 "ring 0.17.8",
 "rust_decimal",
- "thiserror 2.0.4",
+ "thiserror 2.0.6",
 "tokio",
 "tokio-rustls 0.26.0",
 "tokio-util",
@@ -8258,7 +8251,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"

 [[package]]
 name = "pipeline"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "ahash 0.8.11",
 "api",
@@ -8420,7 +8413,7 @@ dependencies = [

 [[package]]
 name = "plugins"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "auth",
 "clap 4.5.19",
@@ -8708,7 +8701,7 @@ dependencies = [

 [[package]]
 name = "promql"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "ahash 0.8.11",
 "async-trait",
@@ -8943,7 +8936,7 @@ dependencies = [

 [[package]]
 name = "puffin"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-compression 0.4.13",
 "async-trait",
@@ -9068,7 +9061,7 @@ dependencies = [

 [[package]]
 name = "query"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "ahash 0.8.11",
 "api",
@@ -9109,8 +9102,10 @@ dependencies = [
 "humantime",
 "itertools 0.10.5",
 "lazy_static",
+ "log-query",
 "meter-core",
 "meter-macros",
+ "nalgebra 0.33.2",
 "num",
 "num-traits",
 "object-store",
@@ -9131,7 +9126,7 @@ dependencies = [
 "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
 "statrs",
 "store-api",
- "substrait 0.11.1",
+ "substrait 0.12.0",
 "table",
 "tokio",
 "tokio-stream",
@@ -9515,9 +9510,9 @@ dependencies = [

 [[package]]
 name = "reqsign"
-version = "0.16.0"
+version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03dd4ba7c3901dd43e6b8c7446a760d45bc1ea4301002e1a6fa48f97c3a796fa"
+checksum = "eb0075a66c8bfbf4cc8b70dca166e722e1f55a3ea9250ecbb85f4d92a5f64149"
 dependencies = [
 "anyhow",
 "async-trait",
@@ -10615,7 +10610,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"

 [[package]]
 name = "script"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "arc-swap",
@@ -10907,7 +10902,7 @@ dependencies = [

 [[package]]
 name = "servers"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "ahash 0.8.11",
 "api",
@@ -10963,6 +10958,7 @@ dependencies = [
 "json5",
 "jsonb",
 "lazy_static",
+ "log-query",
 "loki-api",
 "mime_guess",
 "mysql_async",
@@ -11018,7 +11014,7 @@ dependencies = [

 [[package]]
 name = "session"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "arc-swap",
@@ -11372,7 +11368,7 @@ dependencies = [

 [[package]]
 name = "sql"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "chrono",
@@ -11436,7 +11432,7 @@ dependencies = [

 [[package]]
 name = "sqlness-runner"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-trait",
 "clap 4.5.19",
@@ -11654,7 +11650,7 @@ dependencies = [

 [[package]]
 name = "store-api"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "aquamarine",
@@ -11816,7 +11812,7 @@ dependencies = [

 [[package]]
 name = "substrait"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "async-trait",
 "bytes",
@@ -12015,7 +12011,7 @@ dependencies = [

 [[package]]
 name = "table"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "async-trait",
@@ -12292,7 +12288,7 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"

 [[package]]
 name = "tests-fuzz"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "arbitrary",
 "async-trait",
@@ -12335,7 +12331,7 @@ dependencies = [

 [[package]]
 name = "tests-integration"
-version = "0.11.1"
+version = "0.12.0"
 dependencies = [
 "api",
 "arrow-flight",
@@ -12399,7 +12395,7 @@ dependencies = [
 "sql",
 "sqlx",
 "store-api",
- "substrait 0.11.1",
+ "substrait 0.12.0",
 "table",
 "tempfile",
 "time",
@@ -12445,11 +12441,11 @@ dependencies = [

 [[package]]
 name = "thiserror"
-version = "2.0.4"
+version = "2.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490"
+checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47"
 dependencies = [
- "thiserror-impl 2.0.4",
+ "thiserror-impl 2.0.6",
 ]

 [[package]]
@@ -12465,9 +12461,9 @@ dependencies = [

 [[package]]
 name = "thiserror-impl"
-version = "2.0.4"
+version = "2.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061"
+checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312"
 dependencies = [
 "proc-macro2",
 "quote",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -68,7 +68,7 @@ members = [
 resolver = "2"

 [workspace.package]
-version = "0.11.1"
+version = "0.12.0"
 edition = "2021"
 license = "Apache-2.0"

@@ -126,6 +126,7 @@ futures = "0.3"
 futures-util = "0.3"
 greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "a875e976441188028353f7274a46a7e6e065c5d4" }
 hex = "0.4"
+http = "0.2"
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
@@ -134,6 +135,7 @@ lazy_static = "1.4"
 meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "a10facb353b41460eeb98578868ebf19c2084fac" }
 mockall = "0.11.4"
 moka = "0.12"
+nalgebra = "0.33"
 notify = "6.1"
 num_cpus = "1.16"
 once_cell = "1.18"
@@ -238,6 +240,7 @@ file-engine = { path = "src/file-engine" }
 flow = { path = "src/flow" }
 frontend = { path = "src/frontend", default-features = false }
 index = { path = "src/index" }
+log-query = { path = "src/log-query" }
 log-store = { path = "src/log-store" }
 meta-client = { path = "src/meta-client" }
 meta-srv = { path = "src/meta-srv" }
--- a/README.md
+++ b/README.md
@@ -70,23 +70,23 @@ Our core developers have been building time-series data platforms for years. Bas

 * **Unified Processing of Metrics, Logs, and Events**

-GreptimeDB unifies time series data processing by treating all data - whether metrics, logs, or events - as timestamped events with context. Users can analyze this data using either [SQL](https://docs.greptime.com/user-guide/query-data/sql) or [PromQL](https://docs.greptime.com/user-guide/query-data/promql) and leverage stream processing ([Flow](https://docs.greptime.com/user-guide/continuous-aggregation/overview)) to enable continuous aggregation. [Read more](https://docs.greptime.com/user-guide/concepts/data-model).
+  GreptimeDB unifies time series data processing by treating all data - whether metrics, logs, or events - as timestamped events with context. Users can analyze this data using either [SQL](https://docs.greptime.com/user-guide/query-data/sql) or [PromQL](https://docs.greptime.com/user-guide/query-data/promql) and leverage stream processing ([Flow](https://docs.greptime.com/user-guide/flow-computation/overview)) to enable continuous aggregation. [Read more](https://docs.greptime.com/user-guide/concepts/data-model).

 * **Cloud-native Distributed Database**

-Built for [Kubernetes](https://docs.greptime.com/user-guide/deployments/deploy-on-kubernetes/greptimedb-operator-management). GreptimeDB achieves seamless scalability with its [cloud-native architecture](https://docs.greptime.com/user-guide/concepts/architecture) of separated compute and storage, built on object storage (AWS S3, Azure Blob Storage, etc.) while enabling cross-cloud deployment through a unified data access layer.
+  Built for [Kubernetes](https://docs.greptime.com/user-guide/deployments/deploy-on-kubernetes/greptimedb-operator-management). GreptimeDB achieves seamless scalability with its [cloud-native architecture](https://docs.greptime.com/user-guide/concepts/architecture) of separated compute and storage, built on object storage (AWS S3, Azure Blob Storage, etc.) while enabling cross-cloud deployment through a unified data access layer.

 * **Performance and Cost-effective**

-Written in pure Rust for superior performance and reliability. GreptimeDB features a distributed query engine with intelligent indexing to handle high cardinality data efficiently. Its optimized columnar storage achieves 50x cost efficiency on cloud object storage through advanced compression. [Benchmark reports](https://www.greptime.com/blogs/2024-09-09-report-summary).
+  Written in pure Rust for superior performance and reliability. GreptimeDB features a distributed query engine with intelligent indexing to handle high cardinality data efficiently. Its optimized columnar storage achieves 50x cost efficiency on cloud object storage through advanced compression. [Benchmark reports](https://www.greptime.com/blogs/2024-09-09-report-summary).

 * **Cloud-Edge Collaboration**

-GreptimeDB seamlessly operates across cloud and edge (ARM/Android/Linux), providing consistent APIs and control plane for unified data management and efficient synchronization. [Learn how to run on Android](https://docs.greptime.com/user-guide/deployments/run-on-android/).
+  GreptimeDB seamlessly operates across cloud and edge (ARM/Android/Linux), providing consistent APIs and control plane for unified data management and efficient synchronization. [Learn how to run on Android](https://docs.greptime.com/user-guide/deployments/run-on-android/).

 * **Multi-protocol Ingestion, SQL & PromQL Ready**

-Widely adopted database protocols and APIs, including MySQL, PostgreSQL, InfluxDB, OpenTelemetry, Loki and Prometheus, etc.  Effortless Adoption & Seamless Migration. [Supported Protocols Overview](https://docs.greptime.com/user-guide/protocols/overview).
+  Widely adopted database protocols and APIs, including MySQL, PostgreSQL, InfluxDB, OpenTelemetry, Loki and Prometheus, etc.  Effortless Adoption & Seamless Migration. [Supported Protocols Overview](https://docs.greptime.com/user-guide/protocols/overview).

 For more detailed info please read  [Why GreptimeDB](https://docs.greptime.com/user-guide/concepts/why-greptimedb).

@@ -138,7 +138,7 @@ Check the prerequisite:

 * [Rust toolchain](https://www.rust-lang.org/tools/install) (nightly)
 * [Protobuf compiler](https://grpc.io/docs/protoc-installation/) (>= 3.15)
-* Python toolchain (optional): Required only if built with PyO3 backend. More detail for compiling with PyO3 can be found in its [documentation](https://pyo3.rs/v0.18.1/building_and_distribution#configuring-the-python-version).
+* Python toolchain (optional): Required only if built with PyO3 backend. More details for compiling with PyO3 can be found in its [documentation](https://pyo3.rs/v0.18.1/building_and_distribution#configuring-the-python-version).

 Build GreptimeDB binary:

@@ -154,6 +154,10 @@ cargo run -- standalone start

 ## Tools & Extensions

+### Kubernetes
+
+- [GreptimeDB Operator](https://github.com/GrepTimeTeam/greptimedb-operator)
+
 ### Dashboard

 - [The dashboard UI for GreptimeDB](https://github.com/GreptimeTeam/dashboard)
@@ -173,7 +177,7 @@ Our official Grafana dashboard for monitoring GreptimeDB is available at [grafan

 ## Project Status

-GreptimeDB is currently in Beta. We are targeting GA (General Availability) with v1.0 release by Early 2025. 
+GreptimeDB is currently in Beta. We are targeting GA (General Availability) with v1.0 release by Early 2025.

 While in Beta, GreptimeDB is already:

--- a/config/config.md
+++ b/config/config.md
@@ -18,6 +18,7 @@
 | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
 | `max_concurrent_queries` | Integer | `0` | The maximum current queries allowed to be executed. Zero means unlimited. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. Enabled by default. |
+| `max_in_flight_write_bytes` | String | Unset | The maximum in-flight write bytes. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
 | `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
@@ -156,6 +157,11 @@
 | `region_engine.mito.fulltext_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.apply_on_query` | String | `auto` | Whether to apply the index on query<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.mem_threshold_on_create` | String | `auto` | Memory threshold for index creation.<br/>- `auto`: automatically determine the threshold based on the system memory size (default)<br/>- `unlimited`: no memory limit<br/>- `[size]` e.g. `64MB`: fixed memory threshold |
+| `region_engine.mito.bloom_filter_index` | -- | -- | The options for bloom filter in Mito engine. |
+| `region_engine.mito.bloom_filter_index.create_on_flush` | String | `auto` | Whether to create the bloom filter on flush.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.create_on_compaction` | String | `auto` | Whether to create the bloom filter on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.apply_on_query` | String | `auto` | Whether to apply the bloom filter on query<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.mem_threshold_on_create` | String | `auto` | Memory threshold for bloom filter creation.<br/>- `auto`: automatically determine the threshold based on the system memory size (default)<br/>- `unlimited`: no memory limit<br/>- `[size]` e.g. `64MB`: fixed memory threshold |
 | `region_engine.mito.memtable` | -- | -- | -- |
 | `region_engine.mito.memtable.type` | String | `time_series` | Memtable type.<br/>- `time_series`: time-series memtable<br/>- `partition_tree`: partition tree memtable (experimental) |
 | `region_engine.mito.memtable.index_max_keys_per_shard` | Integer | `8192` | The max number of keys in one shard.<br/>Only available for `partition_tree` memtable. |
@@ -195,6 +201,7 @@
 | Key | Type | Default | Descriptions |
 | --- | -----| ------- | ----------- |
 | `default_timezone` | String | Unset | The default timezone of the server. |
+| `max_in_flight_write_bytes` | String | Unset | The maximum in-flight write bytes. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
 | `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
@@ -421,7 +428,7 @@
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
-| `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.<br/>A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling. |
+| `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.<br/>A local file directory, defaults to `{data_home}`. An empty string means disabling. |
 | `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. |
 | `storage.bucket` | String | Unset | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
 | `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
@@ -460,7 +467,7 @@
 | `region_engine.mito.page_cache_size` | String | Auto | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
 | `region_engine.mito.selector_result_cache_size` | String | Auto | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. |
-| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/object_cache/write`. |
+| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}`. |
 | `region_engine.mito.experimental_write_cache_size` | String | `5GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. |
 | `region_engine.mito.experimental_write_cache_ttl` | String | Unset | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
@@ -484,6 +491,11 @@
 | `region_engine.mito.fulltext_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.apply_on_query` | String | `auto` | Whether to apply the index on query<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.mem_threshold_on_create` | String | `auto` | Memory threshold for index creation.<br/>- `auto`: automatically determine the threshold based on the system memory size (default)<br/>- `unlimited`: no memory limit<br/>- `[size]` e.g. `64MB`: fixed memory threshold |
+| `region_engine.mito.bloom_filter_index` | -- | -- | The options for bloom filter index in Mito engine. |
+| `region_engine.mito.bloom_filter_index.create_on_flush` | String | `auto` | Whether to create the index on flush.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.apply_on_query` | String | `auto` | Whether to apply the index on query<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.mem_threshold_on_create` | String | `auto` | Memory threshold for the index creation.<br/>- `auto`: automatically determine the threshold based on the system memory size (default)<br/>- `unlimited`: no memory limit<br/>- `[size]` e.g. `64MB`: fixed memory threshold |
 | `region_engine.mito.memtable` | -- | -- | -- |
 | `region_engine.mito.memtable.type` | String | `time_series` | Memtable type.<br/>- `time_series`: time-series memtable<br/>- `partition_tree`: partition tree memtable (experimental) |
 | `region_engine.mito.memtable.index_max_keys_per_shard` | Integer | `8192` | The max number of keys in one shard.<br/>Only available for `partition_tree` memtable. |
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -294,7 +294,7 @@ data_home = "/tmp/greptimedb/"
 type = "File"

 ## Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.
-## A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling.
+## A local file directory, defaults to `{data_home}`. An empty string means disabling.
 ## @toml2docs:none-default
 #+ cache_path = ""

@@ -478,7 +478,7 @@ auto_flush_interval = "1h"
 ## Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance.
 enable_experimental_write_cache = false

-## File system path for write cache, defaults to `{data_home}/object_cache/write`.
+## File system path for write cache, defaults to `{data_home}`.
 experimental_write_cache_path = ""

 ## Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger.
@@ -576,6 +576,30 @@ apply_on_query = "auto"
 ## - `[size]` e.g. `64MB`: fixed memory threshold
 mem_threshold_on_create = "auto"

+## The options for bloom filter index in Mito engine.
+[region_engine.mito.bloom_filter_index]
+
+## Whether to create the index on flush.
+## - `auto`: automatically (default)
+## - `disable`: never
+create_on_flush = "auto"
+
+## Whether to create the index on compaction.
+## - `auto`: automatically (default)
+## - `disable`: never
+create_on_compaction = "auto"
+
+## Whether to apply the index on query
+## - `auto`: automatically (default)
+## - `disable`: never
+apply_on_query = "auto"
+
+## Memory threshold for the index creation.
+## - `auto`: automatically determine the threshold based on the system memory size (default)
+## - `unlimited`: no memory limit
+## - `[size]` e.g. `64MB`: fixed memory threshold
+mem_threshold_on_create = "auto"
+
 [region_engine.mito.memtable]
 ## Memtable type.
 ## - `time_series`: time-series memtable
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -2,6 +2,10 @@
 ## @toml2docs:none-default
 default_timezone = "UTC"

+## The maximum in-flight write bytes.
+## @toml2docs:none-default
+#+ max_in_flight_write_bytes = "500MB"
+
 ## The runtime options.
 #+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -18,6 +18,10 @@ max_concurrent_queries = 0
 ## Enable telemetry to collect anonymous usage data. Enabled by default.
 #+ enable_telemetry = true

+## The maximum in-flight write bytes.
+## @toml2docs:none-default
+#+ max_in_flight_write_bytes = "500MB"
+
 ## The runtime options.
 #+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
@@ -615,6 +619,30 @@ apply_on_query = "auto"
 ## - `[size]` e.g. `64MB`: fixed memory threshold
 mem_threshold_on_create = "auto"

+## The options for bloom filter in Mito engine.
+[region_engine.mito.bloom_filter_index]
+
+## Whether to create the bloom filter on flush.
+## - `auto`: automatically (default)
+## - `disable`: never
+create_on_flush = "auto"
+
+## Whether to create the bloom filter on compaction.
+## - `auto`: automatically (default)
+## - `disable`: never
+create_on_compaction = "auto"
+
+## Whether to apply the bloom filter on query
+## - `auto`: automatically (default)
+## - `disable`: never
+apply_on_query = "auto"
+
+## Memory threshold for bloom filter creation.
+## - `auto`: automatically determine the threshold based on the system memory size (default)
+## - `unlimited`: no memory limit
+## - `[size]` e.g. `64MB`: fixed memory threshold
+mem_threshold_on_create = "auto"
+
 [region_engine.mito.memtable]
 ## Memtable type.
 ## - `time_series`: time-series memtable
--- a/src/auth/src/permission.rs
+++ b/src/auth/src/permission.rs
@@ -25,6 +25,7 @@ pub enum PermissionReq<'a> {
    GrpcRequest(&'a Request),
    SqlStatement(&'a Statement),
    PromQuery,
+    LogQuery,
    Opentsdb,
    LineProtocol,
    PromStoreWrite,
--- a/src/catalog/src/kvbackend/table_cache.rs
+++ b/src/catalog/src/kvbackend/table_cache.rs
@@ -38,7 +38,7 @@ pub fn new_table_cache(
 ) -> TableCache {
    let init = init_factory(table_info_cache, table_name_cache);

-    CacheContainer::new(name, cache, Box::new(invalidator), init, Box::new(filter))
+    CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
 }

 fn init_factory(
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -22,6 +22,7 @@ use catalog::information_schema::InformationExtension;
 use catalog::kvbackend::KvBackendCatalogManager;
 use clap::Parser;
 use client::api::v1::meta::RegionRole;
+use common_base::readable_size::ReadableSize;
 use common_base::Plugins;
 use common_catalog::consts::{MIN_USER_FLOW_ID, MIN_USER_TABLE_ID};
 use common_config::{metadata_store_dir, Configurable, KvBackendConfig};
@@ -152,6 +153,7 @@ pub struct StandaloneOptions {
    pub tracing: TracingOptions,
    pub init_regions_in_background: bool,
    pub init_regions_parallelism: usize,
+    pub max_in_flight_write_bytes: Option<ReadableSize>,
 }

 impl Default for StandaloneOptions {
@@ -181,6 +183,7 @@ impl Default for StandaloneOptions {
            tracing: TracingOptions::default(),
            init_regions_in_background: false,
            init_regions_parallelism: 16,
+            max_in_flight_write_bytes: None,
        }
    }
 }
@@ -218,6 +221,7 @@ impl StandaloneOptions {
            user_provider: cloned_opts.user_provider,
            // Handle the export metrics task run by standalone to frontend for execution
            export_metrics: cloned_opts.export_metrics,
+            max_in_flight_write_bytes: cloned_opts.max_in_flight_write_bytes,
            ..Default::default()
        }
    }
--- a/src/common/datasource/src/object_store/fs.rs
+++ b/src/common/datasource/src/object_store/fs.rs
@@ -27,7 +27,7 @@ pub fn build_fs_backend(root: &str) -> Result<ObjectStore> {
            DefaultLoggingInterceptor,
        ))
        .layer(object_store::layers::TracingLayer)
-        .layer(object_store::layers::PrometheusMetricsLayer::new(true))
+        .layer(object_store::layers::build_prometheus_metrics_layer(true))
        .finish();
    Ok(object_store)
 }
--- a/src/common/datasource/src/object_store/s3.rs
+++ b/src/common/datasource/src/object_store/s3.rs
@@ -89,7 +89,7 @@ pub fn build_s3_backend(
            DefaultLoggingInterceptor,
        ))
        .layer(object_store::layers::TracingLayer)
-        .layer(object_store::layers::PrometheusMetricsLayer::new(true))
+        .layer(object_store::layers::build_prometheus_metrics_layer(true))
        .finish())
 }

--- a/src/common/error/Cargo.toml
+++ b/src/common/error/Cargo.toml
@@ -8,6 +8,7 @@ license.workspace = true
 workspace = true

 [dependencies]
+http.workspace = true
 snafu.workspace = true
 strum.workspace = true
 tonic.workspace = true
--- a/src/common/error/src/lib.rs
+++ b/src/common/error/src/lib.rs
@@ -18,9 +18,30 @@ pub mod ext;
 pub mod mock;
 pub mod status_code;

+use http::{HeaderMap, HeaderValue};
 pub use snafu;

 // HACK - these headers are here for shared in gRPC services. For common HTTP headers,
 // please define in `src/servers/src/http/header.rs`.
 pub const GREPTIME_DB_HEADER_ERROR_CODE: &str = "x-greptime-err-code";
 pub const GREPTIME_DB_HEADER_ERROR_MSG: &str = "x-greptime-err-msg";
+
+/// Create a http header map from error code and message.
+/// using `GREPTIME_DB_HEADER_ERROR_CODE` and `GREPTIME_DB_HEADER_ERROR_MSG` as keys.
+pub fn from_err_code_msg_to_header(code: u32, msg: &str) -> HeaderMap {
+    let mut header = HeaderMap::new();
+
+    let msg = HeaderValue::from_str(msg).unwrap_or_else(|_| {
+        HeaderValue::from_bytes(
+            &msg.as_bytes()
+                .iter()
+                .flat_map(|b| std::ascii::escape_default(*b))
+                .collect::<Vec<u8>>(),
+        )
+        .expect("Already escaped string should be valid ascii")
+    });
+
+    header.insert(GREPTIME_DB_HEADER_ERROR_CODE, code.into());
+    header.insert(GREPTIME_DB_HEADER_ERROR_MSG, msg);
+    header
+}
--- a/src/common/function/Cargo.toml
+++ b/src/common/function/Cargo.toml
@@ -33,7 +33,7 @@ geo-types = { version = "0.7", optional = true }
 geohash = { version = "0.13", optional = true }
 h3o = { version = "0.6", optional = true }
 jsonb.workspace = true
-nalgebra = "0.33"
+nalgebra.workspace = true
 num = "0.4"
 num-traits = "0.2"
 once_cell.workspace = true
--- a/src/common/function/src/lib.rs
+++ b/src/common/function/src/lib.rs
@@ -26,3 +26,4 @@ pub mod function_registry;
 pub mod handlers;
 pub mod helper;
 pub mod state;
+pub mod utils;
--- a/src/common/function/src/scalars/aggregate.rs
+++ b/src/common/function/src/scalars/aggregate.rs
@@ -32,6 +32,7 @@ pub use scipy_stats_norm_cdf::ScipyStatsNormCdfAccumulatorCreator;
 pub use scipy_stats_norm_pdf::ScipyStatsNormPdfAccumulatorCreator;

 use crate::function_registry::FunctionRegistry;
+use crate::scalars::vector::sum::VectorSumCreator;

 /// A function creates `AggregateFunctionCreator`.
 /// "Aggregator" *is* AggregatorFunction. Since the later one is long, we named an short alias for it.
@@ -91,6 +92,7 @@ impl AggregateFunctions {
        register_aggr_func!("argmin", 1, ArgminAccumulatorCreator);
        register_aggr_func!("scipystatsnormcdf", 2, ScipyStatsNormCdfAccumulatorCreator);
        register_aggr_func!("scipystatsnormpdf", 2, ScipyStatsNormPdfAccumulatorCreator);
+        register_aggr_func!("vec_sum", 1, VectorSumCreator);

        #[cfg(feature = "geo")]
        register_aggr_func!(
--- a/src/common/function/src/scalars/matches.rs
+++ b/src/common/function/src/scalars/matches.rs
@@ -204,20 +204,10 @@ impl PatternAst {
    fn convert_literal(column: &str, pattern: &str) -> Expr {
        logical_expr::col(column).like(logical_expr::lit(format!(
            "%{}%",
-            Self::escape_pattern(pattern)
+            crate::utils::escape_like_pattern(pattern)
        )))
    }

-    fn escape_pattern(pattern: &str) -> String {
-        pattern
-            .chars()
-            .flat_map(|c| match c {
-                '\\' | '%' | '_' => vec!['\\', c],
-                _ => vec![c],
-            })
-            .collect::<String>()
-    }
-
    /// Transform this AST with preset rules to make it correct.
    fn transform_ast(self) -> Result<Self> {
        self.transform_up(Self::collapse_binary_branch_fn)
--- a/src/common/function/src/scalars/vector.rs
+++ b/src/common/function/src/scalars/vector.rs
@@ -14,9 +14,13 @@

 mod convert;
 mod distance;
-pub(crate) mod impl_conv;
+mod elem_sum;
+pub mod impl_conv;
 mod scalar_add;
 mod scalar_mul;
+mod sub;
+pub(crate) mod sum;
+mod vector_mul;

 use std::sync::Arc;

@@ -38,5 +42,10 @@ impl VectorFunction {
        // scalar calculation
        registry.register(Arc::new(scalar_add::ScalarAddFunction));
        registry.register(Arc::new(scalar_mul::ScalarMulFunction));
+
+        // vector calculation
+        registry.register(Arc::new(vector_mul::VectorMulFunction));
+        registry.register(Arc::new(sub::SubFunction));
+        registry.register(Arc::new(elem_sum::ElemSumFunction));
    }
 }
--- a/src/common/function/src/scalars/vector/elem_sum.rs
+++ b/src/common/function/src/scalars/vector/elem_sum.rs
@@ -0,0 +1,129 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::borrow::Cow;
+use std::fmt::Display;
+
+use common_query::error::InvalidFuncArgsSnafu;
+use common_query::prelude::{Signature, TypeSignature, Volatility};
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{Float32VectorBuilder, MutableVector, VectorRef};
+use nalgebra::DVectorView;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const};
+
+const NAME: &str = "vec_elem_sum";
+
+#[derive(Debug, Clone, Default)]
+pub struct ElemSumFunction;
+
+impl Function for ElemSumFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(
+        &self,
+        _input_types: &[ConcreteDataType],
+    ) -> common_query::error::Result<ConcreteDataType> {
+        Ok(ConcreteDataType::float32_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::one_of(
+            vec![
+                TypeSignature::Exact(vec![ConcreteDataType::string_datatype()]),
+                TypeSignature::Exact(vec![ConcreteDataType::binary_datatype()]),
+            ],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(
+        &self,
+        _func_ctx: FunctionContext,
+        columns: &[VectorRef],
+    ) -> common_query::error::Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly one, have: {}",
+                    columns.len()
+                )
+            }
+        );
+        let arg0 = &columns[0];
+
+        let len = arg0.len();
+        let mut result = Float32VectorBuilder::with_capacity(len);
+        if len == 0 {
+            return Ok(result.to_vector());
+        }
+
+        let arg0_const = as_veclit_if_const(arg0)?;
+
+        for i in 0..len {
+            let arg0 = match arg0_const.as_ref() {
+                Some(arg0) => Some(Cow::Borrowed(arg0.as_ref())),
+                None => as_veclit(arg0.get_ref(i))?,
+            };
+            let Some(arg0) = arg0 else {
+                result.push_null();
+                continue;
+            };
+            result.push(Some(DVectorView::from_slice(&arg0, arg0.len()).sum()));
+        }
+
+        Ok(result.to_vector())
+    }
+}
+
+impl Display for ElemSumFunction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", NAME.to_ascii_uppercase())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use datatypes::vectors::StringVector;
+
+    use super::*;
+    use crate::function::FunctionContext;
+
+    #[test]
+    fn test_elem_sum() {
+        let func = ElemSumFunction;
+
+        let input0 = Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[4.0,5.0,6.0]".to_string()),
+            None,
+        ]));
+
+        let result = func.eval(FunctionContext::default(), &[input0]).unwrap();
+
+        let result = result.as_ref();
+        assert_eq!(result.len(), 3);
+        assert_eq!(result.get_ref(0).as_f32().unwrap(), Some(6.0));
+        assert_eq!(result.get_ref(1).as_f32().unwrap(), Some(15.0));
+        assert_eq!(result.get_ref(2).as_f32().unwrap(), None);
+    }
+}
--- a/src/common/function/src/scalars/vector/sub.rs
+++ b/src/common/function/src/scalars/vector/sub.rs
@@ -0,0 +1,223 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::borrow::Cow;
+use std::fmt::Display;
+
+use common_query::error::InvalidFuncArgsSnafu;
+use common_query::prelude::Signature;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{BinaryVectorBuilder, MutableVector, VectorRef};
+use nalgebra::DVectorView;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::helper;
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const, veclit_to_binlit};
+
+const NAME: &str = "vec_sub";
+
+/// Subtracts corresponding elements of two vectors, returns a vector.
+///
+/// # Example
+///
+/// ```sql
+/// SELECT vec_to_string(vec_sub("[1.0, 1.0]", "[1.0, 2.0]")) as result;
+///
+/// +---------------------------------------------------------------+
+/// | vec_to_string(vec_sub(Utf8("[1.0, 1.0]"),Utf8("[1.0, 2.0]"))) |
+/// +---------------------------------------------------------------+
+/// | [0,-1]                                                        |
+/// +---------------------------------------------------------------+
+///
+/// -- Negative scalar to simulate subtraction
+/// SELECT vec_to_string(vec_sub('[-1.0, -1.0]', '[1.0, 2.0]'));
+///
+/// +-----------------------------------------------------------------+
+/// | vec_to_string(vec_sub(Utf8("[-1.0, -1.0]"),Utf8("[1.0, 2.0]"))) |
+/// +-----------------------------------------------------------------+
+/// | [-2,-3]                                                         |
+/// +-----------------------------------------------------------------+
+///
+#[derive(Debug, Clone, Default)]
+pub struct SubFunction;
+
+impl Function for SubFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(
+        &self,
+        _input_types: &[ConcreteDataType],
+    ) -> common_query::error::Result<ConcreteDataType> {
+        Ok(ConcreteDataType::binary_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        helper::one_of_sigs2(
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::binary_datatype(),
+            ],
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::binary_datatype(),
+            ],
+        )
+    }
+
+    fn eval(
+        &self,
+        _func_ctx: FunctionContext,
+        columns: &[VectorRef],
+    ) -> common_query::error::Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly two, have: {}",
+                    columns.len()
+                )
+            }
+        );
+        let arg0 = &columns[0];
+        let arg1 = &columns[1];
+
+        ensure!(
+            arg0.len() == arg1.len(),
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The lengths of the vector are not aligned, args 0: {}, args 1: {}",
+                    arg0.len(),
+                    arg1.len(),
+                )
+            }
+        );
+
+        let len = arg0.len();
+        let mut result = BinaryVectorBuilder::with_capacity(len);
+        if len == 0 {
+            return Ok(result.to_vector());
+        }
+
+        let arg0_const = as_veclit_if_const(arg0)?;
+        let arg1_const = as_veclit_if_const(arg1)?;
+
+        for i in 0..len {
+            let arg0 = match arg0_const.as_ref() {
+                Some(arg0) => Some(Cow::Borrowed(arg0.as_ref())),
+                None => as_veclit(arg0.get_ref(i))?,
+            };
+            let arg1 = match arg1_const.as_ref() {
+                Some(arg1) => Some(Cow::Borrowed(arg1.as_ref())),
+                None => as_veclit(arg1.get_ref(i))?,
+            };
+            let (Some(arg0), Some(arg1)) = (arg0, arg1) else {
+                result.push_null();
+                continue;
+            };
+            let vec0 = DVectorView::from_slice(&arg0, arg0.len());
+            let vec1 = DVectorView::from_slice(&arg1, arg1.len());
+
+            let vec_res = vec0 - vec1;
+            let veclit = vec_res.as_slice();
+            let binlit = veclit_to_binlit(veclit);
+            result.push(Some(&binlit));
+        }
+
+        Ok(result.to_vector())
+    }
+}
+
+impl Display for SubFunction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", NAME.to_ascii_uppercase())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::error::Error;
+    use datatypes::vectors::StringVector;
+
+    use super::*;
+
+    #[test]
+    fn test_sub() {
+        let func = SubFunction;
+
+        let input0 = Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[4.0,5.0,6.0]".to_string()),
+            None,
+            Some("[2.0,3.0,3.0]".to_string()),
+        ]));
+        let input1 = Arc::new(StringVector::from(vec![
+            Some("[1.0,1.0,1.0]".to_string()),
+            Some("[6.0,5.0,4.0]".to_string()),
+            Some("[3.0,2.0,2.0]".to_string()),
+            None,
+        ]));
+
+        let result = func
+            .eval(FunctionContext::default(), &[input0, input1])
+            .unwrap();
+
+        let result = result.as_ref();
+        assert_eq!(result.len(), 4);
+        assert_eq!(
+            result.get_ref(0).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[0.0, 1.0, 2.0]).as_slice())
+        );
+        assert_eq!(
+            result.get_ref(1).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[-2.0, 0.0, 2.0]).as_slice())
+        );
+        assert!(result.get_ref(2).is_null());
+        assert!(result.get_ref(3).is_null());
+    }
+
+    #[test]
+    fn test_sub_error() {
+        let func = SubFunction;
+
+        let input0 = Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[4.0,5.0,6.0]".to_string()),
+            None,
+            Some("[2.0,3.0,3.0]".to_string()),
+        ]));
+        let input1 = Arc::new(StringVector::from(vec![
+            Some("[1.0,1.0,1.0]".to_string()),
+            Some("[6.0,5.0,4.0]".to_string()),
+            Some("[3.0,2.0,2.0]".to_string()),
+        ]));
+
+        let result = func.eval(FunctionContext::default(), &[input0, input1]);
+
+        match result {
+            Err(Error::InvalidFuncArgs { err_msg, .. }) => {
+                assert_eq!(
+                    err_msg,
+                    "The lengths of the vector are not aligned, args 0: 4, args 1: 3"
+                )
+            }
+            _ => unreachable!(),
+        }
+    }
+}
--- a/src/common/function/src/scalars/vector/sum.rs
+++ b/src/common/function/src/scalars/vector/sum.rs
@@ -0,0 +1,202 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_query::error::{CreateAccumulatorSnafu, Error, InvalidFuncArgsSnafu};
+use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
+use common_query::prelude::AccumulatorCreatorFunction;
+use datatypes::prelude::{ConcreteDataType, Value, *};
+use datatypes::vectors::VectorRef;
+use nalgebra::{Const, DVectorView, Dyn, OVector};
+use snafu::ensure;
+
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const, veclit_to_binlit};
+
+#[derive(Debug, Default)]
+pub struct VectorSum {
+    sum: Option<OVector<f32, Dyn>>,
+    has_null: bool,
+}
+
+#[as_aggr_func_creator]
+#[derive(Debug, Default, AggrFuncTypeStore)]
+pub struct VectorSumCreator {}
+
+impl AggregateFunctionCreator for VectorSumCreator {
+    fn creator(&self) -> AccumulatorCreatorFunction {
+        let creator: AccumulatorCreatorFunction = Arc::new(move |types: &[ConcreteDataType]| {
+            ensure!(
+                types.len() == 1,
+                InvalidFuncArgsSnafu {
+                    err_msg: format!(
+                        "The length of the args is not correct, expect exactly one, have: {}",
+                        types.len()
+                    )
+                }
+            );
+            let input_type = &types[0];
+            match input_type {
+                ConcreteDataType::String(_) | ConcreteDataType::Binary(_) => {
+                    Ok(Box::new(VectorSum::default()))
+                }
+                _ => {
+                    let err_msg = format!(
+                        "\"VEC_SUM\" aggregate function not support data type {:?}",
+                        input_type.logical_type_id(),
+                    );
+                    CreateAccumulatorSnafu { err_msg }.fail()?
+                }
+            }
+        });
+        creator
+    }
+
+    fn output_type(&self) -> common_query::error::Result<ConcreteDataType> {
+        Ok(ConcreteDataType::binary_datatype())
+    }
+
+    fn state_types(&self) -> common_query::error::Result<Vec<ConcreteDataType>> {
+        Ok(vec![self.output_type()?])
+    }
+}
+
+impl VectorSum {
+    fn inner(&mut self, len: usize) -> &mut OVector<f32, Dyn> {
+        self.sum
+            .get_or_insert_with(|| OVector::zeros_generic(Dyn(len), Const::<1>))
+    }
+
+    fn update(&mut self, values: &[VectorRef], is_update: bool) -> Result<(), Error> {
+        if values.is_empty() || self.has_null {
+            return Ok(());
+        };
+        let column = &values[0];
+        let len = column.len();
+
+        match as_veclit_if_const(column)? {
+            Some(column) => {
+                let vec_column = DVectorView::from_slice(&column, column.len()).scale(len as f32);
+                *self.inner(vec_column.len()) += vec_column;
+            }
+            None => {
+                for i in 0..len {
+                    let Some(arg0) = as_veclit(column.get_ref(i))? else {
+                        if is_update {
+                            self.has_null = true;
+                            self.sum = None;
+                        }
+                        return Ok(());
+                    };
+                    let vec_column = DVectorView::from_slice(&arg0, arg0.len());
+                    *self.inner(vec_column.len()) += vec_column;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+impl Accumulator for VectorSum {
+    fn state(&self) -> common_query::error::Result<Vec<Value>> {
+        self.evaluate().map(|v| vec![v])
+    }
+
+    fn update_batch(&mut self, values: &[VectorRef]) -> common_query::error::Result<()> {
+        self.update(values, true)
+    }
+
+    fn merge_batch(&mut self, states: &[VectorRef]) -> common_query::error::Result<()> {
+        self.update(states, false)
+    }
+
+    fn evaluate(&self) -> common_query::error::Result<Value> {
+        match &self.sum {
+            None => Ok(Value::Null),
+            Some(vector) => Ok(Value::from(veclit_to_binlit(vector.as_slice()))),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use datatypes::vectors::{ConstantVector, StringVector};
+
+    use super::*;
+
+    #[test]
+    fn test_update_batch() {
+        // test update empty batch, expect not updating anything
+        let mut vec_sum = VectorSum::default();
+        vec_sum.update_batch(&[]).unwrap();
+        assert!(vec_sum.sum.is_none());
+        assert!(!vec_sum.has_null);
+        assert_eq!(Value::Null, vec_sum.evaluate().unwrap());
+
+        // test update one not-null value
+        let mut vec_sum = VectorSum::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![Some(
+            "[1.0,2.0,3.0]".to_string(),
+        )]))];
+        vec_sum.update_batch(&v).unwrap();
+        assert_eq!(
+            Value::from(veclit_to_binlit(&[1.0, 2.0, 3.0])),
+            vec_sum.evaluate().unwrap()
+        );
+
+        // test update one null value
+        let mut vec_sum = VectorSum::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![Option::<String>::None]))];
+        vec_sum.update_batch(&v).unwrap();
+        assert_eq!(Value::Null, vec_sum.evaluate().unwrap());
+
+        // test update no null-value batch
+        let mut vec_sum = VectorSum::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[4.0,5.0,6.0]".to_string()),
+            Some("[7.0,8.0,9.0]".to_string()),
+        ]))];
+        vec_sum.update_batch(&v).unwrap();
+        assert_eq!(
+            Value::from(veclit_to_binlit(&[12.0, 15.0, 18.0])),
+            vec_sum.evaluate().unwrap()
+        );
+
+        // test update null-value batch
+        let mut vec_sum = VectorSum::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            None,
+            Some("[7.0,8.0,9.0]".to_string()),
+        ]))];
+        vec_sum.update_batch(&v).unwrap();
+        assert_eq!(Value::Null, vec_sum.evaluate().unwrap());
+
+        // test update with constant vector
+        let mut vec_sum = VectorSum::default();
+        let v: Vec<VectorRef> = vec![Arc::new(ConstantVector::new(
+            Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])),
+            4,
+        ))];
+        vec_sum.update_batch(&v).unwrap();
+        assert_eq!(
+            Value::from(veclit_to_binlit(&[4.0, 8.0, 12.0])),
+            vec_sum.evaluate().unwrap()
+        );
+    }
+}
--- a/src/common/function/src/scalars/vector/vector_mul.rs
+++ b/src/common/function/src/scalars/vector/vector_mul.rs
@@ -0,0 +1,205 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::borrow::Cow;
+use std::fmt::Display;
+
+use common_query::error::{InvalidFuncArgsSnafu, Result};
+use common_query::prelude::Signature;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{BinaryVectorBuilder, MutableVector, VectorRef};
+use nalgebra::DVectorView;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::helper;
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const, veclit_to_binlit};
+
+const NAME: &str = "vec_mul";
+
+/// Multiplies corresponding elements of two vectors.
+///
+/// # Example
+///
+/// ```sql
+/// SELECT vec_to_string(vec_mul("[1, 2, 3]", "[1, 2, 3]")) as result;
+///
+/// +---------+
+/// | result  |
+/// +---------+
+/// | [1,4,9] |
+/// +---------+
+///
+/// ```
+#[derive(Debug, Clone, Default)]
+pub struct VectorMulFunction;
+
+impl Function for VectorMulFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::binary_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        helper::one_of_sigs2(
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::binary_datatype(),
+            ],
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::binary_datatype(),
+            ],
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly two, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let arg0 = &columns[0];
+        let arg1 = &columns[1];
+
+        let len = arg0.len();
+        let mut result = BinaryVectorBuilder::with_capacity(len);
+        if len == 0 {
+            return Ok(result.to_vector());
+        }
+
+        let arg0_const = as_veclit_if_const(arg0)?;
+        let arg1_const = as_veclit_if_const(arg1)?;
+
+        for i in 0..len {
+            let arg0 = match arg0_const.as_ref() {
+                Some(arg0) => Some(Cow::Borrowed(arg0.as_ref())),
+                None => as_veclit(arg0.get_ref(i))?,
+            };
+
+            let arg1 = match arg1_const.as_ref() {
+                Some(arg1) => Some(Cow::Borrowed(arg1.as_ref())),
+                None => as_veclit(arg1.get_ref(i))?,
+            };
+
+            if let (Some(arg0), Some(arg1)) = (arg0, arg1) {
+                ensure!(
+                    arg0.len() == arg1.len(),
+                    InvalidFuncArgsSnafu {
+                        err_msg: format!(
+                            "The length of the vectors must match for multiplying, have: {} vs {}",
+                            arg0.len(),
+                            arg1.len()
+                        ),
+                    }
+                );
+                let vec0 = DVectorView::from_slice(&arg0, arg0.len());
+                let vec1 = DVectorView::from_slice(&arg1, arg1.len());
+                let vec_res = vec1.component_mul(&vec0);
+
+                let veclit = vec_res.as_slice();
+                let binlit = veclit_to_binlit(veclit);
+                result.push(Some(&binlit));
+            } else {
+                result.push_null();
+            }
+        }
+
+        Ok(result.to_vector())
+    }
+}
+
+impl Display for VectorMulFunction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", NAME.to_ascii_uppercase())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::error;
+    use datatypes::vectors::StringVector;
+
+    use super::*;
+
+    #[test]
+    fn test_vector_mul() {
+        let func = VectorMulFunction;
+
+        let vec0 = vec![1.0, 2.0, 3.0];
+        let vec1 = vec![1.0, 1.0];
+        let (len0, len1) = (vec0.len(), vec1.len());
+        let input0 = Arc::new(StringVector::from(vec![Some(format!("{vec0:?}"))]));
+        let input1 = Arc::new(StringVector::from(vec![Some(format!("{vec1:?}"))]));
+
+        let err = func
+            .eval(FunctionContext::default(), &[input0, input1])
+            .unwrap_err();
+
+        match err {
+            error::Error::InvalidFuncArgs { err_msg, .. } => {
+                assert_eq!(
+                    err_msg,
+                    format!(
+                        "The length of the vectors must match for multiplying, have: {} vs {}",
+                        len0, len1
+                    )
+                )
+            }
+            _ => unreachable!(),
+        }
+
+        let input0 = Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[8.0,10.0,12.0]".to_string()),
+            Some("[7.0,8.0,9.0]".to_string()),
+            None,
+        ]));
+
+        let input1 = Arc::new(StringVector::from(vec![
+            Some("[1.0,1.0,1.0]".to_string()),
+            Some("[2.0,2.0,2.0]".to_string()),
+            None,
+            Some("[3.0,3.0,3.0]".to_string()),
+        ]));
+
+        let result = func
+            .eval(FunctionContext::default(), &[input0, input1])
+            .unwrap();
+
+        let result = result.as_ref();
+        assert_eq!(result.len(), 4);
+        assert_eq!(
+            result.get_ref(0).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[1.0, 2.0, 3.0]).as_slice())
+        );
+        assert_eq!(
+            result.get_ref(1).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[16.0, 20.0, 24.0]).as_slice())
+        );
+        assert!(result.get_ref(2).is_null());
+        assert!(result.get_ref(3).is_null());
+    }
+}
--- a/src/common/function/src/utils.rs
+++ b/src/common/function/src/utils.rs
@@ -0,0 +1,58 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Escapes special characters in the provided pattern string for `LIKE`.
+///
+/// Specifically, it prefixes the backslash (`\`), percent (`%`), and underscore (`_`)
+/// characters with an additional backslash to ensure they are treated literally.
+///
+/// # Examples
+///
+/// ```rust
+/// let escaped = escape_pattern("100%_some\\path");
+/// assert_eq!(escaped, "100\\%\\_some\\\\path");
+/// ```
+pub fn escape_like_pattern(pattern: &str) -> String {
+    pattern
+        .chars()
+        .flat_map(|c| match c {
+            '\\' | '%' | '_' => vec!['\\', c],
+            _ => vec![c],
+        })
+        .collect::<String>()
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_escape_like_pattern() {
+        assert_eq!(
+            escape_like_pattern("100%_some\\path"),
+            "100\\%\\_some\\\\path"
+        );
+        assert_eq!(escape_like_pattern(""), "");
+        assert_eq!(escape_like_pattern("hello"), "hello");
+        assert_eq!(escape_like_pattern("\\%_"), "\\\\\\%\\_");
+        assert_eq!(escape_like_pattern("%%__\\\\"), "\\%\\%\\_\\_\\\\\\\\");
+        assert_eq!(escape_like_pattern("abc123"), "abc123");
+        assert_eq!(escape_like_pattern("%_\\"), "\\%\\_\\\\");
+        assert_eq!(
+            escape_like_pattern("%%__\\\\another%string"),
+            "\\%\\%\\_\\_\\\\\\\\another\\%string"
+        );
+        assert_eq!(escape_like_pattern("foo%bar_"), "foo\\%bar\\_");
+        assert_eq!(escape_like_pattern("\\_\\%"), "\\\\\\_\\\\\\%");
+    }
+}
--- a/src/common/meta/src/cache/container.rs
+++ b/src/common/meta/src/cache/container.rs
@@ -43,7 +43,7 @@ pub struct CacheContainer<K, V, CacheToken> {
    cache: Cache<K, V>,
    invalidator: Invalidator<K, V, CacheToken>,
    initializer: Initializer<K, V>,
-    token_filter: TokenFilter<CacheToken>,
+    token_filter: fn(&CacheToken) -> bool,
 }

 impl<K, V, CacheToken> CacheContainer<K, V, CacheToken>
@@ -58,7 +58,7 @@ where
        cache: Cache<K, V>,
        invalidator: Invalidator<K, V, CacheToken>,
        initializer: Initializer<K, V>,
-        token_filter: TokenFilter<CacheToken>,
+        token_filter: fn(&CacheToken) -> bool,
    ) -> Self {
        Self {
            name,
@@ -206,10 +206,13 @@ mod tests {
        name: &'a str,
    }

+    fn always_true_filter(_: &String) -> bool {
+        true
+    }
+
    #[tokio::test]
    async fn test_get() {
        let cache: Cache<NameKey, String> = CacheBuilder::new(128).build();
-        let filter: TokenFilter<String> = Box::new(|_| true);
        let counter = Arc::new(AtomicI32::new(0));
        let moved_counter = counter.clone();
        let init: Initializer<NameKey, String> = Arc::new(move |_| {
@@ -219,7 +222,13 @@ mod tests {
        let invalidator: Invalidator<NameKey, String, String> =
            Box::new(|_, _| Box::pin(async { Ok(()) }));

-        let adv_cache = CacheContainer::new("test".to_string(), cache, invalidator, init, filter);
+        let adv_cache = CacheContainer::new(
+            "test".to_string(),
+            cache,
+            invalidator,
+            init,
+            always_true_filter,
+        );
        let key = NameKey { name: "key" };
        let value = adv_cache.get(key).await.unwrap().unwrap();
        assert_eq!(value, "hi");
@@ -233,7 +242,6 @@ mod tests {
    #[tokio::test]
    async fn test_get_by_ref() {
        let cache: Cache<String, String> = CacheBuilder::new(128).build();
-        let filter: TokenFilter<String> = Box::new(|_| true);
        let counter = Arc::new(AtomicI32::new(0));
        let moved_counter = counter.clone();
        let init: Initializer<String, String> = Arc::new(move |_| {
@@ -243,7 +251,13 @@ mod tests {
        let invalidator: Invalidator<String, String, String> =
            Box::new(|_, _| Box::pin(async { Ok(()) }));

-        let adv_cache = CacheContainer::new("test".to_string(), cache, invalidator, init, filter);
+        let adv_cache = CacheContainer::new(
+            "test".to_string(),
+            cache,
+            invalidator,
+            init,
+            always_true_filter,
+        );
        let value = adv_cache.get_by_ref("foo").await.unwrap().unwrap();
        assert_eq!(value, "hi");
        let value = adv_cache.get_by_ref("foo").await.unwrap().unwrap();
@@ -257,13 +271,18 @@ mod tests {
    #[tokio::test]
    async fn test_get_value_not_exits() {
        let cache: Cache<String, String> = CacheBuilder::new(128).build();
-        let filter: TokenFilter<String> = Box::new(|_| true);
        let init: Initializer<String, String> =
            Arc::new(move |_| Box::pin(async { error::ValueNotExistSnafu {}.fail() }));
        let invalidator: Invalidator<String, String, String> =
            Box::new(|_, _| Box::pin(async { Ok(()) }));

-        let adv_cache = CacheContainer::new("test".to_string(), cache, invalidator, init, filter);
+        let adv_cache = CacheContainer::new(
+            "test".to_string(),
+            cache,
+            invalidator,
+            init,
+            always_true_filter,
+        );
        let value = adv_cache.get_by_ref("foo").await.unwrap();
        assert!(value.is_none());
    }
@@ -271,7 +290,6 @@ mod tests {
    #[tokio::test]
    async fn test_invalidate() {
        let cache: Cache<String, String> = CacheBuilder::new(128).build();
-        let filter: TokenFilter<String> = Box::new(|_| true);
        let counter = Arc::new(AtomicI32::new(0));
        let moved_counter = counter.clone();
        let init: Initializer<String, String> = Arc::new(move |_| {
@@ -285,7 +303,13 @@ mod tests {
            })
        });

-        let adv_cache = CacheContainer::new("test".to_string(), cache, invalidator, init, filter);
+        let adv_cache = CacheContainer::new(
+            "test".to_string(),
+            cache,
+            invalidator,
+            init,
+            always_true_filter,
+        );
        let value = adv_cache.get_by_ref("foo").await.unwrap().unwrap();
        assert_eq!(value, "hi");
        let value = adv_cache.get_by_ref("foo").await.unwrap().unwrap();
--- a/src/common/meta/src/cache/flow/table_flownode.rs
+++ b/src/common/meta/src/cache/flow/table_flownode.rs
@@ -45,7 +45,7 @@ pub fn new_table_flownode_set_cache(
    let table_flow_manager = Arc::new(TableFlowManager::new(kv_backend));
    let init = init_factory(table_flow_manager);

-    CacheContainer::new(name, cache, Box::new(invalidator), init, Box::new(filter))
+    CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
 }

 fn init_factory(table_flow_manager: TableFlowManagerRef) -> Initializer<TableId, FlownodeSet> {
--- a/src/common/meta/src/cache/registry.rs
+++ b/src/common/meta/src/cache/registry.rs
@@ -151,12 +151,15 @@ mod tests {
    use crate::cache::*;
    use crate::instruction::CacheIdent;

+    fn always_true_filter(_: &CacheIdent) -> bool {
+        true
+    }
+
    fn test_cache(
        name: &str,
        invalidator: Invalidator<String, String, CacheIdent>,
    ) -> CacheContainer<String, String, CacheIdent> {
        let cache: Cache<String, String> = CacheBuilder::new(128).build();
-        let filter: TokenFilter<CacheIdent> = Box::new(|_| true);
        let counter = Arc::new(AtomicI32::new(0));
        let moved_counter = counter.clone();
        let init: Initializer<String, String> = Arc::new(move |_| {
@@ -164,7 +167,13 @@ mod tests {
            Box::pin(async { Ok(Some("hi".to_string())) })
        });

-        CacheContainer::new(name.to_string(), cache, invalidator, init, filter)
+        CacheContainer::new(
+            name.to_string(),
+            cache,
+            invalidator,
+            init,
+            always_true_filter,
+        )
    }

    fn test_i32_cache(
@@ -172,7 +181,6 @@ mod tests {
        invalidator: Invalidator<i32, String, CacheIdent>,
    ) -> CacheContainer<i32, String, CacheIdent> {
        let cache: Cache<i32, String> = CacheBuilder::new(128).build();
-        let filter: TokenFilter<CacheIdent> = Box::new(|_| true);
        let counter = Arc::new(AtomicI32::new(0));
        let moved_counter = counter.clone();
        let init: Initializer<i32, String> = Arc::new(move |_| {
@@ -180,7 +188,13 @@ mod tests {
            Box::pin(async { Ok(Some("foo".to_string())) })
        });

-        CacheContainer::new(name.to_string(), cache, invalidator, init, filter)
+        CacheContainer::new(
+            name.to_string(),
+            cache,
+            invalidator,
+            init,
+            always_true_filter,
+        )
    }

    #[tokio::test]
--- a/src/common/meta/src/cache/table/schema.rs
+++ b/src/common/meta/src/cache/table/schema.rs
@@ -36,7 +36,7 @@ pub fn new_schema_cache(
    let schema_manager = SchemaManager::new(kv_backend.clone());
    let init = init_factory(schema_manager);

-    CacheContainer::new(name, cache, Box::new(invalidator), init, Box::new(filter))
+    CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
 }

 fn init_factory(schema_manager: SchemaManager) -> Initializer<SchemaName, Arc<SchemaNameValue>> {
--- a/src/common/meta/src/cache/table/table_info.rs
+++ b/src/common/meta/src/cache/table/table_info.rs
@@ -41,7 +41,7 @@ pub fn new_table_info_cache(
    let table_info_manager = Arc::new(TableInfoManager::new(kv_backend));
    let init = init_factory(table_info_manager);

-    CacheContainer::new(name, cache, Box::new(invalidator), init, Box::new(filter))
+    CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
 }

 fn init_factory(table_info_manager: TableInfoManagerRef) -> Initializer<TableId, Arc<TableInfo>> {
--- a/src/common/meta/src/cache/table/table_name.rs
+++ b/src/common/meta/src/cache/table/table_name.rs
@@ -41,7 +41,7 @@ pub fn new_table_name_cache(
    let table_name_manager = Arc::new(TableNameManager::new(kv_backend));
    let init = init_factory(table_name_manager);

-    CacheContainer::new(name, cache, Box::new(invalidator), init, Box::new(filter))
+    CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
 }

 fn init_factory(table_name_manager: TableNameManagerRef) -> Initializer<TableName, TableId> {
--- a/src/common/meta/src/cache/table/table_route.rs
+++ b/src/common/meta/src/cache/table/table_route.rs
@@ -65,7 +65,7 @@ pub fn new_table_route_cache(
    let table_info_manager = Arc::new(TableRouteManager::new(kv_backend));
    let init = init_factory(table_info_manager);

-    CacheContainer::new(name, cache, Box::new(invalidator), init, Box::new(filter))
+    CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
 }

 fn init_factory(
--- a/src/common/meta/src/cache/table/table_schema.rs
+++ b/src/common/meta/src/cache/table/table_schema.rs
@@ -40,7 +40,7 @@ pub fn new_table_schema_cache(
    let table_info_manager = TableInfoManager::new(kv_backend);
    let init = init_factory(table_info_manager);

-    CacheContainer::new(name, cache, Box::new(invalidator), init, Box::new(filter))
+    CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
 }

 fn init_factory(table_info_manager: TableInfoManager) -> Initializer<TableId, Arc<SchemaName>> {
--- a/src/common/meta/src/cache/table/view_info.rs
+++ b/src/common/meta/src/cache/table/view_info.rs
@@ -40,7 +40,7 @@ pub fn new_view_info_cache(
    let view_info_manager = Arc::new(ViewInfoManager::new(kv_backend));
    let init = init_factory(view_info_manager);

-    CacheContainer::new(name, cache, Box::new(invalidator), init, Box::new(filter))
+    CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
 }

 fn init_factory(view_info_manager: ViewInfoManagerRef) -> Initializer<TableId, Arc<ViewInfoValue>> {
--- a/src/common/meta/src/key/catalog_name.rs
+++ b/src/common/meta/src/key/catalog_name.rs
@@ -13,7 +13,6 @@
 // limitations under the License.

 use std::fmt::Display;
-use std::sync::Arc;

 use common_catalog::consts::DEFAULT_CATALOG_NAME;
 use futures::stream::BoxStream;
@@ -146,7 +145,7 @@ impl CatalogManager {
            self.kv_backend.clone(),
            req,
            DEFAULT_PAGE_SIZE,
-            Arc::new(catalog_decoder),
+            catalog_decoder,
        )
        .into_stream();

@@ -156,6 +155,8 @@ impl CatalogManager {

 #[cfg(test)]
 mod tests {
+    use std::sync::Arc;
+
    use super::*;
    use crate::kv_backend::memory::MemoryKvBackend;

--- a/src/common/meta/src/key/datanode_table.rs
+++ b/src/common/meta/src/key/datanode_table.rs
@@ -14,7 +14,6 @@

 use std::collections::HashMap;
 use std::fmt::Display;
-use std::sync::Arc;

 use futures::stream::BoxStream;
 use serde::{Deserialize, Serialize};
@@ -166,7 +165,7 @@ impl DatanodeTableManager {
            self.kv_backend.clone(),
            req,
            DEFAULT_PAGE_SIZE,
-            Arc::new(datanode_table_value_decoder),
+            datanode_table_value_decoder,
        )
        .into_stream();

--- a/src/common/meta/src/key/flow/flow_name.rs
+++ b/src/common/meta/src/key/flow/flow_name.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::sync::Arc;
-
 use futures::stream::BoxStream;
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -201,7 +199,7 @@ impl FlowNameManager {
            self.kv_backend.clone(),
            req,
            DEFAULT_PAGE_SIZE,
-            Arc::new(flow_name_decoder),
+            flow_name_decoder,
        )
        .into_stream();

--- a/src/common/meta/src/key/flow/flow_route.rs
+++ b/src/common/meta/src/key/flow/flow_route.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::sync::Arc;
-
 use futures::stream::BoxStream;
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -179,7 +177,7 @@ impl FlowRouteManager {
            self.kv_backend.clone(),
            req,
            DEFAULT_PAGE_SIZE,
-            Arc::new(flow_route_decoder),
+            flow_route_decoder,
        )
        .into_stream();

--- a/src/common/meta/src/key/flow/flownode_flow.rs
+++ b/src/common/meta/src/key/flow/flownode_flow.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::sync::Arc;
-
 use futures::stream::BoxStream;
 use futures::TryStreamExt;
 use lazy_static::lazy_static;
@@ -179,7 +177,7 @@ impl FlownodeFlowManager {
            self.kv_backend.clone(),
            req,
            DEFAULT_PAGE_SIZE,
-            Arc::new(flownode_flow_key_decoder),
+            flownode_flow_key_decoder,
        )
        .into_stream();

--- a/src/common/meta/src/key/flow/table_flow.rs
+++ b/src/common/meta/src/key/flow/table_flow.rs
@@ -206,7 +206,7 @@ impl TableFlowManager {
            self.kv_backend.clone(),
            req,
            DEFAULT_PAGE_SIZE,
-            Arc::new(table_flow_decoder),
+            table_flow_decoder,
        )
        .into_stream();

--- a/src/common/meta/src/key/schema_name.rs
+++ b/src/common/meta/src/key/schema_name.rs
@@ -14,7 +14,6 @@

 use std::collections::HashMap;
 use std::fmt::Display;
-use std::sync::Arc;

 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_time::DatabaseTimeToLive;
@@ -283,7 +282,7 @@ impl SchemaManager {
            self.kv_backend.clone(),
            req,
            DEFAULT_PAGE_SIZE,
-            Arc::new(schema_decoder),
+            schema_decoder,
        )
        .into_stream();

@@ -308,6 +307,7 @@ impl<'a> From<&'a SchemaName> for SchemaNameKey<'a> {

 #[cfg(test)]
 mod tests {
+    use std::sync::Arc;
    use std::time::Duration;

    use super::*;
--- a/src/common/meta/src/key/table_name.rs
+++ b/src/common/meta/src/key/table_name.rs
@@ -269,7 +269,7 @@ impl TableNameManager {
            self.kv_backend.clone(),
            req,
            DEFAULT_PAGE_SIZE,
-            Arc::new(table_decoder),
+            table_decoder,
        )
        .into_stream();

--- a/src/common/meta/src/kv_backend/postgres.rs
+++ b/src/common/meta/src/kv_backend/postgres.rs
@@ -16,6 +16,7 @@ use std::any::Any;
 use std::borrow::Cow;
 use std::sync::Arc;

+use common_telemetry::error;
 use snafu::ResultExt;
 use tokio_postgres::types::ToSql;
 use tokio_postgres::{Client, NoTls};
@@ -97,7 +98,11 @@ impl PgStore {
        let (client, conn) = tokio_postgres::connect(url, NoTls)
            .await
            .context(ConnectPostgresSnafu)?;
-        tokio::spawn(async move { conn.await.context(ConnectPostgresSnafu) });
+        tokio::spawn(async move {
+            if let Err(e) = conn.await {
+                error!(e; "connection error");
+            }
+        });
        Self::with_pg_client(client).await
    }

--- a/src/common/meta/src/range_stream.rs
+++ b/src/common/meta/src/range_stream.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::sync::Arc;
-
 use async_stream::try_stream;
 use common_telemetry::debug;
 use futures::Stream;
@@ -148,7 +146,7 @@ impl PaginationStreamFactory {
 }

 pub struct PaginationStream<T> {
-    decoder_fn: Arc<KeyValueDecoderFn<T>>,
+    decoder_fn: fn(KeyValue) -> Result<T>,
    factory: PaginationStreamFactory,
 }

@@ -158,7 +156,7 @@ impl<T> PaginationStream<T> {
        kv: KvBackendRef,
        req: RangeRequest,
        page_size: usize,
-        decoder_fn: Arc<KeyValueDecoderFn<T>>,
+        decoder_fn: fn(KeyValue) -> Result<T>,
    ) -> Self {
        Self {
            decoder_fn,
@@ -191,6 +189,7 @@ mod tests {

    use std::assert_matches::assert_matches;
    use std::collections::BTreeMap;
+    use std::sync::Arc;

    use futures::TryStreamExt;

@@ -250,7 +249,7 @@ mod tests {
                ..Default::default()
            },
            DEFAULT_PAGE_SIZE,
-            Arc::new(decoder),
+            decoder,
        )
        .into_stream();
        let kv = stream.try_collect::<Vec<_>>().await.unwrap();
@@ -290,7 +289,7 @@ mod tests {
                ..Default::default()
            },
            2,
-            Arc::new(decoder),
+            decoder,
        );
        let kv = stream
            .into_stream()
--- a/src/common/meta/src/state_store.rs
+++ b/src/common/meta/src/state_store.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::sync::Arc;
-
 use async_trait::async_trait;
 use common_error::ext::BoxedError;
 use common_procedure::error::{DeleteStatesSnafu, ListStateSnafu, PutStateSnafu};
@@ -171,7 +169,7 @@ impl StateStore for KvStateStore {
            self.kv_backend.clone(),
            req,
            self.max_num_per_range_request.unwrap_or_default(),
-            Arc::new(decode_kv),
+            decode_kv,
        )
        .into_stream();

--- a/src/common/procedure/src/local/runner.rs
+++ b/src/common/procedure/src/local/runner.rs
@@ -544,7 +544,7 @@ mod tests {
    use common_test_util::temp_dir::create_temp_dir;
    use futures_util::future::BoxFuture;
    use futures_util::FutureExt;
-    use object_store::ObjectStore;
+    use object_store::{EntryMode, ObjectStore};
    use tokio::sync::mpsc;

    use super::*;
@@ -578,7 +578,11 @@ mod tests {
    ) {
        let dir = proc_path!(procedure_store, "{procedure_id}/");
        let lister = object_store.list(&dir).await.unwrap();
-        let mut files_in_dir: Vec<_> = lister.into_iter().map(|de| de.name().to_string()).collect();
+        let mut files_in_dir: Vec<_> = lister
+            .into_iter()
+            .filter(|x| x.metadata().mode() == EntryMode::FILE)
+            .map(|de| de.name().to_string())
+            .collect();
        files_in_dir.sort_unstable();
        assert_eq!(files, files_in_dir);
    }
--- a/src/datanode/src/error.rs
+++ b/src/datanode/src/error.rs
@@ -193,6 +193,14 @@ pub enum Error {
        location: Location,
    },

+    #[snafu(display("Failed to build http client"))]
+    BuildHttpClient {
+        #[snafu(implicit)]
+        location: Location,
+        #[snafu(source)]
+        error: reqwest::Error,
+    },
+
    #[snafu(display("Missing required field: {}", name))]
    MissingRequiredField {
        name: String,
@@ -406,9 +414,10 @@ impl ErrorExt for Error {
            | MissingKvBackend { .. }
            | TomlFormat { .. } => StatusCode::InvalidArguments,

-            PayloadNotExist { .. } | Unexpected { .. } | WatchAsyncTaskChange { .. } => {
-                StatusCode::Unexpected
-            }
+            PayloadNotExist { .. }
+            | Unexpected { .. }
+            | WatchAsyncTaskChange { .. }
+            | BuildHttpClient { .. } => StatusCode::Unexpected,

            AsyncTaskExecute { source, .. } => source.status_code(),

--- a/src/datanode/src/store.rs
+++ b/src/datanode/src/store.rs
@@ -28,11 +28,11 @@ use common_telemetry::{info, warn};
 use object_store::layers::{LruCacheLayer, RetryInterceptor, RetryLayer};
 use object_store::services::Fs;
 use object_store::util::{join_dir, normalize_dir, with_instrument_layers};
-use object_store::{Access, Error, HttpClient, ObjectStore, ObjectStoreBuilder, OBJECT_CACHE_DIR};
+use object_store::{Access, Error, HttpClient, ObjectStore, ObjectStoreBuilder};
 use snafu::prelude::*;

 use crate::config::{HttpClientConfig, ObjectStoreConfig, DEFAULT_OBJECT_STORE_CACHE_SIZE};
-use crate::error::{self, CreateDirSnafu, Result};
+use crate::error::{self, BuildHttpClientSnafu, CreateDirSnafu, Result};

 pub(crate) async fn new_raw_object_store(
    store: &ObjectStoreConfig,
@@ -147,12 +147,10 @@ async fn build_cache_layer(
    };

    // Enable object cache by default
-    // Set the cache_path to be `${data_home}/object_cache/read/{name}` by default
+    // Set the cache_path to be `${data_home}` by default
    // if it's not present
    if cache_path.is_none() {
-        let object_cache_path = join_dir(data_home, OBJECT_CACHE_DIR);
-        let read_cache_path = join_dir(&object_cache_path, "read");
-        let read_cache_path = join_dir(&read_cache_path, &name.to_lowercase());
+        let read_cache_path = data_home.to_string();
        tokio::fs::create_dir_all(Path::new(&read_cache_path))
            .await
            .context(CreateDirSnafu {
@@ -236,7 +234,8 @@ pub(crate) fn build_http_client(config: &HttpClientConfig) -> Result<HttpClient>
        builder.timeout(config.timeout)
    };

-    HttpClient::build(http_builder).context(error::InitBackendSnafu)
+    let client = http_builder.build().context(BuildHttpClientSnafu)?;
+    Ok(HttpClient::with(client))
 }
 struct PrintDetailedError;

--- a/src/datatypes/src/schema.rs
+++ b/src/datatypes/src/schema.rs
@@ -29,7 +29,7 @@ use crate::error::{self, DuplicateColumnSnafu, Error, ProjectArrowSchemaSnafu, R
 use crate::prelude::ConcreteDataType;
 pub use crate::schema::column_schema::{
    ColumnSchema, FulltextAnalyzer, FulltextOptions, Metadata, SkippingIndexOptions,
-    COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE, COLUMN_FULLTEXT_OPT_KEY_ANALYZER,
+    SkippingIndexType, COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE, COLUMN_FULLTEXT_OPT_KEY_ANALYZER,
    COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY,
    COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE, COMMENT_KEY, FULLTEXT_KEY, INVERTED_INDEX_KEY,
    SKIPPING_INDEX_KEY, TIME_INDEX_KEY,
--- a/src/datatypes/src/schema/column_schema.rs
+++ b/src/datatypes/src/schema/column_schema.rs
@@ -543,7 +543,7 @@ pub struct SkippingIndexOptions {
    pub granularity: u32,
    /// The type of the skip index.
    #[serde(default)]
-    pub index_type: SkipIndexType,
+    pub index_type: SkippingIndexType,
 }

 impl fmt::Display for SkippingIndexOptions {
@@ -556,15 +556,15 @@ impl fmt::Display for SkippingIndexOptions {

 /// Skip index types.
 #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
-pub enum SkipIndexType {
+pub enum SkippingIndexType {
    #[default]
    BloomFilter,
 }

-impl fmt::Display for SkipIndexType {
+impl fmt::Display for SkippingIndexType {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
-            SkipIndexType::BloomFilter => write!(f, "BLOOM"),
+            SkippingIndexType::BloomFilter => write!(f, "BLOOM"),
        }
    }
 }
@@ -587,7 +587,7 @@ impl TryFrom<HashMap<String, String>> for SkippingIndexOptions {
        // Parse index type with default value BloomFilter
        let index_type = match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE) {
            Some(typ) => match typ.to_ascii_uppercase().as_str() {
-                "BLOOM" => SkipIndexType::BloomFilter,
+                "BLOOM" => SkippingIndexType::BloomFilter,
                _ => {
                    return error::InvalidSkippingIndexOptionSnafu {
                        msg: format!("Invalid index type: {typ}, expected: 'BLOOM'"),
@@ -595,7 +595,7 @@ impl TryFrom<HashMap<String, String>> for SkippingIndexOptions {
                    .fail();
                }
            },
-            None => SkipIndexType::default(),
+            None => SkippingIndexType::default(),
        };

        Ok(SkippingIndexOptions {
--- a/src/file-engine/src/manifest.rs
+++ b/src/file-engine/src/manifest.rs
@@ -46,7 +46,7 @@ impl FileRegionManifest {
    pub async fn store(&self, region_dir: &str, object_store: &ObjectStore) -> Result<()> {
        let path = &region_manifest_path(region_dir);
        let exist = object_store
-            .is_exist(path)
+            .exists(path)
            .await
            .context(CheckObjectSnafu { path })?;
        ensure!(!exist, ManifestExistsSnafu { path });
--- a/src/file-engine/src/region.rs
+++ b/src/file-engine/src/region.rs
@@ -130,7 +130,7 @@ mod tests {
        assert_eq!(region.metadata.primary_key, vec![1]);

        assert!(object_store
-            .is_exist("create_region_dir/manifest/_file_manifest")
+            .exists("create_region_dir/manifest/_file_manifest")
            .await
            .unwrap());

@@ -198,13 +198,13 @@ mod tests {
            .unwrap();

        assert!(object_store
-            .is_exist("drop_region_dir/manifest/_file_manifest")
+            .exists("drop_region_dir/manifest/_file_manifest")
            .await
            .unwrap());

        FileRegion::drop(&region, &object_store).await.unwrap();
        assert!(!object_store
-            .is_exist("drop_region_dir/manifest/_file_manifest")
+            .exists("drop_region_dir/manifest/_file_manifest")
            .await
            .unwrap());

--- a/src/flow/Cargo.toml
+++ b/src/flow/Cargo.toml
@@ -45,6 +45,7 @@ get-size2 = "0.1.2"
 greptime-proto.workspace = true
 # This fork of hydroflow is simply for keeping our dependency in our org, and pin the version
 # otherwise it is the same with upstream repo
+http.workspace = true
 hydroflow = { git = "https://github.com/GreptimeTeam/hydroflow.git", branch = "main" }
 itertools.workspace = true
 lazy_static.workspace = true
--- a/src/flow/src/adapter.rs
+++ b/src/flow/src/adapter.rs
@@ -30,7 +30,7 @@ use common_telemetry::{debug, info, trace};
 use datatypes::schema::ColumnSchema;
 use datatypes::value::Value;
 use greptime_proto::v1;
-use itertools::Itertools;
+use itertools::{EitherOrBoth, Itertools};
 use meta_client::MetaClientOptions;
 use query::QueryEngine;
 use serde::{Deserialize, Serialize};
@@ -46,17 +46,19 @@ use tokio::sync::{broadcast, watch, Mutex, RwLock};

 pub(crate) use crate::adapter::node_context::FlownodeContext;
 use crate::adapter::table_source::TableSource;
-use crate::adapter::util::column_schemas_to_proto;
+use crate::adapter::util::{
+    relation_desc_to_column_schemas_with_fallback, table_info_value_to_relation_desc,
+};
 use crate::adapter::worker::{create_worker, Worker, WorkerHandle};
 use crate::compute::ErrCollector;
 use crate::df_optimizer::sql_to_flow_plan;
 use crate::error::{
-    EvalSnafu, ExternalSnafu, FlowAlreadyExistSnafu, InternalSnafu, TableNotFoundSnafu,
+    EvalSnafu, ExternalSnafu, FlowAlreadyExistSnafu, InternalSnafu, InvalidQuerySnafu,
    UnexpectedSnafu,
 };
-use crate::expr::{Batch, GlobalId};
-use crate::metrics::{METRIC_FLOW_INSERT_ELAPSED, METRIC_FLOW_RUN_INTERVAL_MS};
-use crate::repr::{self, DiffRow, Row, BATCH_SIZE};
+use crate::expr::Batch;
+use crate::metrics::{METRIC_FLOW_INSERT_ELAPSED, METRIC_FLOW_ROWS, METRIC_FLOW_RUN_INTERVAL_MS};
+use crate::repr::{self, DiffRow, RelationDesc, Row, BATCH_SIZE};

 mod flownode_impl;
 mod parse_expr;
@@ -245,16 +247,26 @@ impl FlowWorkerManager {
            let (catalog, schema) = (table_name[0].clone(), table_name[1].clone());
            let ctx = Arc::new(QueryContext::with(&catalog, &schema));

-            let (is_ts_placeholder, proto_schema) =
-                self.try_fetch_or_create_table(&table_name).await?;
+            let (is_ts_placeholder, proto_schema) = self
+                .try_fetch_existing_table(&table_name)
+                .await?
+                .context(UnexpectedSnafu {
+                    reason: format!("Table not found: {}", table_name.join(".")),
+                })?;
            let schema_len = proto_schema.len();

+            let total_rows = reqs.iter().map(|r| r.len()).sum::<usize>();
            trace!(
                "Sending {} writeback requests to table {}, reqs total rows={}",
                reqs.len(),
                table_name.join("."),
                reqs.iter().map(|r| r.len()).sum::<usize>()
            );
+
+            METRIC_FLOW_ROWS
+                .with_label_values(&["out"])
+                .inc_by(total_rows as u64);
+
            let now = self.tick_manager.tick();
            for req in reqs {
                match req {
@@ -390,14 +402,12 @@ impl FlowWorkerManager {
        Ok(output)
    }

-    /// Fetch table info or create table from flow's schema if not exist
-    async fn try_fetch_or_create_table(
+    /// Fetch table schema and primary key from table info source, if table not exist return None
+    async fn fetch_table_pk_schema(
        &self,
        table_name: &TableName,
-    ) -> Result<(bool, Vec<api::v1::ColumnSchema>), Error> {
-        // TODO(discord9): instead of auto build table from request schema, actually build table
-        // before `create flow` to be able to assign pk and ts etc.
-        let (primary_keys, schema, is_ts_placeholder) = if let Some(table_id) = self
+    ) -> Result<Option<(Vec<String>, Option<usize>, Vec<ColumnSchema>)>, Error> {
+        if let Some(table_id) = self
            .table_info_source
            .get_table_id_from_name(table_name)
            .await?
@@ -414,97 +424,64 @@ impl FlowWorkerManager {
                .map(|i| meta.schema.column_schemas[i].name.clone())
                .collect_vec();
            let schema = meta.schema.column_schemas;
-            // check if the last column is the auto created timestamp column, hence the table is auto created from
-            // flow's plan type
-            let is_auto_create = {
-                let correct_name = schema
-                    .last()
-                    .map(|s| s.name == AUTO_CREATED_PLACEHOLDER_TS_COL)
-                    .unwrap_or(false);
-                let correct_time_index = meta.schema.timestamp_index == Some(schema.len() - 1);
-                correct_name && correct_time_index
-            };
-            (primary_keys, schema, is_auto_create)
+            let time_index = meta.schema.timestamp_index;
+            Ok(Some((primary_keys, time_index, schema)))
        } else {
-            // TODO(discord9): condiser remove buggy auto create by schema
+            Ok(None)
+        }
+    }

-            let node_ctx = self.node_context.read().await;
-            let gid: GlobalId = node_ctx
-                .table_repr
-                .get_by_name(table_name)
-                .map(|x| x.1)
-                .unwrap();
-            let schema = node_ctx
-                .schema
-                .get(&gid)
-                .with_context(|| TableNotFoundSnafu {
-                    name: format!("Table name = {:?}", table_name),
-                })?
-                .clone();
-            // TODO(discord9): use default key from schema
-            let primary_keys = schema
-                .typ()
-                .keys
-                .first()
-                .map(|v| {
-                    v.column_indices
-                        .iter()
-                        .map(|i| {
-                            schema
-                                .get_name(*i)
-                                .clone()
-                                .unwrap_or_else(|| format!("col_{i}"))
-                        })
-                        .collect_vec()
-                })
-                .unwrap_or_default();
-            let update_at = ColumnSchema::new(
-                UPDATE_AT_TS_COL,
+    /// return (primary keys, schema and if the table have a placeholder timestamp column)
+    /// schema of the table comes from flow's output plan
+    ///
+    /// adjust to add `update_at` column and ts placeholder if needed
+    async fn adjust_auto_created_table_schema(
+        &self,
+        schema: &RelationDesc,
+    ) -> Result<(Vec<String>, Vec<ColumnSchema>, bool), Error> {
+        // TODO(discord9): condiser remove buggy auto create by schema
+
+        // TODO(discord9): use default key from schema
+        let primary_keys = schema
+            .typ()
+            .keys
+            .first()
+            .map(|v| {
+                v.column_indices
+                    .iter()
+                    .map(|i| {
+                        schema
+                            .get_name(*i)
+                            .clone()
+                            .unwrap_or_else(|| format!("col_{i}"))
+                    })
+                    .collect_vec()
+            })
+            .unwrap_or_default();
+        let update_at = ColumnSchema::new(
+            UPDATE_AT_TS_COL,
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            true,
+        );
+
+        let original_schema = relation_desc_to_column_schemas_with_fallback(schema);
+
+        let mut with_auto_added_col = original_schema.clone();
+        with_auto_added_col.push(update_at);
+
+        // if no time index, add one as placeholder
+        let no_time_index = schema.typ().time_index.is_none();
+        if no_time_index {
+            let ts_col = ColumnSchema::new(
+                AUTO_CREATED_PLACEHOLDER_TS_COL,
                ConcreteDataType::timestamp_millisecond_datatype(),
                true,
-            );
+            )
+            .with_time_index(true);
+            with_auto_added_col.push(ts_col);
+        }

-            let original_schema = schema
-                .typ()
-                .column_types
-                .clone()
-                .into_iter()
-                .enumerate()
-                .map(|(idx, typ)| {
-                    let name = schema
-                        .names
-                        .get(idx)
-                        .cloned()
-                        .flatten()
-                        .unwrap_or(format!("col_{}", idx));
-                    let ret = ColumnSchema::new(name, typ.scalar_type, typ.nullable);
-                    if schema.typ().time_index == Some(idx) {
-                        ret.with_time_index(true)
-                    } else {
-                        ret
-                    }
-                })
-                .collect_vec();
-
-            let mut with_auto_added_col = original_schema.clone();
-            with_auto_added_col.push(update_at);
-
-            // if no time index, add one as placeholder
-            let no_time_index = schema.typ().time_index.is_none();
-            if no_time_index {
-                let ts_col = ColumnSchema::new(
-                    AUTO_CREATED_PLACEHOLDER_TS_COL,
-                    ConcreteDataType::timestamp_millisecond_datatype(),
-                    true,
-                )
-                .with_time_index(true);
-                with_auto_added_col.push(ts_col);
-            }
-
-            (primary_keys, with_auto_added_col, no_time_index)
-        };
-        let proto_schema = column_schemas_to_proto(schema, &primary_keys)?;
-        Ok((is_ts_placeholder, proto_schema))
+        Ok((primary_keys, with_auto_added_col, no_time_index))
    }
 }

@@ -807,7 +784,85 @@ impl FlowWorkerManager {
        let flow_plan = sql_to_flow_plan(&mut node_ctx, &self.query_engine, &sql).await?;

        debug!("Flow {:?}'s Plan is {:?}", flow_id, flow_plan);
-        node_ctx.assign_table_schema(&sink_table_name, flow_plan.schema.clone())?;
+
+        // check schema against actual table schema if exists
+        // if not exist create sink table immediately
+        if let Some((_, _, real_schema)) = self.fetch_table_pk_schema(&sink_table_name).await? {
+            let auto_schema = relation_desc_to_column_schemas_with_fallback(&flow_plan.schema);
+
+            // for column schema, only `data_type` need to be check for equality
+            // since one can omit flow's column name when write flow query
+            // print a user friendly error message about mismatch and how to correct them
+            for (idx, zipped) in auto_schema
+                .iter()
+                .zip_longest(real_schema.iter())
+                .enumerate()
+            {
+                match zipped {
+                    EitherOrBoth::Both(auto, real) => {
+                        if auto.data_type != real.data_type {
+                            InvalidQuerySnafu {
+                                    reason: format!(
+                                        "Column {}(name is '{}', flow inferred name is '{}')'s data type mismatch, expect {:?} got {:?}",
+                                        idx,
+                                        real.name,
+                                        auto.name,
+                                        real.data_type,
+                                        auto.data_type
+                                    ),
+                                }
+                                .fail()?;
+                        }
+                    }
+                    EitherOrBoth::Right(real) if real.data_type.is_timestamp() => {
+                        // if table is auto created, the last one or two column should be timestamp(update at and ts placeholder)
+                        continue;
+                    }
+                    _ => InvalidQuerySnafu {
+                        reason: format!(
+                            "schema length mismatched, expected {} found {}",
+                            real_schema.len(),
+                            auto_schema.len()
+                        ),
+                    }
+                    .fail()?,
+                }
+            }
+
+            let table_id = self
+                .table_info_source
+                .get_table_id_from_name(&sink_table_name)
+                .await?
+                .context(UnexpectedSnafu {
+                    reason: format!("Can't get table id for table name {:?}", sink_table_name),
+                })?;
+            let table_info_value = self
+                .table_info_source
+                .get_table_info_value(&table_id)
+                .await?
+                .context(UnexpectedSnafu {
+                    reason: format!("Can't get table info value for table id {:?}", table_id),
+                })?;
+            let real_schema = table_info_value_to_relation_desc(table_info_value)?;
+            node_ctx.assign_table_schema(&sink_table_name, real_schema.clone())?;
+        } else {
+            // assign inferred schema to sink table
+            // create sink table
+            node_ctx.assign_table_schema(&sink_table_name, flow_plan.schema.clone())?;
+            let did_create = self
+                .create_table_from_relation(
+                    &format!("flow-id={flow_id}"),
+                    &sink_table_name,
+                    &flow_plan.schema,
+                )
+                .await?;
+            if !did_create {
+                UnexpectedSnafu {
+                    reason: format!("Failed to create table {:?}", sink_table_name),
+                }
+                .fail()?;
+            }
+        }

        let _ = comment;
        let _ = flow_options;
--- a/src/flow/src/adapter/flownode_impl.rs
+++ b/src/flow/src/adapter/flownode_impl.rs
@@ -138,7 +138,7 @@ impl Flownode for FlowWorkerManager {
    }

    async fn handle_inserts(&self, request: InsertRequests) -> Result<FlowResponse> {
-        // using try_read makesure two things:
+        // using try_read to ensure two things:
        // 1. flush wouldn't happen until inserts before it is inserted
        // 2. inserts happening concurrently with flush wouldn't be block by flush
        let _flush_lock = self.flush_lock.try_read();
--- a/src/flow/src/adapter/node_context.rs
+++ b/src/flow/src/adapter/node_context.rs
@@ -331,12 +331,14 @@ impl FlownodeContext {
        } else {
            let global_id = self.new_global_id();

+            // table id is Some meaning db must have created the table
            if let Some(table_id) = table_id {
                let (known_table_name, schema) = srv_map.get_table_name_schema(&table_id).await?;
                table_name = table_name.or(Some(known_table_name));
                self.schema.insert(global_id, schema);
            } // if we don't have table id, it means database havn't assign one yet or we don't need it

+            // still update the mapping with new global id
            self.table_repr.insert(table_name, table_id, global_id);
            Ok(global_id)
        }
@@ -358,6 +360,7 @@ impl FlownodeContext {
            })?;

        self.schema.insert(gid, schema);
+
        Ok(())
    }

--- a/src/flow/src/adapter/table_source.rs
+++ b/src/flow/src/adapter/table_source.rs
@@ -20,11 +20,12 @@ use common_meta::key::table_name::{TableNameKey, TableNameManager};
 use snafu::{OptionExt, ResultExt};
 use table::metadata::TableId;

+use crate::adapter::util::table_info_value_to_relation_desc;
 use crate::adapter::TableName;
 use crate::error::{
    Error, ExternalSnafu, TableNotFoundMetaSnafu, TableNotFoundSnafu, UnexpectedSnafu,
 };
-use crate::repr::{self, ColumnType, RelationDesc, RelationType};
+use crate::repr::RelationDesc;

 /// mapping of table name <-> table id should be query from tableinfo manager
 pub struct TableSource {
@@ -121,38 +122,7 @@ impl TableSource {
            table_name.table_name,
        ];

-        let raw_schema = table_info_value.table_info.meta.schema;
-        let (column_types, col_names): (Vec<_>, Vec<_>) = raw_schema
-            .column_schemas
-            .clone()
-            .into_iter()
-            .map(|col| {
-                (
-                    ColumnType {
-                        nullable: col.is_nullable(),
-                        scalar_type: col.data_type,
-                    },
-                    Some(col.name),
-                )
-            })
-            .unzip();
-
-        let key = table_info_value.table_info.meta.primary_key_indices;
-        let keys = vec![repr::Key::from(key)];
-
-        let time_index = raw_schema.timestamp_index;
-        Ok((
-            table_name,
-            RelationDesc {
-                typ: RelationType {
-                    column_types,
-                    keys,
-                    time_index,
-                    // by default table schema's column are all non-auto
-                    auto_columns: vec![],
-                },
-                names: col_names,
-            },
-        ))
+        let desc = table_info_value_to_relation_desc(table_info_value)?;
+        Ok((table_name, desc))
    }
 }
--- a/src/flow/src/adapter/util.rs
+++ b/src/flow/src/adapter/util.rs
@@ -12,16 +12,153 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+use std::sync::Arc;
+
 use api::helper::ColumnDataTypeWrapper;
 use api::v1::column_def::options_from_column_schema;
-use api::v1::{ColumnDataType, ColumnDataTypeExtension, SemanticType};
+use api::v1::{ColumnDataType, ColumnDataTypeExtension, CreateTableExpr, SemanticType};
 use common_error::ext::BoxedError;
+use common_meta::key::table_info::TableInfoValue;
 use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::ColumnSchema;
 use itertools::Itertools;
-use snafu::ResultExt;
+use operator::expr_factory::CreateExprFactory;
+use session::context::QueryContextBuilder;
+use snafu::{OptionExt, ResultExt};
+use table::table_reference::TableReference;

-use crate::error::{Error, ExternalSnafu};
+use crate::adapter::{TableName, AUTO_CREATED_PLACEHOLDER_TS_COL};
+use crate::error::{Error, ExternalSnafu, UnexpectedSnafu};
+use crate::repr::{ColumnType, RelationDesc, RelationType};
+use crate::FlowWorkerManager;
+
+impl FlowWorkerManager {
+    /// Create table from given schema(will adjust to add auto column if needed), return true if table is created
+    pub(crate) async fn create_table_from_relation(
+        &self,
+        flow_name: &str,
+        table_name: &TableName,
+        relation_desc: &RelationDesc,
+    ) -> Result<bool, Error> {
+        if self.fetch_table_pk_schema(table_name).await?.is_some() {
+            return Ok(false);
+        }
+        let (pks, tys, _) = self.adjust_auto_created_table_schema(relation_desc).await?;
+
+        //create sink table using pks, column types and is_ts_auto
+
+        let proto_schema = column_schemas_to_proto(tys.clone(), &pks)?;
+
+        // create sink table
+        let create_expr = CreateExprFactory {}
+            .create_table_expr_by_column_schemas(
+                &TableReference {
+                    catalog: &table_name[0],
+                    schema: &table_name[1],
+                    table: &table_name[2],
+                },
+                &proto_schema,
+                "mito",
+                Some(&format!("Sink table for flow {}", flow_name)),
+            )
+            .map_err(BoxedError::new)
+            .context(ExternalSnafu)?;
+
+        self.submit_create_sink_table_ddl(create_expr).await?;
+        Ok(true)
+    }
+
+    /// Try fetch table with adjusted schema(added auto column if needed)
+    pub(crate) async fn try_fetch_existing_table(
+        &self,
+        table_name: &TableName,
+    ) -> Result<Option<(bool, Vec<api::v1::ColumnSchema>)>, Error> {
+        if let Some((primary_keys, time_index, schema)) =
+            self.fetch_table_pk_schema(table_name).await?
+        {
+            // check if the last column is the auto created timestamp column, hence the table is auto created from
+            // flow's plan type
+            let is_auto_create = {
+                let correct_name = schema
+                    .last()
+                    .map(|s| s.name == AUTO_CREATED_PLACEHOLDER_TS_COL)
+                    .unwrap_or(false);
+                let correct_time_index = time_index == Some(schema.len() - 1);
+                correct_name && correct_time_index
+            };
+            let proto_schema = column_schemas_to_proto(schema, &primary_keys)?;
+            Ok(Some((is_auto_create, proto_schema)))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// submit a create table ddl
+    pub(crate) async fn submit_create_sink_table_ddl(
+        &self,
+        mut create_table: CreateTableExpr,
+    ) -> Result<(), Error> {
+        let stmt_exec = {
+            self.frontend_invoker
+                .read()
+                .await
+                .as_ref()
+                .map(|f| f.statement_executor())
+        }
+        .context(UnexpectedSnafu {
+            reason: "Failed to get statement executor",
+        })?;
+        let ctx = Arc::new(
+            QueryContextBuilder::default()
+                .current_catalog(create_table.catalog_name.clone())
+                .current_schema(create_table.schema_name.clone())
+                .build(),
+        );
+        stmt_exec
+            .create_table_inner(&mut create_table, None, ctx)
+            .await
+            .map_err(BoxedError::new)
+            .context(ExternalSnafu)?;
+
+        Ok(())
+    }
+}
+
+pub fn table_info_value_to_relation_desc(
+    table_info_value: TableInfoValue,
+) -> Result<RelationDesc, Error> {
+    let raw_schema = table_info_value.table_info.meta.schema;
+    let (column_types, col_names): (Vec<_>, Vec<_>) = raw_schema
+        .column_schemas
+        .clone()
+        .into_iter()
+        .map(|col| {
+            (
+                ColumnType {
+                    nullable: col.is_nullable(),
+                    scalar_type: col.data_type,
+                },
+                Some(col.name),
+            )
+        })
+        .unzip();
+
+    let key = table_info_value.table_info.meta.primary_key_indices;
+    let keys = vec![crate::repr::Key::from(key)];
+
+    let time_index = raw_schema.timestamp_index;
+
+    Ok(RelationDesc {
+        typ: RelationType {
+            column_types,
+            keys,
+            time_index,
+            // by default table schema's column are all non-auto
+            auto_columns: vec![],
+        },
+        names: col_names,
+    })
+}

 pub fn from_proto_to_data_type(
    column_schema: &api::v1::ColumnSchema,
@@ -75,3 +212,29 @@ pub fn column_schemas_to_proto(
        .collect();
    Ok(ret)
 }
+
+/// Convert `RelationDesc` to `ColumnSchema` list,
+/// if the column name is not present, use `col_{idx}` as the column name
+pub fn relation_desc_to_column_schemas_with_fallback(schema: &RelationDesc) -> Vec<ColumnSchema> {
+    schema
+        .typ()
+        .column_types
+        .clone()
+        .into_iter()
+        .enumerate()
+        .map(|(idx, typ)| {
+            let name = schema
+                .names
+                .get(idx)
+                .cloned()
+                .flatten()
+                .unwrap_or(format!("col_{}", idx));
+            let ret = ColumnSchema::new(name, typ.scalar_type, typ.nullable);
+            if schema.typ().time_index == Some(idx) {
+                ret.with_time_index(true)
+            } else {
+                ret
+            }
+        })
+        .collect_vec()
+}
--- a/src/flow/src/compute/render/reduce.rs
+++ b/src/flow/src/compute/render/reduce.rs
@@ -16,6 +16,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::ops::Range;
 use std::sync::Arc;

+use arrow::array::new_null_array;
 use common_telemetry::trace;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::prelude::DataType;
@@ -398,20 +399,54 @@ fn reduce_batch_subgraph(
                }
            }

-            // TODO: here reduce numbers of eq to minimal by keeping slicing key/val batch
+            let key_data_types = output_type
+                .column_types
+                .iter()
+                .map(|t| t.scalar_type.clone())
+                .collect_vec();
+
+            // TODO(discord9): here reduce numbers of eq to minimal by keeping slicing key/val batch
            for key_row in distinct_keys {
                let key_scalar_value = {
                    let mut key_scalar_value = Vec::with_capacity(key_row.len());
-                    for key in key_row.iter() {
+                    for (key_idx, key) in key_row.iter().enumerate() {
                        let v =
                            key.try_to_scalar_value(&key.data_type())
                                .context(DataTypeSnafu {
                                    msg: "can't convert key values to datafusion value",
                                })?;
-                        let arrow_value =
+
+                        let key_data_type = key_data_types.get(key_idx).context(InternalSnafu {
+                            reason: format!(
+                                "Key index out of bound, expected at most {} but got {}",
+                                output_type.column_types.len(),
+                                key_idx
+                            ),
+                        })?;
+
+                        // if incoming value's datatype is null, it need to be handled specially, see below
+                        if key_data_type.as_arrow_type() != v.data_type()
+                            && !v.data_type().is_null()
+                        {
+                            crate::expr::error::InternalSnafu {
+                                reason: format!(
+                                    "Key data type mismatch, expected {:?} but got {:?}",
+                                    key_data_type.as_arrow_type(),
+                                    v.data_type()
+                                ),
+                            }
+                            .fail()?
+                        }
+
+                        // handle single null key
+                        let arrow_value = if v.data_type().is_null() {
+                            let ret = new_null_array(&arrow::datatypes::DataType::Null, 1);
+                            arrow::array::Scalar::new(ret)
+                        } else {
                            v.to_scalar().context(crate::expr::error::DatafusionSnafu {
                                context: "can't convert key values to arrow value",
-                            })?;
+                            })?
+                        };
                        key_scalar_value.push(arrow_value);
                    }
                    key_scalar_value
@@ -423,7 +458,19 @@ fn reduce_batch_subgraph(
                    .zip(key_batch.batch().iter())
                    .map(|(key, col)| {
                        // TODO(discord9): this takes half of the cpu! And this is redundant amount of `eq`!
-                        arrow::compute::kernels::cmp::eq(&key, &col.to_arrow_array().as_ref() as _)
+
+                        // note that if lhs is a null, we still need to get all rows that are null! But can't use `eq` since
+                        // it will return null if input have null, so we need to use `is_null` instead
+                        if arrow::array::Datum::get(&key).0.data_type().is_null() {
+                            arrow::compute::kernels::boolean::is_null(
+                                col.to_arrow_array().as_ref() as _
+                            )
+                        } else {
+                            arrow::compute::kernels::cmp::eq(
+                                &key,
+                                &col.to_arrow_array().as_ref() as _,
+                            )
+                        }
                    })
                    .try_collect::<_, Vec<_>, _>()
                    .context(ArrowSnafu {
--- a/src/flow/src/compute/types.rs
+++ b/src/flow/src/compute/types.rs
@@ -17,6 +17,7 @@ use std::collections::{BTreeMap, VecDeque};
 use std::rc::Rc;
 use std::sync::Arc;

+use common_error::ext::ErrorExt;
 use hydroflow::scheduled::graph::Hydroflow;
 use hydroflow::scheduled::handoff::TeeingHandoff;
 use hydroflow::scheduled::port::RecvPort;
@@ -25,6 +26,7 @@ use itertools::Itertools;
 use tokio::sync::Mutex;

 use crate::expr::{Batch, EvalError, ScalarExpr};
+use crate::metrics::METRIC_FLOW_ERRORS;
 use crate::repr::DiffRow;
 use crate::utils::ArrangeHandler;

@@ -185,6 +187,9 @@ impl ErrCollector {
    }

    pub fn push_err(&self, err: EvalError) {
+        METRIC_FLOW_ERRORS
+            .with_label_values(&[err.status_code().as_ref()])
+            .inc();
        self.inner.blocking_lock().push_back(err)
    }

--- a/src/flow/src/error.rs
+++ b/src/flow/src/error.rs
@@ -16,12 +16,13 @@

 use std::any::Any;

-use common_error::define_into_tonic_status;
 use common_error::ext::BoxedError;
+use common_error::{define_into_tonic_status, from_err_code_msg_to_header};
 use common_macro::stack_trace_debug;
 use common_telemetry::common_error::ext::ErrorExt;
 use common_telemetry::common_error::status_code::StatusCode;
 use snafu::{Location, Snafu};
+use tonic::metadata::MetadataMap;

 use crate::adapter::FlowId;
 use crate::expr::EvalError;
@@ -186,6 +187,20 @@ pub enum Error {
    },
 }

+/// the outer message is the full error stack, and inner message in header is the last error message that can be show directly to user
+pub fn to_status_with_last_err(err: impl ErrorExt) -> tonic::Status {
+    let msg = err.to_string();
+    let last_err_msg = common_error::ext::StackError::last(&err).to_string();
+    let code = err.status_code() as u32;
+    let header = from_err_code_msg_to_header(code, &last_err_msg);
+
+    tonic::Status::with_metadata(
+        tonic::Code::InvalidArgument,
+        msg,
+        MetadataMap::from_headers(header),
+    )
+}
+
 /// Result type for flow module
 pub type Result<T> = std::result::Result<T, Error>;

@@ -200,9 +215,8 @@ impl ErrorExt for Error {
            | Self::TableNotFoundMeta { .. }
            | Self::FlowNotFound { .. }
            | Self::ListFlows { .. } => StatusCode::TableNotFound,
-            Self::InvalidQuery { .. } | Self::Plan { .. } | Self::Datatypes { .. } => {
-                StatusCode::PlanQuery
-            }
+            Self::Plan { .. } | Self::Datatypes { .. } => StatusCode::PlanQuery,
+            Self::InvalidQuery { .. } => StatusCode::EngineExecuteQuery,
            Self::Unexpected { .. } => StatusCode::Unexpected,
            Self::NotImplemented { .. } | Self::UnsupportedTemporalFilter { .. } => {
                StatusCode::Unsupported
--- a/src/flow/src/expr/error.rs
+++ b/src/flow/src/expr/error.rs
@@ -14,8 +14,11 @@

 //! Error handling for expression evaluation.

+use std::any::Any;
+
 use arrow_schema::ArrowError;
-use common_error::ext::BoxedError;
+use common_error::ext::{BoxedError, ErrorExt};
+use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
 use datafusion_common::DataFusionError;
 use datatypes::data_type::ConcreteDataType;
@@ -126,3 +129,29 @@ pub enum EvalError {
        source: BoxedError,
    },
 }
+
+impl ErrorExt for EvalError {
+    fn status_code(&self) -> StatusCode {
+        use EvalError::*;
+        match self {
+            DivisionByZero { .. }
+            | TypeMismatch { .. }
+            | TryFromValue { .. }
+            | DataAlreadyExpired { .. }
+            | InvalidArgument { .. }
+            | Overflow { .. } => StatusCode::InvalidArguments,
+
+            CastValue { source, .. } | DataType { source, .. } => source.status_code(),
+
+            Internal { .. }
+            | Optimize { .. }
+            | Arrow { .. }
+            | Datafusion { .. }
+            | External { .. } => StatusCode::Internal,
+        }
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
--- a/src/flow/src/metrics.rs
+++ b/src/flow/src/metrics.rs
@@ -30,4 +30,22 @@ lazy_static! {
    .unwrap();
    pub static ref METRIC_FLOW_RUN_INTERVAL_MS: IntGauge =
        register_int_gauge!("greptime_flow_run_interval_ms", "flow run interval in ms").unwrap();
+    pub static ref METRIC_FLOW_ROWS: IntCounterVec = register_int_counter_vec!(
+        "greptime_flow_processed_rows",
+        "Count of rows flowing through the system",
+        &["direction"]
+    )
+    .unwrap();
+    pub static ref METRIC_FLOW_PROCESSING_TIME: HistogramVec = register_histogram_vec!(
+        "greptime_flow_processing_time",
+        "Time spent processing requests",
+        &["type"]
+    )
+    .unwrap();
+    pub static ref METRIC_FLOW_ERRORS: IntCounterVec = register_int_counter_vec!(
+        "greptime_flow_errors",
+        "Count of errors in flow processing",
+        &["code"]
+    )
+    .unwrap();
 }
--- a/src/flow/src/repr/relation.rs
+++ b/src/flow/src/repr/relation.rs
@@ -212,6 +212,8 @@ impl RelationType {
        for key in &mut self.keys {
            key.remove_col(time_index.unwrap_or(usize::MAX));
        }
+        // remove empty keys
+        self.keys.retain(|key| !key.is_empty());
        self
    }

--- a/src/flow/src/server.rs
+++ b/src/flow/src/server.rs
@@ -50,10 +50,11 @@ use tonic::{Request, Response, Status};

 use crate::adapter::{CreateFlowArgs, FlowWorkerManagerRef};
 use crate::error::{
-    CacheRequiredSnafu, ExternalSnafu, FlowNotFoundSnafu, ListFlowsSnafu, ParseAddrSnafu,
-    ShutdownServerSnafu, StartServerSnafu, UnexpectedSnafu,
+    to_status_with_last_err, CacheRequiredSnafu, ExternalSnafu, FlowNotFoundSnafu, ListFlowsSnafu,
+    ParseAddrSnafu, ShutdownServerSnafu, StartServerSnafu, UnexpectedSnafu,
 };
 use crate::heartbeat::HeartbeatTask;
+use crate::metrics::{METRIC_FLOW_PROCESSING_TIME, METRIC_FLOW_ROWS};
 use crate::transform::register_function_to_query_engine;
 use crate::utils::{SizeReportSender, StateReportHandler};
 use crate::{Error, FlowWorkerManager, FlownodeOptions};
@@ -77,41 +78,52 @@ impl flow_server::Flow for FlowService {
        &self,
        request: Request<FlowRequest>,
    ) -> Result<Response<FlowResponse>, Status> {
+        let _timer = METRIC_FLOW_PROCESSING_TIME
+            .with_label_values(&["ddl"])
+            .start_timer();
+
        let request = request.into_inner();
        self.manager
            .handle(request)
            .await
            .map(Response::new)
-            .map_err(|e| {
-                let msg = format!("failed to handle request: {:?}", e);
-                Status::internal(msg)
-            })
+            .map_err(to_status_with_last_err)
    }

    async fn handle_mirror_request(
        &self,
        request: Request<InsertRequests>,
    ) -> Result<Response<FlowResponse>, Status> {
+        let _timer = METRIC_FLOW_PROCESSING_TIME
+            .with_label_values(&["insert"])
+            .start_timer();
+
        let request = request.into_inner();
        // TODO(discord9): fix protobuf import order shenanigans to remove this duplicated define
+        let mut row_count = 0;
        let request = api::v1::region::InsertRequests {
            requests: request
                .requests
                .into_iter()
-                .map(|insert| api::v1::region::InsertRequest {
-                    region_id: insert.region_id,
-                    rows: insert.rows,
+                .map(|insert| {
+                    insert.rows.as_ref().inspect(|x| row_count += x.rows.len());
+                    api::v1::region::InsertRequest {
+                        region_id: insert.region_id,
+                        rows: insert.rows,
+                    }
                })
                .collect_vec(),
        };
+
+        METRIC_FLOW_ROWS
+            .with_label_values(&["in"])
+            .inc_by(row_count as u64);
+
        self.manager
            .handle_inserts(request)
            .await
            .map(Response::new)
-            .map_err(|e| {
-                let msg = format!("failed to handle request: {:?}", e);
-                Status::internal(msg)
-            })
+            .map_err(to_status_with_last_err)
    }
 }

@@ -500,6 +512,10 @@ impl FrontendInvoker {
        requests: RowInsertRequests,
        ctx: QueryContextRef,
    ) -> common_frontend::error::Result<Output> {
+        let _timer = METRIC_FLOW_PROCESSING_TIME
+            .with_label_values(&["output_insert"])
+            .start_timer();
+
        self.inserter
            .handle_row_inserts(requests, ctx, &self.statement_executor)
            .await
@@ -512,10 +528,18 @@ impl FrontendInvoker {
        requests: RowDeleteRequests,
        ctx: QueryContextRef,
    ) -> common_frontend::error::Result<Output> {
+        let _timer = METRIC_FLOW_PROCESSING_TIME
+            .with_label_values(&["output_delete"])
+            .start_timer();
+
        self.deleter
            .handle_row_deletes(requests, ctx)
            .await
            .map_err(BoxedError::new)
            .context(common_frontend::error::ExternalSnafu)
    }
+
+    pub fn statement_executor(&self) -> Arc<StatementExecutor> {
+        self.statement_executor.clone()
+    }
 }
--- a/src/flow/src/transform/aggr.rs
+++ b/src/flow/src/transform/aggr.rs
@@ -216,6 +216,7 @@ impl KeyValPlan {

 /// find out the column that should be time index in group exprs(which is all columns that should be keys)
 /// TODO(discord9): better ways to assign time index
+/// for now, it will found the first column that is timestamp or has a tumble window floor function
 fn find_time_index_in_group_exprs(group_exprs: &[TypedExpr]) -> Option<usize> {
    group_exprs.iter().position(|expr| {
        matches!(
@@ -224,7 +225,7 @@ fn find_time_index_in_group_exprs(group_exprs: &[TypedExpr]) -> Option<usize> {
                func: UnaryFunc::TumbleWindowFloor { .. },
                expr: _
            }
-        )
+        ) || expr.typ.scalar_type.is_timestamp()
    })
 }

@@ -1482,7 +1483,7 @@ mod test {
                ColumnType::new(CDT::float64_datatype(), true),
                ColumnType::new(CDT::timestamp_millisecond_datatype(), true),
            ])
-            .with_key(vec![1])
+            .with_time_index(Some(1))
            .into_named(vec![
                Some(
                    "MAX(numbers_with_ts.number) - MIN(numbers_with_ts.number) / Float64(30)"
@@ -1571,7 +1572,7 @@ mod test {
                            ColumnType::new(ConcreteDataType::uint32_datatype(), true), // max
                            ColumnType::new(ConcreteDataType::uint32_datatype(), true), // min
                        ])
-                        .with_key(vec![0])
+                        .with_time_index(Some(0))
                        .into_unnamed(),
                    ),
                ),
--- a/src/frontend/Cargo.toml
+++ b/src/frontend/Cargo.toml
@@ -41,6 +41,7 @@ datafusion-expr.workspace = true
 datanode.workspace = true
 humantime-serde.workspace = true
 lazy_static.workspace = true
+log-query.workspace = true
 log-store.workspace = true
 meta-client.workspace = true
 opentelemetry-proto.workspace = true
--- a/src/frontend/src/error.rs
+++ b/src/frontend/src/error.rs
@@ -321,6 +321,12 @@ pub enum Error {
        location: Location,
        source: BoxedError,
    },
+
+    #[snafu(display("In-flight write bytes exceeded the maximum limit"))]
+    InFlightWriteBytesExceeded {
+        #[snafu(implicit)]
+        location: Location,
+    },
 }

 pub type Result<T> = std::result::Result<T, Error>;
@@ -392,6 +398,8 @@ impl ErrorExt for Error {
            Error::StartScriptManager { source, .. } => source.status_code(),

            Error::TableOperation { source, .. } => source.status_code(),
+
+            Error::InFlightWriteBytesExceeded { .. } => StatusCode::RateLimited,
        }
    }

--- a/src/frontend/src/frontend.rs
+++ b/src/frontend/src/frontend.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+use common_base::readable_size::ReadableSize;
 use common_config::config::Configurable;
 use common_options::datanode::DatanodeClientOptions;
 use common_telemetry::logging::{LoggingOptions, TracingOptions};
@@ -46,6 +47,7 @@ pub struct FrontendOptions {
    pub user_provider: Option<String>,
    pub export_metrics: ExportMetricsOption,
    pub tracing: TracingOptions,
+    pub max_in_flight_write_bytes: Option<ReadableSize>,
 }

 impl Default for FrontendOptions {
@@ -68,6 +70,7 @@ impl Default for FrontendOptions {
            user_provider: None,
            export_metrics: ExportMetricsOption::default(),
            tracing: TracingOptions::default(),
+            max_in_flight_write_bytes: None,
        }
    }
 }
--- a/src/frontend/src/instance.rs
+++ b/src/frontend/src/instance.rs
@@ -16,6 +16,7 @@ pub mod builder;
 mod grpc;
 mod influxdb;
 mod log_handler;
+mod logs;
 mod opentsdb;
 mod otlp;
 mod prom_store;
@@ -64,8 +65,8 @@ use servers::prometheus_handler::PrometheusHandler;
 use servers::query_handler::grpc::GrpcQueryHandler;
 use servers::query_handler::sql::SqlQueryHandler;
 use servers::query_handler::{
-    InfluxdbLineProtocolHandler, OpenTelemetryProtocolHandler, OpentsdbProtocolHandler,
-    PipelineHandler, PromStoreProtocolHandler, ScriptHandler,
+    InfluxdbLineProtocolHandler, LogQueryHandler, OpenTelemetryProtocolHandler,
+    OpentsdbProtocolHandler, PipelineHandler, PromStoreProtocolHandler, ScriptHandler,
 };
 use servers::server::ServerHandlers;
 use session::context::QueryContextRef;
@@ -86,6 +87,7 @@ use crate::error::{
 };
 use crate::frontend::FrontendOptions;
 use crate::heartbeat::HeartbeatTask;
+use crate::limiter::LimiterRef;
 use crate::script::ScriptExecutor;

 #[async_trait]
@@ -99,6 +101,7 @@ pub trait FrontendInstance:
    + ScriptHandler
    + PrometheusHandler
    + PipelineHandler
+    + LogQueryHandler
    + Send
    + Sync
    + 'static
@@ -124,6 +127,7 @@ pub struct Instance {
    export_metrics_task: Option<ExportMetricsTask>,
    table_metadata_manager: TableMetadataManagerRef,
    stats: StatementStatistics,
+    limiter: Option<LimiterRef>,
 }

 impl Instance {
--- a/src/frontend/src/instance/builder.rs
+++ b/src/frontend/src/instance/builder.rs
@@ -43,6 +43,7 @@ use crate::frontend::FrontendOptions;
 use crate::heartbeat::HeartbeatTask;
 use crate::instance::region_query::FrontendRegionQueryHandler;
 use crate::instance::Instance;
+use crate::limiter::Limiter;
 use crate::script::ScriptExecutor;

 /// The frontend [`Instance`] builder.
@@ -196,6 +197,14 @@ impl FrontendBuilder {

        plugins.insert::<StatementExecutorRef>(statement_executor.clone());

+        // Create the limiter if the max_in_flight_write_bytes is set.
+        let limiter = self
+            .options
+            .max_in_flight_write_bytes
+            .map(|max_in_flight_write_bytes| {
+                Arc::new(Limiter::new(max_in_flight_write_bytes.as_bytes()))
+            });
+
        Ok(Instance {
            options: self.options,
            catalog_manager: self.catalog_manager,
@@ -211,6 +220,7 @@ impl FrontendBuilder {
            export_metrics_task: None,
            table_metadata_manager: Arc::new(TableMetadataManager::new(kv_backend)),
            stats: self.stats,
+            limiter,
        })
    }
 }
--- a/src/frontend/src/instance/grpc.rs
+++ b/src/frontend/src/instance/grpc.rs
@@ -29,8 +29,8 @@ use snafu::{ensure, OptionExt, ResultExt};
 use table::table_name::TableName;

 use crate::error::{
-    Error, IncompleteGrpcRequestSnafu, NotSupportedSnafu, PermissionSnafu, Result,
-    TableOperationSnafu,
+    Error, InFlightWriteBytesExceededSnafu, IncompleteGrpcRequestSnafu, NotSupportedSnafu,
+    PermissionSnafu, Result, TableOperationSnafu,
 };
 use crate::instance::{attach_timer, Instance};
 use crate::metrics::{GRPC_HANDLE_PROMQL_ELAPSED, GRPC_HANDLE_SQL_ELAPSED};
@@ -50,6 +50,16 @@ impl GrpcQueryHandler for Instance {
            .check_permission(ctx.current_user(), PermissionReq::GrpcRequest(&request))
            .context(PermissionSnafu)?;

+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_request(&request);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
        let output = match request {
            Request::Inserts(requests) => self.handle_inserts(requests, ctx.clone()).await?,
            Request::RowInserts(requests) => self.handle_row_inserts(requests, ctx.clone()).await?,
--- a/src/frontend/src/instance/influxdb.rs
+++ b/src/frontend/src/instance/influxdb.rs
@@ -16,7 +16,7 @@ use async_trait::async_trait;
 use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq};
 use client::Output;
 use common_error::ext::BoxedError;
-use servers::error::{AuthSnafu, Error};
+use servers::error::{AuthSnafu, Error, InFlightWriteBytesExceededSnafu};
 use servers::influxdb::InfluxdbRequest;
 use servers::interceptor::{LineProtocolInterceptor, LineProtocolInterceptorRef};
 use servers::query_handler::InfluxdbLineProtocolHandler;
@@ -46,6 +46,16 @@ impl InfluxdbLineProtocolHandler for Instance {
            .post_lines_conversion(requests, ctx.clone())
            .await?;

+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&requests);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
        self.handle_influx_row_inserts(requests, ctx)
            .await
            .map_err(BoxedError::new)
--- a/src/frontend/src/instance/log_handler.rs
+++ b/src/frontend/src/instance/log_handler.rs
@@ -22,7 +22,8 @@ use common_error::ext::BoxedError;
 use pipeline::pipeline_operator::PipelineOperator;
 use pipeline::{GreptimeTransformer, Pipeline, PipelineInfo, PipelineVersion};
 use servers::error::{
-    AuthSnafu, Error as ServerError, ExecuteGrpcRequestSnafu, PipelineSnafu, Result as ServerResult,
+    AuthSnafu, Error as ServerError, ExecuteGrpcRequestSnafu, InFlightWriteBytesExceededSnafu,
+    PipelineSnafu, Result as ServerResult,
 };
 use servers::interceptor::{LogIngestInterceptor, LogIngestInterceptorRef};
 use servers::query_handler::PipelineHandler;
@@ -110,6 +111,16 @@ impl Instance {
        log: RowInsertRequests,
        ctx: QueryContextRef,
    ) -> ServerResult<Output> {
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&log);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
        self.inserter
            .handle_log_inserts(log, ctx, self.statement_executor.as_ref())
            .await
--- a/src/frontend/src/instance/logs.rs
+++ b/src/frontend/src/instance/logs.rs
@@ -0,0 +1,67 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq};
+use client::Output;
+use common_error::ext::BoxedError;
+use log_query::LogQuery;
+use server_error::Result as ServerResult;
+use servers::error::{self as server_error, AuthSnafu, ExecuteQuerySnafu};
+use servers::interceptor::{LogQueryInterceptor, LogQueryInterceptorRef};
+use servers::query_handler::LogQueryHandler;
+use session::context::QueryContextRef;
+use snafu::ResultExt;
+use tonic::async_trait;
+
+use super::Instance;
+
+#[async_trait]
+impl LogQueryHandler for Instance {
+    async fn query(&self, mut request: LogQuery, ctx: QueryContextRef) -> ServerResult<Output> {
+        let interceptor = self
+            .plugins
+            .get::<LogQueryInterceptorRef<server_error::Error>>();
+
+        self.plugins
+            .get::<PermissionCheckerRef>()
+            .as_ref()
+            .check_permission(ctx.current_user(), PermissionReq::LogQuery)
+            .context(AuthSnafu)?;
+
+        interceptor.as_ref().pre_query(&request, ctx.clone())?;
+
+        request
+            .time_filter
+            .canonicalize()
+            .map_err(BoxedError::new)
+            .context(ExecuteQuerySnafu)?;
+
+        let plan = self
+            .query_engine
+            .planner()
+            .plan_logs_query(request, ctx.clone())
+            .await
+            .map_err(BoxedError::new)
+            .context(ExecuteQuerySnafu)?;
+
+        let output = self
+            .statement_executor
+            .exec_plan(plan, ctx.clone())
+            .await
+            .map_err(BoxedError::new)
+            .context(ExecuteQuerySnafu)?;
+
+        Ok(interceptor.as_ref().post_query(output, ctx.clone())?)
+    }
+}
--- a/src/frontend/src/instance/opentsdb.rs
+++ b/src/frontend/src/instance/opentsdb.rs
@@ -17,7 +17,7 @@ use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq};
 use common_error::ext::BoxedError;
 use common_telemetry::tracing;
 use servers::error as server_error;
-use servers::error::AuthSnafu;
+use servers::error::{AuthSnafu, InFlightWriteBytesExceededSnafu};
 use servers::opentsdb::codec::DataPoint;
 use servers::opentsdb::data_point_to_grpc_row_insert_requests;
 use servers::query_handler::OpentsdbProtocolHandler;
@@ -41,6 +41,17 @@ impl OpentsdbProtocolHandler for Instance {
            .context(AuthSnafu)?;

        let (requests, _) = data_point_to_grpc_row_insert_requests(data_points)?;
+
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&requests);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
        let output = self
            .handle_row_inserts(requests, ctx)
            .await
--- a/src/frontend/src/instance/otlp.rs
+++ b/src/frontend/src/instance/otlp.rs
@@ -21,7 +21,7 @@ use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
 use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
 use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
 use pipeline::PipelineWay;
-use servers::error::{self, AuthSnafu, Result as ServerResult};
+use servers::error::{self, AuthSnafu, InFlightWriteBytesExceededSnafu, Result as ServerResult};
 use servers::interceptor::{OpenTelemetryProtocolInterceptor, OpenTelemetryProtocolInterceptorRef};
 use servers::otlp;
 use servers::query_handler::OpenTelemetryProtocolHandler;
@@ -53,6 +53,16 @@ impl OpenTelemetryProtocolHandler for Instance {
        let (requests, rows) = otlp::metrics::to_grpc_insert_requests(request)?;
        OTLP_METRICS_ROWS.inc_by(rows as u64);

+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&requests);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
        self.handle_row_inserts(requests, ctx)
            .await
            .map_err(BoxedError::new)
@@ -83,6 +93,16 @@ impl OpenTelemetryProtocolHandler for Instance {

        OTLP_TRACES_ROWS.inc_by(rows as u64);

+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&requests);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
        self.handle_log_inserts(requests, ctx)
            .await
            .map_err(BoxedError::new)
@@ -109,6 +129,17 @@ impl OpenTelemetryProtocolHandler for Instance {
        interceptor_ref.pre_execute(ctx.clone())?;

        let (requests, rows) = otlp::logs::to_grpc_insert_requests(request, pipeline, table_name)?;
+
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&requests);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
        self.handle_log_inserts(requests, ctx)
            .await
            .inspect(|_| OTLP_LOGS_ROWS.inc_by(rows as u64))
--- a/src/frontend/src/instance/prom_store.rs
+++ b/src/frontend/src/instance/prom_store.rs
@@ -30,7 +30,7 @@ use common_telemetry::{debug, tracing};
 use operator::insert::InserterRef;
 use operator::statement::StatementExecutor;
 use prost::Message;
-use servers::error::{self, AuthSnafu, Result as ServerResult};
+use servers::error::{self, AuthSnafu, InFlightWriteBytesExceededSnafu, Result as ServerResult};
 use servers::http::header::{collect_plan_metrics, CONTENT_ENCODING_SNAPPY, CONTENT_TYPE_PROTOBUF};
 use servers::http::prom_store::PHYSICAL_TABLE_PARAM;
 use servers::interceptor::{PromStoreProtocolInterceptor, PromStoreProtocolInterceptorRef};
@@ -175,6 +175,16 @@ impl PromStoreProtocolHandler for Instance {
            .get::<PromStoreProtocolInterceptorRef<servers::error::Error>>();
        interceptor_ref.pre_write(&request, ctx.clone())?;

+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&request);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
        let output = if with_metric_engine {
            let physical_table = ctx
                .extension(PHYSICAL_TABLE_PARAM)
--- a/src/frontend/src/lib.rs
+++ b/src/frontend/src/lib.rs
@@ -18,6 +18,7 @@ pub mod error;
 pub mod frontend;
 pub mod heartbeat;
 pub mod instance;
+pub(crate) mod limiter;
 pub(crate) mod metrics;
 mod script;
 pub mod server;
--- a/src/frontend/src/limiter.rs
+++ b/src/frontend/src/limiter.rs
@@ -0,0 +1,291 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+
+use api::v1::column::Values;
+use api::v1::greptime_request::Request;
+use api::v1::value::ValueData;
+use api::v1::{Decimal128, InsertRequests, IntervalMonthDayNano, RowInsertRequests};
+use common_telemetry::{debug, warn};
+
+pub(crate) type LimiterRef = Arc<Limiter>;
+
+/// A frontend request limiter that controls the total size of in-flight write requests.
+pub(crate) struct Limiter {
+    // The maximum number of bytes that can be in flight.
+    max_in_flight_write_bytes: u64,
+
+    // The current in-flight write bytes.
+    in_flight_write_bytes: Arc<AtomicU64>,
+}
+
+/// A counter for the in-flight write bytes.
+pub(crate) struct InFlightWriteBytesCounter {
+    // The current in-flight write bytes.
+    in_flight_write_bytes: Arc<AtomicU64>,
+
+    // The write bytes that are being processed.
+    processing_write_bytes: u64,
+}
+
+impl InFlightWriteBytesCounter {
+    /// Creates a new InFlightWriteBytesCounter. It will decrease the in-flight write bytes when dropped.
+    pub fn new(in_flight_write_bytes: Arc<AtomicU64>, processing_write_bytes: u64) -> Self {
+        debug!(
+            "processing write bytes: {}, current in-flight write bytes: {}",
+            processing_write_bytes,
+            in_flight_write_bytes.load(Ordering::Relaxed)
+        );
+        Self {
+            in_flight_write_bytes,
+            processing_write_bytes,
+        }
+    }
+}
+
+impl Drop for InFlightWriteBytesCounter {
+    // When the request is finished, the in-flight write bytes should be decreased.
+    fn drop(&mut self) {
+        self.in_flight_write_bytes
+            .fetch_sub(self.processing_write_bytes, Ordering::Relaxed);
+    }
+}
+
+impl Limiter {
+    pub fn new(max_in_flight_write_bytes: u64) -> Self {
+        Self {
+            max_in_flight_write_bytes,
+            in_flight_write_bytes: Arc::new(AtomicU64::new(0)),
+        }
+    }
+
+    pub fn limit_request(&self, request: &Request) -> Option<InFlightWriteBytesCounter> {
+        let size = match request {
+            Request::Inserts(requests) => self.insert_requests_data_size(requests),
+            Request::RowInserts(requests) => self.rows_insert_requests_data_size(requests),
+            _ => 0,
+        };
+        self.limit_in_flight_write_bytes(size as u64)
+    }
+
+    pub fn limit_row_inserts(
+        &self,
+        requests: &RowInsertRequests,
+    ) -> Option<InFlightWriteBytesCounter> {
+        let size = self.rows_insert_requests_data_size(requests);
+        self.limit_in_flight_write_bytes(size as u64)
+    }
+
+    /// Returns None if the in-flight write bytes exceed the maximum limit.
+    /// Otherwise, returns Some(InFlightWriteBytesCounter) and the in-flight write bytes will be increased.
+    pub fn limit_in_flight_write_bytes(&self, bytes: u64) -> Option<InFlightWriteBytesCounter> {
+        let result = self.in_flight_write_bytes.fetch_update(
+            Ordering::Relaxed,
+            Ordering::Relaxed,
+            |current| {
+                if current + bytes > self.max_in_flight_write_bytes {
+                    warn!(
+                        "in-flight write bytes exceed the maximum limit {}, request with {} bytes will be limited",
+                        self.max_in_flight_write_bytes,
+                        bytes
+                    );
+                    return None;
+                }
+                Some(current + bytes)
+            },
+        );
+
+        match result {
+            // Update the in-flight write bytes successfully.
+            Ok(_) => Some(InFlightWriteBytesCounter::new(
+                self.in_flight_write_bytes.clone(),
+                bytes,
+            )),
+            // It means the in-flight write bytes exceed the maximum limit.
+            Err(_) => None,
+        }
+    }
+
+    /// Returns the current in-flight write bytes.
+    #[allow(dead_code)]
+    pub fn in_flight_write_bytes(&self) -> u64 {
+        self.in_flight_write_bytes.load(Ordering::Relaxed)
+    }
+
+    fn insert_requests_data_size(&self, request: &InsertRequests) -> usize {
+        let mut size: usize = 0;
+        for insert in &request.inserts {
+            for column in &insert.columns {
+                if let Some(values) = &column.values {
+                    size += self.size_of_column_values(values);
+                }
+            }
+        }
+        size
+    }
+
+    fn rows_insert_requests_data_size(&self, request: &RowInsertRequests) -> usize {
+        let mut size: usize = 0;
+        for insert in &request.inserts {
+            if let Some(rows) = &insert.rows {
+                for row in &rows.rows {
+                    for value in &row.values {
+                        if let Some(value) = &value.value_data {
+                            size += self.size_of_value_data(value);
+                        }
+                    }
+                }
+            }
+        }
+        size
+    }
+
+    fn size_of_column_values(&self, values: &Values) -> usize {
+        let mut size: usize = 0;
+        size += values.i8_values.len() * size_of::<i32>();
+        size += values.i16_values.len() * size_of::<i32>();
+        size += values.i32_values.len() * size_of::<i32>();
+        size += values.i64_values.len() * size_of::<i64>();
+        size += values.u8_values.len() * size_of::<u32>();
+        size += values.u16_values.len() * size_of::<u32>();
+        size += values.u32_values.len() * size_of::<u32>();
+        size += values.u64_values.len() * size_of::<u64>();
+        size += values.f32_values.len() * size_of::<f32>();
+        size += values.f64_values.len() * size_of::<f64>();
+        size += values.bool_values.len() * size_of::<bool>();
+        size += values
+            .binary_values
+            .iter()
+            .map(|v| v.len() * size_of::<u8>())
+            .sum::<usize>();
+        size += values.string_values.iter().map(|v| v.len()).sum::<usize>();
+        size += values.date_values.len() * size_of::<i32>();
+        size += values.datetime_values.len() * size_of::<i64>();
+        size += values.timestamp_second_values.len() * size_of::<i64>();
+        size += values.timestamp_millisecond_values.len() * size_of::<i64>();
+        size += values.timestamp_microsecond_values.len() * size_of::<i64>();
+        size += values.timestamp_nanosecond_values.len() * size_of::<i64>();
+        size += values.time_second_values.len() * size_of::<i64>();
+        size += values.time_millisecond_values.len() * size_of::<i64>();
+        size += values.time_microsecond_values.len() * size_of::<i64>();
+        size += values.time_nanosecond_values.len() * size_of::<i64>();
+        size += values.interval_year_month_values.len() * size_of::<i64>();
+        size += values.interval_day_time_values.len() * size_of::<i64>();
+        size += values.interval_month_day_nano_values.len() * size_of::<IntervalMonthDayNano>();
+        size += values.decimal128_values.len() * size_of::<Decimal128>();
+        size
+    }
+
+    fn size_of_value_data(&self, value: &ValueData) -> usize {
+        match value {
+            ValueData::I8Value(_) => size_of::<i32>(),
+            ValueData::I16Value(_) => size_of::<i32>(),
+            ValueData::I32Value(_) => size_of::<i32>(),
+            ValueData::I64Value(_) => size_of::<i64>(),
+            ValueData::U8Value(_) => size_of::<u32>(),
+            ValueData::U16Value(_) => size_of::<u32>(),
+            ValueData::U32Value(_) => size_of::<u32>(),
+            ValueData::U64Value(_) => size_of::<u64>(),
+            ValueData::F32Value(_) => size_of::<f32>(),
+            ValueData::F64Value(_) => size_of::<f64>(),
+            ValueData::BoolValue(_) => size_of::<bool>(),
+            ValueData::BinaryValue(v) => v.len() * size_of::<u8>(),
+            ValueData::StringValue(v) => v.len(),
+            ValueData::DateValue(_) => size_of::<i32>(),
+            ValueData::DatetimeValue(_) => size_of::<i64>(),
+            ValueData::TimestampSecondValue(_) => size_of::<i64>(),
+            ValueData::TimestampMillisecondValue(_) => size_of::<i64>(),
+            ValueData::TimestampMicrosecondValue(_) => size_of::<i64>(),
+            ValueData::TimestampNanosecondValue(_) => size_of::<i64>(),
+            ValueData::TimeSecondValue(_) => size_of::<i64>(),
+            ValueData::TimeMillisecondValue(_) => size_of::<i64>(),
+            ValueData::TimeMicrosecondValue(_) => size_of::<i64>(),
+            ValueData::TimeNanosecondValue(_) => size_of::<i64>(),
+            ValueData::IntervalYearMonthValue(_) => size_of::<i32>(),
+            ValueData::IntervalDayTimeValue(_) => size_of::<i64>(),
+            ValueData::IntervalMonthDayNanoValue(_) => size_of::<IntervalMonthDayNano>(),
+            ValueData::Decimal128Value(_) => size_of::<Decimal128>(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use api::v1::column::Values;
+    use api::v1::greptime_request::Request;
+    use api::v1::{Column, InsertRequest};
+
+    use super::*;
+
+    fn generate_request(size: usize) -> Request {
+        let i8_values = vec![0; size / 4];
+        Request::Inserts(InsertRequests {
+            inserts: vec![InsertRequest {
+                columns: vec![Column {
+                    values: Some(Values {
+                        i8_values,
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                }],
+                ..Default::default()
+            }],
+        })
+    }
+
+    #[tokio::test]
+    async fn test_limiter() {
+        let limiter_ref: LimiterRef = Arc::new(Limiter::new(1024));
+        let tasks_count = 10;
+        let request_data_size = 100;
+        let mut handles = vec![];
+
+        // Generate multiple requests to test the limiter.
+        for _ in 0..tasks_count {
+            let limiter = limiter_ref.clone();
+            let handle = tokio::spawn(async move {
+                let result = limiter.limit_request(&generate_request(request_data_size));
+                assert!(result.is_some());
+            });
+            handles.push(handle);
+        }
+
+        // Wait for all threads to complete.
+        for handle in handles {
+            handle.await.unwrap();
+        }
+    }
+
+    #[test]
+    fn test_in_flight_write_bytes() {
+        let limiter_ref: LimiterRef = Arc::new(Limiter::new(1024));
+        let req1 = generate_request(100);
+        let result1 = limiter_ref.limit_request(&req1);
+        assert!(result1.is_some());
+        assert_eq!(limiter_ref.in_flight_write_bytes(), 100);
+
+        let req2 = generate_request(200);
+        let result2 = limiter_ref.limit_request(&req2);
+        assert!(result2.is_some());
+        assert_eq!(limiter_ref.in_flight_write_bytes(), 300);
+
+        drop(result1.unwrap());
+        assert_eq!(limiter_ref.in_flight_write_bytes(), 200);
+
+        drop(result2.unwrap());
+        assert_eq!(limiter_ref.in_flight_write_bytes(), 0);
+    }
+}
--- a/src/frontend/src/server.rs
+++ b/src/frontend/src/server.rs
@@ -87,6 +87,7 @@ where
        let ingest_interceptor = self.plugins.get::<LogIngestInterceptorRef<ServerError>>();
        builder =
            builder.with_log_ingest_handler(self.instance.clone(), validator, ingest_interceptor);
+        builder = builder.with_logs_handler(self.instance.clone());

        if let Some(user_provider) = self.plugins.get::<UserProviderRef>() {
            builder = builder.with_user_provider(user_provider);
--- a/src/index/Cargo.toml
+++ b/src/index/Cargo.toml
@@ -22,6 +22,7 @@ fst.workspace = true
 futures.workspace = true
 greptime-proto.workspace = true
 mockall.workspace = true
+parquet.workspace = true
 pin-project.workspace = true
 prost.workspace = true
 regex.workspace = true
--- a/src/index/src/bloom_filter.rs
+++ b/src/index/src/bloom_filter.rs
@@ -14,14 +14,19 @@

 use serde::{Deserialize, Serialize};

+pub mod applier;
 pub mod creator;
-mod error;
+pub mod error;
+pub mod reader;

 pub type Bytes = Vec<u8>;
 pub type BytesRef<'a> = &'a [u8];

+/// The seed used for the Bloom filter.
+pub const SEED: u128 = 42;
+
 /// The Meta information of the bloom filter stored in the file.
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Debug, Default, Serialize, Deserialize, Clone)]
 pub struct BloomFilterMeta {
    /// The number of rows per segment.
    pub rows_per_segment: usize,
@@ -40,7 +45,7 @@ pub struct BloomFilterMeta {
 }

 /// The location of the bloom filter segment in the file.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone, Hash, PartialEq, Eq)]
 pub struct BloomFilterSegmentLocation {
    /// The offset of the bloom filter segment in the file.
    pub offset: u64,
--- a/src/index/src/bloom_filter/applier.rs
+++ b/src/index/src/bloom_filter/applier.rs
@@ -0,0 +1,133 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, HashSet};
+
+use parquet::arrow::arrow_reader::RowSelection;
+use parquet::file::metadata::RowGroupMetaData;
+
+use crate::bloom_filter::error::Result;
+use crate::bloom_filter::reader::BloomFilterReader;
+use crate::bloom_filter::{BloomFilterMeta, BloomFilterSegmentLocation, Bytes};
+
+/// Enumerates types of predicates for value filtering.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Predicate {
+    /// Predicate for matching values in a list.
+    InList(InListPredicate),
+}
+
+/// `InListPredicate` contains a list of acceptable values. A value needs to match at least
+/// one of the elements (logical OR semantic) for the predicate to be satisfied.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct InListPredicate {
+    /// List of acceptable values.
+    pub list: HashSet<Bytes>,
+}
+
+pub struct BloomFilterApplier {
+    reader: Box<dyn BloomFilterReader + Send>,
+    meta: BloomFilterMeta,
+}
+
+impl BloomFilterApplier {
+    pub async fn new(mut reader: Box<dyn BloomFilterReader + Send>) -> Result<Self> {
+        let meta = reader.metadata().await?;
+
+        Ok(Self { reader, meta })
+    }
+
+    /// Searches for matching row groups using bloom filters.
+    ///
+    /// This method applies bloom filter index to eliminate row groups that definitely
+    /// don't contain the searched values. It works by:
+    ///
+    /// 1. Computing prefix sums for row counts
+    /// 2. Calculating bloom filter segment locations for each row group
+    ///     1. A row group may span multiple bloom filter segments
+    /// 3. Probing bloom filter segments
+    /// 4. Removing non-matching row groups from the basement
+    ///     1. If a row group doesn't match any bloom filter segment with any probe, it is removed
+    ///
+    /// # Note
+    /// The method modifies the `basement` map in-place by removing row groups that
+    /// don't match the bloom filter criteria.
+    pub async fn search(
+        &mut self,
+        probes: &HashSet<Bytes>,
+        row_group_metas: &[RowGroupMetaData],
+        basement: &mut BTreeMap<usize, Option<RowSelection>>,
+    ) -> Result<()> {
+        // 0. Fast path - if basement is empty return empty vec
+        if basement.is_empty() {
+            return Ok(());
+        }
+
+        // 1. Compute prefix sum for row counts
+        let mut sum = 0usize;
+        let mut prefix_sum = Vec::with_capacity(row_group_metas.len() + 1);
+        prefix_sum.push(0usize);
+        for meta in row_group_metas {
+            sum += meta.num_rows() as usize;
+            prefix_sum.push(sum);
+        }
+
+        // 2. Calculate bloom filter segment locations
+        let mut row_groups_to_remove = HashSet::new();
+        for &row_group_idx in basement.keys() {
+            // TODO(ruihang): support further filter over row selection
+
+            // todo: dedup & overlap
+            let rows_range_start = prefix_sum[row_group_idx] / self.meta.rows_per_segment;
+            let rows_range_end = (prefix_sum[row_group_idx + 1] as f64
+                / self.meta.rows_per_segment as f64)
+                .ceil() as usize;
+
+            let mut is_any_range_hit = false;
+            for i in rows_range_start..rows_range_end {
+                // 3. Probe each bloom filter segment
+                let loc = BloomFilterSegmentLocation {
+                    offset: self.meta.bloom_filter_segments[i].offset,
+                    size: self.meta.bloom_filter_segments[i].size,
+                    elem_count: self.meta.bloom_filter_segments[i].elem_count,
+                };
+                let bloom = self.reader.bloom_filter(&loc).await?;
+
+                // Check if any probe exists in bloom filter
+                let mut matches = false;
+                for probe in probes {
+                    if bloom.contains(probe) {
+                        matches = true;
+                        break;
+                    }
+                }
+
+                is_any_range_hit |= matches;
+                if matches {
+                    break;
+                }
+            }
+            if !is_any_range_hit {
+                row_groups_to_remove.insert(row_group_idx);
+            }
+        }
+
+        // 4. Remove row groups that do not match any bloom filter segment
+        for row_group_idx in row_groups_to_remove {
+            basement.remove(&row_group_idx);
+        }
+
+        Ok(())
+    }
+}
--- a/src/index/src/bloom_filter/creator.rs
+++ b/src/index/src/bloom_filter/creator.rs
@@ -12,21 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::collections::HashSet;
+mod finalize_segment;
+mod intermediate_codec;

-use fastbloom::BloomFilter;
-use futures::{AsyncWrite, AsyncWriteExt};
+use std::collections::HashSet;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
+use finalize_segment::FinalizedBloomFilterStorage;
+use futures::{AsyncWrite, AsyncWriteExt, StreamExt};
 use snafu::ResultExt;

-use super::error::{IoSnafu, SerdeJsonSnafu};
-use crate::bloom_filter::error::Result;
-use crate::bloom_filter::{BloomFilterMeta, BloomFilterSegmentLocation, Bytes};
-
-/// The seed used for the Bloom filter.
-const SEED: u128 = 42;
+use crate::bloom_filter::error::{IoSnafu, Result, SerdeJsonSnafu};
+use crate::bloom_filter::{BloomFilterMeta, BloomFilterSegmentLocation, Bytes, SEED};
+use crate::external_provider::ExternalTempFileProvider;

 /// The false positive rate of the Bloom filter.
-const FALSE_POSITIVE_RATE: f64 = 0.01;
+pub const FALSE_POSITIVE_RATE: f64 = 0.01;

 /// `BloomFilterCreator` is responsible for creating and managing bloom filters
 /// for a set of elements. It divides the rows into segments and creates
@@ -58,6 +60,9 @@ pub struct BloomFilterCreator {

    /// Storage for finalized Bloom filters.
    finalized_bloom_filters: FinalizedBloomFilterStorage,
+
+    /// Global memory usage of the bloom filter creator.
+    global_memory_usage: Arc<AtomicUsize>,
 }

 impl BloomFilterCreator {
@@ -66,7 +71,12 @@ impl BloomFilterCreator {
    /// # PANICS
    ///
    /// `rows_per_segment` <= 0
-    pub fn new(rows_per_segment: usize) -> Self {
+    pub fn new(
+        rows_per_segment: usize,
+        intermediate_provider: Arc<dyn ExternalTempFileProvider>,
+        global_memory_usage: Arc<AtomicUsize>,
+        global_memory_usage_threshold: Option<usize>,
+    ) -> Self {
        assert!(
            rows_per_segment > 0,
            "rows_per_segment must be greater than 0"
@@ -77,54 +87,110 @@ impl BloomFilterCreator {
            accumulated_row_count: 0,
            cur_seg_distinct_elems: HashSet::default(),
            cur_seg_distinct_elems_mem_usage: 0,
-            finalized_bloom_filters: FinalizedBloomFilterStorage::default(),
+            global_memory_usage: global_memory_usage.clone(),
+            finalized_bloom_filters: FinalizedBloomFilterStorage::new(
+                intermediate_provider,
+                global_memory_usage,
+                global_memory_usage_threshold,
+            ),
        }
    }

+    /// Adds multiple rows of elements to the bloom filter. If the number of accumulated rows
+    /// reaches `rows_per_segment`, it finalizes the current segment.
+    pub async fn push_n_row_elems(
+        &mut self,
+        mut nrows: usize,
+        elems: impl IntoIterator<Item = Bytes>,
+    ) -> Result<()> {
+        if nrows == 0 {
+            return Ok(());
+        }
+        if nrows == 1 {
+            return self.push_row_elems(elems).await;
+        }
+
+        let elems = elems.into_iter().collect::<Vec<_>>();
+        while nrows > 0 {
+            let rows_to_seg_end =
+                self.rows_per_segment - (self.accumulated_row_count % self.rows_per_segment);
+            let rows_to_push = nrows.min(rows_to_seg_end);
+            nrows -= rows_to_push;
+
+            self.accumulated_row_count += rows_to_push;
+
+            let mut mem_diff = 0;
+            for elem in &elems {
+                let len = elem.len();
+                let is_new = self.cur_seg_distinct_elems.insert(elem.clone());
+                if is_new {
+                    mem_diff += len;
+                }
+            }
+            self.cur_seg_distinct_elems_mem_usage += mem_diff;
+            self.global_memory_usage
+                .fetch_add(mem_diff, Ordering::Relaxed);
+
+            if self.accumulated_row_count % self.rows_per_segment == 0 {
+                self.finalize_segment().await?;
+            }
+        }
+
+        Ok(())
+    }
+
    /// Adds a row of elements to the bloom filter. If the number of accumulated rows
    /// reaches `rows_per_segment`, it finalizes the current segment.
-    pub fn push_row_elems(&mut self, elems: impl IntoIterator<Item = Bytes>) {
+    pub async fn push_row_elems(&mut self, elems: impl IntoIterator<Item = Bytes>) -> Result<()> {
        self.accumulated_row_count += 1;
+
+        let mut mem_diff = 0;
        for elem in elems.into_iter() {
            let len = elem.len();
            let is_new = self.cur_seg_distinct_elems.insert(elem);
            if is_new {
-                self.cur_seg_distinct_elems_mem_usage += len;
+                mem_diff += len;
            }
        }
+        self.cur_seg_distinct_elems_mem_usage += mem_diff;
+        self.global_memory_usage
+            .fetch_add(mem_diff, Ordering::Relaxed);

        if self.accumulated_row_count % self.rows_per_segment == 0 {
-            self.finalize_segment();
+            self.finalize_segment().await?;
        }
+
+        Ok(())
    }

    /// Finalizes any remaining segments and writes the bloom filters and metadata to the provided writer.
    pub async fn finish(&mut self, mut writer: impl AsyncWrite + Unpin) -> Result<()> {
        if !self.cur_seg_distinct_elems.is_empty() {
-            self.finalize_segment();
+            self.finalize_segment().await?;
        }

        let mut meta = BloomFilterMeta {
            rows_per_segment: self.rows_per_segment,
-            seg_count: self.finalized_bloom_filters.len(),
            row_count: self.accumulated_row_count,
            ..Default::default()
        };

-        let mut buf = Vec::new();
-        for segment in self.finalized_bloom_filters.drain() {
-            let slice = segment.bloom_filter.as_slice();
-            buf.clear();
-            write_u64_slice(&mut buf, slice);
-            writer.write_all(&buf).await.context(IoSnafu)?;
+        let mut segs = self.finalized_bloom_filters.drain().await?;
+        while let Some(segment) = segs.next().await {
+            let segment = segment?;
+            writer
+                .write_all(&segment.bloom_filter_bytes)
+                .await
+                .context(IoSnafu)?;

-            let size = buf.len();
+            let size = segment.bloom_filter_bytes.len();
            meta.bloom_filter_segments.push(BloomFilterSegmentLocation {
                offset: meta.bloom_filter_segments_size as _,
                size: size as _,
                elem_count: segment.element_count,
            });
            meta.bloom_filter_segments_size += size;
+            meta.seg_count += 1;
        }

        let meta_bytes = serde_json::to_vec(&meta).context(SerdeJsonSnafu)?;
@@ -145,91 +211,36 @@ impl BloomFilterCreator {
        self.cur_seg_distinct_elems_mem_usage + self.finalized_bloom_filters.memory_usage()
    }

-    fn finalize_segment(&mut self) {
+    async fn finalize_segment(&mut self) -> Result<()> {
        let elem_count = self.cur_seg_distinct_elems.len();
        self.finalized_bloom_filters
-            .add(self.cur_seg_distinct_elems.drain(), elem_count);
+            .add(self.cur_seg_distinct_elems.drain(), elem_count)
+            .await?;
+
+        self.global_memory_usage
+            .fetch_sub(self.cur_seg_distinct_elems_mem_usage, Ordering::Relaxed);
        self.cur_seg_distinct_elems_mem_usage = 0;
+        Ok(())
    }
 }

-/// Storage for finalized Bloom filters.
-///
-/// TODO(zhongzc): Add support for storing intermediate bloom filters on disk to control memory usage.
-#[derive(Debug, Default)]
-struct FinalizedBloomFilterStorage {
-    /// Bloom filters that are stored in memory.
-    in_memory: Vec<FinalizedBloomFilterSegment>,
-}
-
-impl FinalizedBloomFilterStorage {
-    fn memory_usage(&self) -> usize {
-        self.in_memory.iter().map(|s| s.size).sum()
-    }
-
-    /// Adds a new finalized Bloom filter to the storage.
-    ///
-    /// TODO(zhongzc): Add support for flushing to disk.
-    fn add(&mut self, elems: impl IntoIterator<Item = Bytes>, elem_count: usize) {
-        let mut bf = BloomFilter::with_false_pos(FALSE_POSITIVE_RATE)
-            .seed(&SEED)
-            .expected_items(elem_count);
-        for elem in elems.into_iter() {
-            bf.insert(&elem);
-        }
-
-        let cbf = FinalizedBloomFilterSegment::new(bf, elem_count);
-        self.in_memory.push(cbf);
-    }
-
-    fn len(&self) -> usize {
-        self.in_memory.len()
-    }
-
-    fn drain(&mut self) -> impl Iterator<Item = FinalizedBloomFilterSegment> + '_ {
-        self.in_memory.drain(..)
-    }
-}
-
-/// A finalized Bloom filter segment.
-#[derive(Debug)]
-struct FinalizedBloomFilterSegment {
-    /// The underlying Bloom filter.
-    bloom_filter: BloomFilter,
-
-    /// The number of elements in the Bloom filter.
-    element_count: usize,
-
-    /// The occupied memory size of the Bloom filter.
-    size: usize,
-}
-
-impl FinalizedBloomFilterSegment {
-    fn new(bloom_filter: BloomFilter, elem_count: usize) -> Self {
-        let memory_usage = std::mem::size_of_val(bloom_filter.as_slice());
-        Self {
-            bloom_filter,
-            element_count: elem_count,
-            size: memory_usage,
-        }
-    }
-}
-
-/// Writes a slice of `u64` to the buffer in little-endian order.
-fn write_u64_slice(buf: &mut Vec<u8>, slice: &[u64]) {
-    buf.reserve(std::mem::size_of_val(slice));
-    for &x in slice {
-        buf.extend_from_slice(&x.to_le_bytes());
+impl Drop for BloomFilterCreator {
+    fn drop(&mut self) {
+        self.global_memory_usage
+            .fetch_sub(self.cur_seg_distinct_elems_mem_usage, Ordering::Relaxed);
    }
 }

 #[cfg(test)]
 mod tests {
+    use fastbloom::BloomFilter;
    use futures::io::Cursor;

    use super::*;
+    use crate::external_provider::MockExternalTempFileProvider;

-    fn u64_vec_from_bytes(bytes: &[u8]) -> Vec<u64> {
+    /// Converts a slice of bytes to a vector of `u64`.
+    pub fn u64_vec_from_bytes(bytes: &[u8]) -> Vec<u64> {
        bytes
            .chunks_exact(std::mem::size_of::<u64>())
            .map(|chunk| u64::from_le_bytes(chunk.try_into().unwrap()))
@@ -239,18 +250,32 @@ mod tests {
    #[tokio::test]
    async fn test_bloom_filter_creator() {
        let mut writer = Cursor::new(Vec::new());
-        let mut creator = BloomFilterCreator::new(2);
+        let mut creator = BloomFilterCreator::new(
+            2,
+            Arc::new(MockExternalTempFileProvider::new()),
+            Arc::new(AtomicUsize::new(0)),
+            None,
+        );

-        creator.push_row_elems(vec![b"a".to_vec(), b"b".to_vec()]);
+        creator
+            .push_row_elems(vec![b"a".to_vec(), b"b".to_vec()])
+            .await
+            .unwrap();
        assert!(creator.cur_seg_distinct_elems_mem_usage > 0);
        assert!(creator.memory_usage() > 0);

-        creator.push_row_elems(vec![b"c".to_vec(), b"d".to_vec()]);
+        creator
+            .push_row_elems(vec![b"c".to_vec(), b"d".to_vec()])
+            .await
+            .unwrap();
        // Finalize the first segment
-        assert!(creator.cur_seg_distinct_elems_mem_usage == 0);
+        assert_eq!(creator.cur_seg_distinct_elems_mem_usage, 0);
        assert!(creator.memory_usage() > 0);

-        creator.push_row_elems(vec![b"e".to_vec(), b"f".to_vec()]);
+        creator
+            .push_row_elems(vec![b"e".to_vec(), b"f".to_vec()])
+            .await
+            .unwrap();
        assert!(creator.cur_seg_distinct_elems_mem_usage > 0);
        assert!(creator.memory_usage() > 0);

@@ -291,4 +316,79 @@ mod tests {
        assert!(bfs[1].contains(&b"e"));
        assert!(bfs[1].contains(&b"f"));
    }
+
+    #[tokio::test]
+    async fn test_bloom_filter_creator_batch_push() {
+        let mut writer = Cursor::new(Vec::new());
+        let mut creator: BloomFilterCreator = BloomFilterCreator::new(
+            2,
+            Arc::new(MockExternalTempFileProvider::new()),
+            Arc::new(AtomicUsize::new(0)),
+            None,
+        );
+
+        creator
+            .push_n_row_elems(5, vec![b"a".to_vec(), b"b".to_vec()])
+            .await
+            .unwrap();
+        assert!(creator.cur_seg_distinct_elems_mem_usage > 0);
+        assert!(creator.memory_usage() > 0);
+
+        creator
+            .push_n_row_elems(5, vec![b"c".to_vec(), b"d".to_vec()])
+            .await
+            .unwrap();
+        assert_eq!(creator.cur_seg_distinct_elems_mem_usage, 0);
+        assert!(creator.memory_usage() > 0);
+
+        creator
+            .push_n_row_elems(10, vec![b"e".to_vec(), b"f".to_vec()])
+            .await
+            .unwrap();
+        assert_eq!(creator.cur_seg_distinct_elems_mem_usage, 0);
+        assert!(creator.memory_usage() > 0);
+
+        creator.finish(&mut writer).await.unwrap();
+
+        let bytes = writer.into_inner();
+        let total_size = bytes.len();
+        let meta_size_offset = total_size - 4;
+        let meta_size = u32::from_le_bytes((&bytes[meta_size_offset..]).try_into().unwrap());
+
+        let meta_bytes = &bytes[total_size - meta_size as usize - 4..total_size - 4];
+        let meta: BloomFilterMeta = serde_json::from_slice(meta_bytes).unwrap();
+
+        assert_eq!(meta.rows_per_segment, 2);
+        assert_eq!(meta.seg_count, 10);
+        assert_eq!(meta.row_count, 20);
+        assert_eq!(
+            meta.bloom_filter_segments_size + meta_bytes.len() + 4,
+            total_size
+        );
+
+        let mut bfs = Vec::new();
+        for segment in meta.bloom_filter_segments {
+            let bloom_filter_bytes =
+                &bytes[segment.offset as usize..(segment.offset + segment.size) as usize];
+            let v = u64_vec_from_bytes(bloom_filter_bytes);
+            let bloom_filter = BloomFilter::from_vec(v)
+                .seed(&SEED)
+                .expected_items(segment.elem_count);
+            bfs.push(bloom_filter);
+        }
+
+        assert_eq!(bfs.len(), 10);
+        for bf in bfs.iter().take(3) {
+            assert!(bf.contains(&b"a"));
+            assert!(bf.contains(&b"b"));
+        }
+        for bf in bfs.iter().take(5).skip(2) {
+            assert!(bf.contains(&b"c"));
+            assert!(bf.contains(&b"d"));
+        }
+        for bf in bfs.iter().take(10).skip(5) {
+            assert!(bf.contains(&b"e"));
+            assert!(bf.contains(&b"f"));
+        }
+    }
 }
--- a/src/index/src/bloom_filter/creator/finalize_segment.rs
+++ b/src/index/src/bloom_filter/creator/finalize_segment.rs
@@ -0,0 +1,300 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::pin::Pin;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
+use asynchronous_codec::{FramedRead, FramedWrite};
+use fastbloom::BloomFilter;
+use futures::stream::StreamExt;
+use futures::{stream, AsyncWriteExt, Stream};
+use snafu::ResultExt;
+
+use super::intermediate_codec::IntermediateBloomFilterCodecV1;
+use crate::bloom_filter::creator::{FALSE_POSITIVE_RATE, SEED};
+use crate::bloom_filter::error::{IntermediateSnafu, IoSnafu, Result};
+use crate::bloom_filter::Bytes;
+use crate::external_provider::ExternalTempFileProvider;
+
+/// The minimum memory usage threshold for flushing in-memory Bloom filters to disk.
+const MIN_MEMORY_USAGE_THRESHOLD: usize = 1024 * 1024; // 1MB
+
+/// Storage for finalized Bloom filters.
+pub struct FinalizedBloomFilterStorage {
+    /// Bloom filters that are stored in memory.
+    in_memory: Vec<FinalizedBloomFilterSegment>,
+
+    /// Used to generate unique file IDs for intermediate Bloom filters.
+    intermediate_file_id_counter: usize,
+
+    /// Prefix for intermediate Bloom filter files.
+    intermediate_prefix: String,
+
+    /// The provider for intermediate Bloom filter files.
+    intermediate_provider: Arc<dyn ExternalTempFileProvider>,
+
+    /// The memory usage of the in-memory Bloom filters.
+    memory_usage: usize,
+
+    /// The global memory usage provided by the user to track the
+    /// total memory usage of the creating Bloom filters.
+    global_memory_usage: Arc<AtomicUsize>,
+
+    /// The threshold of the global memory usage of the creating Bloom filters.
+    global_memory_usage_threshold: Option<usize>,
+}
+
+impl FinalizedBloomFilterStorage {
+    /// Creates a new `FinalizedBloomFilterStorage`.
+    pub fn new(
+        intermediate_provider: Arc<dyn ExternalTempFileProvider>,
+        global_memory_usage: Arc<AtomicUsize>,
+        global_memory_usage_threshold: Option<usize>,
+    ) -> Self {
+        let external_prefix = format!("intm-bloom-filters-{}", uuid::Uuid::new_v4());
+        Self {
+            in_memory: Vec::new(),
+            intermediate_file_id_counter: 0,
+            intermediate_prefix: external_prefix,
+            intermediate_provider,
+            memory_usage: 0,
+            global_memory_usage,
+            global_memory_usage_threshold,
+        }
+    }
+
+    /// Returns the memory usage of the storage.
+    pub fn memory_usage(&self) -> usize {
+        self.memory_usage
+    }
+
+    /// Adds a new finalized Bloom filter to the storage.
+    ///
+    /// If the memory usage exceeds the threshold, flushes the in-memory Bloom filters to disk.
+    pub async fn add(
+        &mut self,
+        elems: impl IntoIterator<Item = Bytes>,
+        element_count: usize,
+    ) -> Result<()> {
+        let mut bf = BloomFilter::with_false_pos(FALSE_POSITIVE_RATE)
+            .seed(&SEED)
+            .expected_items(element_count);
+        for elem in elems.into_iter() {
+            bf.insert(&elem);
+        }
+
+        let fbf = FinalizedBloomFilterSegment::from(bf, element_count);
+
+        // Update memory usage.
+        let memory_diff = fbf.bloom_filter_bytes.len();
+        self.memory_usage += memory_diff;
+        self.global_memory_usage
+            .fetch_add(memory_diff, Ordering::Relaxed);
+
+        // Add the finalized Bloom filter to the in-memory storage.
+        self.in_memory.push(fbf);
+
+        // Flush to disk if necessary.
+
+        // Do not flush if memory usage is too low.
+        if self.memory_usage < MIN_MEMORY_USAGE_THRESHOLD {
+            return Ok(());
+        }
+
+        // Check if the global memory usage exceeds the threshold and flush to disk if necessary.
+        if let Some(threshold) = self.global_memory_usage_threshold {
+            let global = self.global_memory_usage.load(Ordering::Relaxed);
+
+            if global > threshold {
+                self.flush_in_memory_to_disk().await?;
+
+                self.global_memory_usage
+                    .fetch_sub(self.memory_usage, Ordering::Relaxed);
+                self.memory_usage = 0;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Drains the storage and returns a stream of finalized Bloom filter segments.
+    pub async fn drain(
+        &mut self,
+    ) -> Result<Pin<Box<dyn Stream<Item = Result<FinalizedBloomFilterSegment>> + Send + '_>>> {
+        // FAST PATH: memory only
+        if self.intermediate_file_id_counter == 0 {
+            return Ok(Box::pin(stream::iter(self.in_memory.drain(..).map(Ok))));
+        }
+
+        // SLOW PATH: memory + disk
+        let mut on_disk = self
+            .intermediate_provider
+            .read_all(&self.intermediate_prefix)
+            .await
+            .context(IntermediateSnafu)?;
+        on_disk.sort_unstable_by(|x, y| x.0.cmp(&y.0));
+
+        let streams = on_disk
+            .into_iter()
+            .map(|(_, reader)| FramedRead::new(reader, IntermediateBloomFilterCodecV1::default()));
+
+        let in_memory_stream = stream::iter(self.in_memory.drain(..)).map(Ok);
+        Ok(Box::pin(
+            stream::iter(streams).flatten().chain(in_memory_stream),
+        ))
+    }
+
+    /// Flushes the in-memory Bloom filters to disk.
+    async fn flush_in_memory_to_disk(&mut self) -> Result<()> {
+        let file_id = self.intermediate_file_id_counter;
+        self.intermediate_file_id_counter += 1;
+
+        let file_id = format!("{:08}", file_id);
+        let mut writer = self
+            .intermediate_provider
+            .create(&self.intermediate_prefix, &file_id)
+            .await
+            .context(IntermediateSnafu)?;
+
+        let fw = FramedWrite::new(&mut writer, IntermediateBloomFilterCodecV1::default());
+        // `forward()` will flush and close the writer when the stream ends
+        if let Err(e) = stream::iter(self.in_memory.drain(..).map(Ok))
+            .forward(fw)
+            .await
+        {
+            writer.close().await.context(IoSnafu)?;
+            writer.flush().await.context(IoSnafu)?;
+            return Err(e);
+        }
+
+        Ok(())
+    }
+}
+
+impl Drop for FinalizedBloomFilterStorage {
+    fn drop(&mut self) {
+        self.global_memory_usage
+            .fetch_sub(self.memory_usage, Ordering::Relaxed);
+    }
+}
+
+/// A finalized Bloom filter segment.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FinalizedBloomFilterSegment {
+    /// The underlying Bloom filter bytes.
+    pub bloom_filter_bytes: Vec<u8>,
+
+    /// The number of elements in the Bloom filter.
+    pub element_count: usize,
+}
+
+impl FinalizedBloomFilterSegment {
+    fn from(bf: BloomFilter, elem_count: usize) -> Self {
+        let bf_slice = bf.as_slice();
+        let mut bloom_filter_bytes = Vec::with_capacity(std::mem::size_of_val(bf_slice));
+        for &x in bf_slice {
+            bloom_filter_bytes.extend_from_slice(&x.to_le_bytes());
+        }
+
+        Self {
+            bloom_filter_bytes,
+            element_count: elem_count,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Mutex;
+
+    use futures::AsyncRead;
+    use tokio::io::duplex;
+    use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
+
+    use super::*;
+    use crate::bloom_filter::creator::tests::u64_vec_from_bytes;
+    use crate::external_provider::MockExternalTempFileProvider;
+
+    #[tokio::test]
+    async fn test_finalized_bloom_filter_storage() {
+        let mut mock_provider = MockExternalTempFileProvider::new();
+
+        let mock_files: Arc<Mutex<HashMap<String, Box<dyn AsyncRead + Unpin + Send>>>> =
+            Arc::new(Mutex::new(HashMap::new()));
+
+        mock_provider.expect_create().returning({
+            let files = Arc::clone(&mock_files);
+            move |file_group, file_id| {
+                assert!(file_group.starts_with("intm-bloom-filters-"));
+                let mut files = files.lock().unwrap();
+                let (writer, reader) = duplex(2 * 1024 * 1024);
+                files.insert(file_id.to_string(), Box::new(reader.compat()));
+                Ok(Box::new(writer.compat_write()))
+            }
+        });
+
+        mock_provider.expect_read_all().returning({
+            let files = Arc::clone(&mock_files);
+            move |file_group| {
+                assert!(file_group.starts_with("intm-bloom-filters-"));
+                let mut files = files.lock().unwrap();
+                Ok(files.drain().collect::<Vec<_>>())
+            }
+        });
+
+        let global_memory_usage = Arc::new(AtomicUsize::new(0));
+        let global_memory_usage_threshold = Some(1024 * 1024); // 1MB
+        let provider = Arc::new(mock_provider);
+        let mut storage = FinalizedBloomFilterStorage::new(
+            provider,
+            global_memory_usage.clone(),
+            global_memory_usage_threshold,
+        );
+
+        let elem_count = 2000;
+        let batch = 1000;
+
+        for i in 0..batch {
+            let elems = (elem_count * i..elem_count * (i + 1)).map(|x| x.to_string().into_bytes());
+            storage.add(elems, elem_count).await.unwrap();
+        }
+
+        // Flush happens.
+        assert!(storage.intermediate_file_id_counter > 0);
+
+        // Drain the storage.
+        let mut stream = storage.drain().await.unwrap();
+
+        let mut i = 0;
+        while let Some(segment) = stream.next().await {
+            let segment = segment.unwrap();
+            assert_eq!(segment.element_count, elem_count);
+
+            let v = u64_vec_from_bytes(&segment.bloom_filter_bytes);
+
+            // Check the correctness of the Bloom filter.
+            let bf = BloomFilter::from_vec(v)
+                .seed(&SEED)
+                .expected_items(segment.element_count);
+            for elem in (elem_count * i..elem_count * (i + 1)).map(|x| x.to_string().into_bytes()) {
+                assert!(bf.contains(&elem));
+            }
+            i += 1;
+        }
+
+        assert_eq!(i, batch);
+    }
+}
--- a/src/index/src/bloom_filter/creator/intermediate_codec.rs
+++ b/src/index/src/bloom_filter/creator/intermediate_codec.rs
@@ -0,0 +1,248 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use asynchronous_codec::{BytesMut, Decoder, Encoder};
+use bytes::{Buf, BufMut};
+use snafu::{ensure, ResultExt};
+
+use crate::bloom_filter::creator::finalize_segment::FinalizedBloomFilterSegment;
+use crate::bloom_filter::error::{Error, InvalidIntermediateMagicSnafu, IoSnafu, Result};
+
+/// The magic number for the codec version 1 of the intermediate bloom filter.
+const CODEC_V1_MAGIC: &[u8; 4] = b"bi01";
+
+/// Codec of the intermediate finalized bloom filter segment.
+///
+/// # Format
+///
+/// [ magic ][ elem count ][    size    ][ bloom filter ][ elem count ][    size    ][ bloom filter ]...
+///    [4]       [8]            [8]           [size]         [8]            [8]           [size]
+#[derive(Debug, Default)]
+pub struct IntermediateBloomFilterCodecV1 {
+    handled_header_magic: bool,
+}
+
+impl Encoder for IntermediateBloomFilterCodecV1 {
+    type Item<'a> = FinalizedBloomFilterSegment;
+    type Error = Error;
+
+    fn encode(&mut self, item: FinalizedBloomFilterSegment, dst: &mut BytesMut) -> Result<()> {
+        if !self.handled_header_magic {
+            dst.extend_from_slice(CODEC_V1_MAGIC);
+            self.handled_header_magic = true;
+        }
+
+        let segment_bytes = item.bloom_filter_bytes;
+        let elem_count = item.element_count;
+
+        dst.reserve(2 * std::mem::size_of::<u64>() + segment_bytes.len());
+        dst.put_u64_le(elem_count as u64);
+        dst.put_u64_le(segment_bytes.len() as u64);
+        dst.extend_from_slice(&segment_bytes);
+        Ok(())
+    }
+}
+
+impl Decoder for IntermediateBloomFilterCodecV1 {
+    type Item = FinalizedBloomFilterSegment;
+    type Error = Error;
+
+    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>> {
+        if !self.handled_header_magic {
+            let m_len = CODEC_V1_MAGIC.len();
+            if src.remaining() < m_len {
+                return Ok(None);
+            }
+            let magic_bytes = &src[..m_len];
+            ensure!(
+                magic_bytes == CODEC_V1_MAGIC,
+                InvalidIntermediateMagicSnafu {
+                    invalid: magic_bytes,
+                }
+            );
+            self.handled_header_magic = true;
+            src.advance(m_len);
+        }
+
+        let s = &src[..];
+
+        let u64_size = std::mem::size_of::<u64>();
+        let n_size = u64_size * 2;
+        if s.len() < n_size {
+            return Ok(None);
+        }
+
+        let element_count = u64::from_le_bytes(s[0..u64_size].try_into().unwrap()) as usize;
+        let segment_size = u64::from_le_bytes(s[u64_size..n_size].try_into().unwrap()) as usize;
+
+        if s.len() < n_size + segment_size {
+            return Ok(None);
+        }
+
+        let bloom_filter_bytes = s[n_size..n_size + segment_size].to_vec();
+        src.advance(n_size + segment_size);
+        Ok(Some(FinalizedBloomFilterSegment {
+            element_count,
+            bloom_filter_bytes,
+        }))
+    }
+}
+
+/// Required for [`Encoder`] and [`Decoder`] implementations.
+impl From<std::io::Error> for Error {
+    fn from(error: std::io::Error) -> Self {
+        Err::<(), std::io::Error>(error)
+            .context(IoSnafu)
+            .unwrap_err()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use asynchronous_codec::{FramedRead, FramedWrite};
+    use futures::io::Cursor;
+    use futures::{SinkExt, StreamExt};
+
+    use super::*;
+    use crate::bloom_filter::creator::finalize_segment::FinalizedBloomFilterSegment;
+
+    #[test]
+    fn test_intermediate_bloom_filter_codec_v1_basic() {
+        let mut encoder = IntermediateBloomFilterCodecV1::default();
+        let mut buf = BytesMut::new();
+
+        let item1 = FinalizedBloomFilterSegment {
+            element_count: 2,
+            bloom_filter_bytes: vec![1, 2, 3, 4],
+        };
+        let item2 = FinalizedBloomFilterSegment {
+            element_count: 3,
+            bloom_filter_bytes: vec![5, 6, 7, 8],
+        };
+        let item3 = FinalizedBloomFilterSegment {
+            element_count: 4,
+            bloom_filter_bytes: vec![9, 10, 11, 12],
+        };
+
+        encoder.encode(item1.clone(), &mut buf).unwrap();
+        encoder.encode(item2.clone(), &mut buf).unwrap();
+        encoder.encode(item3.clone(), &mut buf).unwrap();
+
+        let mut buf = buf.freeze().try_into_mut().unwrap();
+
+        let mut decoder = IntermediateBloomFilterCodecV1::default();
+        let decoded_item1 = decoder.decode(&mut buf).unwrap().unwrap();
+        let decoded_item2 = decoder.decode(&mut buf).unwrap().unwrap();
+        let decoded_item3 = decoder.decode(&mut buf).unwrap().unwrap();
+
+        assert_eq!(item1, decoded_item1);
+        assert_eq!(item2, decoded_item2);
+        assert_eq!(item3, decoded_item3);
+    }
+
+    #[tokio::test]
+    async fn test_intermediate_bloom_filter_codec_v1_frame_read_write() {
+        let item1 = FinalizedBloomFilterSegment {
+            element_count: 2,
+            bloom_filter_bytes: vec![1, 2, 3, 4],
+        };
+        let item2 = FinalizedBloomFilterSegment {
+            element_count: 3,
+            bloom_filter_bytes: vec![5, 6, 7, 8],
+        };
+        let item3 = FinalizedBloomFilterSegment {
+            element_count: 4,
+            bloom_filter_bytes: vec![9, 10, 11, 12],
+        };
+
+        let mut bytes = Cursor::new(vec![]);
+
+        let mut writer = FramedWrite::new(&mut bytes, IntermediateBloomFilterCodecV1::default());
+        writer.send(item1.clone()).await.unwrap();
+        writer.send(item2.clone()).await.unwrap();
+        writer.send(item3.clone()).await.unwrap();
+        writer.flush().await.unwrap();
+        writer.close().await.unwrap();
+
+        let bytes = bytes.into_inner();
+        let mut reader =
+            FramedRead::new(bytes.as_slice(), IntermediateBloomFilterCodecV1::default());
+        let decoded_item1 = reader.next().await.unwrap().unwrap();
+        let decoded_item2 = reader.next().await.unwrap().unwrap();
+        let decoded_item3 = reader.next().await.unwrap().unwrap();
+        assert!(reader.next().await.is_none());
+
+        assert_eq!(item1, decoded_item1);
+        assert_eq!(item2, decoded_item2);
+        assert_eq!(item3, decoded_item3);
+    }
+
+    #[tokio::test]
+    async fn test_intermediate_bloom_filter_codec_v1_frame_read_write_only_magic() {
+        let bytes = CODEC_V1_MAGIC.to_vec();
+        let mut reader =
+            FramedRead::new(bytes.as_slice(), IntermediateBloomFilterCodecV1::default());
+        assert!(reader.next().await.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_intermediate_bloom_filter_codec_v1_frame_read_write_partial_magic() {
+        let bytes = CODEC_V1_MAGIC[..3].to_vec();
+        let mut reader =
+            FramedRead::new(bytes.as_slice(), IntermediateBloomFilterCodecV1::default());
+        let e = reader.next().await.unwrap();
+        assert!(e.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_intermediate_bloom_filter_codec_v1_frame_read_write_partial_item() {
+        let mut bytes = vec![];
+        bytes.extend_from_slice(CODEC_V1_MAGIC);
+        bytes.extend_from_slice(&2u64.to_le_bytes());
+        bytes.extend_from_slice(&4u64.to_le_bytes());
+
+        let mut reader =
+            FramedRead::new(bytes.as_slice(), IntermediateBloomFilterCodecV1::default());
+        let e = reader.next().await.unwrap();
+        assert!(e.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_intermediate_bloom_filter_codec_v1_frame_read_write_corrupted_magic() {
+        let mut bytes = vec![];
+        bytes.extend_from_slice(b"bi02");
+        bytes.extend_from_slice(&2u64.to_le_bytes());
+        bytes.extend_from_slice(&4u64.to_le_bytes());
+        bytes.extend_from_slice(&[1, 2, 3, 4]);
+
+        let mut reader =
+            FramedRead::new(bytes.as_slice(), IntermediateBloomFilterCodecV1::default());
+        let e = reader.next().await.unwrap();
+        assert!(e.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_intermediate_bloom_filter_codec_v1_frame_read_write_corrupted_length() {
+        let mut bytes = vec![];
+        bytes.extend_from_slice(CODEC_V1_MAGIC);
+        bytes.extend_from_slice(&2u64.to_le_bytes());
+        bytes.extend_from_slice(&4u64.to_le_bytes());
+        bytes.extend_from_slice(&[1, 2, 3]);
+
+        let mut reader =
+            FramedRead::new(bytes.as_slice(), IntermediateBloomFilterCodecV1::default());
+        let e = reader.next().await.unwrap();
+        assert!(e.is_err());
+    }
+}
--- a/src/index/src/bloom_filter/error.rs
+++ b/src/index/src/bloom_filter/error.rs
@@ -39,6 +39,43 @@ pub enum Error {
        location: Location,
    },

+    #[snafu(display("Failed to deserialize json"))]
+    DeserializeJson {
+        #[snafu(source)]
+        error: serde_json::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Intermediate error"))]
+    Intermediate {
+        source: crate::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("File size too small for bloom filter"))]
+    FileSizeTooSmall {
+        size: u64,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Unexpected bloom filter meta size"))]
+    UnexpectedMetaSize {
+        max_meta_size: u64,
+        actual_meta_size: u64,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Invalid intermediate magic"))]
+    InvalidIntermediateMagic {
+        invalid: Vec<u8>,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
    #[snafu(display("External error"))]
    External {
        source: BoxedError,
@@ -52,8 +89,14 @@ impl ErrorExt for Error {
        use Error::*;

        match self {
-            Io { .. } | Self::SerdeJson { .. } => StatusCode::Unexpected,
+            Io { .. }
+            | SerdeJson { .. }
+            | FileSizeTooSmall { .. }
+            | UnexpectedMetaSize { .. }
+            | DeserializeJson { .. }
+            | InvalidIntermediateMagic { .. } => StatusCode::Unexpected,

+            Intermediate { source, .. } => source.status_code(),
            External { source, .. } => source.status_code(),
        }
    }
--- a/src/index/src/bloom_filter/reader.rs
+++ b/src/index/src/bloom_filter/reader.rs
@@ -0,0 +1,273 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::ops::Range;
+
+use async_trait::async_trait;
+use bytes::Bytes;
+use common_base::range_read::RangeReader;
+use fastbloom::BloomFilter;
+use snafu::{ensure, ResultExt};
+
+use crate::bloom_filter::error::{
+    DeserializeJsonSnafu, FileSizeTooSmallSnafu, IoSnafu, Result, UnexpectedMetaSizeSnafu,
+};
+use crate::bloom_filter::{BloomFilterMeta, BloomFilterSegmentLocation, SEED};
+
+/// Minimum size of the bloom filter, which is the size of the length of the bloom filter.
+const BLOOM_META_LEN_SIZE: u64 = 4;
+
+/// Default prefetch size of bloom filter meta.
+pub const DEFAULT_PREFETCH_SIZE: u64 = 1024; // 1KiB
+
+/// `BloomFilterReader` reads the bloom filter from the file.
+#[async_trait]
+pub trait BloomFilterReader {
+    /// Reads range of bytes from the file.
+    async fn range_read(&mut self, offset: u64, size: u32) -> Result<Bytes>;
+
+    /// Reads bunch of ranges from the file.
+    async fn read_vec(&mut self, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
+        let mut results = Vec::with_capacity(ranges.len());
+        for range in ranges {
+            let size = (range.end - range.start) as u32;
+            let data = self.range_read(range.start, size).await?;
+            results.push(data);
+        }
+        Ok(results)
+    }
+
+    /// Reads the meta information of the bloom filter.
+    async fn metadata(&mut self) -> Result<BloomFilterMeta>;
+
+    /// Reads a bloom filter with the given location.
+    async fn bloom_filter(&mut self, loc: &BloomFilterSegmentLocation) -> Result<BloomFilter> {
+        let bytes = self.range_read(loc.offset, loc.size as _).await?;
+        let vec = bytes
+            .chunks_exact(std::mem::size_of::<u64>())
+            .map(|chunk| u64::from_le_bytes(chunk.try_into().unwrap()))
+            .collect();
+        let bm = BloomFilter::from_vec(vec)
+            .seed(&SEED)
+            .expected_items(loc.elem_count);
+        Ok(bm)
+    }
+}
+
+/// `BloomFilterReaderImpl` reads the bloom filter from the file.
+pub struct BloomFilterReaderImpl<R: RangeReader> {
+    /// The underlying reader.
+    reader: R,
+}
+
+impl<R: RangeReader> BloomFilterReaderImpl<R> {
+    /// Creates a new `BloomFilterReaderImpl` with the given reader.
+    pub fn new(reader: R) -> Self {
+        Self { reader }
+    }
+}
+
+#[async_trait]
+impl<R: RangeReader> BloomFilterReader for BloomFilterReaderImpl<R> {
+    async fn range_read(&mut self, offset: u64, size: u32) -> Result<Bytes> {
+        self.reader
+            .read(offset..offset + size as u64)
+            .await
+            .context(IoSnafu)
+    }
+
+    async fn read_vec(&mut self, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
+        self.reader.read_vec(ranges).await.context(IoSnafu)
+    }
+
+    async fn metadata(&mut self) -> Result<BloomFilterMeta> {
+        let metadata = self.reader.metadata().await.context(IoSnafu)?;
+        let file_size = metadata.content_length;
+
+        let mut meta_reader =
+            BloomFilterMetaReader::new(&mut self.reader, file_size, Some(DEFAULT_PREFETCH_SIZE));
+        meta_reader.metadata().await
+    }
+}
+
+/// `BloomFilterMetaReader` reads the metadata of the bloom filter.
+struct BloomFilterMetaReader<R: RangeReader> {
+    reader: R,
+    file_size: u64,
+    prefetch_size: u64,
+}
+
+impl<R: RangeReader> BloomFilterMetaReader<R> {
+    pub fn new(reader: R, file_size: u64, prefetch_size: Option<u64>) -> Self {
+        Self {
+            reader,
+            file_size,
+            prefetch_size: prefetch_size
+                .unwrap_or(BLOOM_META_LEN_SIZE)
+                .max(BLOOM_META_LEN_SIZE),
+        }
+    }
+
+    /// Reads the metadata of the bloom filter.
+    ///
+    /// It will first prefetch some bytes from the end of the file,
+    /// then parse the metadata from the prefetch bytes.
+    pub async fn metadata(&mut self) -> Result<BloomFilterMeta> {
+        ensure!(
+            self.file_size >= BLOOM_META_LEN_SIZE,
+            FileSizeTooSmallSnafu {
+                size: self.file_size,
+            }
+        );
+
+        let meta_start = self.file_size.saturating_sub(self.prefetch_size);
+        let suffix = self
+            .reader
+            .read(meta_start..self.file_size)
+            .await
+            .context(IoSnafu)?;
+        let suffix_len = suffix.len();
+        let length = u32::from_le_bytes(Self::read_tailing_four_bytes(&suffix)?) as u64;
+        self.validate_meta_size(length)?;
+
+        if length > suffix_len as u64 - BLOOM_META_LEN_SIZE {
+            let metadata_start = self.file_size - length - BLOOM_META_LEN_SIZE;
+            let meta = self
+                .reader
+                .read(metadata_start..self.file_size - BLOOM_META_LEN_SIZE)
+                .await
+                .context(IoSnafu)?;
+            serde_json::from_slice(&meta).context(DeserializeJsonSnafu)
+        } else {
+            let metadata_start = self.file_size - length - BLOOM_META_LEN_SIZE - meta_start;
+            let meta = &suffix[metadata_start as usize..suffix_len - BLOOM_META_LEN_SIZE as usize];
+            serde_json::from_slice(meta).context(DeserializeJsonSnafu)
+        }
+    }
+
+    fn read_tailing_four_bytes(suffix: &[u8]) -> Result<[u8; 4]> {
+        let suffix_len = suffix.len();
+        ensure!(
+            suffix_len >= 4,
+            FileSizeTooSmallSnafu {
+                size: suffix_len as u64
+            }
+        );
+        let mut bytes = [0; 4];
+        bytes.copy_from_slice(&suffix[suffix_len - 4..suffix_len]);
+
+        Ok(bytes)
+    }
+
+    fn validate_meta_size(&self, length: u64) -> Result<()> {
+        let max_meta_size = self.file_size - BLOOM_META_LEN_SIZE;
+        ensure!(
+            length <= max_meta_size,
+            UnexpectedMetaSizeSnafu {
+                max_meta_size,
+                actual_meta_size: length,
+            }
+        );
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::atomic::AtomicUsize;
+    use std::sync::Arc;
+
+    use futures::io::Cursor;
+
+    use super::*;
+    use crate::bloom_filter::creator::BloomFilterCreator;
+    use crate::external_provider::MockExternalTempFileProvider;
+
+    async fn mock_bloom_filter_bytes() -> Vec<u8> {
+        let mut writer = Cursor::new(vec![]);
+        let mut creator = BloomFilterCreator::new(
+            2,
+            Arc::new(MockExternalTempFileProvider::new()),
+            Arc::new(AtomicUsize::new(0)),
+            None,
+        );
+
+        creator
+            .push_row_elems(vec![b"a".to_vec(), b"b".to_vec()])
+            .await
+            .unwrap();
+        creator
+            .push_row_elems(vec![b"c".to_vec(), b"d".to_vec()])
+            .await
+            .unwrap();
+        creator
+            .push_row_elems(vec![b"e".to_vec(), b"f".to_vec()])
+            .await
+            .unwrap();
+
+        creator.finish(&mut writer).await.unwrap();
+
+        writer.into_inner()
+    }
+
+    #[tokio::test]
+    async fn test_bloom_filter_meta_reader() {
+        let bytes = mock_bloom_filter_bytes().await;
+        let file_size = bytes.len() as u64;
+
+        for prefetch in [0u64, file_size / 2, file_size, file_size + 10] {
+            let mut reader =
+                BloomFilterMetaReader::new(bytes.clone(), file_size as _, Some(prefetch));
+            let meta = reader.metadata().await.unwrap();
+
+            assert_eq!(meta.rows_per_segment, 2);
+            assert_eq!(meta.seg_count, 2);
+            assert_eq!(meta.row_count, 3);
+            assert_eq!(meta.bloom_filter_segments.len(), 2);
+
+            assert_eq!(meta.bloom_filter_segments[0].offset, 0);
+            assert_eq!(meta.bloom_filter_segments[0].elem_count, 4);
+            assert_eq!(
+                meta.bloom_filter_segments[1].offset,
+                meta.bloom_filter_segments[0].size
+            );
+            assert_eq!(meta.bloom_filter_segments[1].elem_count, 2);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_bloom_filter_reader() {
+        let bytes = mock_bloom_filter_bytes().await;
+
+        let mut reader = BloomFilterReaderImpl::new(bytes);
+        let meta = reader.metadata().await.unwrap();
+
+        assert_eq!(meta.bloom_filter_segments.len(), 2);
+        let bf = reader
+            .bloom_filter(&meta.bloom_filter_segments[0])
+            .await
+            .unwrap();
+        assert!(bf.contains(&b"a"));
+        assert!(bf.contains(&b"b"));
+        assert!(bf.contains(&b"c"));
+        assert!(bf.contains(&b"d"));
+
+        let bf = reader
+            .bloom_filter(&meta.bloom_filter_segments[1])
+            .await
+            .unwrap();
+        assert!(bf.contains(&b"e"));
+        assert!(bf.contains(&b"f"));
+    }
+}
--- a/Show More
+++ b/Show More