feat: add sqlness test for bloom filter index (#5240)

* feat: add sqlness test for bloom filter index

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* drop table after finished

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* redact more variables

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

.github/workflows/develop.yml

@@ -697,7 +697,7 @@ jobs:
         working-directory: tests-integration/fixtures/postgres
         run: docker compose -f docker-compose-standalone.yml up -d --wait
       - name: Run nextest cases
-        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
+        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard -F pg_kvbackend
         env:
           CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
           RUST_BACKTRACE: 1

Cargo.lock

@@ -2016,6 +2016,7 @@ dependencies = [
 name = "common-error"
 version = "0.12.0"
 dependencies = [
+ "http 0.2.12",
  "snafu 0.8.5",
  "strum 0.25.0",
  "tonic 0.11.0",
@@ -4061,6 +4062,7 @@ dependencies = [
  "get-size-derive2",
  "get-size2",
  "greptime-proto",
+ "http 0.2.12",
  "hydroflow",
  "itertools 0.10.5",
  "lazy_static",
@@ -5286,6 +5288,7 @@ dependencies = [
  "futures",
  "greptime-proto",
  "mockall",
+ "parquet",
  "pin-project",
  "prost 0.12.6",
  "rand",
@@ -9102,6 +9105,7 @@ dependencies = [
  "log-query",
  "meter-core",
  "meter-macros",
+ "nalgebra 0.33.2",
  "num",
  "num-traits",
  "object-store",

Cargo.toml

@@ -126,6 +126,7 @@ futures = "0.3"
 futures-util = "0.3"
 greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "a875e976441188028353f7274a46a7e6e065c5d4" }
 hex = "0.4"
+http = "0.2"
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
@@ -134,6 +135,7 @@ lazy_static = "1.4"
 meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "a10facb353b41460eeb98578868ebf19c2084fac" }
 mockall = "0.11.4"
 moka = "0.12"
+nalgebra = "0.33"
 notify = "6.1"
 num_cpus = "1.16"
 once_cell = "1.18"

config/config.md

@@ -18,6 +18,7 @@
 | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
 | `max_concurrent_queries` | Integer | `0` | The maximum current queries allowed to be executed. Zero means unlimited. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. Enabled by default. |
+| `max_in_flight_write_bytes` | String | Unset | The maximum in-flight write bytes. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
 | `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
@@ -156,6 +157,11 @@
 | `region_engine.mito.fulltext_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.apply_on_query` | String | `auto` | Whether to apply the index on query<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.mem_threshold_on_create` | String | `auto` | Memory threshold for index creation.<br/>- `auto`: automatically determine the threshold based on the system memory size (default)<br/>- `unlimited`: no memory limit<br/>- `[size]` e.g. `64MB`: fixed memory threshold |
+| `region_engine.mito.bloom_filter_index` | -- | -- | The options for bloom filter in Mito engine. |
+| `region_engine.mito.bloom_filter_index.create_on_flush` | String | `auto` | Whether to create the bloom filter on flush.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.create_on_compaction` | String | `auto` | Whether to create the bloom filter on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.apply_on_query` | String | `auto` | Whether to apply the bloom filter on query<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.mem_threshold_on_create` | String | `auto` | Memory threshold for bloom filter creation.<br/>- `auto`: automatically determine the threshold based on the system memory size (default)<br/>- `unlimited`: no memory limit<br/>- `[size]` e.g. `64MB`: fixed memory threshold |
 | `region_engine.mito.memtable` | -- | -- | -- |
 | `region_engine.mito.memtable.type` | String | `time_series` | Memtable type.<br/>- `time_series`: time-series memtable<br/>- `partition_tree`: partition tree memtable (experimental) |
 | `region_engine.mito.memtable.index_max_keys_per_shard` | Integer | `8192` | The max number of keys in one shard.<br/>Only available for `partition_tree` memtable. |
@@ -195,6 +201,7 @@
 | Key | Type | Default | Descriptions |
 | --- | -----| ------- | ----------- |
 | `default_timezone` | String | Unset | The default timezone of the server. |
+| `max_in_flight_write_bytes` | String | Unset | The maximum in-flight write bytes. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
 | `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
@@ -421,7 +428,7 @@
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
-| `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.<br/>A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling. |
+| `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.<br/>A local file directory, defaults to `{data_home}`. An empty string means disabling. |
 | `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. |
 | `storage.bucket` | String | Unset | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
 | `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
@@ -460,7 +467,7 @@
 | `region_engine.mito.page_cache_size` | String | Auto | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
 | `region_engine.mito.selector_result_cache_size` | String | Auto | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. |
-| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/object_cache/write`. |
+| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}`. |
 | `region_engine.mito.experimental_write_cache_size` | String | `5GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. |
 | `region_engine.mito.experimental_write_cache_ttl` | String | Unset | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
@@ -484,6 +491,11 @@
 | `region_engine.mito.fulltext_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.apply_on_query` | String | `auto` | Whether to apply the index on query<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.mem_threshold_on_create` | String | `auto` | Memory threshold for index creation.<br/>- `auto`: automatically determine the threshold based on the system memory size (default)<br/>- `unlimited`: no memory limit<br/>- `[size]` e.g. `64MB`: fixed memory threshold |
+| `region_engine.mito.bloom_filter_index` | -- | -- | The options for bloom filter index in Mito engine. |
+| `region_engine.mito.bloom_filter_index.create_on_flush` | String | `auto` | Whether to create the index on flush.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.apply_on_query` | String | `auto` | Whether to apply the index on query<br/>- `auto`: automatically (default)<br/>- `disable`: never |
+| `region_engine.mito.bloom_filter_index.mem_threshold_on_create` | String | `auto` | Memory threshold for the index creation.<br/>- `auto`: automatically determine the threshold based on the system memory size (default)<br/>- `unlimited`: no memory limit<br/>- `[size]` e.g. `64MB`: fixed memory threshold |
 | `region_engine.mito.memtable` | -- | -- | -- |
 | `region_engine.mito.memtable.type` | String | `time_series` | Memtable type.<br/>- `time_series`: time-series memtable<br/>- `partition_tree`: partition tree memtable (experimental) |
 | `region_engine.mito.memtable.index_max_keys_per_shard` | Integer | `8192` | The max number of keys in one shard.<br/>Only available for `partition_tree` memtable. |

config/datanode.example.toml

@@ -294,7 +294,7 @@ data_home = "/tmp/greptimedb/"
 type = "File"
 
 ## Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.
-## A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling.
+## A local file directory, defaults to `{data_home}`. An empty string means disabling.
 ## @toml2docs:none-default
 #+ cache_path = ""
 
@@ -478,7 +478,7 @@ auto_flush_interval = "1h"
 ## Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance.
 enable_experimental_write_cache = false
 
-## File system path for write cache, defaults to `{data_home}/object_cache/write`.
+## File system path for write cache, defaults to `{data_home}`.
 experimental_write_cache_path = ""
 
 ## Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger.
@@ -576,6 +576,30 @@ apply_on_query = "auto"
 ## - `[size]` e.g. `64MB`: fixed memory threshold
 mem_threshold_on_create = "auto"
 
+## The options for bloom filter index in Mito engine.
+[region_engine.mito.bloom_filter_index]
+
+## Whether to create the index on flush.
+## - `auto`: automatically (default)
+## - `disable`: never
+create_on_flush = "auto"
+
+## Whether to create the index on compaction.
+## - `auto`: automatically (default)
+## - `disable`: never
+create_on_compaction = "auto"
+
+## Whether to apply the index on query
+## - `auto`: automatically (default)
+## - `disable`: never
+apply_on_query = "auto"
+
+## Memory threshold for the index creation.
+## - `auto`: automatically determine the threshold based on the system memory size (default)
+## - `unlimited`: no memory limit
+## - `[size]` e.g. `64MB`: fixed memory threshold
+mem_threshold_on_create = "auto"
+
 [region_engine.mito.memtable]
 ## Memtable type.
 ## - `time_series`: time-series memtable

config/frontend.example.toml

@@ -2,6 +2,10 @@
 ## @toml2docs:none-default
 default_timezone = "UTC"
 
+## The maximum in-flight write bytes.
+## @toml2docs:none-default
+#+ max_in_flight_write_bytes = "500MB"
+
 ## The runtime options.
 #+ [runtime]
 ## The number of threads to execute the runtime for global read operations.

config/standalone.example.toml

@@ -18,6 +18,10 @@ max_concurrent_queries = 0
 ## Enable telemetry to collect anonymous usage data. Enabled by default.
 #+ enable_telemetry = true
 
+## The maximum in-flight write bytes.
+## @toml2docs:none-default
+#+ max_in_flight_write_bytes = "500MB"
+
 ## The runtime options.
 #+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
@@ -615,6 +619,30 @@ apply_on_query = "auto"
 ## - `[size]` e.g. `64MB`: fixed memory threshold
 mem_threshold_on_create = "auto"
 
+## The options for bloom filter in Mito engine.
+[region_engine.mito.bloom_filter_index]
+
+## Whether to create the bloom filter on flush.
+## - `auto`: automatically (default)
+## - `disable`: never
+create_on_flush = "auto"
+
+## Whether to create the bloom filter on compaction.
+## - `auto`: automatically (default)
+## - `disable`: never
+create_on_compaction = "auto"
+
+## Whether to apply the bloom filter on query
+## - `auto`: automatically (default)
+## - `disable`: never
+apply_on_query = "auto"
+
+## Memory threshold for bloom filter creation.
+## - `auto`: automatically determine the threshold based on the system memory size (default)
+## - `unlimited`: no memory limit
+## - `[size]` e.g. `64MB`: fixed memory threshold
+mem_threshold_on_create = "auto"
+
 [region_engine.mito.memtable]
 ## Memtable type.
 ## - `time_series`: time-series memtable

src/cmd/src/standalone.rs

@@ -22,6 +22,7 @@ use catalog::information_schema::InformationExtension;
 use catalog::kvbackend::KvBackendCatalogManager;
 use clap::Parser;
 use client::api::v1::meta::RegionRole;
+use common_base::readable_size::ReadableSize;
 use common_base::Plugins;
 use common_catalog::consts::{MIN_USER_FLOW_ID, MIN_USER_TABLE_ID};
 use common_config::{metadata_store_dir, Configurable, KvBackendConfig};
@@ -152,6 +153,7 @@ pub struct StandaloneOptions {
     pub tracing: TracingOptions,
     pub init_regions_in_background: bool,
     pub init_regions_parallelism: usize,
+    pub max_in_flight_write_bytes: Option<ReadableSize>,
 }
 
 impl Default for StandaloneOptions {
@@ -181,6 +183,7 @@ impl Default for StandaloneOptions {
             tracing: TracingOptions::default(),
             init_regions_in_background: false,
             init_regions_parallelism: 16,
+            max_in_flight_write_bytes: None,
         }
     }
 }
@@ -218,6 +221,7 @@ impl StandaloneOptions {
             user_provider: cloned_opts.user_provider,
             // Handle the export metrics task run by standalone to frontend for execution
             export_metrics: cloned_opts.export_metrics,
+            max_in_flight_write_bytes: cloned_opts.max_in_flight_write_bytes,
             ..Default::default()
         }
     }

src/common/error/Cargo.toml

@@ -8,6 +8,7 @@ license.workspace = true
 workspace = true
 
 [dependencies]
+http.workspace = true
 snafu.workspace = true
 strum.workspace = true
 tonic.workspace = true

src/common/error/src/lib.rs

@@ -18,9 +18,30 @@ pub mod ext;
 pub mod mock;
 pub mod status_code;
 
+use http::{HeaderMap, HeaderValue};
 pub use snafu;
 
 // HACK - these headers are here for shared in gRPC services. For common HTTP headers,
 // please define in `src/servers/src/http/header.rs`.
 pub const GREPTIME_DB_HEADER_ERROR_CODE: &str = "x-greptime-err-code";
 pub const GREPTIME_DB_HEADER_ERROR_MSG: &str = "x-greptime-err-msg";
+
+/// Create a http header map from error code and message.
+/// using `GREPTIME_DB_HEADER_ERROR_CODE` and `GREPTIME_DB_HEADER_ERROR_MSG` as keys.
+pub fn from_err_code_msg_to_header(code: u32, msg: &str) -> HeaderMap {
+    let mut header = HeaderMap::new();
+
+    let msg = HeaderValue::from_str(msg).unwrap_or_else(|_| {
+        HeaderValue::from_bytes(
+            &msg.as_bytes()
+                .iter()
+                .flat_map(|b| std::ascii::escape_default(*b))
+                .collect::<Vec<u8>>(),
+        )
+        .expect("Already escaped string should be valid ascii")
+    });
+
+    header.insert(GREPTIME_DB_HEADER_ERROR_CODE, code.into());
+    header.insert(GREPTIME_DB_HEADER_ERROR_MSG, msg);
+    header
+}

src/common/function/Cargo.toml

@@ -33,7 +33,7 @@ geo-types = { version = "0.7", optional = true }
 geohash = { version = "0.13", optional = true }
 h3o = { version = "0.6", optional = true }
 jsonb.workspace = true
-nalgebra = "0.33"
+nalgebra.workspace = true
 num = "0.4"
 num-traits = "0.2"
 once_cell.workspace = true

src/common/function/src/scalars/aggregate.rs

@@ -32,6 +32,7 @@ pub use scipy_stats_norm_cdf::ScipyStatsNormCdfAccumulatorCreator;
 pub use scipy_stats_norm_pdf::ScipyStatsNormPdfAccumulatorCreator;
 
 use crate::function_registry::FunctionRegistry;
+use crate::scalars::vector::sum::VectorSumCreator;
 
 /// A function creates `AggregateFunctionCreator`.
 /// "Aggregator" *is* AggregatorFunction. Since the later one is long, we named an short alias for it.
@@ -91,6 +92,7 @@ impl AggregateFunctions {
         register_aggr_func!("argmin", 1, ArgminAccumulatorCreator);
         register_aggr_func!("scipystatsnormcdf", 2, ScipyStatsNormCdfAccumulatorCreator);
         register_aggr_func!("scipystatsnormpdf", 2, ScipyStatsNormPdfAccumulatorCreator);
+        register_aggr_func!("vec_sum", 1, VectorSumCreator);
 
         #[cfg(feature = "geo")]
         register_aggr_func!(

src/common/function/src/scalars/vector.rs

@@ -14,9 +14,13 @@
 
 mod convert;
 mod distance;
-pub(crate) mod impl_conv;
+mod elem_sum;
+pub mod impl_conv;
 mod scalar_add;
 mod scalar_mul;
+mod sub;
+pub(crate) mod sum;
+mod vector_mul;
 
 use std::sync::Arc;
 
@@ -38,5 +42,10 @@ impl VectorFunction {
         // scalar calculation
         registry.register(Arc::new(scalar_add::ScalarAddFunction));
         registry.register(Arc::new(scalar_mul::ScalarMulFunction));
+
+        // vector calculation
+        registry.register(Arc::new(vector_mul::VectorMulFunction));
+        registry.register(Arc::new(sub::SubFunction));
+        registry.register(Arc::new(elem_sum::ElemSumFunction));
     }
 }

src/common/function/src/scalars/vector/elem_sum.rs

@@ -0,0 +1,129 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::borrow::Cow;
+use std::fmt::Display;
+
+use common_query::error::InvalidFuncArgsSnafu;
+use common_query::prelude::{Signature, TypeSignature, Volatility};
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{Float32VectorBuilder, MutableVector, VectorRef};
+use nalgebra::DVectorView;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const};
+
+const NAME: &str = "vec_elem_sum";
+
+#[derive(Debug, Clone, Default)]
+pub struct ElemSumFunction;
+
+impl Function for ElemSumFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(
+        &self,
+        _input_types: &[ConcreteDataType],
+    ) -> common_query::error::Result<ConcreteDataType> {
+        Ok(ConcreteDataType::float32_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::one_of(
+            vec![
+                TypeSignature::Exact(vec![ConcreteDataType::string_datatype()]),
+                TypeSignature::Exact(vec![ConcreteDataType::binary_datatype()]),
+            ],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(
+        &self,
+        _func_ctx: FunctionContext,
+        columns: &[VectorRef],
+    ) -> common_query::error::Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly one, have: {}",
+                    columns.len()
+                )
+            }
+        );
+        let arg0 = &columns[0];
+
+        let len = arg0.len();
+        let mut result = Float32VectorBuilder::with_capacity(len);
+        if len == 0 {
+            return Ok(result.to_vector());
+        }
+
+        let arg0_const = as_veclit_if_const(arg0)?;
+
+        for i in 0..len {
+            let arg0 = match arg0_const.as_ref() {
+                Some(arg0) => Some(Cow::Borrowed(arg0.as_ref())),
+                None => as_veclit(arg0.get_ref(i))?,
+            };
+            let Some(arg0) = arg0 else {
+                result.push_null();
+                continue;
+            };
+            result.push(Some(DVectorView::from_slice(&arg0, arg0.len()).sum()));
+        }
+
+        Ok(result.to_vector())
+    }
+}
+
+impl Display for ElemSumFunction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", NAME.to_ascii_uppercase())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use datatypes::vectors::StringVector;
+
+    use super::*;
+    use crate::function::FunctionContext;
+
+    #[test]
+    fn test_elem_sum() {
+        let func = ElemSumFunction;
+
+        let input0 = Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[4.0,5.0,6.0]".to_string()),
+            None,
+        ]));
+
+        let result = func.eval(FunctionContext::default(), &[input0]).unwrap();
+
+        let result = result.as_ref();
+        assert_eq!(result.len(), 3);
+        assert_eq!(result.get_ref(0).as_f32().unwrap(), Some(6.0));
+        assert_eq!(result.get_ref(1).as_f32().unwrap(), Some(15.0));
+        assert_eq!(result.get_ref(2).as_f32().unwrap(), None);
+    }
+}

src/common/function/src/scalars/vector/sub.rs

@@ -0,0 +1,223 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::borrow::Cow;
+use std::fmt::Display;
+
+use common_query::error::InvalidFuncArgsSnafu;
+use common_query::prelude::Signature;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{BinaryVectorBuilder, MutableVector, VectorRef};
+use nalgebra::DVectorView;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::helper;
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const, veclit_to_binlit};
+
+const NAME: &str = "vec_sub";
+
+/// Subtracts corresponding elements of two vectors, returns a vector.
+///
+/// # Example
+///
+/// ```sql
+/// SELECT vec_to_string(vec_sub("[1.0, 1.0]", "[1.0, 2.0]")) as result;
+///
+/// +---------------------------------------------------------------+
+/// | vec_to_string(vec_sub(Utf8("[1.0, 1.0]"),Utf8("[1.0, 2.0]"))) |
+/// +---------------------------------------------------------------+
+/// | [0,-1]                                                        |
+/// +---------------------------------------------------------------+
+///
+/// -- Negative scalar to simulate subtraction
+/// SELECT vec_to_string(vec_sub('[-1.0, -1.0]', '[1.0, 2.0]'));
+///
+/// +-----------------------------------------------------------------+
+/// | vec_to_string(vec_sub(Utf8("[-1.0, -1.0]"),Utf8("[1.0, 2.0]"))) |
+/// +-----------------------------------------------------------------+
+/// | [-2,-3]                                                         |
+/// +-----------------------------------------------------------------+
+///
+#[derive(Debug, Clone, Default)]
+pub struct SubFunction;
+
+impl Function for SubFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(
+        &self,
+        _input_types: &[ConcreteDataType],
+    ) -> common_query::error::Result<ConcreteDataType> {
+        Ok(ConcreteDataType::binary_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        helper::one_of_sigs2(
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::binary_datatype(),
+            ],
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::binary_datatype(),
+            ],
+        )
+    }
+
+    fn eval(
+        &self,
+        _func_ctx: FunctionContext,
+        columns: &[VectorRef],
+    ) -> common_query::error::Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly two, have: {}",
+                    columns.len()
+                )
+            }
+        );
+        let arg0 = &columns[0];
+        let arg1 = &columns[1];
+
+        ensure!(
+            arg0.len() == arg1.len(),
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The lengths of the vector are not aligned, args 0: {}, args 1: {}",
+                    arg0.len(),
+                    arg1.len(),
+                )
+            }
+        );
+
+        let len = arg0.len();
+        let mut result = BinaryVectorBuilder::with_capacity(len);
+        if len == 0 {
+            return Ok(result.to_vector());
+        }
+
+        let arg0_const = as_veclit_if_const(arg0)?;
+        let arg1_const = as_veclit_if_const(arg1)?;
+
+        for i in 0..len {
+            let arg0 = match arg0_const.as_ref() {
+                Some(arg0) => Some(Cow::Borrowed(arg0.as_ref())),
+                None => as_veclit(arg0.get_ref(i))?,
+            };
+            let arg1 = match arg1_const.as_ref() {
+                Some(arg1) => Some(Cow::Borrowed(arg1.as_ref())),
+                None => as_veclit(arg1.get_ref(i))?,
+            };
+            let (Some(arg0), Some(arg1)) = (arg0, arg1) else {
+                result.push_null();
+                continue;
+            };
+            let vec0 = DVectorView::from_slice(&arg0, arg0.len());
+            let vec1 = DVectorView::from_slice(&arg1, arg1.len());
+
+            let vec_res = vec0 - vec1;
+            let veclit = vec_res.as_slice();
+            let binlit = veclit_to_binlit(veclit);
+            result.push(Some(&binlit));
+        }
+
+        Ok(result.to_vector())
+    }
+}
+
+impl Display for SubFunction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", NAME.to_ascii_uppercase())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::error::Error;
+    use datatypes::vectors::StringVector;
+
+    use super::*;
+
+    #[test]
+    fn test_sub() {
+        let func = SubFunction;
+
+        let input0 = Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[4.0,5.0,6.0]".to_string()),
+            None,
+            Some("[2.0,3.0,3.0]".to_string()),
+        ]));
+        let input1 = Arc::new(StringVector::from(vec![
+            Some("[1.0,1.0,1.0]".to_string()),
+            Some("[6.0,5.0,4.0]".to_string()),
+            Some("[3.0,2.0,2.0]".to_string()),
+            None,
+        ]));
+
+        let result = func
+            .eval(FunctionContext::default(), &[input0, input1])
+            .unwrap();
+
+        let result = result.as_ref();
+        assert_eq!(result.len(), 4);
+        assert_eq!(
+            result.get_ref(0).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[0.0, 1.0, 2.0]).as_slice())
+        );
+        assert_eq!(
+            result.get_ref(1).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[-2.0, 0.0, 2.0]).as_slice())
+        );
+        assert!(result.get_ref(2).is_null());
+        assert!(result.get_ref(3).is_null());
+    }
+
+    #[test]
+    fn test_sub_error() {
+        let func = SubFunction;
+
+        let input0 = Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[4.0,5.0,6.0]".to_string()),
+            None,
+            Some("[2.0,3.0,3.0]".to_string()),
+        ]));
+        let input1 = Arc::new(StringVector::from(vec![
+            Some("[1.0,1.0,1.0]".to_string()),
+            Some("[6.0,5.0,4.0]".to_string()),
+            Some("[3.0,2.0,2.0]".to_string()),
+        ]));
+
+        let result = func.eval(FunctionContext::default(), &[input0, input1]);
+
+        match result {
+            Err(Error::InvalidFuncArgs { err_msg, .. }) => {
+                assert_eq!(
+                    err_msg,
+                    "The lengths of the vector are not aligned, args 0: 4, args 1: 3"
+                )
+            }
+            _ => unreachable!(),
+        }
+    }
+}

src/common/function/src/scalars/vector/sum.rs

@@ -0,0 +1,202 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_query::error::{CreateAccumulatorSnafu, Error, InvalidFuncArgsSnafu};
+use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
+use common_query::prelude::AccumulatorCreatorFunction;
+use datatypes::prelude::{ConcreteDataType, Value, *};
+use datatypes::vectors::VectorRef;
+use nalgebra::{Const, DVectorView, Dyn, OVector};
+use snafu::ensure;
+
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const, veclit_to_binlit};
+
+#[derive(Debug, Default)]
+pub struct VectorSum {
+    sum: Option<OVector<f32, Dyn>>,
+    has_null: bool,
+}
+
+#[as_aggr_func_creator]
+#[derive(Debug, Default, AggrFuncTypeStore)]
+pub struct VectorSumCreator {}
+
+impl AggregateFunctionCreator for VectorSumCreator {
+    fn creator(&self) -> AccumulatorCreatorFunction {
+        let creator: AccumulatorCreatorFunction = Arc::new(move |types: &[ConcreteDataType]| {
+            ensure!(
+                types.len() == 1,
+                InvalidFuncArgsSnafu {
+                    err_msg: format!(
+                        "The length of the args is not correct, expect exactly one, have: {}",
+                        types.len()
+                    )
+                }
+            );
+            let input_type = &types[0];
+            match input_type {
+                ConcreteDataType::String(_) | ConcreteDataType::Binary(_) => {
+                    Ok(Box::new(VectorSum::default()))
+                }
+                _ => {
+                    let err_msg = format!(
+                        "\"VEC_SUM\" aggregate function not support data type {:?}",
+                        input_type.logical_type_id(),
+                    );
+                    CreateAccumulatorSnafu { err_msg }.fail()?
+                }
+            }
+        });
+        creator
+    }
+
+    fn output_type(&self) -> common_query::error::Result<ConcreteDataType> {
+        Ok(ConcreteDataType::binary_datatype())
+    }
+
+    fn state_types(&self) -> common_query::error::Result<Vec<ConcreteDataType>> {
+        Ok(vec![self.output_type()?])
+    }
+}
+
+impl VectorSum {
+    fn inner(&mut self, len: usize) -> &mut OVector<f32, Dyn> {
+        self.sum
+            .get_or_insert_with(|| OVector::zeros_generic(Dyn(len), Const::<1>))
+    }
+
+    fn update(&mut self, values: &[VectorRef], is_update: bool) -> Result<(), Error> {
+        if values.is_empty() || self.has_null {
+            return Ok(());
+        };
+        let column = &values[0];
+        let len = column.len();
+
+        match as_veclit_if_const(column)? {
+            Some(column) => {
+                let vec_column = DVectorView::from_slice(&column, column.len()).scale(len as f32);
+                *self.inner(vec_column.len()) += vec_column;
+            }
+            None => {
+                for i in 0..len {
+                    let Some(arg0) = as_veclit(column.get_ref(i))? else {
+                        if is_update {
+                            self.has_null = true;
+                            self.sum = None;
+                        }
+                        return Ok(());
+                    };
+                    let vec_column = DVectorView::from_slice(&arg0, arg0.len());
+                    *self.inner(vec_column.len()) += vec_column;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+impl Accumulator for VectorSum {
+    fn state(&self) -> common_query::error::Result<Vec<Value>> {
+        self.evaluate().map(|v| vec![v])
+    }
+
+    fn update_batch(&mut self, values: &[VectorRef]) -> common_query::error::Result<()> {
+        self.update(values, true)
+    }
+
+    fn merge_batch(&mut self, states: &[VectorRef]) -> common_query::error::Result<()> {
+        self.update(states, false)
+    }
+
+    fn evaluate(&self) -> common_query::error::Result<Value> {
+        match &self.sum {
+            None => Ok(Value::Null),
+            Some(vector) => Ok(Value::from(veclit_to_binlit(vector.as_slice()))),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use datatypes::vectors::{ConstantVector, StringVector};
+
+    use super::*;
+
+    #[test]
+    fn test_update_batch() {
+        // test update empty batch, expect not updating anything
+        let mut vec_sum = VectorSum::default();
+        vec_sum.update_batch(&[]).unwrap();
+        assert!(vec_sum.sum.is_none());
+        assert!(!vec_sum.has_null);
+        assert_eq!(Value::Null, vec_sum.evaluate().unwrap());
+
+        // test update one not-null value
+        let mut vec_sum = VectorSum::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![Some(
+            "[1.0,2.0,3.0]".to_string(),
+        )]))];
+        vec_sum.update_batch(&v).unwrap();
+        assert_eq!(
+            Value::from(veclit_to_binlit(&[1.0, 2.0, 3.0])),
+            vec_sum.evaluate().unwrap()
+        );
+
+        // test update one null value
+        let mut vec_sum = VectorSum::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![Option::<String>::None]))];
+        vec_sum.update_batch(&v).unwrap();
+        assert_eq!(Value::Null, vec_sum.evaluate().unwrap());
+
+        // test update no null-value batch
+        let mut vec_sum = VectorSum::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[4.0,5.0,6.0]".to_string()),
+            Some("[7.0,8.0,9.0]".to_string()),
+        ]))];
+        vec_sum.update_batch(&v).unwrap();
+        assert_eq!(
+            Value::from(veclit_to_binlit(&[12.0, 15.0, 18.0])),
+            vec_sum.evaluate().unwrap()
+        );
+
+        // test update null-value batch
+        let mut vec_sum = VectorSum::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            None,
+            Some("[7.0,8.0,9.0]".to_string()),
+        ]))];
+        vec_sum.update_batch(&v).unwrap();
+        assert_eq!(Value::Null, vec_sum.evaluate().unwrap());
+
+        // test update with constant vector
+        let mut vec_sum = VectorSum::default();
+        let v: Vec<VectorRef> = vec![Arc::new(ConstantVector::new(
+            Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])),
+            4,
+        ))];
+        vec_sum.update_batch(&v).unwrap();
+        assert_eq!(
+            Value::from(veclit_to_binlit(&[4.0, 8.0, 12.0])),
+            vec_sum.evaluate().unwrap()
+        );
+    }
+}

src/common/function/src/scalars/vector/vector_mul.rs

@@ -0,0 +1,205 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::borrow::Cow;
+use std::fmt::Display;
+
+use common_query::error::{InvalidFuncArgsSnafu, Result};
+use common_query::prelude::Signature;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{BinaryVectorBuilder, MutableVector, VectorRef};
+use nalgebra::DVectorView;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::helper;
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const, veclit_to_binlit};
+
+const NAME: &str = "vec_mul";
+
+/// Multiplies corresponding elements of two vectors.
+///
+/// # Example
+///
+/// ```sql
+/// SELECT vec_to_string(vec_mul("[1, 2, 3]", "[1, 2, 3]")) as result;
+///
+/// +---------+
+/// | result  |
+/// +---------+
+/// | [1,4,9] |
+/// +---------+
+///
+/// ```
+#[derive(Debug, Clone, Default)]
+pub struct VectorMulFunction;
+
+impl Function for VectorMulFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::binary_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        helper::one_of_sigs2(
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::binary_datatype(),
+            ],
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::binary_datatype(),
+            ],
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly two, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let arg0 = &columns[0];
+        let arg1 = &columns[1];
+
+        let len = arg0.len();
+        let mut result = BinaryVectorBuilder::with_capacity(len);
+        if len == 0 {
+            return Ok(result.to_vector());
+        }
+
+        let arg0_const = as_veclit_if_const(arg0)?;
+        let arg1_const = as_veclit_if_const(arg1)?;
+
+        for i in 0..len {
+            let arg0 = match arg0_const.as_ref() {
+                Some(arg0) => Some(Cow::Borrowed(arg0.as_ref())),
+                None => as_veclit(arg0.get_ref(i))?,
+            };
+
+            let arg1 = match arg1_const.as_ref() {
+                Some(arg1) => Some(Cow::Borrowed(arg1.as_ref())),
+                None => as_veclit(arg1.get_ref(i))?,
+            };
+
+            if let (Some(arg0), Some(arg1)) = (arg0, arg1) {
+                ensure!(
+                    arg0.len() == arg1.len(),
+                    InvalidFuncArgsSnafu {
+                        err_msg: format!(
+                            "The length of the vectors must match for multiplying, have: {} vs {}",
+                            arg0.len(),
+                            arg1.len()
+                        ),
+                    }
+                );
+                let vec0 = DVectorView::from_slice(&arg0, arg0.len());
+                let vec1 = DVectorView::from_slice(&arg1, arg1.len());
+                let vec_res = vec1.component_mul(&vec0);
+
+                let veclit = vec_res.as_slice();
+                let binlit = veclit_to_binlit(veclit);
+                result.push(Some(&binlit));
+            } else {
+                result.push_null();
+            }
+        }
+
+        Ok(result.to_vector())
+    }
+}
+
+impl Display for VectorMulFunction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", NAME.to_ascii_uppercase())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::error;
+    use datatypes::vectors::StringVector;
+
+    use super::*;
+
+    #[test]
+    fn test_vector_mul() {
+        let func = VectorMulFunction;
+
+        let vec0 = vec![1.0, 2.0, 3.0];
+        let vec1 = vec![1.0, 1.0];
+        let (len0, len1) = (vec0.len(), vec1.len());
+        let input0 = Arc::new(StringVector::from(vec![Some(format!("{vec0:?}"))]));
+        let input1 = Arc::new(StringVector::from(vec![Some(format!("{vec1:?}"))]));
+
+        let err = func
+            .eval(FunctionContext::default(), &[input0, input1])
+            .unwrap_err();
+
+        match err {
+            error::Error::InvalidFuncArgs { err_msg, .. } => {
+                assert_eq!(
+                    err_msg,
+                    format!(
+                        "The length of the vectors must match for multiplying, have: {} vs {}",
+                        len0, len1
+                    )
+                )
+            }
+            _ => unreachable!(),
+        }
+
+        let input0 = Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[8.0,10.0,12.0]".to_string()),
+            Some("[7.0,8.0,9.0]".to_string()),
+            None,
+        ]));
+
+        let input1 = Arc::new(StringVector::from(vec![
+            Some("[1.0,1.0,1.0]".to_string()),
+            Some("[2.0,2.0,2.0]".to_string()),
+            None,
+            Some("[3.0,3.0,3.0]".to_string()),
+        ]));
+
+        let result = func
+            .eval(FunctionContext::default(), &[input0, input1])
+            .unwrap();
+
+        let result = result.as_ref();
+        assert_eq!(result.len(), 4);
+        assert_eq!(
+            result.get_ref(0).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[1.0, 2.0, 3.0]).as_slice())
+        );
+        assert_eq!(
+            result.get_ref(1).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[16.0, 20.0, 24.0]).as_slice())
+        );
+        assert!(result.get_ref(2).is_null());
+        assert!(result.get_ref(3).is_null());
+    }
+}

src/common/meta/src/kv_backend/postgres.rs

@@ -16,6 +16,7 @@ use std::any::Any;
 use std::borrow::Cow;
 use std::sync::Arc;
 
+use common_telemetry::error;
 use snafu::ResultExt;
 use tokio_postgres::types::ToSql;
 use tokio_postgres::{Client, NoTls};
@@ -97,7 +98,11 @@ impl PgStore {
         let (client, conn) = tokio_postgres::connect(url, NoTls)
             .await
             .context(ConnectPostgresSnafu)?;
-        tokio::spawn(async move { conn.await.context(ConnectPostgresSnafu) });
+        tokio::spawn(async move {
+            if let Err(e) = conn.await {
+                error!(e; "connection error");
+            }
+        });
         Self::with_pg_client(client).await
     }
 

src/datanode/src/store.rs

@@ -28,7 +28,7 @@ use common_telemetry::{info, warn};
 use object_store::layers::{LruCacheLayer, RetryInterceptor, RetryLayer};
 use object_store::services::Fs;
 use object_store::util::{join_dir, normalize_dir, with_instrument_layers};
-use object_store::{Access, Error, HttpClient, ObjectStore, ObjectStoreBuilder, OBJECT_CACHE_DIR};
+use object_store::{Access, Error, HttpClient, ObjectStore, ObjectStoreBuilder};
 use snafu::prelude::*;
 
 use crate::config::{HttpClientConfig, ObjectStoreConfig, DEFAULT_OBJECT_STORE_CACHE_SIZE};
@@ -147,12 +147,10 @@ async fn build_cache_layer(
     };
 
     // Enable object cache by default
-    // Set the cache_path to be `${data_home}/object_cache/read/{name}` by default
+    // Set the cache_path to be `${data_home}` by default
     // if it's not present
     if cache_path.is_none() {
-        let object_cache_path = join_dir(data_home, OBJECT_CACHE_DIR);
-        let read_cache_path = join_dir(&object_cache_path, "read");
-        let read_cache_path = join_dir(&read_cache_path, &name.to_lowercase());
+        let read_cache_path = data_home.to_string();
         tokio::fs::create_dir_all(Path::new(&read_cache_path))
             .await
             .context(CreateDirSnafu {

src/datatypes/src/schema.rs

@@ -29,7 +29,7 @@ use crate::error::{self, DuplicateColumnSnafu, Error, ProjectArrowSchemaSnafu, R
 use crate::prelude::ConcreteDataType;
 pub use crate::schema::column_schema::{
     ColumnSchema, FulltextAnalyzer, FulltextOptions, Metadata, SkippingIndexOptions,
-    COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE, COLUMN_FULLTEXT_OPT_KEY_ANALYZER,
+    SkippingIndexType, COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE, COLUMN_FULLTEXT_OPT_KEY_ANALYZER,
     COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY,
     COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE, COMMENT_KEY, FULLTEXT_KEY, INVERTED_INDEX_KEY,
     SKIPPING_INDEX_KEY, TIME_INDEX_KEY,

src/datatypes/src/schema/column_schema.rs

@@ -543,7 +543,7 @@ pub struct SkippingIndexOptions {
     pub granularity: u32,
     /// The type of the skip index.
     #[serde(default)]
-    pub index_type: SkipIndexType,
+    pub index_type: SkippingIndexType,
 }
 
 impl fmt::Display for SkippingIndexOptions {
@@ -556,15 +556,15 @@ impl fmt::Display for SkippingIndexOptions {
 
 /// Skip index types.
 #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
-pub enum SkipIndexType {
+pub enum SkippingIndexType {
     #[default]
     BloomFilter,
 }
 
-impl fmt::Display for SkipIndexType {
+impl fmt::Display for SkippingIndexType {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            SkipIndexType::BloomFilter => write!(f, "BLOOM"),
+            SkippingIndexType::BloomFilter => write!(f, "BLOOM"),
         }
     }
 }
@@ -587,7 +587,7 @@ impl TryFrom<HashMap<String, String>> for SkippingIndexOptions {
         // Parse index type with default value BloomFilter
         let index_type = match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE) {
             Some(typ) => match typ.to_ascii_uppercase().as_str() {
-                "BLOOM" => SkipIndexType::BloomFilter,
+                "BLOOM" => SkippingIndexType::BloomFilter,
                 _ => {
                     return error::InvalidSkippingIndexOptionSnafu {
                         msg: format!("Invalid index type: {typ}, expected: 'BLOOM'"),
@@ -595,7 +595,7 @@ impl TryFrom<HashMap<String, String>> for SkippingIndexOptions {
                     .fail();
                 }
             },
-            None => SkipIndexType::default(),
+            None => SkippingIndexType::default(),
         };
 
         Ok(SkippingIndexOptions {

src/flow/Cargo.toml

@@ -45,6 +45,7 @@ get-size2 = "0.1.2"
 greptime-proto.workspace = true
 # This fork of hydroflow is simply for keeping our dependency in our org, and pin the version
 # otherwise it is the same with upstream repo
+http.workspace = true
 hydroflow = { git = "https://github.com/GreptimeTeam/hydroflow.git", branch = "main" }
 itertools.workspace = true
 lazy_static.workspace = true

src/flow/src/adapter.rs

@@ -30,7 +30,7 @@ use common_telemetry::{debug, info, trace};
 use datatypes::schema::ColumnSchema;
 use datatypes::value::Value;
 use greptime_proto::v1;
-use itertools::Itertools;
+use itertools::{EitherOrBoth, Itertools};
 use meta_client::MetaClientOptions;
 use query::QueryEngine;
 use serde::{Deserialize, Serialize};
@@ -46,17 +46,19 @@ use tokio::sync::{broadcast, watch, Mutex, RwLock};
 
 pub(crate) use crate::adapter::node_context::FlownodeContext;
 use crate::adapter::table_source::TableSource;
-use crate::adapter::util::column_schemas_to_proto;
+use crate::adapter::util::{
+    relation_desc_to_column_schemas_with_fallback, table_info_value_to_relation_desc,
+};
 use crate::adapter::worker::{create_worker, Worker, WorkerHandle};
 use crate::compute::ErrCollector;
 use crate::df_optimizer::sql_to_flow_plan;
 use crate::error::{
-    EvalSnafu, ExternalSnafu, FlowAlreadyExistSnafu, InternalSnafu, TableNotFoundSnafu,
+    EvalSnafu, ExternalSnafu, FlowAlreadyExistSnafu, InternalSnafu, InvalidQuerySnafu,
     UnexpectedSnafu,
 };
-use crate::expr::{Batch, GlobalId};
-use crate::metrics::{METRIC_FLOW_INSERT_ELAPSED, METRIC_FLOW_RUN_INTERVAL_MS};
-use crate::repr::{self, DiffRow, Row, BATCH_SIZE};
+use crate::expr::Batch;
+use crate::metrics::{METRIC_FLOW_INSERT_ELAPSED, METRIC_FLOW_ROWS, METRIC_FLOW_RUN_INTERVAL_MS};
+use crate::repr::{self, DiffRow, RelationDesc, Row, BATCH_SIZE};
 
 mod flownode_impl;
 mod parse_expr;
@@ -245,16 +247,26 @@ impl FlowWorkerManager {
             let (catalog, schema) = (table_name[0].clone(), table_name[1].clone());
             let ctx = Arc::new(QueryContext::with(&catalog, &schema));
 
-            let (is_ts_placeholder, proto_schema) =
-                self.try_fetch_or_create_table(&table_name).await?;
+            let (is_ts_placeholder, proto_schema) = self
+                .try_fetch_existing_table(&table_name)
+                .await?
+                .context(UnexpectedSnafu {
+                    reason: format!("Table not found: {}", table_name.join(".")),
+                })?;
             let schema_len = proto_schema.len();
 
+            let total_rows = reqs.iter().map(|r| r.len()).sum::<usize>();
             trace!(
                 "Sending {} writeback requests to table {}, reqs total rows={}",
                 reqs.len(),
                 table_name.join("."),
                 reqs.iter().map(|r| r.len()).sum::<usize>()
             );
+
+            METRIC_FLOW_ROWS
+                .with_label_values(&["out"])
+                .inc_by(total_rows as u64);
+
             let now = self.tick_manager.tick();
             for req in reqs {
                 match req {
@@ -390,14 +402,12 @@ impl FlowWorkerManager {
         Ok(output)
     }
 
-    /// Fetch table info or create table from flow's schema if not exist
-    async fn try_fetch_or_create_table(
+    /// Fetch table schema and primary key from table info source, if table not exist return None
+    async fn fetch_table_pk_schema(
         &self,
         table_name: &TableName,
-    ) -> Result<(bool, Vec<api::v1::ColumnSchema>), Error> {
-        // TODO(discord9): instead of auto build table from request schema, actually build table
-        // before `create flow` to be able to assign pk and ts etc.
-        let (primary_keys, schema, is_ts_placeholder) = if let Some(table_id) = self
+    ) -> Result<Option<(Vec<String>, Option<usize>, Vec<ColumnSchema>)>, Error> {
+        if let Some(table_id) = self
             .table_info_source
             .get_table_id_from_name(table_name)
             .await?
@@ -414,97 +424,64 @@ impl FlowWorkerManager {
                 .map(|i| meta.schema.column_schemas[i].name.clone())
                 .collect_vec();
             let schema = meta.schema.column_schemas;
-            // check if the last column is the auto created timestamp column, hence the table is auto created from
-            // flow's plan type
-            let is_auto_create = {
-                let correct_name = schema
-                    .last()
-                    .map(|s| s.name == AUTO_CREATED_PLACEHOLDER_TS_COL)
-                    .unwrap_or(false);
-                let correct_time_index = meta.schema.timestamp_index == Some(schema.len() - 1);
-                correct_name && correct_time_index
-            };
-            (primary_keys, schema, is_auto_create)
+            let time_index = meta.schema.timestamp_index;
+            Ok(Some((primary_keys, time_index, schema)))
         } else {
-            // TODO(discord9): condiser remove buggy auto create by schema
+            Ok(None)
+        }
+    }
 
-            let node_ctx = self.node_context.read().await;
-            let gid: GlobalId = node_ctx
-                .table_repr
-                .get_by_name(table_name)
-                .map(|x| x.1)
-                .unwrap();
-            let schema = node_ctx
-                .schema
-                .get(&gid)
-                .with_context(|| TableNotFoundSnafu {
-                    name: format!("Table name = {:?}", table_name),
-                })?
-                .clone();
-            // TODO(discord9): use default key from schema
-            let primary_keys = schema
-                .typ()
-                .keys
-                .first()
-                .map(|v| {
-                    v.column_indices
-                        .iter()
-                        .map(|i| {
-                            schema
-                                .get_name(*i)
-                                .clone()
-                                .unwrap_or_else(|| format!("col_{i}"))
-                        })
-                        .collect_vec()
-                })
-                .unwrap_or_default();
-            let update_at = ColumnSchema::new(
-                UPDATE_AT_TS_COL,
+    /// return (primary keys, schema and if the table have a placeholder timestamp column)
+    /// schema of the table comes from flow's output plan
+    ///
+    /// adjust to add `update_at` column and ts placeholder if needed
+    async fn adjust_auto_created_table_schema(
+        &self,
+        schema: &RelationDesc,
+    ) -> Result<(Vec<String>, Vec<ColumnSchema>, bool), Error> {
+        // TODO(discord9): condiser remove buggy auto create by schema
+
+        // TODO(discord9): use default key from schema
+        let primary_keys = schema
+            .typ()
+            .keys
+            .first()
+            .map(|v| {
+                v.column_indices
+                    .iter()
+                    .map(|i| {
+                        schema
+                            .get_name(*i)
+                            .clone()
+                            .unwrap_or_else(|| format!("col_{i}"))
+                    })
+                    .collect_vec()
+            })
+            .unwrap_or_default();
+        let update_at = ColumnSchema::new(
+            UPDATE_AT_TS_COL,
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            true,
+        );
+
+        let original_schema = relation_desc_to_column_schemas_with_fallback(schema);
+
+        let mut with_auto_added_col = original_schema.clone();
+        with_auto_added_col.push(update_at);
+
+        // if no time index, add one as placeholder
+        let no_time_index = schema.typ().time_index.is_none();
+        if no_time_index {
+            let ts_col = ColumnSchema::new(
+                AUTO_CREATED_PLACEHOLDER_TS_COL,
                 ConcreteDataType::timestamp_millisecond_datatype(),
                 true,
-            );
+            )
+            .with_time_index(true);
+            with_auto_added_col.push(ts_col);
+        }
 
-            let original_schema = schema
-                .typ()
-                .column_types
-                .clone()
-                .into_iter()
-                .enumerate()
-                .map(|(idx, typ)| {
-                    let name = schema
-                        .names
-                        .get(idx)
-                        .cloned()
-                        .flatten()
-                        .unwrap_or(format!("col_{}", idx));
-                    let ret = ColumnSchema::new(name, typ.scalar_type, typ.nullable);
-                    if schema.typ().time_index == Some(idx) {
-                        ret.with_time_index(true)
-                    } else {
-                        ret
-                    }
-                })
-                .collect_vec();
-
-            let mut with_auto_added_col = original_schema.clone();
-            with_auto_added_col.push(update_at);
-
-            // if no time index, add one as placeholder
-            let no_time_index = schema.typ().time_index.is_none();
-            if no_time_index {
-                let ts_col = ColumnSchema::new(
-                    AUTO_CREATED_PLACEHOLDER_TS_COL,
-                    ConcreteDataType::timestamp_millisecond_datatype(),
-                    true,
-                )
-                .with_time_index(true);
-                with_auto_added_col.push(ts_col);
-            }
-
-            (primary_keys, with_auto_added_col, no_time_index)
-        };
-        let proto_schema = column_schemas_to_proto(schema, &primary_keys)?;
-        Ok((is_ts_placeholder, proto_schema))
+        Ok((primary_keys, with_auto_added_col, no_time_index))
     }
 }
 
@@ -807,7 +784,85 @@ impl FlowWorkerManager {
         let flow_plan = sql_to_flow_plan(&mut node_ctx, &self.query_engine, &sql).await?;
 
         debug!("Flow {:?}'s Plan is {:?}", flow_id, flow_plan);
-        node_ctx.assign_table_schema(&sink_table_name, flow_plan.schema.clone())?;
+
+        // check schema against actual table schema if exists
+        // if not exist create sink table immediately
+        if let Some((_, _, real_schema)) = self.fetch_table_pk_schema(&sink_table_name).await? {
+            let auto_schema = relation_desc_to_column_schemas_with_fallback(&flow_plan.schema);
+
+            // for column schema, only `data_type` need to be check for equality
+            // since one can omit flow's column name when write flow query
+            // print a user friendly error message about mismatch and how to correct them
+            for (idx, zipped) in auto_schema
+                .iter()
+                .zip_longest(real_schema.iter())
+                .enumerate()
+            {
+                match zipped {
+                    EitherOrBoth::Both(auto, real) => {
+                        if auto.data_type != real.data_type {
+                            InvalidQuerySnafu {
+                                    reason: format!(
+                                        "Column {}(name is '{}', flow inferred name is '{}')'s data type mismatch, expect {:?} got {:?}",
+                                        idx,
+                                        real.name,
+                                        auto.name,
+                                        real.data_type,
+                                        auto.data_type
+                                    ),
+                                }
+                                .fail()?;
+                        }
+                    }
+                    EitherOrBoth::Right(real) if real.data_type.is_timestamp() => {
+                        // if table is auto created, the last one or two column should be timestamp(update at and ts placeholder)
+                        continue;
+                    }
+                    _ => InvalidQuerySnafu {
+                        reason: format!(
+                            "schema length mismatched, expected {} found {}",
+                            real_schema.len(),
+                            auto_schema.len()
+                        ),
+                    }
+                    .fail()?,
+                }
+            }
+
+            let table_id = self
+                .table_info_source
+                .get_table_id_from_name(&sink_table_name)
+                .await?
+                .context(UnexpectedSnafu {
+                    reason: format!("Can't get table id for table name {:?}", sink_table_name),
+                })?;
+            let table_info_value = self
+                .table_info_source
+                .get_table_info_value(&table_id)
+                .await?
+                .context(UnexpectedSnafu {
+                    reason: format!("Can't get table info value for table id {:?}", table_id),
+                })?;
+            let real_schema = table_info_value_to_relation_desc(table_info_value)?;
+            node_ctx.assign_table_schema(&sink_table_name, real_schema.clone())?;
+        } else {
+            // assign inferred schema to sink table
+            // create sink table
+            node_ctx.assign_table_schema(&sink_table_name, flow_plan.schema.clone())?;
+            let did_create = self
+                .create_table_from_relation(
+                    &format!("flow-id={flow_id}"),
+                    &sink_table_name,
+                    &flow_plan.schema,
+                )
+                .await?;
+            if !did_create {
+                UnexpectedSnafu {
+                    reason: format!("Failed to create table {:?}", sink_table_name),
+                }
+                .fail()?;
+            }
+        }
 
         let _ = comment;
         let _ = flow_options;

src/flow/src/adapter/flownode_impl.rs

@@ -138,7 +138,7 @@ impl Flownode for FlowWorkerManager {
     }
 
     async fn handle_inserts(&self, request: InsertRequests) -> Result<FlowResponse> {
-        // using try_read makesure two things:
+        // using try_read to ensure two things:
         // 1. flush wouldn't happen until inserts before it is inserted
         // 2. inserts happening concurrently with flush wouldn't be block by flush
         let _flush_lock = self.flush_lock.try_read();

src/flow/src/adapter/node_context.rs

@@ -331,12 +331,14 @@ impl FlownodeContext {
         } else {
             let global_id = self.new_global_id();
 
+            // table id is Some meaning db must have created the table
             if let Some(table_id) = table_id {
                 let (known_table_name, schema) = srv_map.get_table_name_schema(&table_id).await?;
                 table_name = table_name.or(Some(known_table_name));
                 self.schema.insert(global_id, schema);
             } // if we don't have table id, it means database havn't assign one yet or we don't need it
 
+            // still update the mapping with new global id
             self.table_repr.insert(table_name, table_id, global_id);
             Ok(global_id)
         }
@@ -358,6 +360,7 @@ impl FlownodeContext {
             })?;
 
         self.schema.insert(gid, schema);
+
         Ok(())
     }
 

src/flow/src/adapter/table_source.rs

@@ -20,11 +20,12 @@ use common_meta::key::table_name::{TableNameKey, TableNameManager};
 use snafu::{OptionExt, ResultExt};
 use table::metadata::TableId;
 
+use crate::adapter::util::table_info_value_to_relation_desc;
 use crate::adapter::TableName;
 use crate::error::{
     Error, ExternalSnafu, TableNotFoundMetaSnafu, TableNotFoundSnafu, UnexpectedSnafu,
 };
-use crate::repr::{self, ColumnType, RelationDesc, RelationType};
+use crate::repr::RelationDesc;
 
 /// mapping of table name <-> table id should be query from tableinfo manager
 pub struct TableSource {
@@ -121,38 +122,7 @@ impl TableSource {
             table_name.table_name,
         ];
 
-        let raw_schema = table_info_value.table_info.meta.schema;
-        let (column_types, col_names): (Vec<_>, Vec<_>) = raw_schema
-            .column_schemas
-            .clone()
-            .into_iter()
-            .map(|col| {
-                (
-                    ColumnType {
-                        nullable: col.is_nullable(),
-                        scalar_type: col.data_type,
-                    },
-                    Some(col.name),
-                )
-            })
-            .unzip();
-
-        let key = table_info_value.table_info.meta.primary_key_indices;
-        let keys = vec![repr::Key::from(key)];
-
-        let time_index = raw_schema.timestamp_index;
-        Ok((
-            table_name,
-            RelationDesc {
-                typ: RelationType {
-                    column_types,
-                    keys,
-                    time_index,
-                    // by default table schema's column are all non-auto
-                    auto_columns: vec![],
-                },
-                names: col_names,
-            },
-        ))
+        let desc = table_info_value_to_relation_desc(table_info_value)?;
+        Ok((table_name, desc))
     }
 }

src/flow/src/adapter/util.rs

@@ -12,16 +12,153 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+
 use api::helper::ColumnDataTypeWrapper;
 use api::v1::column_def::options_from_column_schema;
-use api::v1::{ColumnDataType, ColumnDataTypeExtension, SemanticType};
+use api::v1::{ColumnDataType, ColumnDataTypeExtension, CreateTableExpr, SemanticType};
 use common_error::ext::BoxedError;
+use common_meta::key::table_info::TableInfoValue;
 use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::ColumnSchema;
 use itertools::Itertools;
-use snafu::ResultExt;
+use operator::expr_factory::CreateExprFactory;
+use session::context::QueryContextBuilder;
+use snafu::{OptionExt, ResultExt};
+use table::table_reference::TableReference;
 
-use crate::error::{Error, ExternalSnafu};
+use crate::adapter::{TableName, AUTO_CREATED_PLACEHOLDER_TS_COL};
+use crate::error::{Error, ExternalSnafu, UnexpectedSnafu};
+use crate::repr::{ColumnType, RelationDesc, RelationType};
+use crate::FlowWorkerManager;
+
+impl FlowWorkerManager {
+    /// Create table from given schema(will adjust to add auto column if needed), return true if table is created
+    pub(crate) async fn create_table_from_relation(
+        &self,
+        flow_name: &str,
+        table_name: &TableName,
+        relation_desc: &RelationDesc,
+    ) -> Result<bool, Error> {
+        if self.fetch_table_pk_schema(table_name).await?.is_some() {
+            return Ok(false);
+        }
+        let (pks, tys, _) = self.adjust_auto_created_table_schema(relation_desc).await?;
+
+        //create sink table using pks, column types and is_ts_auto
+
+        let proto_schema = column_schemas_to_proto(tys.clone(), &pks)?;
+
+        // create sink table
+        let create_expr = CreateExprFactory {}
+            .create_table_expr_by_column_schemas(
+                &TableReference {
+                    catalog: &table_name[0],
+                    schema: &table_name[1],
+                    table: &table_name[2],
+                },
+                &proto_schema,
+                "mito",
+                Some(&format!("Sink table for flow {}", flow_name)),
+            )
+            .map_err(BoxedError::new)
+            .context(ExternalSnafu)?;
+
+        self.submit_create_sink_table_ddl(create_expr).await?;
+        Ok(true)
+    }
+
+    /// Try fetch table with adjusted schema(added auto column if needed)
+    pub(crate) async fn try_fetch_existing_table(
+        &self,
+        table_name: &TableName,
+    ) -> Result<Option<(bool, Vec<api::v1::ColumnSchema>)>, Error> {
+        if let Some((primary_keys, time_index, schema)) =
+            self.fetch_table_pk_schema(table_name).await?
+        {
+            // check if the last column is the auto created timestamp column, hence the table is auto created from
+            // flow's plan type
+            let is_auto_create = {
+                let correct_name = schema
+                    .last()
+                    .map(|s| s.name == AUTO_CREATED_PLACEHOLDER_TS_COL)
+                    .unwrap_or(false);
+                let correct_time_index = time_index == Some(schema.len() - 1);
+                correct_name && correct_time_index
+            };
+            let proto_schema = column_schemas_to_proto(schema, &primary_keys)?;
+            Ok(Some((is_auto_create, proto_schema)))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// submit a create table ddl
+    pub(crate) async fn submit_create_sink_table_ddl(
+        &self,
+        mut create_table: CreateTableExpr,
+    ) -> Result<(), Error> {
+        let stmt_exec = {
+            self.frontend_invoker
+                .read()
+                .await
+                .as_ref()
+                .map(|f| f.statement_executor())
+        }
+        .context(UnexpectedSnafu {
+            reason: "Failed to get statement executor",
+        })?;
+        let ctx = Arc::new(
+            QueryContextBuilder::default()
+                .current_catalog(create_table.catalog_name.clone())
+                .current_schema(create_table.schema_name.clone())
+                .build(),
+        );
+        stmt_exec
+            .create_table_inner(&mut create_table, None, ctx)
+            .await
+            .map_err(BoxedError::new)
+            .context(ExternalSnafu)?;
+
+        Ok(())
+    }
+}
+
+pub fn table_info_value_to_relation_desc(
+    table_info_value: TableInfoValue,
+) -> Result<RelationDesc, Error> {
+    let raw_schema = table_info_value.table_info.meta.schema;
+    let (column_types, col_names): (Vec<_>, Vec<_>) = raw_schema
+        .column_schemas
+        .clone()
+        .into_iter()
+        .map(|col| {
+            (
+                ColumnType {
+                    nullable: col.is_nullable(),
+                    scalar_type: col.data_type,
+                },
+                Some(col.name),
+            )
+        })
+        .unzip();
+
+    let key = table_info_value.table_info.meta.primary_key_indices;
+    let keys = vec![crate::repr::Key::from(key)];
+
+    let time_index = raw_schema.timestamp_index;
+
+    Ok(RelationDesc {
+        typ: RelationType {
+            column_types,
+            keys,
+            time_index,
+            // by default table schema's column are all non-auto
+            auto_columns: vec![],
+        },
+        names: col_names,
+    })
+}
 
 pub fn from_proto_to_data_type(
     column_schema: &api::v1::ColumnSchema,
@@ -75,3 +212,29 @@ pub fn column_schemas_to_proto(
         .collect();
     Ok(ret)
 }
+
+/// Convert `RelationDesc` to `ColumnSchema` list,
+/// if the column name is not present, use `col_{idx}` as the column name
+pub fn relation_desc_to_column_schemas_with_fallback(schema: &RelationDesc) -> Vec<ColumnSchema> {
+    schema
+        .typ()
+        .column_types
+        .clone()
+        .into_iter()
+        .enumerate()
+        .map(|(idx, typ)| {
+            let name = schema
+                .names
+                .get(idx)
+                .cloned()
+                .flatten()
+                .unwrap_or(format!("col_{}", idx));
+            let ret = ColumnSchema::new(name, typ.scalar_type, typ.nullable);
+            if schema.typ().time_index == Some(idx) {
+                ret.with_time_index(true)
+            } else {
+                ret
+            }
+        })
+        .collect_vec()
+}

src/flow/src/compute/render/reduce.rs

@@ -16,6 +16,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::ops::Range;
 use std::sync::Arc;
 
+use arrow::array::new_null_array;
 use common_telemetry::trace;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::prelude::DataType;
@@ -398,20 +399,54 @@ fn reduce_batch_subgraph(
                 }
             }
 
-            // TODO: here reduce numbers of eq to minimal by keeping slicing key/val batch
+            let key_data_types = output_type
+                .column_types
+                .iter()
+                .map(|t| t.scalar_type.clone())
+                .collect_vec();
+
+            // TODO(discord9): here reduce numbers of eq to minimal by keeping slicing key/val batch
             for key_row in distinct_keys {
                 let key_scalar_value = {
                     let mut key_scalar_value = Vec::with_capacity(key_row.len());
-                    for key in key_row.iter() {
+                    for (key_idx, key) in key_row.iter().enumerate() {
                         let v =
                             key.try_to_scalar_value(&key.data_type())
                                 .context(DataTypeSnafu {
                                     msg: "can't convert key values to datafusion value",
                                 })?;
-                        let arrow_value =
+
+                        let key_data_type = key_data_types.get(key_idx).context(InternalSnafu {
+                            reason: format!(
+                                "Key index out of bound, expected at most {} but got {}",
+                                output_type.column_types.len(),
+                                key_idx
+                            ),
+                        })?;
+
+                        // if incoming value's datatype is null, it need to be handled specially, see below
+                        if key_data_type.as_arrow_type() != v.data_type()
+                            && !v.data_type().is_null()
+                        {
+                            crate::expr::error::InternalSnafu {
+                                reason: format!(
+                                    "Key data type mismatch, expected {:?} but got {:?}",
+                                    key_data_type.as_arrow_type(),
+                                    v.data_type()
+                                ),
+                            }
+                            .fail()?
+                        }
+
+                        // handle single null key
+                        let arrow_value = if v.data_type().is_null() {
+                            let ret = new_null_array(&arrow::datatypes::DataType::Null, 1);
+                            arrow::array::Scalar::new(ret)
+                        } else {
                             v.to_scalar().context(crate::expr::error::DatafusionSnafu {
                                 context: "can't convert key values to arrow value",
-                            })?;
+                            })?
+                        };
                         key_scalar_value.push(arrow_value);
                     }
                     key_scalar_value
@@ -423,7 +458,19 @@ fn reduce_batch_subgraph(
                     .zip(key_batch.batch().iter())
                     .map(|(key, col)| {
                         // TODO(discord9): this takes half of the cpu! And this is redundant amount of `eq`!
-                        arrow::compute::kernels::cmp::eq(&key, &col.to_arrow_array().as_ref() as _)
+
+                        // note that if lhs is a null, we still need to get all rows that are null! But can't use `eq` since
+                        // it will return null if input have null, so we need to use `is_null` instead
+                        if arrow::array::Datum::get(&key).0.data_type().is_null() {
+                            arrow::compute::kernels::boolean::is_null(
+                                col.to_arrow_array().as_ref() as _
+                            )
+                        } else {
+                            arrow::compute::kernels::cmp::eq(
+                                &key,
+                                &col.to_arrow_array().as_ref() as _,
+                            )
+                        }
                     })
                     .try_collect::<_, Vec<_>, _>()
                     .context(ArrowSnafu {

src/flow/src/compute/types.rs

@@ -17,6 +17,7 @@ use std::collections::{BTreeMap, VecDeque};
 use std::rc::Rc;
 use std::sync::Arc;
 
+use common_error::ext::ErrorExt;
 use hydroflow::scheduled::graph::Hydroflow;
 use hydroflow::scheduled::handoff::TeeingHandoff;
 use hydroflow::scheduled::port::RecvPort;
@@ -25,6 +26,7 @@ use itertools::Itertools;
 use tokio::sync::Mutex;
 
 use crate::expr::{Batch, EvalError, ScalarExpr};
+use crate::metrics::METRIC_FLOW_ERRORS;
 use crate::repr::DiffRow;
 use crate::utils::ArrangeHandler;
 
@@ -185,6 +187,9 @@ impl ErrCollector {
     }
 
     pub fn push_err(&self, err: EvalError) {
+        METRIC_FLOW_ERRORS
+            .with_label_values(&[err.status_code().as_ref()])
+            .inc();
         self.inner.blocking_lock().push_back(err)
     }
 

src/flow/src/error.rs

@@ -16,12 +16,13 @@
 
 use std::any::Any;
 
-use common_error::define_into_tonic_status;
 use common_error::ext::BoxedError;
+use common_error::{define_into_tonic_status, from_err_code_msg_to_header};
 use common_macro::stack_trace_debug;
 use common_telemetry::common_error::ext::ErrorExt;
 use common_telemetry::common_error::status_code::StatusCode;
 use snafu::{Location, Snafu};
+use tonic::metadata::MetadataMap;
 
 use crate::adapter::FlowId;
 use crate::expr::EvalError;
@@ -186,6 +187,20 @@ pub enum Error {
     },
 }
 
+/// the outer message is the full error stack, and inner message in header is the last error message that can be show directly to user
+pub fn to_status_with_last_err(err: impl ErrorExt) -> tonic::Status {
+    let msg = err.to_string();
+    let last_err_msg = common_error::ext::StackError::last(&err).to_string();
+    let code = err.status_code() as u32;
+    let header = from_err_code_msg_to_header(code, &last_err_msg);
+
+    tonic::Status::with_metadata(
+        tonic::Code::InvalidArgument,
+        msg,
+        MetadataMap::from_headers(header),
+    )
+}
+
 /// Result type for flow module
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -200,9 +215,8 @@ impl ErrorExt for Error {
             | Self::TableNotFoundMeta { .. }
             | Self::FlowNotFound { .. }
             | Self::ListFlows { .. } => StatusCode::TableNotFound,
-            Self::InvalidQuery { .. } | Self::Plan { .. } | Self::Datatypes { .. } => {
-                StatusCode::PlanQuery
-            }
+            Self::Plan { .. } | Self::Datatypes { .. } => StatusCode::PlanQuery,
+            Self::InvalidQuery { .. } => StatusCode::EngineExecuteQuery,
             Self::Unexpected { .. } => StatusCode::Unexpected,
             Self::NotImplemented { .. } | Self::UnsupportedTemporalFilter { .. } => {
                 StatusCode::Unsupported

src/flow/src/expr/error.rs

@@ -14,8 +14,11 @@
 
 //! Error handling for expression evaluation.
 
+use std::any::Any;
+
 use arrow_schema::ArrowError;
-use common_error::ext::BoxedError;
+use common_error::ext::{BoxedError, ErrorExt};
+use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
 use datafusion_common::DataFusionError;
 use datatypes::data_type::ConcreteDataType;
@@ -126,3 +129,29 @@ pub enum EvalError {
         source: BoxedError,
     },
 }
+
+impl ErrorExt for EvalError {
+    fn status_code(&self) -> StatusCode {
+        use EvalError::*;
+        match self {
+            DivisionByZero { .. }
+            | TypeMismatch { .. }
+            | TryFromValue { .. }
+            | DataAlreadyExpired { .. }
+            | InvalidArgument { .. }
+            | Overflow { .. } => StatusCode::InvalidArguments,
+
+            CastValue { source, .. } | DataType { source, .. } => source.status_code(),
+
+            Internal { .. }
+            | Optimize { .. }
+            | Arrow { .. }
+            | Datafusion { .. }
+            | External { .. } => StatusCode::Internal,
+        }
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/flow/src/metrics.rs

@@ -30,4 +30,22 @@ lazy_static! {
     .unwrap();
     pub static ref METRIC_FLOW_RUN_INTERVAL_MS: IntGauge =
         register_int_gauge!("greptime_flow_run_interval_ms", "flow run interval in ms").unwrap();
+    pub static ref METRIC_FLOW_ROWS: IntCounterVec = register_int_counter_vec!(
+        "greptime_flow_processed_rows",
+        "Count of rows flowing through the system",
+        &["direction"]
+    )
+    .unwrap();
+    pub static ref METRIC_FLOW_PROCESSING_TIME: HistogramVec = register_histogram_vec!(
+        "greptime_flow_processing_time",
+        "Time spent processing requests",
+        &["type"]
+    )
+    .unwrap();
+    pub static ref METRIC_FLOW_ERRORS: IntCounterVec = register_int_counter_vec!(
+        "greptime_flow_errors",
+        "Count of errors in flow processing",
+        &["code"]
+    )
+    .unwrap();
 }

src/flow/src/repr/relation.rs

@@ -212,6 +212,8 @@ impl RelationType {
         for key in &mut self.keys {
             key.remove_col(time_index.unwrap_or(usize::MAX));
         }
+        // remove empty keys
+        self.keys.retain(|key| !key.is_empty());
         self
     }
 

src/flow/src/server.rs

@@ -50,10 +50,11 @@ use tonic::{Request, Response, Status};
 
 use crate::adapter::{CreateFlowArgs, FlowWorkerManagerRef};
 use crate::error::{
-    CacheRequiredSnafu, ExternalSnafu, FlowNotFoundSnafu, ListFlowsSnafu, ParseAddrSnafu,
-    ShutdownServerSnafu, StartServerSnafu, UnexpectedSnafu,
+    to_status_with_last_err, CacheRequiredSnafu, ExternalSnafu, FlowNotFoundSnafu, ListFlowsSnafu,
+    ParseAddrSnafu, ShutdownServerSnafu, StartServerSnafu, UnexpectedSnafu,
 };
 use crate::heartbeat::HeartbeatTask;
+use crate::metrics::{METRIC_FLOW_PROCESSING_TIME, METRIC_FLOW_ROWS};
 use crate::transform::register_function_to_query_engine;
 use crate::utils::{SizeReportSender, StateReportHandler};
 use crate::{Error, FlowWorkerManager, FlownodeOptions};
@@ -77,41 +78,52 @@ impl flow_server::Flow for FlowService {
         &self,
         request: Request<FlowRequest>,
     ) -> Result<Response<FlowResponse>, Status> {
+        let _timer = METRIC_FLOW_PROCESSING_TIME
+            .with_label_values(&["ddl"])
+            .start_timer();
+
         let request = request.into_inner();
         self.manager
             .handle(request)
             .await
             .map(Response::new)
-            .map_err(|e| {
-                let msg = format!("failed to handle request: {:?}", e);
-                Status::internal(msg)
-            })
+            .map_err(to_status_with_last_err)
     }
 
     async fn handle_mirror_request(
         &self,
         request: Request<InsertRequests>,
     ) -> Result<Response<FlowResponse>, Status> {
+        let _timer = METRIC_FLOW_PROCESSING_TIME
+            .with_label_values(&["insert"])
+            .start_timer();
+
         let request = request.into_inner();
         // TODO(discord9): fix protobuf import order shenanigans to remove this duplicated define
+        let mut row_count = 0;
         let request = api::v1::region::InsertRequests {
             requests: request
                 .requests
                 .into_iter()
-                .map(|insert| api::v1::region::InsertRequest {
-                    region_id: insert.region_id,
-                    rows: insert.rows,
+                .map(|insert| {
+                    insert.rows.as_ref().inspect(|x| row_count += x.rows.len());
+                    api::v1::region::InsertRequest {
+                        region_id: insert.region_id,
+                        rows: insert.rows,
+                    }
                 })
                 .collect_vec(),
         };
+
+        METRIC_FLOW_ROWS
+            .with_label_values(&["in"])
+            .inc_by(row_count as u64);
+
         self.manager
             .handle_inserts(request)
             .await
             .map(Response::new)
-            .map_err(|e| {
-                let msg = format!("failed to handle request: {:?}", e);
-                Status::internal(msg)
-            })
+            .map_err(to_status_with_last_err)
     }
 }
 
@@ -500,6 +512,10 @@ impl FrontendInvoker {
         requests: RowInsertRequests,
         ctx: QueryContextRef,
     ) -> common_frontend::error::Result<Output> {
+        let _timer = METRIC_FLOW_PROCESSING_TIME
+            .with_label_values(&["output_insert"])
+            .start_timer();
+
         self.inserter
             .handle_row_inserts(requests, ctx, &self.statement_executor)
             .await
@@ -512,10 +528,18 @@ impl FrontendInvoker {
         requests: RowDeleteRequests,
         ctx: QueryContextRef,
     ) -> common_frontend::error::Result<Output> {
+        let _timer = METRIC_FLOW_PROCESSING_TIME
+            .with_label_values(&["output_delete"])
+            .start_timer();
+
         self.deleter
             .handle_row_deletes(requests, ctx)
             .await
             .map_err(BoxedError::new)
             .context(common_frontend::error::ExternalSnafu)
     }
+
+    pub fn statement_executor(&self) -> Arc<StatementExecutor> {
+        self.statement_executor.clone()
+    }
 }

src/flow/src/transform/aggr.rs

@@ -216,6 +216,7 @@ impl KeyValPlan {
 
 /// find out the column that should be time index in group exprs(which is all columns that should be keys)
 /// TODO(discord9): better ways to assign time index
+/// for now, it will found the first column that is timestamp or has a tumble window floor function
 fn find_time_index_in_group_exprs(group_exprs: &[TypedExpr]) -> Option<usize> {
     group_exprs.iter().position(|expr| {
         matches!(
@@ -224,7 +225,7 @@ fn find_time_index_in_group_exprs(group_exprs: &[TypedExpr]) -> Option<usize> {
                 func: UnaryFunc::TumbleWindowFloor { .. },
                 expr: _
             }
-        )
+        ) || expr.typ.scalar_type.is_timestamp()
     })
 }
 
@@ -1482,7 +1483,7 @@ mod test {
                 ColumnType::new(CDT::float64_datatype(), true),
                 ColumnType::new(CDT::timestamp_millisecond_datatype(), true),
             ])
-            .with_key(vec![1])
+            .with_time_index(Some(1))
             .into_named(vec![
                 Some(
                     "MAX(numbers_with_ts.number) - MIN(numbers_with_ts.number) / Float64(30)"
@@ -1571,7 +1572,7 @@ mod test {
                             ColumnType::new(ConcreteDataType::uint32_datatype(), true), // max
                             ColumnType::new(ConcreteDataType::uint32_datatype(), true), // min
                         ])
-                        .with_key(vec![0])
+                        .with_time_index(Some(0))
                         .into_unnamed(),
                     ),
                 ),

src/frontend/src/error.rs

@@ -321,6 +321,12 @@ pub enum Error {
         location: Location,
         source: BoxedError,
     },
+
+    #[snafu(display("In-flight write bytes exceeded the maximum limit"))]
+    InFlightWriteBytesExceeded {
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -392,6 +398,8 @@ impl ErrorExt for Error {
             Error::StartScriptManager { source, .. } => source.status_code(),
 
             Error::TableOperation { source, .. } => source.status_code(),
+
+            Error::InFlightWriteBytesExceeded { .. } => StatusCode::RateLimited,
         }
     }
 

src/frontend/src/frontend.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use common_base::readable_size::ReadableSize;
 use common_config::config::Configurable;
 use common_options::datanode::DatanodeClientOptions;
 use common_telemetry::logging::{LoggingOptions, TracingOptions};
@@ -46,6 +47,7 @@ pub struct FrontendOptions {
     pub user_provider: Option<String>,
     pub export_metrics: ExportMetricsOption,
     pub tracing: TracingOptions,
+    pub max_in_flight_write_bytes: Option<ReadableSize>,
 }
 
 impl Default for FrontendOptions {
@@ -68,6 +70,7 @@ impl Default for FrontendOptions {
             user_provider: None,
             export_metrics: ExportMetricsOption::default(),
             tracing: TracingOptions::default(),
+            max_in_flight_write_bytes: None,
         }
     }
 }

src/frontend/src/instance.rs

@@ -87,6 +87,7 @@ use crate::error::{
 };
 use crate::frontend::FrontendOptions;
 use crate::heartbeat::HeartbeatTask;
+use crate::limiter::LimiterRef;
 use crate::script::ScriptExecutor;
 
 #[async_trait]
@@ -126,6 +127,7 @@ pub struct Instance {
     export_metrics_task: Option<ExportMetricsTask>,
     table_metadata_manager: TableMetadataManagerRef,
     stats: StatementStatistics,
+    limiter: Option<LimiterRef>,
 }
 
 impl Instance {

src/frontend/src/instance/builder.rs

@@ -43,6 +43,7 @@ use crate::frontend::FrontendOptions;
 use crate::heartbeat::HeartbeatTask;
 use crate::instance::region_query::FrontendRegionQueryHandler;
 use crate::instance::Instance;
+use crate::limiter::Limiter;
 use crate::script::ScriptExecutor;
 
 /// The frontend [`Instance`] builder.
@@ -196,6 +197,14 @@ impl FrontendBuilder {
 
         plugins.insert::<StatementExecutorRef>(statement_executor.clone());
 
+        // Create the limiter if the max_in_flight_write_bytes is set.
+        let limiter = self
+            .options
+            .max_in_flight_write_bytes
+            .map(|max_in_flight_write_bytes| {
+                Arc::new(Limiter::new(max_in_flight_write_bytes.as_bytes()))
+            });
+
         Ok(Instance {
             options: self.options,
             catalog_manager: self.catalog_manager,
@@ -211,6 +220,7 @@ impl FrontendBuilder {
             export_metrics_task: None,
             table_metadata_manager: Arc::new(TableMetadataManager::new(kv_backend)),
             stats: self.stats,
+            limiter,
         })
     }
 }

src/frontend/src/instance/grpc.rs

@@ -29,8 +29,8 @@ use snafu::{ensure, OptionExt, ResultExt};
 use table::table_name::TableName;
 
 use crate::error::{
-    Error, IncompleteGrpcRequestSnafu, NotSupportedSnafu, PermissionSnafu, Result,
-    TableOperationSnafu,
+    Error, InFlightWriteBytesExceededSnafu, IncompleteGrpcRequestSnafu, NotSupportedSnafu,
+    PermissionSnafu, Result, TableOperationSnafu,
 };
 use crate::instance::{attach_timer, Instance};
 use crate::metrics::{GRPC_HANDLE_PROMQL_ELAPSED, GRPC_HANDLE_SQL_ELAPSED};
@@ -50,6 +50,16 @@ impl GrpcQueryHandler for Instance {
             .check_permission(ctx.current_user(), PermissionReq::GrpcRequest(&request))
             .context(PermissionSnafu)?;
 
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_request(&request);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
         let output = match request {
             Request::Inserts(requests) => self.handle_inserts(requests, ctx.clone()).await?,
             Request::RowInserts(requests) => self.handle_row_inserts(requests, ctx.clone()).await?,

src/frontend/src/instance/influxdb.rs

@@ -16,7 +16,7 @@ use async_trait::async_trait;
 use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq};
 use client::Output;
 use common_error::ext::BoxedError;
-use servers::error::{AuthSnafu, Error};
+use servers::error::{AuthSnafu, Error, InFlightWriteBytesExceededSnafu};
 use servers::influxdb::InfluxdbRequest;
 use servers::interceptor::{LineProtocolInterceptor, LineProtocolInterceptorRef};
 use servers::query_handler::InfluxdbLineProtocolHandler;
@@ -46,6 +46,16 @@ impl InfluxdbLineProtocolHandler for Instance {
             .post_lines_conversion(requests, ctx.clone())
             .await?;
 
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&requests);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
         self.handle_influx_row_inserts(requests, ctx)
             .await
             .map_err(BoxedError::new)

src/frontend/src/instance/log_handler.rs

@@ -22,7 +22,8 @@ use common_error::ext::BoxedError;
 use pipeline::pipeline_operator::PipelineOperator;
 use pipeline::{GreptimeTransformer, Pipeline, PipelineInfo, PipelineVersion};
 use servers::error::{
-    AuthSnafu, Error as ServerError, ExecuteGrpcRequestSnafu, PipelineSnafu, Result as ServerResult,
+    AuthSnafu, Error as ServerError, ExecuteGrpcRequestSnafu, InFlightWriteBytesExceededSnafu,
+    PipelineSnafu, Result as ServerResult,
 };
 use servers::interceptor::{LogIngestInterceptor, LogIngestInterceptorRef};
 use servers::query_handler::PipelineHandler;
@@ -110,6 +111,16 @@ impl Instance {
         log: RowInsertRequests,
         ctx: QueryContextRef,
     ) -> ServerResult<Output> {
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&log);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
         self.inserter
             .handle_log_inserts(log, ctx, self.statement_executor.as_ref())
             .await

src/frontend/src/instance/opentsdb.rs

@@ -17,7 +17,7 @@ use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq};
 use common_error::ext::BoxedError;
 use common_telemetry::tracing;
 use servers::error as server_error;
-use servers::error::AuthSnafu;
+use servers::error::{AuthSnafu, InFlightWriteBytesExceededSnafu};
 use servers::opentsdb::codec::DataPoint;
 use servers::opentsdb::data_point_to_grpc_row_insert_requests;
 use servers::query_handler::OpentsdbProtocolHandler;
@@ -41,6 +41,17 @@ impl OpentsdbProtocolHandler for Instance {
             .context(AuthSnafu)?;
 
         let (requests, _) = data_point_to_grpc_row_insert_requests(data_points)?;
+
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&requests);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
         let output = self
             .handle_row_inserts(requests, ctx)
             .await

src/frontend/src/instance/otlp.rs

@@ -21,7 +21,7 @@ use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
 use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
 use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
 use pipeline::PipelineWay;
-use servers::error::{self, AuthSnafu, Result as ServerResult};
+use servers::error::{self, AuthSnafu, InFlightWriteBytesExceededSnafu, Result as ServerResult};
 use servers::interceptor::{OpenTelemetryProtocolInterceptor, OpenTelemetryProtocolInterceptorRef};
 use servers::otlp;
 use servers::query_handler::OpenTelemetryProtocolHandler;
@@ -53,6 +53,16 @@ impl OpenTelemetryProtocolHandler for Instance {
         let (requests, rows) = otlp::metrics::to_grpc_insert_requests(request)?;
         OTLP_METRICS_ROWS.inc_by(rows as u64);
 
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&requests);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
         self.handle_row_inserts(requests, ctx)
             .await
             .map_err(BoxedError::new)
@@ -83,6 +93,16 @@ impl OpenTelemetryProtocolHandler for Instance {
 
         OTLP_TRACES_ROWS.inc_by(rows as u64);
 
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&requests);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
         self.handle_log_inserts(requests, ctx)
             .await
             .map_err(BoxedError::new)
@@ -109,6 +129,17 @@ impl OpenTelemetryProtocolHandler for Instance {
         interceptor_ref.pre_execute(ctx.clone())?;
 
         let (requests, rows) = otlp::logs::to_grpc_insert_requests(request, pipeline, table_name)?;
+
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&requests);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
         self.handle_log_inserts(requests, ctx)
             .await
             .inspect(|_| OTLP_LOGS_ROWS.inc_by(rows as u64))

src/frontend/src/instance/prom_store.rs

@@ -30,7 +30,7 @@ use common_telemetry::{debug, tracing};
 use operator::insert::InserterRef;
 use operator::statement::StatementExecutor;
 use prost::Message;
-use servers::error::{self, AuthSnafu, Result as ServerResult};
+use servers::error::{self, AuthSnafu, InFlightWriteBytesExceededSnafu, Result as ServerResult};
 use servers::http::header::{collect_plan_metrics, CONTENT_ENCODING_SNAPPY, CONTENT_TYPE_PROTOBUF};
 use servers::http::prom_store::PHYSICAL_TABLE_PARAM;
 use servers::interceptor::{PromStoreProtocolInterceptor, PromStoreProtocolInterceptorRef};
@@ -175,6 +175,16 @@ impl PromStoreProtocolHandler for Instance {
             .get::<PromStoreProtocolInterceptorRef<servers::error::Error>>();
         interceptor_ref.pre_write(&request, ctx.clone())?;
 
+        let _guard = if let Some(limiter) = &self.limiter {
+            let result = limiter.limit_row_inserts(&request);
+            if result.is_none() {
+                return InFlightWriteBytesExceededSnafu.fail();
+            }
+            result
+        } else {
+            None
+        };
+
         let output = if with_metric_engine {
             let physical_table = ctx
                 .extension(PHYSICAL_TABLE_PARAM)

src/frontend/src/lib.rs

@@ -18,6 +18,7 @@ pub mod error;
 pub mod frontend;
 pub mod heartbeat;
 pub mod instance;
+pub(crate) mod limiter;
 pub(crate) mod metrics;
 mod script;
 pub mod server;

src/frontend/src/limiter.rs

@@ -0,0 +1,291 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+
+use api::v1::column::Values;
+use api::v1::greptime_request::Request;
+use api::v1::value::ValueData;
+use api::v1::{Decimal128, InsertRequests, IntervalMonthDayNano, RowInsertRequests};
+use common_telemetry::{debug, warn};
+
+pub(crate) type LimiterRef = Arc<Limiter>;
+
+/// A frontend request limiter that controls the total size of in-flight write requests.
+pub(crate) struct Limiter {
+    // The maximum number of bytes that can be in flight.
+    max_in_flight_write_bytes: u64,
+
+    // The current in-flight write bytes.
+    in_flight_write_bytes: Arc<AtomicU64>,
+}
+
+/// A counter for the in-flight write bytes.
+pub(crate) struct InFlightWriteBytesCounter {
+    // The current in-flight write bytes.
+    in_flight_write_bytes: Arc<AtomicU64>,
+
+    // The write bytes that are being processed.
+    processing_write_bytes: u64,
+}
+
+impl InFlightWriteBytesCounter {
+    /// Creates a new InFlightWriteBytesCounter. It will decrease the in-flight write bytes when dropped.
+    pub fn new(in_flight_write_bytes: Arc<AtomicU64>, processing_write_bytes: u64) -> Self {
+        debug!(
+            "processing write bytes: {}, current in-flight write bytes: {}",
+            processing_write_bytes,
+            in_flight_write_bytes.load(Ordering::Relaxed)
+        );
+        Self {
+            in_flight_write_bytes,
+            processing_write_bytes,
+        }
+    }
+}
+
+impl Drop for InFlightWriteBytesCounter {
+    // When the request is finished, the in-flight write bytes should be decreased.
+    fn drop(&mut self) {
+        self.in_flight_write_bytes
+            .fetch_sub(self.processing_write_bytes, Ordering::Relaxed);
+    }
+}
+
+impl Limiter {
+    pub fn new(max_in_flight_write_bytes: u64) -> Self {
+        Self {
+            max_in_flight_write_bytes,
+            in_flight_write_bytes: Arc::new(AtomicU64::new(0)),
+        }
+    }
+
+    pub fn limit_request(&self, request: &Request) -> Option<InFlightWriteBytesCounter> {
+        let size = match request {
+            Request::Inserts(requests) => self.insert_requests_data_size(requests),
+            Request::RowInserts(requests) => self.rows_insert_requests_data_size(requests),
+            _ => 0,
+        };
+        self.limit_in_flight_write_bytes(size as u64)
+    }
+
+    pub fn limit_row_inserts(
+        &self,
+        requests: &RowInsertRequests,
+    ) -> Option<InFlightWriteBytesCounter> {
+        let size = self.rows_insert_requests_data_size(requests);
+        self.limit_in_flight_write_bytes(size as u64)
+    }
+
+    /// Returns None if the in-flight write bytes exceed the maximum limit.
+    /// Otherwise, returns Some(InFlightWriteBytesCounter) and the in-flight write bytes will be increased.
+    pub fn limit_in_flight_write_bytes(&self, bytes: u64) -> Option<InFlightWriteBytesCounter> {
+        let result = self.in_flight_write_bytes.fetch_update(
+            Ordering::Relaxed,
+            Ordering::Relaxed,
+            |current| {
+                if current + bytes > self.max_in_flight_write_bytes {
+                    warn!(
+                        "in-flight write bytes exceed the maximum limit {}, request with {} bytes will be limited",
+                        self.max_in_flight_write_bytes,
+                        bytes
+                    );
+                    return None;
+                }
+                Some(current + bytes)
+            },
+        );
+
+        match result {
+            // Update the in-flight write bytes successfully.
+            Ok(_) => Some(InFlightWriteBytesCounter::new(
+                self.in_flight_write_bytes.clone(),
+                bytes,
+            )),
+            // It means the in-flight write bytes exceed the maximum limit.
+            Err(_) => None,
+        }
+    }
+
+    /// Returns the current in-flight write bytes.
+    #[allow(dead_code)]
+    pub fn in_flight_write_bytes(&self) -> u64 {
+        self.in_flight_write_bytes.load(Ordering::Relaxed)
+    }
+
+    fn insert_requests_data_size(&self, request: &InsertRequests) -> usize {
+        let mut size: usize = 0;
+        for insert in &request.inserts {
+            for column in &insert.columns {
+                if let Some(values) = &column.values {
+                    size += self.size_of_column_values(values);
+                }
+            }
+        }
+        size
+    }
+
+    fn rows_insert_requests_data_size(&self, request: &RowInsertRequests) -> usize {
+        let mut size: usize = 0;
+        for insert in &request.inserts {
+            if let Some(rows) = &insert.rows {
+                for row in &rows.rows {
+                    for value in &row.values {
+                        if let Some(value) = &value.value_data {
+                            size += self.size_of_value_data(value);
+                        }
+                    }
+                }
+            }
+        }
+        size
+    }
+
+    fn size_of_column_values(&self, values: &Values) -> usize {
+        let mut size: usize = 0;
+        size += values.i8_values.len() * size_of::<i32>();
+        size += values.i16_values.len() * size_of::<i32>();
+        size += values.i32_values.len() * size_of::<i32>();
+        size += values.i64_values.len() * size_of::<i64>();
+        size += values.u8_values.len() * size_of::<u32>();
+        size += values.u16_values.len() * size_of::<u32>();
+        size += values.u32_values.len() * size_of::<u32>();
+        size += values.u64_values.len() * size_of::<u64>();
+        size += values.f32_values.len() * size_of::<f32>();
+        size += values.f64_values.len() * size_of::<f64>();
+        size += values.bool_values.len() * size_of::<bool>();
+        size += values
+            .binary_values
+            .iter()
+            .map(|v| v.len() * size_of::<u8>())
+            .sum::<usize>();
+        size += values.string_values.iter().map(|v| v.len()).sum::<usize>();
+        size += values.date_values.len() * size_of::<i32>();
+        size += values.datetime_values.len() * size_of::<i64>();
+        size += values.timestamp_second_values.len() * size_of::<i64>();
+        size += values.timestamp_millisecond_values.len() * size_of::<i64>();
+        size += values.timestamp_microsecond_values.len() * size_of::<i64>();
+        size += values.timestamp_nanosecond_values.len() * size_of::<i64>();
+        size += values.time_second_values.len() * size_of::<i64>();
+        size += values.time_millisecond_values.len() * size_of::<i64>();
+        size += values.time_microsecond_values.len() * size_of::<i64>();
+        size += values.time_nanosecond_values.len() * size_of::<i64>();
+        size += values.interval_year_month_values.len() * size_of::<i64>();
+        size += values.interval_day_time_values.len() * size_of::<i64>();
+        size += values.interval_month_day_nano_values.len() * size_of::<IntervalMonthDayNano>();
+        size += values.decimal128_values.len() * size_of::<Decimal128>();
+        size
+    }
+
+    fn size_of_value_data(&self, value: &ValueData) -> usize {
+        match value {
+            ValueData::I8Value(_) => size_of::<i32>(),
+            ValueData::I16Value(_) => size_of::<i32>(),
+            ValueData::I32Value(_) => size_of::<i32>(),
+            ValueData::I64Value(_) => size_of::<i64>(),
+            ValueData::U8Value(_) => size_of::<u32>(),
+            ValueData::U16Value(_) => size_of::<u32>(),
+            ValueData::U32Value(_) => size_of::<u32>(),
+            ValueData::U64Value(_) => size_of::<u64>(),
+            ValueData::F32Value(_) => size_of::<f32>(),
+            ValueData::F64Value(_) => size_of::<f64>(),
+            ValueData::BoolValue(_) => size_of::<bool>(),
+            ValueData::BinaryValue(v) => v.len() * size_of::<u8>(),
+            ValueData::StringValue(v) => v.len(),
+            ValueData::DateValue(_) => size_of::<i32>(),
+            ValueData::DatetimeValue(_) => size_of::<i64>(),
+            ValueData::TimestampSecondValue(_) => size_of::<i64>(),
+            ValueData::TimestampMillisecondValue(_) => size_of::<i64>(),
+            ValueData::TimestampMicrosecondValue(_) => size_of::<i64>(),
+            ValueData::TimestampNanosecondValue(_) => size_of::<i64>(),
+            ValueData::TimeSecondValue(_) => size_of::<i64>(),
+            ValueData::TimeMillisecondValue(_) => size_of::<i64>(),
+            ValueData::TimeMicrosecondValue(_) => size_of::<i64>(),
+            ValueData::TimeNanosecondValue(_) => size_of::<i64>(),
+            ValueData::IntervalYearMonthValue(_) => size_of::<i32>(),
+            ValueData::IntervalDayTimeValue(_) => size_of::<i64>(),
+            ValueData::IntervalMonthDayNanoValue(_) => size_of::<IntervalMonthDayNano>(),
+            ValueData::Decimal128Value(_) => size_of::<Decimal128>(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use api::v1::column::Values;
+    use api::v1::greptime_request::Request;
+    use api::v1::{Column, InsertRequest};
+
+    use super::*;
+
+    fn generate_request(size: usize) -> Request {
+        let i8_values = vec![0; size / 4];
+        Request::Inserts(InsertRequests {
+            inserts: vec![InsertRequest {
+                columns: vec![Column {
+                    values: Some(Values {
+                        i8_values,
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                }],
+                ..Default::default()
+            }],
+        })
+    }
+
+    #[tokio::test]
+    async fn test_limiter() {
+        let limiter_ref: LimiterRef = Arc::new(Limiter::new(1024));
+        let tasks_count = 10;
+        let request_data_size = 100;
+        let mut handles = vec![];
+
+        // Generate multiple requests to test the limiter.
+        for _ in 0..tasks_count {
+            let limiter = limiter_ref.clone();
+            let handle = tokio::spawn(async move {
+                let result = limiter.limit_request(&generate_request(request_data_size));
+                assert!(result.is_some());
+            });
+            handles.push(handle);
+        }
+
+        // Wait for all threads to complete.
+        for handle in handles {
+            handle.await.unwrap();
+        }
+    }
+
+    #[test]
+    fn test_in_flight_write_bytes() {
+        let limiter_ref: LimiterRef = Arc::new(Limiter::new(1024));
+        let req1 = generate_request(100);
+        let result1 = limiter_ref.limit_request(&req1);
+        assert!(result1.is_some());
+        assert_eq!(limiter_ref.in_flight_write_bytes(), 100);
+
+        let req2 = generate_request(200);
+        let result2 = limiter_ref.limit_request(&req2);
+        assert!(result2.is_some());
+        assert_eq!(limiter_ref.in_flight_write_bytes(), 300);
+
+        drop(result1.unwrap());
+        assert_eq!(limiter_ref.in_flight_write_bytes(), 200);
+
+        drop(result2.unwrap());
+        assert_eq!(limiter_ref.in_flight_write_bytes(), 0);
+    }
+}

src/index/Cargo.toml

@@ -22,6 +22,7 @@ fst.workspace = true
 futures.workspace = true
 greptime-proto.workspace = true
 mockall.workspace = true
+parquet.workspace = true
 pin-project.workspace = true
 prost.workspace = true
 regex.workspace = true

src/index/src/bloom_filter.rs

@@ -14,6 +14,7 @@
 
 use serde::{Deserialize, Serialize};
 
+pub mod applier;
 pub mod creator;
 pub mod error;
 pub mod reader;
@@ -25,7 +26,7 @@ pub type BytesRef<'a> = &'a [u8];
 pub const SEED: u128 = 42;
 
 /// The Meta information of the bloom filter stored in the file.
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Debug, Default, Serialize, Deserialize, Clone)]
 pub struct BloomFilterMeta {
     /// The number of rows per segment.
     pub rows_per_segment: usize,
@@ -44,7 +45,7 @@ pub struct BloomFilterMeta {
 }
 
 /// The location of the bloom filter segment in the file.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone, Hash, PartialEq, Eq)]
 pub struct BloomFilterSegmentLocation {
     /// The offset of the bloom filter segment in the file.
     pub offset: u64,

src/index/src/bloom_filter/applier.rs

@@ -0,0 +1,133 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, HashSet};
+
+use parquet::arrow::arrow_reader::RowSelection;
+use parquet::file::metadata::RowGroupMetaData;
+
+use crate::bloom_filter::error::Result;
+use crate::bloom_filter::reader::BloomFilterReader;
+use crate::bloom_filter::{BloomFilterMeta, BloomFilterSegmentLocation, Bytes};
+
+/// Enumerates types of predicates for value filtering.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Predicate {
+    /// Predicate for matching values in a list.
+    InList(InListPredicate),
+}
+
+/// `InListPredicate` contains a list of acceptable values. A value needs to match at least
+/// one of the elements (logical OR semantic) for the predicate to be satisfied.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct InListPredicate {
+    /// List of acceptable values.
+    pub list: HashSet<Bytes>,
+}
+
+pub struct BloomFilterApplier {
+    reader: Box<dyn BloomFilterReader + Send>,
+    meta: BloomFilterMeta,
+}
+
+impl BloomFilterApplier {
+    pub async fn new(mut reader: Box<dyn BloomFilterReader + Send>) -> Result<Self> {
+        let meta = reader.metadata().await?;
+
+        Ok(Self { reader, meta })
+    }
+
+    /// Searches for matching row groups using bloom filters.
+    ///
+    /// This method applies bloom filter index to eliminate row groups that definitely
+    /// don't contain the searched values. It works by:
+    ///
+    /// 1. Computing prefix sums for row counts
+    /// 2. Calculating bloom filter segment locations for each row group
+    ///     1. A row group may span multiple bloom filter segments
+    /// 3. Probing bloom filter segments
+    /// 4. Removing non-matching row groups from the basement
+    ///     1. If a row group doesn't match any bloom filter segment with any probe, it is removed
+    ///
+    /// # Note
+    /// The method modifies the `basement` map in-place by removing row groups that
+    /// don't match the bloom filter criteria.
+    pub async fn search(
+        &mut self,
+        probes: &HashSet<Bytes>,
+        row_group_metas: &[RowGroupMetaData],
+        basement: &mut BTreeMap<usize, Option<RowSelection>>,
+    ) -> Result<()> {
+        // 0. Fast path - if basement is empty return empty vec
+        if basement.is_empty() {
+            return Ok(());
+        }
+
+        // 1. Compute prefix sum for row counts
+        let mut sum = 0usize;
+        let mut prefix_sum = Vec::with_capacity(row_group_metas.len() + 1);
+        prefix_sum.push(0usize);
+        for meta in row_group_metas {
+            sum += meta.num_rows() as usize;
+            prefix_sum.push(sum);
+        }
+
+        // 2. Calculate bloom filter segment locations
+        let mut row_groups_to_remove = HashSet::new();
+        for &row_group_idx in basement.keys() {
+            // TODO(ruihang): support further filter over row selection
+
+            // todo: dedup & overlap
+            let rows_range_start = prefix_sum[row_group_idx] / self.meta.rows_per_segment;
+            let rows_range_end = (prefix_sum[row_group_idx + 1] as f64
+                / self.meta.rows_per_segment as f64)
+                .ceil() as usize;
+
+            let mut is_any_range_hit = false;
+            for i in rows_range_start..rows_range_end {
+                // 3. Probe each bloom filter segment
+                let loc = BloomFilterSegmentLocation {
+                    offset: self.meta.bloom_filter_segments[i].offset,
+                    size: self.meta.bloom_filter_segments[i].size,
+                    elem_count: self.meta.bloom_filter_segments[i].elem_count,
+                };
+                let bloom = self.reader.bloom_filter(&loc).await?;
+
+                // Check if any probe exists in bloom filter
+                let mut matches = false;
+                for probe in probes {
+                    if bloom.contains(probe) {
+                        matches = true;
+                        break;
+                    }
+                }
+
+                is_any_range_hit |= matches;
+                if matches {
+                    break;
+                }
+            }
+            if !is_any_range_hit {
+                row_groups_to_remove.insert(row_group_idx);
+            }
+        }
+
+        // 4. Remove row groups that do not match any bloom filter segment
+        for row_group_idx in row_groups_to_remove {
+            basement.remove(&row_group_idx);
+        }
+
+        Ok(())
+    }
+}

src/index/src/bloom_filter/creator.rs

@@ -73,7 +73,7 @@ impl BloomFilterCreator {
     /// `rows_per_segment` <= 0
     pub fn new(
         rows_per_segment: usize,
-        intermediate_provider: Box<dyn ExternalTempFileProvider>,
+        intermediate_provider: Arc<dyn ExternalTempFileProvider>,
         global_memory_usage: Arc<AtomicUsize>,
         global_memory_usage_threshold: Option<usize>,
     ) -> Self {
@@ -96,6 +96,49 @@ impl BloomFilterCreator {
         }
     }
 
+    /// Adds multiple rows of elements to the bloom filter. If the number of accumulated rows
+    /// reaches `rows_per_segment`, it finalizes the current segment.
+    pub async fn push_n_row_elems(
+        &mut self,
+        mut nrows: usize,
+        elems: impl IntoIterator<Item = Bytes>,
+    ) -> Result<()> {
+        if nrows == 0 {
+            return Ok(());
+        }
+        if nrows == 1 {
+            return self.push_row_elems(elems).await;
+        }
+
+        let elems = elems.into_iter().collect::<Vec<_>>();
+        while nrows > 0 {
+            let rows_to_seg_end =
+                self.rows_per_segment - (self.accumulated_row_count % self.rows_per_segment);
+            let rows_to_push = nrows.min(rows_to_seg_end);
+            nrows -= rows_to_push;
+
+            self.accumulated_row_count += rows_to_push;
+
+            let mut mem_diff = 0;
+            for elem in &elems {
+                let len = elem.len();
+                let is_new = self.cur_seg_distinct_elems.insert(elem.clone());
+                if is_new {
+                    mem_diff += len;
+                }
+            }
+            self.cur_seg_distinct_elems_mem_usage += mem_diff;
+            self.global_memory_usage
+                .fetch_add(mem_diff, Ordering::Relaxed);
+
+            if self.accumulated_row_count % self.rows_per_segment == 0 {
+                self.finalize_segment().await?;
+            }
+        }
+
+        Ok(())
+    }
+
     /// Adds a row of elements to the bloom filter. If the number of accumulated rows
     /// reaches `rows_per_segment`, it finalizes the current segment.
     pub async fn push_row_elems(&mut self, elems: impl IntoIterator<Item = Bytes>) -> Result<()> {
@@ -181,6 +224,13 @@ impl BloomFilterCreator {
     }
 }
 
+impl Drop for BloomFilterCreator {
+    fn drop(&mut self) {
+        self.global_memory_usage
+            .fetch_sub(self.cur_seg_distinct_elems_mem_usage, Ordering::Relaxed);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use fastbloom::BloomFilter;
@@ -202,7 +252,7 @@ mod tests {
         let mut writer = Cursor::new(Vec::new());
         let mut creator = BloomFilterCreator::new(
             2,
-            Box::new(MockExternalTempFileProvider::new()),
+            Arc::new(MockExternalTempFileProvider::new()),
             Arc::new(AtomicUsize::new(0)),
             None,
         );
@@ -266,4 +316,79 @@ mod tests {
         assert!(bfs[1].contains(&b"e"));
         assert!(bfs[1].contains(&b"f"));
     }
+
+    #[tokio::test]
+    async fn test_bloom_filter_creator_batch_push() {
+        let mut writer = Cursor::new(Vec::new());
+        let mut creator: BloomFilterCreator = BloomFilterCreator::new(
+            2,
+            Arc::new(MockExternalTempFileProvider::new()),
+            Arc::new(AtomicUsize::new(0)),
+            None,
+        );
+
+        creator
+            .push_n_row_elems(5, vec![b"a".to_vec(), b"b".to_vec()])
+            .await
+            .unwrap();
+        assert!(creator.cur_seg_distinct_elems_mem_usage > 0);
+        assert!(creator.memory_usage() > 0);
+
+        creator
+            .push_n_row_elems(5, vec![b"c".to_vec(), b"d".to_vec()])
+            .await
+            .unwrap();
+        assert_eq!(creator.cur_seg_distinct_elems_mem_usage, 0);
+        assert!(creator.memory_usage() > 0);
+
+        creator
+            .push_n_row_elems(10, vec![b"e".to_vec(), b"f".to_vec()])
+            .await
+            .unwrap();
+        assert_eq!(creator.cur_seg_distinct_elems_mem_usage, 0);
+        assert!(creator.memory_usage() > 0);
+
+        creator.finish(&mut writer).await.unwrap();
+
+        let bytes = writer.into_inner();
+        let total_size = bytes.len();
+        let meta_size_offset = total_size - 4;
+        let meta_size = u32::from_le_bytes((&bytes[meta_size_offset..]).try_into().unwrap());
+
+        let meta_bytes = &bytes[total_size - meta_size as usize - 4..total_size - 4];
+        let meta: BloomFilterMeta = serde_json::from_slice(meta_bytes).unwrap();
+
+        assert_eq!(meta.rows_per_segment, 2);
+        assert_eq!(meta.seg_count, 10);
+        assert_eq!(meta.row_count, 20);
+        assert_eq!(
+            meta.bloom_filter_segments_size + meta_bytes.len() + 4,
+            total_size
+        );
+
+        let mut bfs = Vec::new();
+        for segment in meta.bloom_filter_segments {
+            let bloom_filter_bytes =
+                &bytes[segment.offset as usize..(segment.offset + segment.size) as usize];
+            let v = u64_vec_from_bytes(bloom_filter_bytes);
+            let bloom_filter = BloomFilter::from_vec(v)
+                .seed(&SEED)
+                .expected_items(segment.elem_count);
+            bfs.push(bloom_filter);
+        }
+
+        assert_eq!(bfs.len(), 10);
+        for bf in bfs.iter().take(3) {
+            assert!(bf.contains(&b"a"));
+            assert!(bf.contains(&b"b"));
+        }
+        for bf in bfs.iter().take(5).skip(2) {
+            assert!(bf.contains(&b"c"));
+            assert!(bf.contains(&b"d"));
+        }
+        for bf in bfs.iter().take(10).skip(5) {
+            assert!(bf.contains(&b"e"));
+            assert!(bf.contains(&b"f"));
+        }
+    }
 }

src/index/src/bloom_filter/creator/finalize_segment.rs

@@ -43,7 +43,7 @@ pub struct FinalizedBloomFilterStorage {
     intermediate_prefix: String,
 
     /// The provider for intermediate Bloom filter files.
-    intermediate_provider: Box<dyn ExternalTempFileProvider>,
+    intermediate_provider: Arc<dyn ExternalTempFileProvider>,
 
     /// The memory usage of the in-memory Bloom filters.
     memory_usage: usize,
@@ -59,7 +59,7 @@ pub struct FinalizedBloomFilterStorage {
 impl FinalizedBloomFilterStorage {
     /// Creates a new `FinalizedBloomFilterStorage`.
     pub fn new(
-        intermediate_provider: Box<dyn ExternalTempFileProvider>,
+        intermediate_provider: Arc<dyn ExternalTempFileProvider>,
         global_memory_usage: Arc<AtomicUsize>,
         global_memory_usage_threshold: Option<usize>,
     ) -> Self {
@@ -132,7 +132,7 @@ impl FinalizedBloomFilterStorage {
     /// Drains the storage and returns a stream of finalized Bloom filter segments.
     pub async fn drain(
         &mut self,
-    ) -> Result<Pin<Box<dyn Stream<Item = Result<FinalizedBloomFilterSegment>> + '_>>> {
+    ) -> Result<Pin<Box<dyn Stream<Item = Result<FinalizedBloomFilterSegment>> + Send + '_>>> {
         // FAST PATH: memory only
         if self.intermediate_file_id_counter == 0 {
             return Ok(Box::pin(stream::iter(self.in_memory.drain(..).map(Ok))));
@@ -183,6 +183,13 @@ impl FinalizedBloomFilterStorage {
     }
 }
 
+impl Drop for FinalizedBloomFilterStorage {
+    fn drop(&mut self) {
+        self.global_memory_usage
+            .fetch_sub(self.memory_usage, Ordering::Relaxed);
+    }
+}
+
 /// A finalized Bloom filter segment.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct FinalizedBloomFilterSegment {
@@ -250,7 +257,7 @@ mod tests {
 
         let global_memory_usage = Arc::new(AtomicUsize::new(0));
         let global_memory_usage_threshold = Some(1024 * 1024); // 1MB
-        let provider = Box::new(mock_provider);
+        let provider = Arc::new(mock_provider);
         let mut storage = FinalizedBloomFilterStorage::new(
             provider,
             global_memory_usage.clone(),

src/index/src/bloom_filter/reader.rs

@@ -38,7 +38,15 @@ pub trait BloomFilterReader {
     async fn range_read(&mut self, offset: u64, size: u32) -> Result<Bytes>;
 
     /// Reads bunch of ranges from the file.
-    async fn read_vec(&mut self, ranges: &[Range<u64>]) -> Result<Vec<Bytes>>;
+    async fn read_vec(&mut self, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
+        let mut results = Vec::with_capacity(ranges.len());
+        for range in ranges {
+            let size = (range.end - range.start) as u32;
+            let data = self.range_read(range.start, size).await?;
+            results.push(data);
+        }
+        Ok(results)
+    }
 
     /// Reads the meta information of the bloom filter.
     async fn metadata(&mut self) -> Result<BloomFilterMeta>;
@@ -190,7 +198,7 @@ mod tests {
         let mut writer = Cursor::new(vec![]);
         let mut creator = BloomFilterCreator::new(
             2,
-            Box::new(MockExternalTempFileProvider::new()),
+            Arc::new(MockExternalTempFileProvider::new()),
             Arc::new(AtomicUsize::new(0)),
             None,
         );

src/meta-srv/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 
 [features]
 mock = []
-pg_kvbackend = ["dep:tokio-postgres"]
+pg_kvbackend = ["dep:tokio-postgres", "common-meta/pg_kvbackend"]
 
 [lints]
 workspace = true
@@ -14,6 +14,7 @@ workspace = true
 [dependencies]
 api.workspace = true
 async-trait = "0.1"
+chrono.workspace = true
 clap.workspace = true
 client.workspace = true
 common-base.workspace = true
@@ -55,7 +56,7 @@ snafu.workspace = true
 store-api.workspace = true
 table.workspace = true
 tokio.workspace = true
-tokio-postgres = { workspace = true, optional = true }
+tokio-postgres = { workspace = true, optional = true, features = ["with-chrono-0_4"] }
 tokio-stream = { workspace = true, features = ["net"] }
 toml.workspace = true
 tonic.workspace = true

src/meta-srv/src/bootstrap.rs

@@ -26,6 +26,8 @@ use common_meta::kv_backend::memory::MemoryKvBackend;
 #[cfg(feature = "pg_kvbackend")]
 use common_meta::kv_backend::postgres::PgStore;
 use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef};
+#[cfg(feature = "pg_kvbackend")]
+use common_telemetry::error;
 use common_telemetry::info;
 use etcd_client::Client;
 use futures::future;
@@ -224,8 +226,9 @@ pub async fn metasrv_builder(
         #[cfg(feature = "pg_kvbackend")]
         (None, BackendImpl::PostgresStore) => {
             let pg_client = create_postgres_client(opts).await?;
-            let kv_backend = PgStore::with_pg_client(pg_client).await.unwrap();
-            // TODO(jeremy, weny): implement election for postgres
+            let kv_backend = PgStore::with_pg_client(pg_client)
+                .await
+                .context(error::KvBackendSnafu)?;
             (kv_backend, None)
         }
     };
@@ -275,8 +278,14 @@ async fn create_postgres_client(opts: &MetasrvOptions) -> Result<tokio_postgres:
     let postgres_url = opts.store_addrs.first().context(InvalidArgumentsSnafu {
         err_msg: "empty store addrs",
     })?;
-    let (client, _) = tokio_postgres::connect(postgres_url, NoTls)
+    let (client, connection) = tokio_postgres::connect(postgres_url, NoTls)
         .await
         .context(error::ConnectPostgresSnafu)?;
+
+    tokio::spawn(async move {
+        if let Err(e) = connection.await {
+            error!(e; "connection error");
+        }
+    });
     Ok(client)
 }

src/meta-srv/src/election.rs

@@ -13,11 +13,12 @@
 // limitations under the License.
 
 pub mod etcd;
+#[cfg(feature = "pg_kvbackend")]
+pub mod postgres;
 
-use std::fmt;
+use std::fmt::{self, Debug};
 use std::sync::Arc;
 
-use etcd_client::LeaderKey;
 use tokio::sync::broadcast::Receiver;
 
 use crate::error::Result;
@@ -26,10 +27,31 @@ use crate::metasrv::MetasrvNodeInfo;
 pub const ELECTION_KEY: &str = "__metasrv_election";
 pub const CANDIDATES_ROOT: &str = "__metasrv_election_candidates/";
 
+pub(crate) const CANDIDATE_LEASE_SECS: u64 = 600;
+const KEEP_ALIVE_INTERVAL_SECS: u64 = CANDIDATE_LEASE_SECS / 2;
+
+/// Messages sent when the leader changes.
 #[derive(Debug, Clone)]
 pub enum LeaderChangeMessage {
-    Elected(Arc<LeaderKey>),
-    StepDown(Arc<LeaderKey>),
+    Elected(Arc<dyn LeaderKey>),
+    StepDown(Arc<dyn LeaderKey>),
+}
+
+/// LeaderKey is a key that represents the leader of metasrv.
+/// The structure is corresponding to [etcd_client::LeaderKey].
+pub trait LeaderKey: Send + Sync + Debug {
+    /// The name in byte. name is the election identifier that corresponds to the leadership key.
+    fn name(&self) -> &[u8];
+
+    /// The key in byte. key is an opaque key representing the ownership of the election. If the key
+    /// is deleted, then leadership is lost.
+    fn key(&self) -> &[u8];
+
+    /// The creation revision of the key.
+    fn revision(&self) -> i64;
+
+    /// The lease ID of the election leader.
+    fn lease_id(&self) -> i64;
 }
 
 impl fmt::Display for LeaderChangeMessage {
@@ -47,8 +69,8 @@ impl fmt::Display for LeaderChangeMessage {
         write!(f, "LeaderKey {{ ")?;
         write!(f, "name: {}", String::from_utf8_lossy(leader_key.name()))?;
         write!(f, ", key: {}", String::from_utf8_lossy(leader_key.key()))?;
-        write!(f, ", rev: {}", leader_key.rev())?;
-        write!(f, ", lease: {}", leader_key.lease())?;
+        write!(f, ", rev: {}", leader_key.revision())?;
+        write!(f, ", lease: {}", leader_key.lease_id())?;
         write!(f, " }})")
     }
 }
@@ -65,7 +87,7 @@ pub trait Election: Send + Sync {
     /// initialization operations can be performed.
     ///
     /// note: a new leader will only return true on the first call.
-    fn in_infancy(&self) -> bool;
+    fn in_leader_infancy(&self) -> bool;
 
     /// Registers a candidate for the election.
     async fn register_candidate(&self, node_info: &MetasrvNodeInfo) -> Result<()>;

src/meta-srv/src/election/etcd.rs

@@ -18,18 +18,41 @@ use std::time::Duration;
 
 use common_meta::distributed_time_constants::{META_KEEP_ALIVE_INTERVAL_SECS, META_LEASE_SECS};
 use common_telemetry::{error, info, warn};
-use etcd_client::{Client, GetOptions, LeaderKey, LeaseKeepAliveStream, LeaseKeeper, PutOptions};
+use etcd_client::{
+    Client, GetOptions, LeaderKey as EtcdLeaderKey, LeaseKeepAliveStream, LeaseKeeper, PutOptions,
+};
 use snafu::{ensure, OptionExt, ResultExt};
 use tokio::sync::broadcast;
 use tokio::sync::broadcast::error::RecvError;
 use tokio::sync::broadcast::Receiver;
 use tokio::time::{timeout, MissedTickBehavior};
 
-use crate::election::{Election, LeaderChangeMessage, CANDIDATES_ROOT, ELECTION_KEY};
+use crate::election::{
+    Election, LeaderChangeMessage, LeaderKey, CANDIDATES_ROOT, CANDIDATE_LEASE_SECS, ELECTION_KEY,
+    KEEP_ALIVE_INTERVAL_SECS,
+};
 use crate::error;
 use crate::error::Result;
 use crate::metasrv::{ElectionRef, LeaderValue, MetasrvNodeInfo};
 
+impl LeaderKey for EtcdLeaderKey {
+    fn name(&self) -> &[u8] {
+        self.name()
+    }
+
+    fn key(&self) -> &[u8] {
+        self.key()
+    }
+
+    fn revision(&self) -> i64 {
+        self.rev()
+    }
+
+    fn lease_id(&self) -> i64 {
+        self.lease()
+    }
+}
+
 pub struct EtcdElection {
     leader_value: String,
     client: Client,
@@ -75,15 +98,15 @@ impl EtcdElection {
                         LeaderChangeMessage::Elected(key) => {
                             info!(
                                 "[{leader_ident}] is elected as leader: {:?}, lease: {}",
-                                key.name_str(),
-                                key.lease()
+                                String::from_utf8_lossy(key.name()),
+                                key.lease_id()
                             );
                         }
                         LeaderChangeMessage::StepDown(key) => {
                             warn!(
                                 "[{leader_ident}] is stepping down: {:?}, lease: {}",
-                                key.name_str(),
-                                key.lease()
+                                String::from_utf8_lossy(key.name()),
+                                key.lease_id()
                             );
                         }
                     },
@@ -126,16 +149,13 @@ impl Election for EtcdElection {
         self.is_leader.load(Ordering::Relaxed)
     }
 
-    fn in_infancy(&self) -> bool {
+    fn in_leader_infancy(&self) -> bool {
         self.infancy
             .compare_exchange(true, false, Ordering::Relaxed, Ordering::Relaxed)
             .is_ok()
     }
 
     async fn register_candidate(&self, node_info: &MetasrvNodeInfo) -> Result<()> {
-        const CANDIDATE_LEASE_SECS: u64 = 600;
-        const KEEP_ALIVE_INTERVAL_SECS: u64 = CANDIDATE_LEASE_SECS / 2;
-
         let mut lease_client = self.client.lease_client();
         let res = lease_client
             .grant(CANDIDATE_LEASE_SECS as i64, None)
@@ -239,7 +259,7 @@ impl Election for EtcdElection {
                 // The keep alive operation MUST be done in `META_KEEP_ALIVE_INTERVAL_SECS`.
                 match timeout(
                     keep_lease_duration,
-                    self.keep_alive(&mut keeper, &mut receiver, leader),
+                    self.keep_alive(&mut keeper, &mut receiver, leader.clone()),
                 )
                 .await
                 {
@@ -303,7 +323,7 @@ impl EtcdElection {
         &self,
         keeper: &mut LeaseKeeper,
         receiver: &mut LeaseKeepAliveStream,
-        leader: &LeaderKey,
+        leader: EtcdLeaderKey,
     ) -> Result<()> {
         keeper.keep_alive().await.context(error::EtcdFailedSnafu)?;
         if let Some(res) = receiver.message().await.context(error::EtcdFailedSnafu)? {
@@ -324,7 +344,7 @@ impl EtcdElection {
 
                 if let Err(e) = self
                     .leader_watcher
-                    .send(LeaderChangeMessage::Elected(Arc::new(leader.clone())))
+                    .send(LeaderChangeMessage::Elected(Arc::new(leader)))
                 {
                     error!(e; "Failed to send leader change message");
                 }

src/meta-srv/src/election/postgres.rs

@@ -0,0 +1,519 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
+use common_time::Timestamp;
+use itertools::Itertools;
+use snafu::{ensure, OptionExt, ResultExt};
+use tokio::sync::broadcast;
+use tokio_postgres::Client;
+
+use crate::election::{Election, LeaderChangeMessage, CANDIDATES_ROOT, ELECTION_KEY};
+use crate::error::{
+    DeserializeFromJsonSnafu, PostgresExecutionSnafu, Result, SerializeToJsonSnafu, UnexpectedSnafu,
+};
+use crate::metasrv::{ElectionRef, LeaderValue, MetasrvNodeInfo};
+
+// Separator between value and expire time.
+const LEASE_SEP: &str = r#"||__metadata_lease_sep||"#;
+
+// SQL to put a value with expire time. Parameters: key, value, LEASE_SEP, expire_time
+const PUT_IF_NOT_EXISTS_WITH_EXPIRE_TIME: &str = r#"
+WITH prev AS (
+    SELECT k, v FROM greptime_metakv WHERE k = $1
+), insert AS (
+    INSERT INTO greptime_metakv
+    VALUES($1, $2 || $3 || TO_CHAR(CURRENT_TIMESTAMP + INTERVAL '1 second' * $4, 'YYYY-MM-DD HH24:MI:SS.MS'))
+    ON CONFLICT (k) DO NOTHING
+)
+
+SELECT k, v FROM prev;
+"#;
+
+// SQL to update a value with expire time. Parameters: key, prev_value_with_lease, updated_value, LEASE_SEP, expire_time
+const CAS_WITH_EXPIRE_TIME: &str = r#"
+UPDATE greptime_metakv
+SET k=$1,
+v=$3 || $4 || TO_CHAR(CURRENT_TIMESTAMP + INTERVAL '1 second' * $5, 'YYYY-MM-DD HH24:MI:SS.MS')
+WHERE 
+    k=$1 AND v=$2
+"#;
+
+const GET_WITH_CURRENT_TIMESTAMP: &str = r#"SELECT v, TO_CHAR(CURRENT_TIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.MS') FROM greptime_metakv WHERE k = $1"#;
+
+const PREFIX_GET_WITH_CURRENT_TIMESTAMP: &str = r#"SELECT v, TO_CHAR(CURRENT_TIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.MS') FROM greptime_metakv WHERE k LIKE $1"#;
+
+const POINT_DELETE: &str = "DELETE FROM greptime_metakv WHERE k = $1 RETURNING k,v;";
+
+/// Parse the value and expire time from the given string. The value should be in the format "value || LEASE_SEP || expire_time".
+fn parse_value_and_expire_time(value: &str) -> Result<(String, Timestamp)> {
+    let (value, expire_time) = value
+        .split(LEASE_SEP)
+        .collect_tuple()
+        .context(UnexpectedSnafu {
+            violated: format!(
+                "Invalid value {}, expect node info || {} || expire time",
+                value, LEASE_SEP
+            ),
+        })?;
+    // Given expire_time is in the format 'YYYY-MM-DD HH24:MI:SS.MS'
+    let expire_time = match Timestamp::from_str(expire_time, None) {
+        Ok(ts) => ts,
+        Err(_) => UnexpectedSnafu {
+            violated: format!("Invalid timestamp: {}", expire_time),
+        }
+        .fail()?,
+    };
+    Ok((value.to_string(), expire_time))
+}
+
+/// PostgreSql implementation of Election.
+/// TODO(CookiePie): Currently only support candidate registration. Add election logic.
+pub struct PgElection {
+    leader_value: String,
+    client: Client,
+    is_leader: AtomicBool,
+    leader_infancy: AtomicBool,
+    leader_watcher: broadcast::Sender<LeaderChangeMessage>,
+    store_key_prefix: String,
+    candidate_lease_ttl_secs: u64,
+}
+
+impl PgElection {
+    pub async fn with_pg_client(
+        leader_value: String,
+        client: Client,
+        store_key_prefix: String,
+        candidate_lease_ttl_secs: u64,
+    ) -> Result<ElectionRef> {
+        let (tx, _) = broadcast::channel(100);
+        Ok(Arc::new(Self {
+            leader_value,
+            client,
+            is_leader: AtomicBool::new(false),
+            leader_infancy: AtomicBool::new(false),
+            leader_watcher: tx,
+            store_key_prefix,
+            candidate_lease_ttl_secs,
+        }))
+    }
+
+    fn _election_key(&self) -> String {
+        format!("{}{}", self.store_key_prefix, ELECTION_KEY)
+    }
+
+    fn candidate_root(&self) -> String {
+        format!("{}{}", self.store_key_prefix, CANDIDATES_ROOT)
+    }
+
+    fn candidate_key(&self) -> String {
+        format!("{}{}", self.candidate_root(), self.leader_value)
+    }
+}
+
+#[async_trait::async_trait]
+impl Election for PgElection {
+    type Leader = LeaderValue;
+
+    fn is_leader(&self) -> bool {
+        self.is_leader.load(Ordering::Relaxed)
+    }
+
+    fn in_leader_infancy(&self) -> bool {
+        self.leader_infancy
+            .compare_exchange(true, false, Ordering::Relaxed, Ordering::Relaxed)
+            .is_ok()
+    }
+
+    /// TODO(CookiePie): Split the candidate registration and keep alive logic into separate methods, so that upper layers can call them separately.
+    async fn register_candidate(&self, node_info: &MetasrvNodeInfo) -> Result<()> {
+        let key = self.candidate_key();
+        let node_info =
+            serde_json::to_string(node_info).with_context(|_| SerializeToJsonSnafu {
+                input: format!("{node_info:?}"),
+            })?;
+        let res = self.put_value_with_lease(&key, &node_info).await?;
+        // May registered before, just update the lease.
+        if !res {
+            self.delete_value(&key).await?;
+            self.put_value_with_lease(&key, &node_info).await?;
+        }
+
+        // Check if the current lease has expired and renew the lease.
+        let mut keep_alive_interval =
+            tokio::time::interval(Duration::from_secs(self.candidate_lease_ttl_secs / 2));
+        loop {
+            let _ = keep_alive_interval.tick().await;
+
+            let (_, prev_expire_time, current_time, origin) = self
+                .get_value_with_lease(&key, true)
+                .await?
+                .unwrap_or_default();
+
+            ensure!(
+                prev_expire_time > current_time,
+                UnexpectedSnafu {
+                    violated: format!(
+                        "Candidate lease expired, key: {:?}",
+                        String::from_utf8_lossy(&key.into_bytes())
+                    ),
+                }
+            );
+
+            // Safety: origin is Some since we are using `get_value_with_lease` with `true`.
+            let origin = origin.unwrap();
+            self.update_value_with_lease(&key, &origin, &node_info)
+                .await?;
+        }
+    }
+
+    async fn all_candidates(&self) -> Result<Vec<MetasrvNodeInfo>> {
+        let key_prefix = self.candidate_root();
+        let (mut candidates, current) = self.get_value_with_lease_by_prefix(&key_prefix).await?;
+        // Remove expired candidates
+        candidates.retain(|c| c.1 > current);
+        let mut valid_candidates = Vec::with_capacity(candidates.len());
+        for (c, _) in candidates {
+            let node_info: MetasrvNodeInfo =
+                serde_json::from_str(&c).with_context(|_| DeserializeFromJsonSnafu {
+                    input: format!("{:?}", c),
+                })?;
+            valid_candidates.push(node_info);
+        }
+        Ok(valid_candidates)
+    }
+
+    async fn campaign(&self) -> Result<()> {
+        todo!()
+    }
+
+    async fn leader(&self) -> Result<Self::Leader> {
+        todo!()
+    }
+
+    async fn resign(&self) -> Result<()> {
+        todo!()
+    }
+
+    fn subscribe_leader_change(&self) -> broadcast::Receiver<LeaderChangeMessage> {
+        self.leader_watcher.subscribe()
+    }
+}
+
+impl PgElection {
+    /// Returns value, expire time and current time. If `with_origin` is true, the origin string is also returned.
+    async fn get_value_with_lease(
+        &self,
+        key: &String,
+        with_origin: bool,
+    ) -> Result<Option<(String, Timestamp, Timestamp, Option<String>)>> {
+        let res = self
+            .client
+            .query(GET_WITH_CURRENT_TIMESTAMP, &[&key])
+            .await
+            .context(PostgresExecutionSnafu)?;
+
+        if res.is_empty() {
+            Ok(None)
+        } else {
+            // Safety: Checked if res is empty above.
+            let current_time_str = res[0].get(1);
+            let current_time = match Timestamp::from_str(current_time_str, None) {
+                Ok(ts) => ts,
+                Err(_) => UnexpectedSnafu {
+                    violated: format!("Invalid timestamp: {}", current_time_str),
+                }
+                .fail()?,
+            };
+            // Safety: Checked if res is empty above.
+            let value_and_expire_time = res[0].get(0);
+            let (value, expire_time) = parse_value_and_expire_time(value_and_expire_time)?;
+
+            if with_origin {
+                Ok(Some((
+                    value,
+                    expire_time,
+                    current_time,
+                    Some(value_and_expire_time.to_string()),
+                )))
+            } else {
+                Ok(Some((value, expire_time, current_time, None)))
+            }
+        }
+    }
+
+    /// Returns all values and expire time with the given key prefix. Also returns the current time.
+    async fn get_value_with_lease_by_prefix(
+        &self,
+        key_prefix: &str,
+    ) -> Result<(Vec<(String, Timestamp)>, Timestamp)> {
+        let key_prefix = format!("{}%", key_prefix);
+        let res = self
+            .client
+            .query(PREFIX_GET_WITH_CURRENT_TIMESTAMP, &[&key_prefix])
+            .await
+            .context(PostgresExecutionSnafu)?;
+
+        let mut values_with_leases = vec![];
+        let mut current = Timestamp::default();
+        for row in res {
+            let current_time_str = row.get(1);
+            current = match Timestamp::from_str(current_time_str, None) {
+                Ok(ts) => ts,
+                Err(_) => UnexpectedSnafu {
+                    violated: format!("Invalid timestamp: {}", current_time_str),
+                }
+                .fail()?,
+            };
+
+            let value_and_expire_time = row.get(0);
+            let (value, expire_time) = parse_value_and_expire_time(value_and_expire_time)?;
+
+            values_with_leases.push((value, expire_time));
+        }
+        Ok((values_with_leases, current))
+    }
+
+    async fn update_value_with_lease(&self, key: &str, prev: &str, updated: &str) -> Result<()> {
+        let res = self
+            .client
+            .execute(
+                CAS_WITH_EXPIRE_TIME,
+                &[
+                    &key,
+                    &prev,
+                    &updated,
+                    &LEASE_SEP,
+                    &(self.candidate_lease_ttl_secs as f64),
+                ],
+            )
+            .await
+            .context(PostgresExecutionSnafu)?;
+
+        ensure!(
+            res == 1,
+            UnexpectedSnafu {
+                violated: format!("Failed to update key: {}", key),
+            }
+        );
+
+        Ok(())
+    }
+
+    /// Returns `true` if the insertion is successful
+    async fn put_value_with_lease(&self, key: &str, value: &str) -> Result<bool> {
+        let res = self
+            .client
+            .query(
+                PUT_IF_NOT_EXISTS_WITH_EXPIRE_TIME,
+                &[
+                    &key,
+                    &value,
+                    &LEASE_SEP,
+                    &(self.candidate_lease_ttl_secs as f64),
+                ],
+            )
+            .await
+            .context(PostgresExecutionSnafu)?;
+        Ok(res.is_empty())
+    }
+
+    /// Returns `true` if the deletion is successful.
+    /// Caution: Should only delete the key if the lease is expired.
+    async fn delete_value(&self, key: &String) -> Result<bool> {
+        let res = self
+            .client
+            .query(POINT_DELETE, &[&key])
+            .await
+            .context(PostgresExecutionSnafu)?;
+
+        Ok(res.len() == 1)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::env;
+
+    use tokio_postgres::{Client, NoTls};
+
+    use super::*;
+    use crate::error::PostgresExecutionSnafu;
+
+    async fn create_postgres_client() -> Result<Client> {
+        let endpoint = env::var("GT_POSTGRES_ENDPOINTS").unwrap_or_default();
+        if endpoint.is_empty() {
+            return UnexpectedSnafu {
+                violated: "Postgres endpoint is empty".to_string(),
+            }
+            .fail();
+        }
+        let (client, connection) = tokio_postgres::connect(&endpoint, NoTls)
+            .await
+            .context(PostgresExecutionSnafu)?;
+        tokio::spawn(async move {
+            connection.await.context(PostgresExecutionSnafu).unwrap();
+        });
+        Ok(client)
+    }
+
+    #[tokio::test]
+    async fn test_postgres_crud() {
+        let client = create_postgres_client().await.unwrap();
+
+        let key = "test_key".to_string();
+        let value = "test_value".to_string();
+
+        let (tx, _) = broadcast::channel(100);
+        let pg_election = PgElection {
+            leader_value: "test_leader".to_string(),
+            client,
+            is_leader: AtomicBool::new(false),
+            leader_infancy: AtomicBool::new(true),
+            leader_watcher: tx,
+            store_key_prefix: "test_prefix".to_string(),
+            candidate_lease_ttl_secs: 10,
+        };
+
+        let res = pg_election
+            .put_value_with_lease(&key, &value)
+            .await
+            .unwrap();
+        assert!(res);
+
+        let (value, _, _, prev) = pg_election
+            .get_value_with_lease(&key, true)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(value, value);
+
+        let prev = prev.unwrap();
+        pg_election
+            .update_value_with_lease(&key, &prev, &value)
+            .await
+            .unwrap();
+
+        let res = pg_election.delete_value(&key).await.unwrap();
+        assert!(res);
+
+        let res = pg_election.get_value_with_lease(&key, false).await.unwrap();
+        assert!(res.is_none());
+
+        for i in 0..10 {
+            let key = format!("test_key_{}", i);
+            let value = format!("test_value_{}", i);
+            pg_election
+                .put_value_with_lease(&key, &value)
+                .await
+                .unwrap();
+        }
+
+        let key_prefix = "test_key".to_string();
+        let (res, _) = pg_election
+            .get_value_with_lease_by_prefix(&key_prefix)
+            .await
+            .unwrap();
+        assert_eq!(res.len(), 10);
+
+        for i in 0..10 {
+            let key = format!("test_key_{}", i);
+            let res = pg_election.delete_value(&key).await.unwrap();
+            assert!(res);
+        }
+
+        let (res, current) = pg_election
+            .get_value_with_lease_by_prefix(&key_prefix)
+            .await
+            .unwrap();
+        assert!(res.is_empty());
+        assert!(current == Timestamp::default());
+    }
+
+    async fn candidate(leader_value: String, candidate_lease_ttl_secs: u64) {
+        let client = create_postgres_client().await.unwrap();
+
+        let (tx, _) = broadcast::channel(100);
+        let pg_election = PgElection {
+            leader_value,
+            client,
+            is_leader: AtomicBool::new(false),
+            leader_infancy: AtomicBool::new(true),
+            leader_watcher: tx,
+            store_key_prefix: "test_prefix".to_string(),
+            candidate_lease_ttl_secs,
+        };
+
+        let node_info = MetasrvNodeInfo {
+            addr: "test_addr".to_string(),
+            version: "test_version".to_string(),
+            git_commit: "test_git_commit".to_string(),
+            start_time_ms: 0,
+        };
+        pg_election.register_candidate(&node_info).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_candidate_registration() {
+        let leader_value_prefix = "test_leader".to_string();
+        let candidate_lease_ttl_secs = 5;
+        let mut handles = vec![];
+        for i in 0..10 {
+            let leader_value = format!("{}{}", leader_value_prefix, i);
+            let handle = tokio::spawn(candidate(leader_value, candidate_lease_ttl_secs));
+            handles.push(handle);
+        }
+        // Wait for candidates to registrate themselves and renew their leases at least once.
+        tokio::time::sleep(Duration::from_secs(6)).await;
+
+        let client = create_postgres_client().await.unwrap();
+
+        let (tx, _) = broadcast::channel(100);
+        let leader_value = "test_leader".to_string();
+        let pg_election = PgElection {
+            leader_value,
+            client,
+            is_leader: AtomicBool::new(false),
+            leader_infancy: AtomicBool::new(true),
+            leader_watcher: tx,
+            store_key_prefix: "test_prefix".to_string(),
+            candidate_lease_ttl_secs,
+        };
+
+        let candidates = pg_election.all_candidates().await.unwrap();
+        assert_eq!(candidates.len(), 10);
+
+        for handle in handles {
+            handle.abort();
+        }
+
+        // Wait for the candidate leases to expire.
+        tokio::time::sleep(Duration::from_secs(5)).await;
+        let candidates = pg_election.all_candidates().await.unwrap();
+        assert!(candidates.is_empty());
+
+        // Garbage collection
+        for i in 0..10 {
+            let key = format!(
+                "{}{}{}{}",
+                "test_prefix", CANDIDATES_ROOT, leader_value_prefix, i
+            );
+            let res = pg_election.delete_value(&key).await.unwrap();
+            assert!(res);
+        }
+    }
+}

src/meta-srv/src/error.rs

@@ -697,6 +697,8 @@ pub enum Error {
     #[cfg(feature = "pg_kvbackend")]
     #[snafu(display("Failed to execute via postgres"))]
     PostgresExecution {
+        #[snafu(source)]
+        error: tokio_postgres::Error,
         #[snafu(implicit)]
         location: Location,
     },

src/meta-srv/src/handler/on_leader_start_handler.rs

@@ -36,7 +36,7 @@ impl HeartbeatHandler for OnLeaderStartHandler {
             return Ok(HandleControl::Continue);
         };
 
-        if election.in_infancy() {
+        if election.in_leader_infancy() {
             ctx.is_infancy = true;
             // TODO(weny): Unifies the multiple leader state between Context and Metasrv.
             // we can't ensure the in-memory kv has already been reset in the outside loop.

src/mito2/src/access_layer.rs

@@ -22,7 +22,7 @@ use store_api::metadata::RegionMetadataRef;
 
 use crate::cache::write_cache::SstUploadRequest;
 use crate::cache::CacheManagerRef;
-use crate::config::{FulltextIndexConfig, InvertedIndexConfig};
+use crate::config::{BloomFilterConfig, FulltextIndexConfig, InvertedIndexConfig};
 use crate::error::{CleanDirSnafu, DeleteIndexSnafu, DeleteSstSnafu, OpenDalSnafu, Result};
 use crate::read::Source;
 use crate::region::options::IndexOptions;
@@ -154,6 +154,7 @@ impl AccessLayer {
                 index_options: request.index_options,
                 inverted_index_config: request.inverted_index_config,
                 fulltext_index_config: request.fulltext_index_config,
+                bloom_filter_index_config: request.bloom_filter_index_config,
             }
             .build()
             .await;
@@ -198,6 +199,7 @@ pub(crate) struct SstWriteRequest {
     pub(crate) index_options: IndexOptions,
     pub(crate) inverted_index_config: InvertedIndexConfig,
     pub(crate) fulltext_index_config: FulltextIndexConfig,
+    pub(crate) bloom_filter_index_config: BloomFilterConfig,
 }
 
 pub(crate) async fn new_fs_cache_store(root: &str) -> Result<ObjectStore> {

src/mito2/src/cache.rs

@@ -28,6 +28,7 @@ use std::sync::Arc;
 use bytes::Bytes;
 use datatypes::value::Value;
 use datatypes::vectors::VectorRef;
+use index::bloom_filter_index::{BloomFilterIndexCache, BloomFilterIndexCacheRef};
 use moka::notification::RemovalCause;
 use moka::sync::Cache;
 use parquet::column::page::Page;
@@ -69,6 +70,8 @@ pub struct CacheManager {
     write_cache: Option<WriteCacheRef>,
     /// Cache for inverted index.
     index_cache: Option<InvertedIndexCacheRef>,
+    /// Cache for bloom filter index.
+    bloom_filter_index_cache: Option<BloomFilterIndexCacheRef>,
     /// Puffin metadata cache.
     puffin_metadata_cache: Option<PuffinMetadataCacheRef>,
     /// Cache for time series selectors.
@@ -221,6 +224,10 @@ impl CacheManager {
         self.index_cache.as_ref()
     }
 
+    pub(crate) fn bloom_filter_index_cache(&self) -> Option<&BloomFilterIndexCacheRef> {
+        self.bloom_filter_index_cache.as_ref()
+    }
+
     pub(crate) fn puffin_metadata_cache(&self) -> Option<&PuffinMetadataCacheRef> {
         self.puffin_metadata_cache.as_ref()
     }
@@ -364,6 +371,12 @@ impl CacheManagerBuilder {
             self.index_content_size,
             self.index_content_page_size,
         );
+        // TODO(ruihang): check if it's ok to reuse the same param with inverted index
+        let bloom_filter_index_cache = BloomFilterIndexCache::new(
+            self.index_metadata_size,
+            self.index_content_size,
+            self.index_content_page_size,
+        );
         let puffin_metadata_cache =
             PuffinMetadataCache::new(self.puffin_metadata_size, &CACHE_BYTES);
         let selector_result_cache = (self.selector_result_cache_size != 0).then(|| {
@@ -387,6 +400,7 @@ impl CacheManagerBuilder {
             page_cache,
             write_cache: self.write_cache,
             index_cache: Some(Arc::new(inverted_index_cache)),
+            bloom_filter_index_cache: Some(Arc::new(bloom_filter_index_cache)),
             puffin_metadata_cache: Some(Arc::new(puffin_metadata_cache)),
             selector_result_cache,
         }

src/mito2/src/cache/file_cache.rs

@@ -37,8 +37,10 @@ use crate::sst::file::FileId;
 use crate::sst::parquet::helper::fetch_byte_ranges;
 use crate::sst::parquet::metadata::MetadataLoader;
 
-/// Subdirectory of cached files.
-const FILE_DIR: &str = "files/";
+/// Subdirectory of cached files for write.
+///
+/// This must contain three layers, corresponding to [`build_prometheus_metrics_layer`](object_store::layers::build_prometheus_metrics_layer).
+const FILE_DIR: &str = "cache/object/write/";
 
 /// A file cache manages files on local store and evict files based
 /// on size.

src/mito2/src/cache/index.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod bloom_filter_index;
 pub mod inverted_index;
 
 use std::future::Future;

src/mito2/src/cache/index/bloom_filter_index.rs

@@ -0,0 +1,167 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Bytes;
+use index::bloom_filter::error::Result;
+use index::bloom_filter::reader::BloomFilterReader;
+use index::bloom_filter::BloomFilterMeta;
+use store_api::storage::ColumnId;
+
+use crate::cache::index::{IndexCache, PageKey, INDEX_METADATA_TYPE};
+use crate::metrics::{CACHE_HIT, CACHE_MISS};
+use crate::sst::file::FileId;
+
+const INDEX_TYPE_BLOOM_FILTER_INDEX: &str = "bloom_filter_index";
+
+/// Cache for bloom filter index.
+pub type BloomFilterIndexCache = IndexCache<(FileId, ColumnId), BloomFilterMeta>;
+pub type BloomFilterIndexCacheRef = Arc<BloomFilterIndexCache>;
+
+impl BloomFilterIndexCache {
+    /// Creates a new bloom filter index cache.
+    pub fn new(index_metadata_cap: u64, index_content_cap: u64, page_size: u64) -> Self {
+        Self::new_with_weighter(
+            index_metadata_cap,
+            index_content_cap,
+            page_size,
+            INDEX_TYPE_BLOOM_FILTER_INDEX,
+            bloom_filter_index_metadata_weight,
+            bloom_filter_index_content_weight,
+        )
+    }
+}
+
+/// Calculates weight for bloom filter index metadata.
+fn bloom_filter_index_metadata_weight(k: &(FileId, ColumnId), _: &Arc<BloomFilterMeta>) -> u32 {
+    (k.0.as_bytes().len()
+        + std::mem::size_of::<ColumnId>()
+        + std::mem::size_of::<BloomFilterMeta>()) as u32
+}
+
+/// Calculates weight for bloom filter index content.
+fn bloom_filter_index_content_weight((k, _): &((FileId, ColumnId), PageKey), v: &Bytes) -> u32 {
+    (k.0.as_bytes().len() + std::mem::size_of::<ColumnId>() + v.len()) as u32
+}
+
+/// Bloom filter index blob reader with cache.
+pub struct CachedBloomFilterIndexBlobReader<R> {
+    file_id: FileId,
+    column_id: ColumnId,
+    file_size: u64,
+    inner: R,
+    cache: BloomFilterIndexCacheRef,
+}
+
+impl<R> CachedBloomFilterIndexBlobReader<R> {
+    /// Creates a new bloom filter index blob reader with cache.
+    pub fn new(
+        file_id: FileId,
+        column_id: ColumnId,
+        file_size: u64,
+        inner: R,
+        cache: BloomFilterIndexCacheRef,
+    ) -> Self {
+        Self {
+            file_id,
+            column_id,
+            file_size,
+            inner,
+            cache,
+        }
+    }
+}
+
+#[async_trait]
+impl<R: BloomFilterReader + Send> BloomFilterReader for CachedBloomFilterIndexBlobReader<R> {
+    async fn range_read(&mut self, offset: u64, size: u32) -> Result<Bytes> {
+        let inner = &mut self.inner;
+        self.cache
+            .get_or_load(
+                (self.file_id, self.column_id),
+                self.file_size,
+                offset,
+                size,
+                move |ranges| async move { inner.read_vec(&ranges).await },
+            )
+            .await
+            .map(|b| b.into())
+    }
+
+    /// Reads the meta information of the bloom filter.
+    async fn metadata(&mut self) -> Result<BloomFilterMeta> {
+        if let Some(cached) = self.cache.get_metadata((self.file_id, self.column_id)) {
+            CACHE_HIT.with_label_values(&[INDEX_METADATA_TYPE]).inc();
+            Ok((*cached).clone())
+        } else {
+            let meta = self.inner.metadata().await?;
+            self.cache
+                .put_metadata((self.file_id, self.column_id), Arc::new(meta.clone()));
+            CACHE_MISS.with_label_values(&[INDEX_METADATA_TYPE]).inc();
+            Ok(meta)
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use rand::{Rng, RngCore};
+
+    use super::*;
+
+    const FUZZ_REPEAT_TIMES: usize = 100;
+
+    #[test]
+    fn fuzz_index_calculation() {
+        let mut rng = rand::thread_rng();
+        let mut data = vec![0u8; 1024 * 1024];
+        rng.fill_bytes(&mut data);
+
+        for _ in 0..FUZZ_REPEAT_TIMES {
+            let offset = rng.gen_range(0..data.len() as u64);
+            let size = rng.gen_range(0..data.len() as u32 - offset as u32);
+            let page_size: usize = rng.gen_range(1..1024);
+
+            let indexes =
+                PageKey::generate_page_keys(offset, size, page_size as u64).collect::<Vec<_>>();
+            let page_num = indexes.len();
+            let mut read = Vec::with_capacity(size as usize);
+            for key in indexes.into_iter() {
+                let start = key.page_id as usize * page_size;
+                let page = if start + page_size < data.len() {
+                    &data[start..start + page_size]
+                } else {
+                    &data[start..]
+                };
+                read.extend_from_slice(page);
+            }
+            let expected_range = offset as usize..(offset + size as u64 as u64) as usize;
+            let read = read[PageKey::calculate_range(offset, size, page_size as u64)].to_vec();
+            assert_eq!(
+                read,
+                data.get(expected_range).unwrap(),
+                "fuzz_read_index failed, offset: {}, size: {}, page_size: {}\nread len: {}, expected len: {}\nrange: {:?}, page num: {}",
+                offset,
+                size,
+                page_size,
+                read.len(),
+                size as usize,
+                PageKey::calculate_range(offset, size, page_size as u64),
+                page_num
+            );
+        }
+    }
+}

src/mito2/src/cache/write_cache.rs

@@ -20,7 +20,6 @@ use std::time::Duration;
 use common_base::readable_size::ReadableSize;
 use common_telemetry::{debug, info};
 use futures::AsyncWriteExt;
-use object_store::manager::ObjectStoreManagerRef;
 use object_store::ObjectStore;
 use snafu::ResultExt;
 
@@ -44,10 +43,6 @@ use crate::sst::{DEFAULT_WRITE_BUFFER_SIZE, DEFAULT_WRITE_CONCURRENCY};
 pub struct WriteCache {
     /// Local file cache.
     file_cache: FileCacheRef,
-    /// Object store manager.
-    #[allow(unused)]
-    /// TODO: Remove unused after implementing async write cache
-    object_store_manager: ObjectStoreManagerRef,
     /// Puffin manager factory for index.
     puffin_manager_factory: PuffinManagerFactory,
     /// Intermediate manager for index.
@@ -61,7 +56,6 @@ impl WriteCache {
     /// `object_store_manager` for all object stores.
     pub async fn new(
         local_store: ObjectStore,
-        object_store_manager: ObjectStoreManagerRef,
         cache_capacity: ReadableSize,
         ttl: Option<Duration>,
         puffin_manager_factory: PuffinManagerFactory,
@@ -72,7 +66,6 @@ impl WriteCache {
 
         Ok(Self {
             file_cache,
-            object_store_manager,
             puffin_manager_factory,
             intermediate_manager,
         })
@@ -81,7 +74,6 @@ impl WriteCache {
     /// Creates a write cache based on local fs.
     pub async fn new_fs(
         cache_dir: &str,
-        object_store_manager: ObjectStoreManagerRef,
         cache_capacity: ReadableSize,
         ttl: Option<Duration>,
         puffin_manager_factory: PuffinManagerFactory,
@@ -92,7 +84,6 @@ impl WriteCache {
         let local_store = new_fs_cache_store(cache_dir).await?;
         Self::new(
             local_store,
-            object_store_manager,
             cache_capacity,
             ttl,
             puffin_manager_factory,
@@ -134,6 +125,7 @@ impl WriteCache {
             index_options: write_request.index_options,
             inverted_index_config: write_request.inverted_index_config,
             fulltext_index_config: write_request.fulltext_index_config,
+            bloom_filter_index_config: write_request.bloom_filter_index_config,
         }
         .build()
         .await;
@@ -387,6 +379,7 @@ mod tests {
             index_options: IndexOptions::default(),
             inverted_index_config: Default::default(),
             fulltext_index_config: Default::default(),
+            bloom_filter_index_config: Default::default(),
         };
 
         let upload_request = SstUploadRequest {
@@ -479,6 +472,7 @@ mod tests {
             index_options: IndexOptions::default(),
             inverted_index_config: Default::default(),
             fulltext_index_config: Default::default(),
+            bloom_filter_index_config: Default::default(),
         };
         let write_opts = WriteOptions {
             row_group_size: 512,

src/mito2/src/compaction/compactor.rs

@@ -21,7 +21,6 @@ use common_telemetry::{info, warn};
 use common_time::TimeToLive;
 use object_store::manager::ObjectStoreManagerRef;
 use serde::{Deserialize, Serialize};
-use smallvec::SmallVec;
 use snafu::{OptionExt, ResultExt};
 use store_api::metadata::RegionMetadataRef;
 use store_api::storage::RegionId;
@@ -41,7 +40,7 @@ use crate::region::options::RegionOptions;
 use crate::region::version::VersionRef;
 use crate::region::{ManifestContext, RegionLeaderState, RegionRoleState};
 use crate::schedule::scheduler::LocalScheduler;
-use crate::sst::file::{FileMeta, IndexType};
+use crate::sst::file::FileMeta;
 use crate::sst::file_purger::LocalFilePurger;
 use crate::sst::index::intermediate::IntermediateManager;
 use crate::sst::index::puffin_manager::PuffinManagerFactory;
@@ -302,6 +301,8 @@ impl Compactor for DefaultCompactor {
             let merge_mode = compaction_region.current_version.options.merge_mode();
             let inverted_index_config = compaction_region.engine_config.inverted_index.clone();
             let fulltext_index_config = compaction_region.engine_config.fulltext_index.clone();
+            let bloom_filter_index_config =
+                compaction_region.engine_config.bloom_filter_index.clone();
             futs.push(async move {
                 let reader = CompactionSstReaderBuilder {
                     metadata: region_metadata.clone(),
@@ -326,6 +327,7 @@ impl Compactor for DefaultCompactor {
                             index_options,
                             inverted_index_config,
                             fulltext_index_config,
+                            bloom_filter_index_config,
                         },
                         &write_opts,
                     )
@@ -336,16 +338,7 @@ impl Compactor for DefaultCompactor {
                         time_range: sst_info.time_range,
                         level: output.output_level,
                         file_size: sst_info.file_size,
-                        available_indexes: {
-                            let mut indexes = SmallVec::new();
-                            if sst_info.index_metadata.inverted_index.is_available() {
-                                indexes.push(IndexType::InvertedIndex);
-                            }
-                            if sst_info.index_metadata.fulltext_index.is_available() {
-                                indexes.push(IndexType::FulltextIndex);
-                            }
-                            indexes
-                        },
+                        available_indexes: sst_info.index_metadata.build_available_indexes(),
                         index_file_size: sst_info.index_metadata.file_size,
                         num_rows: sst_info.num_rows as u64,
                         num_row_groups: sst_info.num_row_groups,

src/mito2/src/config.rs

@@ -20,8 +20,6 @@ use std::time::Duration;
 
 use common_base::readable_size::ReadableSize;
 use common_telemetry::warn;
-use object_store::util::join_dir;
-use object_store::OBJECT_CACHE_DIR;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 
@@ -97,7 +95,7 @@ pub struct MitoConfig {
     pub selector_result_cache_size: ReadableSize,
     /// Whether to enable the experimental write cache.
     pub enable_experimental_write_cache: bool,
-    /// File system path for write cache, defaults to `{data_home}/object_cache/write`.
+    /// File system path for write cache dir's root, defaults to `{data_home}`.
     pub experimental_write_cache_path: String,
     /// Capacity for write cache.
     pub experimental_write_cache_size: ReadableSize,
@@ -119,6 +117,8 @@ pub struct MitoConfig {
     pub inverted_index: InvertedIndexConfig,
     /// Full-text index configs.
     pub fulltext_index: FulltextIndexConfig,
+    /// Bloom filter index configs.
+    pub bloom_filter_index: BloomFilterConfig,
 
     /// Memtable config
     pub memtable: MemtableConfig,
@@ -157,6 +157,7 @@ impl Default for MitoConfig {
             index: IndexConfig::default(),
             inverted_index: InvertedIndexConfig::default(),
             fulltext_index: FulltextIndexConfig::default(),
+            bloom_filter_index: BloomFilterConfig::default(),
             memtable: MemtableConfig::default(),
             min_compaction_interval: Duration::from_secs(0),
         };
@@ -234,8 +235,7 @@ impl MitoConfig {
 
         // Sets write cache path if it is empty.
         if self.experimental_write_cache_path.trim().is_empty() {
-            let object_cache_path = join_dir(data_home, OBJECT_CACHE_DIR);
-            self.experimental_write_cache_path = join_dir(&object_cache_path, "write");
+            self.experimental_write_cache_path = data_home.to_string();
         }
 
         self.index.sanitize(data_home, &self.inverted_index)?;
@@ -514,6 +514,48 @@ impl FulltextIndexConfig {
     }
 }
 
+/// Configuration options for the bloom filter.
+#[serde_as]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+pub struct BloomFilterConfig {
+    /// Whether to create the index on flush: automatically or never.
+    pub create_on_flush: Mode,
+    /// Whether to create the index on compaction: automatically or never.
+    pub create_on_compaction: Mode,
+    /// Whether to apply the index on query: automatically or never.
+    pub apply_on_query: Mode,
+    /// Memory threshold for creating the index.
+    pub mem_threshold_on_create: MemoryThreshold,
+}
+
+impl Default for BloomFilterConfig {
+    fn default() -> Self {
+        Self {
+            create_on_flush: Mode::Auto,
+            create_on_compaction: Mode::Auto,
+            apply_on_query: Mode::Auto,
+            mem_threshold_on_create: MemoryThreshold::Auto,
+        }
+    }
+}
+
+impl BloomFilterConfig {
+    pub fn mem_threshold_on_create(&self) -> Option<usize> {
+        match self.mem_threshold_on_create {
+            MemoryThreshold::Auto => {
+                if let Some(sys_memory) = common_config::utils::get_sys_total_memory() {
+                    Some((sys_memory / INDEX_CREATE_MEM_THRESHOLD_FACTOR).as_bytes() as usize)
+                } else {
+                    Some(ReadableSize::mb(64).as_bytes() as usize)
+                }
+            }
+            MemoryThreshold::Unlimited => None,
+            MemoryThreshold::Size(size) => Some(size.as_bytes() as usize),
+        }
+    }
+}
+
 /// Divide cpu num by a non-zero `divisor` and returns at least 1.
 fn divide_num_cpus(divisor: usize) -> usize {
     debug_assert!(divisor > 0);

src/mito2/src/engine.rs

@@ -433,6 +433,7 @@ impl EngineInner {
         .with_parallel_scan_channel_size(self.config.parallel_scan_channel_size)
         .with_ignore_inverted_index(self.config.inverted_index.apply_on_query.disabled())
         .with_ignore_fulltext_index(self.config.fulltext_index.apply_on_query.disabled())
+        .with_ignore_bloom_filter(self.config.bloom_filter_index.apply_on_query.disabled())
         .with_start_time(query_start);
 
         Ok(scan_region)

src/mito2/src/error.rs

@@ -576,6 +576,13 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to apply bloom filter index"))]
+    ApplyBloomFilterIndex {
+        source: index::bloom_filter::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Failed to push index value"))]
     PushIndexValue {
         source: index::inverted_index::error::Error,
@@ -816,8 +823,8 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to retrieve fulltext options from column metadata"))]
-    FulltextOptions {
+    #[snafu(display("Failed to retrieve index options from column metadata"))]
+    IndexOptions {
         #[snafu(implicit)]
         location: Location,
         source: datatypes::error::Error,
@@ -904,6 +911,20 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Failed to push value to bloom filter"))]
+    PushBloomFilterValue {
+        source: index::bloom_filter::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Failed to finish bloom filter"))]
+    BloomFilterFinish {
+        source: index::bloom_filter::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T, E = Error> = std::result::Result<T, E>;
@@ -1008,6 +1029,7 @@ impl ErrorExt for Error {
             EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound,
             ArrowReader { .. } => StatusCode::StorageUnavailable,
             ConvertValue { source, .. } => source.status_code(),
+            ApplyBloomFilterIndex { source, .. } => source.status_code(),
             BuildIndexApplier { source, .. }
             | PushIndexValue { source, .. }
             | ApplyInvertedIndex { source, .. }
@@ -1029,7 +1051,7 @@ impl ErrorExt for Error {
             UnsupportedOperation { .. } => StatusCode::Unsupported,
             RemoteCompaction { .. } => StatusCode::Unexpected,
 
-            FulltextOptions { source, .. } => source.status_code(),
+            IndexOptions { source, .. } => source.status_code(),
             CreateFulltextCreator { source, .. } => source.status_code(),
             CastVector { source, .. } => source.status_code(),
             FulltextPushText { source, .. }
@@ -1039,7 +1061,12 @@ impl ErrorExt for Error {
             RegionBusy { .. } => StatusCode::RegionBusy,
             GetSchemaMetadata { source, .. } => source.status_code(),
             Timeout { .. } => StatusCode::Cancelled,
+
             DecodeArrowRowGroup { .. } => StatusCode::Internal,
+
+            PushBloomFilterValue { source, .. } | BloomFilterFinish { source, .. } => {
+                source.status_code()
+            }
         }
     }
 

src/mito2/src/flush.rs

@@ -19,7 +19,6 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 
 use common_telemetry::{debug, error, info, trace};
-use smallvec::SmallVec;
 use snafu::ResultExt;
 use store_api::storage::RegionId;
 use strum::IntoStaticStr;
@@ -45,7 +44,7 @@ use crate::request::{
     SenderWriteRequest, WorkerRequest,
 };
 use crate::schedule::scheduler::{Job, SchedulerRef};
-use crate::sst::file::{FileId, FileMeta, IndexType};
+use crate::sst::file::{FileId, FileMeta};
 use crate::sst::parquet::WriteOptions;
 use crate::worker::WorkerListener;
 
@@ -361,6 +360,7 @@ impl RegionFlushTask {
                 index_options: self.index_options.clone(),
                 inverted_index_config: self.engine_config.inverted_index.clone(),
                 fulltext_index_config: self.engine_config.fulltext_index.clone(),
+                bloom_filter_index_config: self.engine_config.bloom_filter_index.clone(),
             };
             let Some(sst_info) = self
                 .access_layer
@@ -378,16 +378,7 @@ impl RegionFlushTask {
                 time_range: sst_info.time_range,
                 level: 0,
                 file_size: sst_info.file_size,
-                available_indexes: {
-                    let mut indexes = SmallVec::new();
-                    if sst_info.index_metadata.inverted_index.is_available() {
-                        indexes.push(IndexType::InvertedIndex);
-                    }
-                    if sst_info.index_metadata.fulltext_index.is_available() {
-                        indexes.push(IndexType::FulltextIndex);
-                    }
-                    indexes
-                },
+                available_indexes: sst_info.index_metadata.build_available_indexes(),
                 index_file_size: sst_info.index_metadata.file_size,
                 num_rows: sst_info.num_rows as u64,
                 num_row_groups: sst_info.num_row_groups,

src/mito2/src/read/prune.rs

@@ -113,13 +113,13 @@ impl PruneReader {
         let num_rows_before_filter = batch.num_rows();
         let Some(batch_filtered) = self.context.precise_filter(batch)? else {
             // the entire batch is filtered out
-            self.metrics.filter_metrics.num_rows_precise_filtered += num_rows_before_filter;
+            self.metrics.filter_metrics.rows_precise_filtered += num_rows_before_filter;
             return Ok(None);
         };
 
         // update metric
         let filtered_rows = num_rows_before_filter - batch_filtered.num_rows();
-        self.metrics.filter_metrics.num_rows_precise_filtered += filtered_rows;
+        self.metrics.filter_metrics.rows_precise_filtered += filtered_rows;
 
         if !batch_filtered.is_empty() {
             Ok(Some(batch_filtered))

src/mito2/src/read/scan_region.rs

@@ -47,6 +47,9 @@ use crate::read::{Batch, Source};
 use crate::region::options::MergeMode;
 use crate::region::version::VersionRef;
 use crate::sst::file::FileHandle;
+use crate::sst::index::bloom_filter::applier::{
+    BloomFilterIndexApplierBuilder, BloomFilterIndexApplierRef,
+};
 use crate::sst::index::fulltext_index::applier::builder::FulltextIndexApplierBuilder;
 use crate::sst::index::fulltext_index::applier::FulltextIndexApplierRef;
 use crate::sst::index::inverted_index::applier::builder::InvertedIndexApplierBuilder;
@@ -175,6 +178,8 @@ pub(crate) struct ScanRegion {
     ignore_inverted_index: bool,
     /// Whether to ignore fulltext index.
     ignore_fulltext_index: bool,
+    /// Whether to ignore bloom filter.
+    ignore_bloom_filter: bool,
     /// Start time of the scan task.
     start_time: Option<Instant>,
 }
@@ -195,6 +200,7 @@ impl ScanRegion {
             parallel_scan_channel_size: DEFAULT_SCAN_CHANNEL_SIZE,
             ignore_inverted_index: false,
             ignore_fulltext_index: false,
+            ignore_bloom_filter: false,
             start_time: None,
         }
     }
@@ -223,6 +229,13 @@ impl ScanRegion {
         self
     }
 
+    /// Sets whether to ignore bloom filter.
+    #[must_use]
+    pub(crate) fn with_ignore_bloom_filter(mut self, ignore: bool) -> Self {
+        self.ignore_bloom_filter = ignore;
+        self
+    }
+
     #[must_use]
     pub(crate) fn with_start_time(mut self, now: Instant) -> Self {
         self.start_time = Some(now);
@@ -322,6 +335,7 @@ impl ScanRegion {
         self.maybe_remove_field_filters();
 
         let inverted_index_applier = self.build_invereted_index_applier();
+        let bloom_filter_applier = self.build_bloom_filter_applier();
         let fulltext_index_applier = self.build_fulltext_index_applier();
         let predicate = Predicate::new(self.request.filters.clone());
         // The mapper always computes projected column ids as the schema of SSTs may change.
@@ -345,6 +359,7 @@ impl ScanRegion {
             .with_files(files)
             .with_cache(self.cache_manager)
             .with_inverted_index_applier(inverted_index_applier)
+            .with_bloom_filter_index_applier(bloom_filter_applier)
             .with_fulltext_index_applier(fulltext_index_applier)
             .with_parallel_scan_channel_size(self.parallel_scan_channel_size)
             .with_start_time(self.start_time)
@@ -448,6 +463,47 @@ impl ScanRegion {
         .map(Arc::new)
     }
 
+    /// Use the latest schema to build the bloom filter index applier.
+    fn build_bloom_filter_applier(&self) -> Option<BloomFilterIndexApplierRef> {
+        if self.ignore_bloom_filter {
+            return None;
+        }
+
+        let file_cache = || -> Option<FileCacheRef> {
+            let cache_manager = self.cache_manager.as_ref()?;
+            let write_cache = cache_manager.write_cache()?;
+            let file_cache = write_cache.file_cache();
+            Some(file_cache)
+        }();
+
+        let index_cache = self
+            .cache_manager
+            .as_ref()
+            .and_then(|c| c.bloom_filter_index_cache())
+            .cloned();
+
+        let puffin_metadata_cache = self
+            .cache_manager
+            .as_ref()
+            .and_then(|c| c.puffin_metadata_cache())
+            .cloned();
+
+        BloomFilterIndexApplierBuilder::new(
+            self.access_layer.region_dir().to_string(),
+            self.access_layer.object_store().clone(),
+            self.version.metadata.as_ref(),
+            self.access_layer.puffin_manager_factory().clone(),
+        )
+        .with_file_cache(file_cache)
+        .with_bloom_filter_index_cache(index_cache)
+        .with_puffin_metadata_cache(puffin_metadata_cache)
+        .build(&self.request.filters)
+        .inspect_err(|err| warn!(err; "Failed to build bloom filter index applier"))
+        .ok()
+        .flatten()
+        .map(Arc::new)
+    }
+
     /// Use the latest schema to build the fulltext index applier.
     fn build_fulltext_index_applier(&self) -> Option<FulltextIndexApplierRef> {
         if self.ignore_fulltext_index {
@@ -501,6 +557,7 @@ pub(crate) struct ScanInput {
     pub(crate) parallel_scan_channel_size: usize,
     /// Index appliers.
     inverted_index_applier: Option<InvertedIndexApplierRef>,
+    bloom_filter_index_applier: Option<BloomFilterIndexApplierRef>,
     fulltext_index_applier: Option<FulltextIndexApplierRef>,
     /// Start time of the query.
     pub(crate) query_start: Option<Instant>,
@@ -529,6 +586,7 @@ impl ScanInput {
             ignore_file_not_found: false,
             parallel_scan_channel_size: DEFAULT_SCAN_CHANNEL_SIZE,
             inverted_index_applier: None,
+            bloom_filter_index_applier: None,
             fulltext_index_applier: None,
             query_start: None,
             append_mode: false,
@@ -600,6 +658,16 @@ impl ScanInput {
         self
     }
 
+    /// Sets bloom filter applier.
+    #[must_use]
+    pub(crate) fn with_bloom_filter_index_applier(
+        mut self,
+        applier: Option<BloomFilterIndexApplierRef>,
+    ) -> Self {
+        self.bloom_filter_index_applier = applier;
+        self
+    }
+
     /// Sets fulltext index applier.
     #[must_use]
     pub(crate) fn with_fulltext_index_applier(
@@ -694,6 +762,7 @@ impl ScanInput {
             .projection(Some(self.mapper.column_ids().to_vec()))
             .cache(self.cache_manager.clone())
             .inverted_index_applier(self.inverted_index_applier.clone())
+            .bloom_filter_index_applier(self.bloom_filter_index_applier.clone())
             .fulltext_index_applier(self.fulltext_index_applier.clone())
             .expected_metadata(Some(self.mapper.metadata().clone()))
             .build_reader_input(reader_metrics)

src/mito2/src/sst/file.rs

@@ -143,6 +143,8 @@ pub enum IndexType {
     InvertedIndex,
     /// Full-text index.
     FulltextIndex,
+    /// Bloom Filter index
+    BloomFilterIndex,
 }
 
 impl FileMeta {
@@ -156,6 +158,12 @@ impl FileMeta {
         self.available_indexes.contains(&IndexType::FulltextIndex)
     }
 
+    /// Returns true if the file has a bloom filter index.
+    pub fn bloom_filter_index_available(&self) -> bool {
+        self.available_indexes
+            .contains(&IndexType::BloomFilterIndex)
+    }
+
     /// Returns the size of the inverted index file
     pub fn inverted_index_size(&self) -> Option<u64> {
         if self.available_indexes.len() == 1 && self.inverted_index_available() {
@@ -173,6 +181,15 @@ impl FileMeta {
             None
         }
     }
+
+    /// Returns the size of the bloom filter index file
+    pub fn bloom_filter_index_size(&self) -> Option<u64> {
+        if self.available_indexes.len() == 1 && self.bloom_filter_index_available() {
+            Some(self.index_file_size)
+        } else {
+            None
+        }
+    }
 }
 
 /// Handle to a SST file.

src/mito2/src/sst/index.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub(crate) mod bloom_filter;
+mod codec;
 pub(crate) mod fulltext_index;
 mod indexer;
 pub(crate) mod intermediate;
@@ -22,24 +24,27 @@ pub(crate) mod store;
 
 use std::num::NonZeroUsize;
 
+use bloom_filter::creator::BloomFilterIndexer;
 use common_telemetry::{debug, warn};
 use puffin_manager::SstPuffinManager;
+use smallvec::SmallVec;
 use statistics::{ByteCount, RowCount};
 use store_api::metadata::RegionMetadataRef;
 use store_api::storage::{ColumnId, RegionId};
 
 use crate::access_layer::OperationType;
-use crate::config::{FulltextIndexConfig, InvertedIndexConfig};
+use crate::config::{BloomFilterConfig, FulltextIndexConfig, InvertedIndexConfig};
 use crate::metrics::INDEX_CREATE_MEMORY_USAGE;
 use crate::read::Batch;
 use crate::region::options::IndexOptions;
-use crate::sst::file::FileId;
+use crate::sst::file::{FileId, IndexType};
 use crate::sst::index::fulltext_index::creator::FulltextIndexer;
 use crate::sst::index::intermediate::IntermediateManager;
 use crate::sst::index::inverted_index::creator::InvertedIndexer;
 
 pub(crate) const TYPE_INVERTED_INDEX: &str = "inverted_index";
 pub(crate) const TYPE_FULLTEXT_INDEX: &str = "fulltext_index";
+pub(crate) const TYPE_BLOOM_FILTER_INDEX: &str = "bloom_filter_index";
 
 /// Output of the index creation.
 #[derive(Debug, Clone, Default)]
@@ -50,6 +55,24 @@ pub struct IndexOutput {
     pub inverted_index: InvertedIndexOutput,
     /// Fulltext index output.
     pub fulltext_index: FulltextIndexOutput,
+    /// Bloom filter output.
+    pub bloom_filter: BloomFilterOutput,
+}
+
+impl IndexOutput {
+    pub fn build_available_indexes(&self) -> SmallVec<[IndexType; 4]> {
+        let mut indexes = SmallVec::new();
+        if self.inverted_index.is_available() {
+            indexes.push(IndexType::InvertedIndex);
+        }
+        if self.fulltext_index.is_available() {
+            indexes.push(IndexType::FulltextIndex);
+        }
+        if self.bloom_filter.is_available() {
+            indexes.push(IndexType::BloomFilterIndex);
+        }
+        indexes
+    }
 }
 
 /// Base output of the index creation.
@@ -73,6 +96,8 @@ impl IndexBaseOutput {
 pub type InvertedIndexOutput = IndexBaseOutput;
 /// Output of the fulltext index creation.
 pub type FulltextIndexOutput = IndexBaseOutput;
+/// Output of the bloom filter creation.
+pub type BloomFilterOutput = IndexBaseOutput;
 
 /// The index creator that hides the error handling details.
 #[derive(Default)]
@@ -86,6 +111,8 @@ pub struct Indexer {
     last_mem_inverted_index: usize,
     fulltext_indexer: Option<FulltextIndexer>,
     last_mem_fulltext_index: usize,
+    bloom_filter_indexer: Option<BloomFilterIndexer>,
+    last_mem_bloom_filter: usize,
 }
 
 impl Indexer {
@@ -129,6 +156,15 @@ impl Indexer {
             .with_label_values(&[TYPE_FULLTEXT_INDEX])
             .add(fulltext_mem as i64 - self.last_mem_fulltext_index as i64);
         self.last_mem_fulltext_index = fulltext_mem;
+
+        let bloom_filter_mem = self
+            .bloom_filter_indexer
+            .as_ref()
+            .map_or(0, |creator| creator.memory_usage());
+        INDEX_CREATE_MEMORY_USAGE
+            .with_label_values(&[TYPE_BLOOM_FILTER_INDEX])
+            .add(bloom_filter_mem as i64 - self.last_mem_bloom_filter as i64);
+        self.last_mem_bloom_filter = bloom_filter_mem;
     }
 }
 
@@ -143,6 +179,7 @@ pub(crate) struct IndexerBuilder<'a> {
     pub(crate) index_options: IndexOptions,
     pub(crate) inverted_index_config: InvertedIndexConfig,
     pub(crate) fulltext_index_config: FulltextIndexConfig,
+    pub(crate) bloom_filter_index_config: BloomFilterConfig,
 }
 
 impl<'a> IndexerBuilder<'a> {
@@ -158,7 +195,11 @@ impl<'a> IndexerBuilder<'a> {
 
         indexer.inverted_indexer = self.build_inverted_indexer();
         indexer.fulltext_indexer = self.build_fulltext_indexer().await;
-        if indexer.inverted_indexer.is_none() && indexer.fulltext_indexer.is_none() {
+        indexer.bloom_filter_indexer = self.build_bloom_filter_indexer();
+        if indexer.inverted_indexer.is_none()
+            && indexer.fulltext_indexer.is_none()
+            && indexer.bloom_filter_indexer.is_none()
+        {
             indexer.abort().await;
             return Indexer::default();
         }
@@ -266,7 +307,7 @@ impl<'a> IndexerBuilder<'a> {
 
         if cfg!(any(test, feature = "test")) {
             panic!(
-                "Failed to create full-text indexer, region_id: {}, file_id: {}, err: {}",
+                "Failed to create full-text indexer, region_id: {}, file_id: {}, err: {:?}",
                 self.metadata.region_id, self.file_id, err
             );
         } else {
@@ -278,6 +319,56 @@ impl<'a> IndexerBuilder<'a> {
 
         None
     }
+
+    fn build_bloom_filter_indexer(&self) -> Option<BloomFilterIndexer> {
+        let create = match self.op_type {
+            OperationType::Flush => self.bloom_filter_index_config.create_on_flush.auto(),
+            OperationType::Compact => self.bloom_filter_index_config.create_on_compaction.auto(),
+        };
+
+        if !create {
+            debug!(
+                "Skip creating bloom filter due to config, region_id: {}, file_id: {}",
+                self.metadata.region_id, self.file_id,
+            );
+            return None;
+        }
+
+        let mem_limit = self.bloom_filter_index_config.mem_threshold_on_create();
+        let indexer = BloomFilterIndexer::new(
+            self.file_id,
+            self.metadata,
+            self.intermediate_manager.clone(),
+            mem_limit,
+        );
+
+        let err = match indexer {
+            Ok(indexer) => {
+                if indexer.is_none() {
+                    debug!(
+                        "Skip creating bloom filter due to no columns require indexing, region_id: {}, file_id: {}",
+                        self.metadata.region_id, self.file_id,
+                    );
+                }
+                return indexer;
+            }
+            Err(err) => err,
+        };
+
+        if cfg!(any(test, feature = "test")) {
+            panic!(
+                "Failed to create bloom filter, region_id: {}, file_id: {}, err: {:?}",
+                self.metadata.region_id, self.file_id, err
+            );
+        } else {
+            warn!(
+                err; "Failed to create bloom filter, region_id: {}, file_id: {}",
+                self.metadata.region_id, self.file_id,
+            );
+        }
+
+        None
+    }
 }
 
 #[cfg(test)]
@@ -286,7 +377,9 @@ mod tests {
 
     use api::v1::SemanticType;
     use datatypes::data_type::ConcreteDataType;
-    use datatypes::schema::{ColumnSchema, FulltextOptions};
+    use datatypes::schema::{
+        ColumnSchema, FulltextOptions, SkippingIndexOptions, SkippingIndexType,
+    };
     use object_store::services::Memory;
     use object_store::ObjectStore;
     use puffin_manager::PuffinManagerFactory;
@@ -298,12 +391,14 @@ mod tests {
     struct MetaConfig {
         with_tag: bool,
         with_fulltext: bool,
+        with_skipping_bloom: bool,
     }
 
     fn mock_region_metadata(
         MetaConfig {
             with_tag,
             with_fulltext,
+            with_skipping_bloom,
         }: MetaConfig,
     ) -> RegionMetadataRef {
         let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 2));
@@ -354,6 +449,24 @@ mod tests {
             builder.push_column_metadata(column);
         }
 
+        if with_skipping_bloom {
+            let column_schema =
+                ColumnSchema::new("bloom", ConcreteDataType::string_datatype(), false)
+                    .with_skipping_options(SkippingIndexOptions {
+                        granularity: 42,
+                        index_type: SkippingIndexType::BloomFilter,
+                    })
+                    .unwrap();
+
+            let column = ColumnMetadata {
+                column_schema,
+                semantic_type: SemanticType::Field,
+                column_id: 5,
+            };
+
+            builder.push_column_metadata(column);
+        }
+
         Arc::new(builder.build().unwrap())
     }
 
@@ -374,6 +487,7 @@ mod tests {
         let metadata = mock_region_metadata(MetaConfig {
             with_tag: true,
             with_fulltext: true,
+            with_skipping_bloom: true,
         });
         let indexer = IndexerBuilder {
             op_type: OperationType::Flush,
@@ -386,12 +500,14 @@ mod tests {
             index_options: IndexOptions::default(),
             inverted_index_config: InvertedIndexConfig::default(),
             fulltext_index_config: FulltextIndexConfig::default(),
+            bloom_filter_index_config: BloomFilterConfig::default(),
         }
         .build()
         .await;
 
         assert!(indexer.inverted_indexer.is_some());
         assert!(indexer.fulltext_indexer.is_some());
+        assert!(indexer.bloom_filter_indexer.is_some());
     }
 
     #[tokio::test]
@@ -403,6 +519,7 @@ mod tests {
         let metadata = mock_region_metadata(MetaConfig {
             with_tag: true,
             with_fulltext: true,
+            with_skipping_bloom: true,
         });
         let indexer = IndexerBuilder {
             op_type: OperationType::Flush,
@@ -418,12 +535,37 @@ mod tests {
                 ..Default::default()
             },
             fulltext_index_config: FulltextIndexConfig::default(),
+            bloom_filter_index_config: BloomFilterConfig::default(),
         }
         .build()
         .await;
 
         assert!(indexer.inverted_indexer.is_none());
         assert!(indexer.fulltext_indexer.is_some());
+        assert!(indexer.bloom_filter_indexer.is_some());
+
+        let indexer = IndexerBuilder {
+            op_type: OperationType::Compact,
+            file_id: FileId::random(),
+            file_path: "test".to_string(),
+            metadata: &metadata,
+            row_group_size: 1024,
+            puffin_manager: factory.build(mock_object_store()),
+            intermediate_manager: intm_manager.clone(),
+            index_options: IndexOptions::default(),
+            inverted_index_config: InvertedIndexConfig::default(),
+            fulltext_index_config: FulltextIndexConfig {
+                create_on_compaction: Mode::Disable,
+                ..Default::default()
+            },
+            bloom_filter_index_config: BloomFilterConfig::default(),
+        }
+        .build()
+        .await;
+
+        assert!(indexer.inverted_indexer.is_some());
+        assert!(indexer.fulltext_indexer.is_none());
+        assert!(indexer.bloom_filter_indexer.is_some());
 
         let indexer = IndexerBuilder {
             op_type: OperationType::Compact,
@@ -435,7 +577,8 @@ mod tests {
             intermediate_manager: intm_manager,
             index_options: IndexOptions::default(),
             inverted_index_config: InvertedIndexConfig::default(),
-            fulltext_index_config: FulltextIndexConfig {
+            fulltext_index_config: FulltextIndexConfig::default(),
+            bloom_filter_index_config: BloomFilterConfig {
                 create_on_compaction: Mode::Disable,
                 ..Default::default()
             },
@@ -444,7 +587,8 @@ mod tests {
         .await;
 
         assert!(indexer.inverted_indexer.is_some());
-        assert!(indexer.fulltext_indexer.is_none());
+        assert!(indexer.fulltext_indexer.is_some());
+        assert!(indexer.bloom_filter_indexer.is_none());
     }
 
     #[tokio::test]
@@ -456,6 +600,7 @@ mod tests {
         let metadata = mock_region_metadata(MetaConfig {
             with_tag: false,
             with_fulltext: true,
+            with_skipping_bloom: true,
         });
         let indexer = IndexerBuilder {
             op_type: OperationType::Flush,
@@ -468,16 +613,44 @@ mod tests {
             index_options: IndexOptions::default(),
             inverted_index_config: InvertedIndexConfig::default(),
             fulltext_index_config: FulltextIndexConfig::default(),
+            bloom_filter_index_config: BloomFilterConfig::default(),
         }
         .build()
         .await;
 
         assert!(indexer.inverted_indexer.is_none());
         assert!(indexer.fulltext_indexer.is_some());
+        assert!(indexer.bloom_filter_indexer.is_some());
 
         let metadata = mock_region_metadata(MetaConfig {
             with_tag: true,
             with_fulltext: false,
+            with_skipping_bloom: true,
+        });
+        let indexer = IndexerBuilder {
+            op_type: OperationType::Flush,
+            file_id: FileId::random(),
+            file_path: "test".to_string(),
+            metadata: &metadata,
+            row_group_size: 1024,
+            puffin_manager: factory.build(mock_object_store()),
+            intermediate_manager: intm_manager.clone(),
+            index_options: IndexOptions::default(),
+            inverted_index_config: InvertedIndexConfig::default(),
+            fulltext_index_config: FulltextIndexConfig::default(),
+            bloom_filter_index_config: BloomFilterConfig::default(),
+        }
+        .build()
+        .await;
+
+        assert!(indexer.inverted_indexer.is_some());
+        assert!(indexer.fulltext_indexer.is_none());
+        assert!(indexer.bloom_filter_indexer.is_some());
+
+        let metadata = mock_region_metadata(MetaConfig {
+            with_tag: true,
+            with_fulltext: true,
+            with_skipping_bloom: false,
         });
         let indexer = IndexerBuilder {
             op_type: OperationType::Flush,
@@ -490,12 +663,14 @@ mod tests {
             index_options: IndexOptions::default(),
             inverted_index_config: InvertedIndexConfig::default(),
             fulltext_index_config: FulltextIndexConfig::default(),
+            bloom_filter_index_config: BloomFilterConfig::default(),
         }
         .build()
         .await;
 
         assert!(indexer.inverted_indexer.is_some());
-        assert!(indexer.fulltext_indexer.is_none());
+        assert!(indexer.fulltext_indexer.is_some());
+        assert!(indexer.bloom_filter_indexer.is_none());
     }
 
     #[tokio::test]
@@ -507,6 +682,7 @@ mod tests {
         let metadata = mock_region_metadata(MetaConfig {
             with_tag: true,
             with_fulltext: true,
+            with_skipping_bloom: true,
         });
         let indexer = IndexerBuilder {
             op_type: OperationType::Flush,
@@ -519,6 +695,7 @@ mod tests {
             index_options: IndexOptions::default(),
             inverted_index_config: InvertedIndexConfig::default(),
             fulltext_index_config: FulltextIndexConfig::default(),
+            bloom_filter_index_config: BloomFilterConfig::default(),
         }
         .build()
         .await;

src/mito2/src/sst/index/bloom_filter.rs

@@ -0,0 +1,18 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) mod applier;
+pub(crate) mod creator;
+
+const INDEX_BLOB_TYPE: &str = "greptime-bloom-filter-v1";

src/mito2/src/sst/index/bloom_filter/applier.rs

@@ -0,0 +1,722 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::sync::Arc;
+
+use common_base::range_read::RangeReader;
+use common_telemetry::warn;
+use datafusion_common::ScalarValue;
+use datafusion_expr::expr::InList;
+use datafusion_expr::{BinaryExpr, Expr, Operator};
+use datatypes::data_type::ConcreteDataType;
+use datatypes::value::Value;
+use index::bloom_filter::applier::{BloomFilterApplier, InListPredicate, Predicate};
+use index::bloom_filter::reader::{BloomFilterReader, BloomFilterReaderImpl};
+use object_store::ObjectStore;
+use parquet::arrow::arrow_reader::RowSelection;
+use parquet::file::metadata::RowGroupMetaData;
+use puffin::puffin_manager::cache::PuffinMetadataCacheRef;
+use puffin::puffin_manager::{BlobGuard, PuffinManager, PuffinReader};
+use snafu::{OptionExt, ResultExt};
+use store_api::metadata::RegionMetadata;
+use store_api::storage::{ColumnId, RegionId};
+
+use super::INDEX_BLOB_TYPE;
+use crate::cache::file_cache::{FileCacheRef, FileType, IndexKey};
+use crate::cache::index::bloom_filter_index::{
+    BloomFilterIndexCacheRef, CachedBloomFilterIndexBlobReader,
+};
+use crate::error::{
+    ApplyBloomFilterIndexSnafu, ColumnNotFoundSnafu, ConvertValueSnafu, MetadataSnafu,
+    PuffinBuildReaderSnafu, PuffinReadBlobSnafu, Result,
+};
+use crate::metrics::INDEX_APPLY_ELAPSED;
+use crate::row_converter::SortField;
+use crate::sst::file::FileId;
+use crate::sst::index::codec::IndexValueCodec;
+use crate::sst::index::puffin_manager::{BlobReader, PuffinManagerFactory};
+use crate::sst::index::TYPE_BLOOM_FILTER_INDEX;
+use crate::sst::location;
+
+pub(crate) type BloomFilterIndexApplierRef = Arc<BloomFilterIndexApplier>;
+
+pub struct BloomFilterIndexApplier {
+    region_dir: String,
+    region_id: RegionId,
+    object_store: ObjectStore,
+    file_cache: Option<FileCacheRef>,
+    puffin_manager_factory: PuffinManagerFactory,
+    puffin_metadata_cache: Option<PuffinMetadataCacheRef>,
+    bloom_filter_index_cache: Option<BloomFilterIndexCacheRef>,
+    filters: HashMap<ColumnId, Vec<Predicate>>,
+}
+
+impl BloomFilterIndexApplier {
+    pub fn new(
+        region_dir: String,
+        region_id: RegionId,
+        object_store: ObjectStore,
+        puffin_manager_factory: PuffinManagerFactory,
+        filters: HashMap<ColumnId, Vec<Predicate>>,
+    ) -> Self {
+        Self {
+            region_dir,
+            region_id,
+            object_store,
+            file_cache: None,
+            puffin_manager_factory,
+            puffin_metadata_cache: None,
+            bloom_filter_index_cache: None,
+            filters,
+        }
+    }
+
+    pub fn with_file_cache(mut self, file_cache: Option<FileCacheRef>) -> Self {
+        self.file_cache = file_cache;
+        self
+    }
+
+    pub fn with_puffin_metadata_cache(
+        mut self,
+        puffin_metadata_cache: Option<PuffinMetadataCacheRef>,
+    ) -> Self {
+        self.puffin_metadata_cache = puffin_metadata_cache;
+        self
+    }
+
+    pub fn with_bloom_filter_cache(
+        mut self,
+        bloom_filter_index_cache: Option<BloomFilterIndexCacheRef>,
+    ) -> Self {
+        self.bloom_filter_index_cache = bloom_filter_index_cache;
+        self
+    }
+
+    /// Applies bloom filter predicates to the provided SST file and returns a bitmap
+    /// indicating which segments may contain matching rows
+    pub async fn apply(
+        &self,
+        file_id: FileId,
+        file_size_hint: Option<u64>,
+        row_group_metas: &[RowGroupMetaData],
+        basement: &mut BTreeMap<usize, Option<RowSelection>>,
+    ) -> Result<()> {
+        let _timer = INDEX_APPLY_ELAPSED
+            .with_label_values(&[TYPE_BLOOM_FILTER_INDEX])
+            .start_timer();
+
+        for (column_id, predicates) in &self.filters {
+            let mut blob = match self.cached_blob_reader(file_id, *column_id).await {
+                Ok(Some(puffin_reader)) => puffin_reader,
+                other => {
+                    if let Err(err) = other {
+                        warn!(err; "An unexpected error occurred while reading the cached index file. Fallback to remote index file.")
+                    }
+                    self.remote_blob_reader(file_id, *column_id, file_size_hint)
+                        .await?
+                }
+            };
+
+            // Create appropriate reader based on whether we have caching enabled
+            if let Some(bloom_filter_cache) = &self.bloom_filter_index_cache {
+                let file_size = if let Some(file_size) = file_size_hint {
+                    file_size
+                } else {
+                    blob.metadata().await.context(MetadataSnafu)?.content_length
+                };
+                let reader = CachedBloomFilterIndexBlobReader::new(
+                    file_id,
+                    *column_id,
+                    file_size,
+                    BloomFilterReaderImpl::new(blob),
+                    bloom_filter_cache.clone(),
+                );
+                self.apply_filters(reader, predicates, row_group_metas, basement)
+                    .await
+                    .context(ApplyBloomFilterIndexSnafu)?;
+            } else {
+                let reader = BloomFilterReaderImpl::new(blob);
+                self.apply_filters(reader, predicates, row_group_metas, basement)
+                    .await
+                    .context(ApplyBloomFilterIndexSnafu)?;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Creates a blob reader from the cached index file
+    async fn cached_blob_reader(
+        &self,
+        file_id: FileId,
+        column_id: ColumnId,
+    ) -> Result<Option<BlobReader>> {
+        let Some(file_cache) = &self.file_cache else {
+            return Ok(None);
+        };
+
+        let index_key = IndexKey::new(self.region_id, file_id, FileType::Puffin);
+        if file_cache.get(index_key).await.is_none() {
+            return Ok(None);
+        };
+
+        let puffin_manager = self.puffin_manager_factory.build(file_cache.local_store());
+        let puffin_file_name = file_cache.cache_file_path(index_key);
+
+        let reader = puffin_manager
+            .reader(&puffin_file_name)
+            .await
+            .context(PuffinBuildReaderSnafu)?
+            .blob(&Self::column_blob_name(column_id))
+            .await
+            .context(PuffinReadBlobSnafu)?
+            .reader()
+            .await
+            .context(PuffinBuildReaderSnafu)?;
+        Ok(Some(reader))
+    }
+
+    // TODO(ruihang): use the same util with the code in creator
+    fn column_blob_name(column_id: ColumnId) -> String {
+        format!("{INDEX_BLOB_TYPE}-{column_id}")
+    }
+
+    /// Creates a blob reader from the remote index file
+    async fn remote_blob_reader(
+        &self,
+        file_id: FileId,
+        column_id: ColumnId,
+        file_size_hint: Option<u64>,
+    ) -> Result<BlobReader> {
+        let puffin_manager = self
+            .puffin_manager_factory
+            .build(self.object_store.clone())
+            .with_puffin_metadata_cache(self.puffin_metadata_cache.clone());
+
+        let file_path = location::index_file_path(&self.region_dir, file_id);
+        puffin_manager
+            .reader(&file_path)
+            .await
+            .context(PuffinBuildReaderSnafu)?
+            .with_file_size_hint(file_size_hint)
+            .blob(&Self::column_blob_name(column_id))
+            .await
+            .context(PuffinReadBlobSnafu)?
+            .reader()
+            .await
+            .context(PuffinBuildReaderSnafu)
+    }
+
+    async fn apply_filters<R: BloomFilterReader + Send + 'static>(
+        &self,
+        reader: R,
+        predicates: &[Predicate],
+        row_group_metas: &[RowGroupMetaData],
+        basement: &mut BTreeMap<usize, Option<RowSelection>>,
+    ) -> std::result::Result<(), index::bloom_filter::error::Error> {
+        let mut applier = BloomFilterApplier::new(Box::new(reader)).await?;
+
+        for predicate in predicates {
+            match predicate {
+                Predicate::InList(in_list) => {
+                    applier
+                        .search(&in_list.list, row_group_metas, basement)
+                        .await?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+pub struct BloomFilterIndexApplierBuilder<'a> {
+    region_dir: String,
+    object_store: ObjectStore,
+    metadata: &'a RegionMetadata,
+    puffin_manager_factory: PuffinManagerFactory,
+    file_cache: Option<FileCacheRef>,
+    puffin_metadata_cache: Option<PuffinMetadataCacheRef>,
+    bloom_filter_index_cache: Option<BloomFilterIndexCacheRef>,
+    output: HashMap<ColumnId, Vec<Predicate>>,
+}
+
+impl<'a> BloomFilterIndexApplierBuilder<'a> {
+    pub fn new(
+        region_dir: String,
+        object_store: ObjectStore,
+        metadata: &'a RegionMetadata,
+        puffin_manager_factory: PuffinManagerFactory,
+    ) -> Self {
+        Self {
+            region_dir,
+            object_store,
+            metadata,
+            puffin_manager_factory,
+            file_cache: None,
+            puffin_metadata_cache: None,
+            bloom_filter_index_cache: None,
+            output: HashMap::default(),
+        }
+    }
+
+    pub fn with_file_cache(mut self, file_cache: Option<FileCacheRef>) -> Self {
+        self.file_cache = file_cache;
+        self
+    }
+
+    pub fn with_puffin_metadata_cache(
+        mut self,
+        puffin_metadata_cache: Option<PuffinMetadataCacheRef>,
+    ) -> Self {
+        self.puffin_metadata_cache = puffin_metadata_cache;
+        self
+    }
+
+    pub fn with_bloom_filter_index_cache(
+        mut self,
+        bloom_filter_index_cache: Option<BloomFilterIndexCacheRef>,
+    ) -> Self {
+        self.bloom_filter_index_cache = bloom_filter_index_cache;
+        self
+    }
+
+    /// Builds the applier with given filter expressions
+    pub fn build(mut self, exprs: &[Expr]) -> Result<Option<BloomFilterIndexApplier>> {
+        for expr in exprs {
+            self.traverse_and_collect(expr);
+        }
+
+        if self.output.is_empty() {
+            return Ok(None);
+        }
+
+        let applier = BloomFilterIndexApplier::new(
+            self.region_dir,
+            self.metadata.region_id,
+            self.object_store,
+            self.puffin_manager_factory,
+            self.output,
+        )
+        .with_file_cache(self.file_cache)
+        .with_puffin_metadata_cache(self.puffin_metadata_cache)
+        .with_bloom_filter_cache(self.bloom_filter_index_cache);
+
+        Ok(Some(applier))
+    }
+
+    /// Recursively traverses expressions to collect bloom filter predicates
+    fn traverse_and_collect(&mut self, expr: &Expr) {
+        let res = match expr {
+            Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op {
+                Operator::And => {
+                    self.traverse_and_collect(left);
+                    self.traverse_and_collect(right);
+                    Ok(())
+                }
+                Operator::Eq => self.collect_eq(left, right),
+                _ => Ok(()),
+            },
+            Expr::InList(in_list) => self.collect_in_list(in_list),
+            _ => Ok(()),
+        };
+
+        if let Err(err) = res {
+            warn!(err; "Failed to collect bloom filter predicates, ignore it. expr: {expr}");
+        }
+    }
+
+    /// Helper function to get the column id and type
+    fn column_id_and_type(
+        &self,
+        column_name: &str,
+    ) -> Result<Option<(ColumnId, ConcreteDataType)>> {
+        let column = self
+            .metadata
+            .column_by_name(column_name)
+            .context(ColumnNotFoundSnafu {
+                column: column_name,
+            })?;
+
+        Ok(Some((
+            column.column_id,
+            column.column_schema.data_type.clone(),
+        )))
+    }
+
+    /// Collects an equality expression (column = value)
+    fn collect_eq(&mut self, left: &Expr, right: &Expr) -> Result<()> {
+        let (col, lit) = match (left, right) {
+            (Expr::Column(col), Expr::Literal(lit)) => (col, lit),
+            (Expr::Literal(lit), Expr::Column(col)) => (col, lit),
+            _ => return Ok(()),
+        };
+        if lit.is_null() {
+            return Ok(());
+        }
+        let Some((column_id, data_type)) = self.column_id_and_type(&col.name)? else {
+            return Ok(());
+        };
+        let value = encode_lit(lit, data_type)?;
+
+        // Create bloom filter predicate
+        let mut set = HashSet::new();
+        set.insert(value);
+        let predicate = Predicate::InList(InListPredicate { list: set });
+
+        // Add to output predicates
+        self.output.entry(column_id).or_default().push(predicate);
+
+        Ok(())
+    }
+
+    /// Collects an in list expression in the form of `column IN (lit, lit, ...)`.
+    fn collect_in_list(&mut self, in_list: &InList) -> Result<()> {
+        // Only collect InList predicates if they reference a column
+        let Expr::Column(column) = &in_list.expr.as_ref() else {
+            return Ok(());
+        };
+        if in_list.list.is_empty() || in_list.negated {
+            return Ok(());
+        }
+
+        let Some((column_id, data_type)) = self.column_id_and_type(&column.name)? else {
+            return Ok(());
+        };
+
+        // Convert all non-null literals to predicates
+        let predicates = in_list
+            .list
+            .iter()
+            .filter_map(Self::nonnull_lit)
+            .map(|lit| encode_lit(lit, data_type.clone()));
+
+        // Collect successful conversions
+        let mut valid_predicates = HashSet::new();
+        for predicate in predicates {
+            match predicate {
+                Ok(p) => {
+                    valid_predicates.insert(p);
+                }
+                Err(e) => warn!(e; "Failed to convert value in InList"),
+            }
+        }
+
+        if !valid_predicates.is_empty() {
+            self.output
+                .entry(column_id)
+                .or_default()
+                .push(Predicate::InList(InListPredicate {
+                    list: valid_predicates,
+                }));
+        }
+
+        Ok(())
+    }
+
+    /// Helper function to get non-null literal value
+    fn nonnull_lit(expr: &Expr) -> Option<&ScalarValue> {
+        match expr {
+            Expr::Literal(lit) if !lit.is_null() => Some(lit),
+            _ => None,
+        }
+    }
+}
+
+// TODO(ruihang): extract this and the one under inverted_index into a common util mod.
+/// Helper function to encode a literal into bytes.
+fn encode_lit(lit: &ScalarValue, data_type: ConcreteDataType) -> Result<Vec<u8>> {
+    let value = Value::try_from(lit.clone()).context(ConvertValueSnafu)?;
+    let mut bytes = vec![];
+    let field = SortField::new(data_type);
+    IndexValueCodec::encode_nonnull_value(value.as_value_ref(), &field, &mut bytes)?;
+    Ok(bytes)
+}
+
+#[cfg(test)]
+mod tests {
+    use api::v1::SemanticType;
+    use datafusion_common::Column;
+    use datatypes::schema::ColumnSchema;
+    use object_store::services::Memory;
+    use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder};
+
+    use super::*;
+
+    fn test_region_metadata() -> RegionMetadata {
+        let mut builder = RegionMetadataBuilder::new(RegionId::new(1234, 5678));
+        builder
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new(
+                    "column1",
+                    ConcreteDataType::string_datatype(),
+                    false,
+                ),
+                semantic_type: SemanticType::Tag,
+                column_id: 1,
+            })
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new(
+                    "column2",
+                    ConcreteDataType::int64_datatype(),
+                    false,
+                ),
+                semantic_type: SemanticType::Field,
+                column_id: 2,
+            })
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new(
+                    "column3",
+                    ConcreteDataType::timestamp_millisecond_datatype(),
+                    false,
+                ),
+                semantic_type: SemanticType::Timestamp,
+                column_id: 3,
+            })
+            .primary_key(vec![1]);
+        builder.build().unwrap()
+    }
+
+    fn test_object_store() -> ObjectStore {
+        ObjectStore::new(Memory::default()).unwrap().finish()
+    }
+
+    fn column(name: &str) -> Expr {
+        Expr::Column(Column {
+            relation: None,
+            name: name.to_string(),
+        })
+    }
+
+    fn string_lit(s: impl Into<String>) -> Expr {
+        Expr::Literal(ScalarValue::Utf8(Some(s.into())))
+    }
+
+    #[test]
+    fn test_build_with_exprs() {
+        let (_d, factory) = PuffinManagerFactory::new_for_test_block("test_build_with_exprs_");
+        let metadata = test_region_metadata();
+        let builder = BloomFilterIndexApplierBuilder::new(
+            "test".to_string(),
+            test_object_store(),
+            &metadata,
+            factory,
+        );
+
+        let exprs = vec![Expr::BinaryExpr(BinaryExpr {
+            left: Box::new(column("column1")),
+            op: Operator::Eq,
+            right: Box::new(string_lit("value1")),
+        })];
+
+        let result = builder.build(&exprs).unwrap();
+        assert!(result.is_some());
+
+        let filters = result.unwrap().filters;
+        assert_eq!(filters.len(), 1);
+
+        let column_predicates = filters.get(&1).unwrap();
+        assert_eq!(column_predicates.len(), 1);
+
+        let expected = encode_lit(
+            &ScalarValue::Utf8(Some("value1".to_string())),
+            ConcreteDataType::string_datatype(),
+        )
+        .unwrap();
+        match &column_predicates[0] {
+            Predicate::InList(p) => {
+                assert_eq!(p.list.iter().next().unwrap(), &expected);
+            }
+        }
+    }
+
+    fn int64_lit(i: i64) -> Expr {
+        Expr::Literal(ScalarValue::Int64(Some(i)))
+    }
+
+    #[test]
+    fn test_build_with_in_list() {
+        let (_d, factory) = PuffinManagerFactory::new_for_test_block("test_build_with_in_list_");
+        let metadata = test_region_metadata();
+        let builder = BloomFilterIndexApplierBuilder::new(
+            "test".to_string(),
+            test_object_store(),
+            &metadata,
+            factory,
+        );
+
+        let exprs = vec![Expr::InList(InList {
+            expr: Box::new(column("column2")),
+            list: vec![int64_lit(1), int64_lit(2), int64_lit(3)],
+            negated: false,
+        })];
+
+        let result = builder.build(&exprs).unwrap();
+        assert!(result.is_some());
+
+        let filters = result.unwrap().filters;
+        let column_predicates = filters.get(&2).unwrap();
+        assert_eq!(column_predicates.len(), 1);
+
+        match &column_predicates[0] {
+            Predicate::InList(p) => {
+                assert_eq!(p.list.len(), 3);
+            }
+        }
+    }
+
+    #[test]
+    fn test_build_with_and_expressions() {
+        let (_d, factory) = PuffinManagerFactory::new_for_test_block("test_build_with_and_");
+        let metadata = test_region_metadata();
+        let builder = BloomFilterIndexApplierBuilder::new(
+            "test".to_string(),
+            test_object_store(),
+            &metadata,
+            factory,
+        );
+
+        let exprs = vec![Expr::BinaryExpr(BinaryExpr {
+            left: Box::new(Expr::BinaryExpr(BinaryExpr {
+                left: Box::new(column("column1")),
+                op: Operator::Eq,
+                right: Box::new(string_lit("value1")),
+            })),
+            op: Operator::And,
+            right: Box::new(Expr::BinaryExpr(BinaryExpr {
+                left: Box::new(column("column2")),
+                op: Operator::Eq,
+                right: Box::new(int64_lit(42)),
+            })),
+        })];
+
+        let result = builder.build(&exprs).unwrap();
+        assert!(result.is_some());
+
+        let filters = result.unwrap().filters;
+        assert_eq!(filters.len(), 2);
+        assert!(filters.contains_key(&1));
+        assert!(filters.contains_key(&2));
+    }
+
+    #[test]
+    fn test_build_with_null_values() {
+        let (_d, factory) = PuffinManagerFactory::new_for_test_block("test_build_with_null_");
+        let metadata = test_region_metadata();
+        let builder = BloomFilterIndexApplierBuilder::new(
+            "test".to_string(),
+            test_object_store(),
+            &metadata,
+            factory,
+        );
+
+        let exprs = vec![
+            Expr::BinaryExpr(BinaryExpr {
+                left: Box::new(column("column1")),
+                op: Operator::Eq,
+                right: Box::new(Expr::Literal(ScalarValue::Utf8(None))),
+            }),
+            Expr::InList(InList {
+                expr: Box::new(column("column2")),
+                list: vec![
+                    int64_lit(1),
+                    Expr::Literal(ScalarValue::Int64(None)),
+                    int64_lit(3),
+                ],
+                negated: false,
+            }),
+        ];
+
+        let result = builder.build(&exprs).unwrap();
+        assert!(result.is_some());
+
+        let filters = result.unwrap().filters;
+        assert!(!filters.contains_key(&1)); // Null equality should be ignored
+        let column2_predicates = filters.get(&2).unwrap();
+        match &column2_predicates[0] {
+            Predicate::InList(p) => {
+                assert_eq!(p.list.len(), 2); // Only non-null values should be included
+            }
+        }
+    }
+
+    #[test]
+    fn test_build_with_invalid_expressions() {
+        let (_d, factory) = PuffinManagerFactory::new_for_test_block("test_build_with_invalid_");
+        let metadata = test_region_metadata();
+        let builder = BloomFilterIndexApplierBuilder::new(
+            "test".to_string(),
+            test_object_store(),
+            &metadata,
+            factory,
+        );
+
+        let exprs = vec![
+            // Non-equality operator
+            Expr::BinaryExpr(BinaryExpr {
+                left: Box::new(column("column1")),
+                op: Operator::Gt,
+                right: Box::new(string_lit("value1")),
+            }),
+            // Non-existent column
+            Expr::BinaryExpr(BinaryExpr {
+                left: Box::new(column("non_existent")),
+                op: Operator::Eq,
+                right: Box::new(string_lit("value")),
+            }),
+            // Negated IN list
+            Expr::InList(InList {
+                expr: Box::new(column("column2")),
+                list: vec![int64_lit(1), int64_lit(2)],
+                negated: true,
+            }),
+        ];
+
+        let result = builder.build(&exprs).unwrap();
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_build_with_multiple_predicates_same_column() {
+        let (_d, factory) = PuffinManagerFactory::new_for_test_block("test_build_with_multiple_");
+        let metadata = test_region_metadata();
+        let builder = BloomFilterIndexApplierBuilder::new(
+            "test".to_string(),
+            test_object_store(),
+            &metadata,
+            factory,
+        );
+
+        let exprs = vec![
+            Expr::BinaryExpr(BinaryExpr {
+                left: Box::new(column("column1")),
+                op: Operator::Eq,
+                right: Box::new(string_lit("value1")),
+            }),
+            Expr::InList(InList {
+                expr: Box::new(column("column1")),
+                list: vec![string_lit("value2"), string_lit("value3")],
+                negated: false,
+            }),
+        ];
+
+        let result = builder.build(&exprs).unwrap();
+        assert!(result.is_some());
+
+        let filters = result.unwrap().filters;
+        let column_predicates = filters.get(&1).unwrap();
+        assert_eq!(column_predicates.len(), 2);
+    }
+}

src/mito2/src/sst/index/bloom_filter/creator.rs

@@ -0,0 +1,530 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::atomic::AtomicUsize;
+use std::sync::Arc;
+
+use common_telemetry::warn;
+use datatypes::schema::SkippingIndexType;
+use index::bloom_filter::creator::BloomFilterCreator;
+use puffin::puffin_manager::{PuffinWriter, PutOptions};
+use snafu::{ensure, ResultExt};
+use store_api::metadata::RegionMetadataRef;
+use store_api::storage::ColumnId;
+use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
+
+use crate::error::{
+    BiErrorsSnafu, BloomFilterFinishSnafu, IndexOptionsSnafu, OperateAbortedIndexSnafu,
+    PuffinAddBlobSnafu, PushBloomFilterValueSnafu, Result,
+};
+use crate::read::Batch;
+use crate::row_converter::SortField;
+use crate::sst::file::FileId;
+use crate::sst::index::bloom_filter::INDEX_BLOB_TYPE;
+use crate::sst::index::codec::{IndexValueCodec, IndexValuesCodec};
+use crate::sst::index::intermediate::{
+    IntermediateLocation, IntermediateManager, TempFileProvider,
+};
+use crate::sst::index::puffin_manager::SstPuffinWriter;
+use crate::sst::index::statistics::{ByteCount, RowCount, Statistics};
+use crate::sst::index::TYPE_BLOOM_FILTER_INDEX;
+
+/// The buffer size for the pipe used to send index data to the puffin blob.
+const PIPE_BUFFER_SIZE_FOR_SENDING_BLOB: usize = 8192;
+
+/// The indexer for the bloom filter index.
+pub struct BloomFilterIndexer {
+    /// The bloom filter creators.
+    creators: HashMap<ColumnId, BloomFilterCreator>,
+
+    /// The provider for intermediate files.
+    temp_file_provider: Arc<TempFileProvider>,
+
+    /// Codec for decoding primary keys.
+    codec: IndexValuesCodec,
+
+    /// Whether the indexing process has been aborted.
+    aborted: bool,
+
+    /// The statistics of the indexer.
+    stats: Statistics,
+
+    /// The global memory usage.
+    global_memory_usage: Arc<AtomicUsize>,
+}
+
+impl BloomFilterIndexer {
+    /// Creates a new bloom filter indexer.
+    pub fn new(
+        sst_file_id: FileId,
+        metadata: &RegionMetadataRef,
+        intermediate_manager: IntermediateManager,
+        memory_usage_threshold: Option<usize>,
+    ) -> Result<Option<Self>> {
+        let mut creators = HashMap::new();
+
+        let temp_file_provider = Arc::new(TempFileProvider::new(
+            IntermediateLocation::new(&metadata.region_id, &sst_file_id),
+            intermediate_manager,
+        ));
+        let global_memory_usage = Arc::new(AtomicUsize::new(0));
+
+        for column in &metadata.column_metadatas {
+            let options =
+                column
+                    .column_schema
+                    .skipping_index_options()
+                    .context(IndexOptionsSnafu {
+                        column_name: &column.column_schema.name,
+                    })?;
+
+            let options = match options {
+                Some(options) if options.index_type == SkippingIndexType::BloomFilter => options,
+                _ => continue,
+            };
+
+            let creator = BloomFilterCreator::new(
+                options.granularity as _,
+                temp_file_provider.clone(),
+                global_memory_usage.clone(),
+                memory_usage_threshold,
+            );
+            creators.insert(column.column_id, creator);
+        }
+
+        if creators.is_empty() {
+            return Ok(None);
+        }
+
+        let codec = IndexValuesCodec::from_tag_columns(metadata.primary_key_columns());
+        let indexer = Self {
+            creators,
+            temp_file_provider,
+            codec,
+            aborted: false,
+            stats: Statistics::new(TYPE_BLOOM_FILTER_INDEX),
+            global_memory_usage,
+        };
+        Ok(Some(indexer))
+    }
+
+    /// Updates index with a batch of rows.
+    /// Garbage will be cleaned up if failed to update.
+    ///
+    /// TODO(zhongzc): duplicate with `mito2::sst::index::inverted_index::creator::InvertedIndexCreator`
+    pub async fn update(&mut self, batch: &Batch) -> Result<()> {
+        ensure!(!self.aborted, OperateAbortedIndexSnafu);
+
+        if self.creators.is_empty() {
+            return Ok(());
+        }
+
+        if let Err(update_err) = self.do_update(batch).await {
+            // clean up garbage if failed to update
+            if let Err(err) = self.do_cleanup().await {
+                if cfg!(any(test, feature = "test")) {
+                    panic!("Failed to clean up index creator, err: {err:?}",);
+                } else {
+                    warn!(err; "Failed to clean up index creator");
+                }
+            }
+            return Err(update_err);
+        }
+
+        Ok(())
+    }
+
+    /// Finishes index creation and cleans up garbage.
+    /// Returns the number of rows and bytes written.
+    ///
+    /// TODO(zhongzc): duplicate with `mito2::sst::index::inverted_index::creator::InvertedIndexCreator`
+    pub async fn finish(
+        &mut self,
+        puffin_writer: &mut SstPuffinWriter,
+    ) -> Result<(RowCount, ByteCount)> {
+        ensure!(!self.aborted, OperateAbortedIndexSnafu);
+
+        if self.stats.row_count() == 0 {
+            // no IO is performed, no garbage to clean up, just return
+            return Ok((0, 0));
+        }
+
+        let finish_res = self.do_finish(puffin_writer).await;
+        // clean up garbage no matter finish successfully or not
+        if let Err(err) = self.do_cleanup().await {
+            if cfg!(any(test, feature = "test")) {
+                panic!("Failed to clean up index creator, err: {err:?}",);
+            } else {
+                warn!(err; "Failed to clean up index creator");
+            }
+        }
+
+        finish_res.map(|_| (self.stats.row_count(), self.stats.byte_count()))
+    }
+
+    /// Aborts index creation and clean up garbage.
+    ///
+    /// TODO(zhongzc): duplicate with `mito2::sst::index::inverted_index::creator::InvertedIndexCreator`
+    pub async fn abort(&mut self) -> Result<()> {
+        if self.aborted {
+            return Ok(());
+        }
+        self.aborted = true;
+
+        self.do_cleanup().await
+    }
+
+    async fn do_update(&mut self, batch: &Batch) -> Result<()> {
+        let mut guard = self.stats.record_update();
+
+        let n = batch.num_rows();
+        guard.inc_row_count(n);
+
+        // Tags
+        for ((col_id, _), field, value) in self.codec.decode(batch.primary_key())? {
+            let Some(creator) = self.creators.get_mut(col_id) else {
+                continue;
+            };
+            let elems = value
+                .map(|v| {
+                    let mut buf = vec![];
+                    IndexValueCodec::encode_nonnull_value(v.as_value_ref(), field, &mut buf)?;
+                    Ok(buf)
+                })
+                .transpose()?;
+            creator
+                .push_n_row_elems(n, elems)
+                .await
+                .context(PushBloomFilterValueSnafu)?;
+        }
+
+        // Fields
+        for field in batch.fields() {
+            let Some(creator) = self.creators.get_mut(&field.column_id) else {
+                continue;
+            };
+
+            let sort_field = SortField::new(field.data.data_type());
+            for i in 0..n {
+                let value = field.data.get_ref(i);
+                let elems = (!value.is_null())
+                    .then(|| {
+                        let mut buf = vec![];
+                        IndexValueCodec::encode_nonnull_value(value, &sort_field, &mut buf)?;
+                        Ok(buf)
+                    })
+                    .transpose()?;
+
+                creator
+                    .push_row_elems(elems)
+                    .await
+                    .context(PushBloomFilterValueSnafu)?;
+            }
+        }
+        Ok(())
+    }
+
+    /// TODO(zhongzc): duplicate with `mito2::sst::index::inverted_index::creator::InvertedIndexCreator`
+    async fn do_finish(&mut self, puffin_writer: &mut SstPuffinWriter) -> Result<()> {
+        let mut guard = self.stats.record_finish();
+
+        for (id, creator) in &mut self.creators {
+            let written_bytes = Self::do_finish_single_creator(id, creator, puffin_writer).await?;
+            guard.inc_byte_count(written_bytes);
+        }
+
+        Ok(())
+    }
+
+    async fn do_cleanup(&mut self) -> Result<()> {
+        let mut _guard = self.stats.record_cleanup();
+
+        self.creators.clear();
+        self.temp_file_provider.cleanup().await
+    }
+
+    /// Data flow of finishing index:
+    ///
+    /// ```text
+    ///                               (In Memory Buffer)
+    ///                                    ┌──────┐
+    ///  ┌─────────────┐                   │ PIPE │
+    ///  │             │ write index data  │      │
+    ///  │ IndexWriter ├──────────────────►│ tx   │
+    ///  │             │                   │      │
+    ///  └─────────────┘                   │      │
+    ///                  ┌─────────────────┤ rx   │
+    ///  ┌─────────────┐ │ read as blob    └──────┘
+    ///  │             │ │
+    ///  │ PuffinWriter├─┤
+    ///  │             │ │ copy to file    ┌──────┐
+    ///  └─────────────┘ └────────────────►│ File │
+    ///                                    └──────┘
+    /// ```
+    ///
+    /// TODO(zhongzc): duplicate with `mito2::sst::index::inverted_index::creator::InvertedIndexCreator`
+    async fn do_finish_single_creator(
+        col_id: &ColumnId,
+        creator: &mut BloomFilterCreator,
+        puffin_writer: &mut SstPuffinWriter,
+    ) -> Result<ByteCount> {
+        let (tx, rx) = tokio::io::duplex(PIPE_BUFFER_SIZE_FOR_SENDING_BLOB);
+
+        let blob_name = format!("{}-{}", INDEX_BLOB_TYPE, col_id);
+        let (index_finish, puffin_add_blob) = futures::join!(
+            creator.finish(tx.compat_write()),
+            puffin_writer.put_blob(&blob_name, rx.compat(), PutOptions::default())
+        );
+
+        match (
+            puffin_add_blob.context(PuffinAddBlobSnafu),
+            index_finish.context(BloomFilterFinishSnafu),
+        ) {
+            (Err(e1), Err(e2)) => BiErrorsSnafu {
+                first: Box::new(e1),
+                second: Box::new(e2),
+            }
+            .fail()?,
+
+            (Ok(_), e @ Err(_)) => e?,
+            (e @ Err(_), Ok(_)) => e.map(|_| ())?,
+            (Ok(written_bytes), Ok(_)) => {
+                return Ok(written_bytes);
+            }
+        }
+
+        Ok(0)
+    }
+
+    /// Returns the memory usage of the indexer.
+    pub fn memory_usage(&self) -> usize {
+        self.global_memory_usage
+            .load(std::sync::atomic::Ordering::Relaxed)
+    }
+
+    /// Returns the column ids to be indexed.
+    pub fn column_ids(&self) -> impl Iterator<Item = ColumnId> + use<'_> {
+        self.creators.keys().copied()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::iter;
+
+    use api::v1::SemanticType;
+    use datatypes::data_type::ConcreteDataType;
+    use datatypes::schema::{ColumnSchema, SkippingIndexOptions};
+    use datatypes::value::ValueRef;
+    use datatypes::vectors::{UInt64Vector, UInt8Vector};
+    use index::bloom_filter::reader::{BloomFilterReader, BloomFilterReaderImpl};
+    use object_store::services::Memory;
+    use object_store::ObjectStore;
+    use puffin::puffin_manager::{BlobGuard, PuffinManager, PuffinReader};
+    use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
+    use store_api::storage::RegionId;
+
+    use super::*;
+    use crate::read::BatchColumn;
+    use crate::row_converter::{McmpRowCodec, RowCodec, SortField};
+    use crate::sst::index::puffin_manager::PuffinManagerFactory;
+
+    fn mock_object_store() -> ObjectStore {
+        ObjectStore::new(Memory::default()).unwrap().finish()
+    }
+
+    async fn new_intm_mgr(path: impl AsRef<str>) -> IntermediateManager {
+        IntermediateManager::init_fs(path).await.unwrap()
+    }
+
+    /// tag_str:
+    ///   - type: string
+    ///   - index: bloom filter
+    ///   - granularity: 2
+    ///   - column_id: 1
+    ///
+    /// ts:
+    ///   - type: timestamp
+    ///   - index: time index
+    ///   - column_id: 2
+    ///
+    /// field_u64:
+    ///   - type: uint64
+    ///   - index: bloom filter
+    ///   - granularity: 4
+    ///   - column_id: 3
+    fn mock_region_metadata() -> RegionMetadataRef {
+        let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 2));
+        builder
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new(
+                    "tag_str",
+                    ConcreteDataType::string_datatype(),
+                    false,
+                )
+                .with_skipping_options(SkippingIndexOptions {
+                    index_type: SkippingIndexType::BloomFilter,
+                    granularity: 2,
+                })
+                .unwrap(),
+                semantic_type: SemanticType::Tag,
+                column_id: 1,
+            })
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new(
+                    "ts",
+                    ConcreteDataType::timestamp_millisecond_datatype(),
+                    false,
+                ),
+                semantic_type: SemanticType::Timestamp,
+                column_id: 2,
+            })
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new(
+                    "field_u64",
+                    ConcreteDataType::uint64_datatype(),
+                    false,
+                )
+                .with_skipping_options(SkippingIndexOptions {
+                    index_type: SkippingIndexType::BloomFilter,
+                    granularity: 4,
+                })
+                .unwrap(),
+                semantic_type: SemanticType::Field,
+                column_id: 3,
+            })
+            .primary_key(vec![1]);
+
+        Arc::new(builder.build().unwrap())
+    }
+
+    fn new_batch(str_tag: impl AsRef<str>, u64_field: impl IntoIterator<Item = u64>) -> Batch {
+        let fields = vec![SortField::new(ConcreteDataType::string_datatype())];
+        let codec = McmpRowCodec::new(fields);
+        let row: [ValueRef; 1] = [str_tag.as_ref().into()];
+        let primary_key = codec.encode(row.into_iter()).unwrap();
+
+        let u64_field = BatchColumn {
+            column_id: 3,
+            data: Arc::new(UInt64Vector::from_iter_values(u64_field)),
+        };
+        let num_rows = u64_field.data.len();
+
+        Batch::new(
+            primary_key,
+            Arc::new(UInt64Vector::from_iter_values(
+                iter::repeat(0).take(num_rows),
+            )),
+            Arc::new(UInt64Vector::from_iter_values(
+                iter::repeat(0).take(num_rows),
+            )),
+            Arc::new(UInt8Vector::from_iter_values(
+                iter::repeat(1).take(num_rows),
+            )),
+            vec![u64_field],
+        )
+        .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_bloom_filter_indexer() {
+        let prefix = "test_bloom_filter_indexer_";
+        let object_store = mock_object_store();
+        let intm_mgr = new_intm_mgr(prefix).await;
+        let region_metadata = mock_region_metadata();
+        let memory_usage_threshold = Some(1024);
+
+        let mut indexer = BloomFilterIndexer::new(
+            FileId::random(),
+            &region_metadata,
+            intm_mgr,
+            memory_usage_threshold,
+        )
+        .unwrap()
+        .unwrap();
+
+        // push 20 rows
+        let batch = new_batch("tag1", 0..10);
+        indexer.update(&batch).await.unwrap();
+
+        let batch = new_batch("tag2", 10..20);
+        indexer.update(&batch).await.unwrap();
+
+        let (_d, factory) = PuffinManagerFactory::new_for_test_async(prefix).await;
+        let puffin_manager = factory.build(object_store);
+
+        let index_file_name = "index_file";
+        let mut puffin_writer = puffin_manager.writer(index_file_name).await.unwrap();
+        let (row_count, byte_count) = indexer.finish(&mut puffin_writer).await.unwrap();
+        assert_eq!(row_count, 20);
+        assert!(byte_count > 0);
+        puffin_writer.finish().await.unwrap();
+
+        let puffin_reader = puffin_manager.reader(index_file_name).await.unwrap();
+
+        // tag_str
+        {
+            let blob_guard = puffin_reader
+                .blob("greptime-bloom-filter-v1-1")
+                .await
+                .unwrap();
+            let reader = blob_guard.reader().await.unwrap();
+            let mut bloom_filter = BloomFilterReaderImpl::new(reader);
+            let metadata = bloom_filter.metadata().await.unwrap();
+
+            assert_eq!(metadata.bloom_filter_segments.len(), 10);
+            for i in 0..5 {
+                let bf = bloom_filter
+                    .bloom_filter(&metadata.bloom_filter_segments[i])
+                    .await
+                    .unwrap();
+                assert!(bf.contains(b"tag1"));
+            }
+            for i in 5..10 {
+                let bf = bloom_filter
+                    .bloom_filter(&metadata.bloom_filter_segments[i])
+                    .await
+                    .unwrap();
+                assert!(bf.contains(b"tag2"));
+            }
+        }
+
+        // field_u64
+        {
+            let sort_field = SortField::new(ConcreteDataType::uint64_datatype());
+
+            let blob_guard = puffin_reader
+                .blob("greptime-bloom-filter-v1-3")
+                .await
+                .unwrap();
+            let reader = blob_guard.reader().await.unwrap();
+            let mut bloom_filter = BloomFilterReaderImpl::new(reader);
+            let metadata = bloom_filter.metadata().await.unwrap();
+
+            assert_eq!(metadata.bloom_filter_segments.len(), 5);
+            for i in 0u64..20 {
+                let bf = bloom_filter
+                    .bloom_filter(&metadata.bloom_filter_segments[i as usize / 4])
+                    .await
+                    .unwrap();
+                let mut buf = vec![];
+                IndexValueCodec::encode_nonnull_value(ValueRef::UInt64(i), &sort_field, &mut buf)
+                    .unwrap();
+
+                assert!(bf.contains(&buf));
+            }
+        }
+    }
+}

src/mito2/src/sst/index/fulltext_index/creator.rs

@@ -27,8 +27,7 @@ use store_api::storage::{ColumnId, ConcreteDataType, RegionId};
 
 use crate::error::{
     CastVectorSnafu, CreateFulltextCreatorSnafu, FieldTypeMismatchSnafu, FulltextFinishSnafu,
-    FulltextOptionsSnafu, FulltextPushTextSnafu, OperateAbortedIndexSnafu, PuffinAddBlobSnafu,
-    Result,
+    FulltextPushTextSnafu, IndexOptionsSnafu, OperateAbortedIndexSnafu, PuffinAddBlobSnafu, Result,
 };
 use crate::read::Batch;
 use crate::sst::file::FileId;
@@ -61,13 +60,12 @@ impl FulltextIndexer {
         let mut creators = HashMap::new();
 
         for column in &metadata.column_metadatas {
-            let options =
-                column
-                    .column_schema
-                    .fulltext_options()
-                    .context(FulltextOptionsSnafu {
-                        column_name: &column.column_schema.name,
-                    })?;
+            let options = column
+                .column_schema
+                .fulltext_options()
+                .context(IndexOptionsSnafu {
+                    column_name: &column.column_schema.name,
+                })?;
 
             // Relax the type constraint here as many types can be casted to string.
 

src/mito2/src/sst/index/indexer/abort.rs

@@ -20,6 +20,7 @@ impl Indexer {
     pub(crate) async fn do_abort(&mut self) {
         self.do_abort_inverted_index().await;
         self.do_abort_fulltext_index().await;
+        self.do_abort_bloom_filter().await;
         self.puffin_manager = None;
     }
 
@@ -33,7 +34,7 @@ impl Indexer {
 
         if cfg!(any(test, feature = "test")) {
             panic!(
-                "Failed to abort inverted index, region_id: {}, file_id: {}, err: {}",
+                "Failed to abort inverted index, region_id: {}, file_id: {}, err: {:?}",
                 self.region_id, self.file_id, err
             );
         } else {
@@ -54,7 +55,7 @@ impl Indexer {
 
         if cfg!(any(test, feature = "test")) {
             panic!(
-                "Failed to abort full-text index, region_id: {}, file_id: {}, err: {}",
+                "Failed to abort full-text index, region_id: {}, file_id: {}, err: {:?}",
                 self.region_id, self.file_id, err
             );
         } else {
@@ -64,4 +65,25 @@ impl Indexer {
             );
         }
     }
+
+    async fn do_abort_bloom_filter(&mut self) {
+        let Some(mut indexer) = self.bloom_filter_indexer.take() else {
+            return;
+        };
+        let Err(err) = indexer.abort().await else {
+            return;
+        };
+
+        if cfg!(any(test, feature = "test")) {
+            panic!(
+                "Failed to abort bloom filter, region_id: {}, file_id: {}, err: {:?}",
+                self.region_id, self.file_id, err
+            );
+        } else {
+            warn!(
+                err; "Failed to abort bloom filter, region_id: {}, file_id: {}",
+                self.region_id, self.file_id,
+            );
+        }
+    }
 }

src/mito2/src/sst/index/indexer/finish.rs

@@ -15,11 +15,14 @@
 use common_telemetry::{debug, warn};
 use puffin::puffin_manager::{PuffinManager, PuffinWriter};
 
+use crate::sst::index::bloom_filter::creator::BloomFilterIndexer;
 use crate::sst::index::fulltext_index::creator::FulltextIndexer;
 use crate::sst::index::inverted_index::creator::InvertedIndexer;
 use crate::sst::index::puffin_manager::SstPuffinWriter;
 use crate::sst::index::statistics::{ByteCount, RowCount};
-use crate::sst::index::{FulltextIndexOutput, IndexOutput, Indexer, InvertedIndexOutput};
+use crate::sst::index::{
+    BloomFilterOutput, FulltextIndexOutput, IndexOutput, Indexer, InvertedIndexOutput,
+};
 
 impl Indexer {
     pub(crate) async fn do_finish(&mut self) -> IndexOutput {
@@ -46,6 +49,12 @@ impl Indexer {
             return IndexOutput::default();
         }
 
+        let success = self.do_finish_bloom_filter(&mut writer, &mut output).await;
+        if !success {
+            self.do_abort().await;
+            return IndexOutput::default();
+        }
+
         output.file_size = self.do_finish_puffin_writer(writer).await;
         output
     }
@@ -60,7 +69,7 @@ impl Indexer {
 
         if cfg!(any(test, feature = "test")) {
             panic!(
-                "Failed to create puffin writer, region_id: {}, file_id: {}, err: {}",
+                "Failed to create puffin writer, region_id: {}, file_id: {}, err: {:?}",
                 self.region_id, self.file_id, err
             );
         } else {
@@ -81,7 +90,7 @@ impl Indexer {
 
         if cfg!(any(test, feature = "test")) {
             panic!(
-                "Failed to finish puffin writer, region_id: {}, file_id: {}, err: {}",
+                "Failed to finish puffin writer, region_id: {}, file_id: {}, err: {:?}",
                 self.region_id, self.file_id, err
             );
         } else {
@@ -119,7 +128,7 @@ impl Indexer {
 
         if cfg!(any(test, feature = "test")) {
             panic!(
-                "Failed to finish inverted index, region_id: {}, file_id: {}, err: {}",
+                "Failed to finish inverted index, region_id: {}, file_id: {}, err: {:?}",
                 self.region_id, self.file_id, err
             );
         } else {
@@ -156,7 +165,7 @@ impl Indexer {
 
         if cfg!(any(test, feature = "test")) {
             panic!(
-                "Failed to finish full-text index, region_id: {}, file_id: {}, err: {}",
+                "Failed to finish full-text index, region_id: {}, file_id: {}, err: {:?}",
                 self.region_id, self.file_id, err
             );
         } else {
@@ -169,6 +178,43 @@ impl Indexer {
         false
     }
 
+    async fn do_finish_bloom_filter(
+        &mut self,
+        puffin_writer: &mut SstPuffinWriter,
+        index_output: &mut IndexOutput,
+    ) -> bool {
+        let Some(mut indexer) = self.bloom_filter_indexer.take() else {
+            return true;
+        };
+
+        let err = match indexer.finish(puffin_writer).await {
+            Ok((row_count, byte_count)) => {
+                self.fill_bloom_filter_output(
+                    &mut index_output.bloom_filter,
+                    row_count,
+                    byte_count,
+                    &indexer,
+                );
+                return true;
+            }
+            Err(err) => err,
+        };
+
+        if cfg!(any(test, feature = "test")) {
+            panic!(
+                "Failed to finish bloom filter, region_id: {}, file_id: {}, err: {:?}",
+                self.region_id, self.file_id, err
+            );
+        } else {
+            warn!(
+                err; "Failed to finish bloom filter, region_id: {}, file_id: {}",
+                self.region_id, self.file_id,
+            );
+        }
+
+        false
+    }
+
     fn fill_inverted_index_output(
         &mut self,
         output: &mut InvertedIndexOutput,
@@ -202,4 +248,21 @@ impl Indexer {
         output.row_count = row_count;
         output.columns = indexer.column_ids().collect();
     }
+
+    fn fill_bloom_filter_output(
+        &mut self,
+        output: &mut BloomFilterOutput,
+        row_count: RowCount,
+        byte_count: ByteCount,
+        indexer: &BloomFilterIndexer,
+    ) {
+        debug!(
+            "Bloom filter created, region_id: {}, file_id: {}, written_bytes: {}, written_rows: {}",
+            self.region_id, self.file_id, byte_count, row_count
+        );
+
+        output.index_size = byte_count;
+        output.row_count = row_count;
+        output.columns = indexer.column_ids().collect();
+    }
 }

src/mito2/src/sst/index/indexer/update.rs

@@ -29,6 +29,9 @@ impl Indexer {
         if !self.do_update_fulltext_index(batch).await {
             self.do_abort().await;
         }
+        if !self.do_update_bloom_filter(batch).await {
+            self.do_abort().await;
+        }
     }
 
     /// Returns false if the update failed.
@@ -43,7 +46,7 @@ impl Indexer {
 
         if cfg!(any(test, feature = "test")) {
             panic!(
-                "Failed to update inverted index, region_id: {}, file_id: {}, err: {}",
+                "Failed to update inverted index, region_id: {}, file_id: {}, err: {:?}",
                 self.region_id, self.file_id, err
             );
         } else {
@@ -68,7 +71,7 @@ impl Indexer {
 
         if cfg!(any(test, feature = "test")) {
             panic!(
-                "Failed to update full-text index, region_id: {}, file_id: {}, err: {}",
+                "Failed to update full-text index, region_id: {}, file_id: {}, err: {:?}",
                 self.region_id, self.file_id, err
             );
         } else {
@@ -80,4 +83,29 @@ impl Indexer {
 
         false
     }
+
+    /// Returns false if the update failed.
+    async fn do_update_bloom_filter(&mut self, batch: &Batch) -> bool {
+        let Some(creator) = self.bloom_filter_indexer.as_mut() else {
+            return true;
+        };
+
+        let Err(err) = creator.update(batch).await else {
+            return true;
+        };
+
+        if cfg!(any(test, feature = "test")) {
+            panic!(
+                "Failed to update bloom filter, region_id: {}, file_id: {}, err: {:?}",
+                self.region_id, self.file_id, err
+            );
+        } else {
+            warn!(
+                err; "Failed to update bloom filter, region_id: {}, file_id: {}",
+                self.region_id, self.file_id,
+            );
+        }
+
+        false
+    }
 }

src/mito2/src/sst/index/intermediate.rs

@@ -14,13 +14,25 @@
 
 use std::path::PathBuf;
 
+use async_trait::async_trait;
+use common_error::ext::BoxedError;
 use common_telemetry::warn;
+use futures::{AsyncRead, AsyncWrite};
+use index::error as index_error;
+use index::error::Result as IndexResult;
+use index::external_provider::ExternalTempFileProvider;
 use object_store::util::{self, normalize_dir};
+use snafu::ResultExt;
 use store_api::storage::{ColumnId, RegionId};
 use uuid::Uuid;
 
 use crate::access_layer::new_fs_cache_store;
 use crate::error::Result;
+use crate::metrics::{
+    INDEX_INTERMEDIATE_FLUSH_OP_TOTAL, INDEX_INTERMEDIATE_READ_BYTES_TOTAL,
+    INDEX_INTERMEDIATE_READ_OP_TOTAL, INDEX_INTERMEDIATE_SEEK_OP_TOTAL,
+    INDEX_INTERMEDIATE_WRITE_BYTES_TOTAL, INDEX_INTERMEDIATE_WRITE_OP_TOTAL,
+};
 use crate::sst::file::FileId;
 use crate::sst::index::store::InstrumentedStore;
 
@@ -129,14 +141,105 @@ impl IntermediateLocation {
     }
 }
 
+/// `TempFileProvider` implements `ExternalTempFileProvider`.
+/// It uses `InstrumentedStore` to create and read intermediate files.
+pub(crate) struct TempFileProvider {
+    /// Provides the location of intermediate files.
+    location: IntermediateLocation,
+    /// Provides store to access to intermediate files.
+    manager: IntermediateManager,
+}
+
+#[async_trait]
+impl ExternalTempFileProvider for TempFileProvider {
+    async fn create(
+        &self,
+        file_group: &str,
+        file_id: &str,
+    ) -> IndexResult<Box<dyn AsyncWrite + Unpin + Send>> {
+        let path = self.location.file_path(file_group, file_id);
+        let writer = self
+            .manager
+            .store()
+            .writer(
+                &path,
+                &INDEX_INTERMEDIATE_WRITE_BYTES_TOTAL,
+                &INDEX_INTERMEDIATE_WRITE_OP_TOTAL,
+                &INDEX_INTERMEDIATE_FLUSH_OP_TOTAL,
+            )
+            .await
+            .map_err(BoxedError::new)
+            .context(index_error::ExternalSnafu)?;
+        Ok(Box::new(writer))
+    }
+
+    async fn read_all(
+        &self,
+        file_group: &str,
+    ) -> IndexResult<Vec<(String, Box<dyn AsyncRead + Unpin + Send>)>> {
+        let file_group_path = self.location.file_group_path(file_group);
+        let entries = self
+            .manager
+            .store()
+            .list(&file_group_path)
+            .await
+            .map_err(BoxedError::new)
+            .context(index_error::ExternalSnafu)?;
+        let mut readers = Vec::with_capacity(entries.len());
+
+        for entry in entries {
+            if entry.metadata().is_dir() {
+                warn!("Unexpected entry in index creation dir: {:?}", entry.path());
+                continue;
+            }
+
+            let im_file_id = self.location.im_file_id_from_path(entry.path());
+
+            let reader = self
+                .manager
+                .store()
+                .reader(
+                    entry.path(),
+                    &INDEX_INTERMEDIATE_READ_BYTES_TOTAL,
+                    &INDEX_INTERMEDIATE_READ_OP_TOTAL,
+                    &INDEX_INTERMEDIATE_SEEK_OP_TOTAL,
+                )
+                .await
+                .map_err(BoxedError::new)
+                .context(index_error::ExternalSnafu)?;
+            readers.push((im_file_id, Box::new(reader) as _));
+        }
+
+        Ok(readers)
+    }
+}
+
+impl TempFileProvider {
+    /// Creates a new `TempFileProvider`.
+    pub fn new(location: IntermediateLocation, manager: IntermediateManager) -> Self {
+        Self { location, manager }
+    }
+
+    /// Removes all intermediate files.
+    pub async fn cleanup(&self) -> Result<()> {
+        self.manager
+            .store()
+            .remove_all(self.location.dir_to_cleanup())
+            .await
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::ffi::OsStr;
 
     use common_test_util::temp_dir;
+    use futures::{AsyncReadExt, AsyncWriteExt};
     use regex::Regex;
+    use store_api::storage::RegionId;
 
     use super::*;
+    use crate::sst::file::FileId;
 
     #[tokio::test]
     async fn test_manager() {
@@ -212,4 +315,58 @@ mod tests {
             .is_match(&pi.next().unwrap().to_string_lossy())); // fulltext path
         assert!(pi.next().is_none());
     }
+
+    #[tokio::test]
+    async fn test_temp_file_provider_basic() {
+        let temp_dir = temp_dir::create_temp_dir("intermediate");
+        let path = temp_dir.path().display().to_string();
+
+        let location = IntermediateLocation::new(&RegionId::new(0, 0), &FileId::random());
+        let store = IntermediateManager::init_fs(path).await.unwrap();
+        let provider = TempFileProvider::new(location.clone(), store);
+
+        let file_group = "tag0";
+        let file_id = "0000000010";
+        let mut writer = provider.create(file_group, file_id).await.unwrap();
+        writer.write_all(b"hello").await.unwrap();
+        writer.flush().await.unwrap();
+        writer.close().await.unwrap();
+
+        let file_id = "0000000100";
+        let mut writer = provider.create(file_group, file_id).await.unwrap();
+        writer.write_all(b"world").await.unwrap();
+        writer.flush().await.unwrap();
+        writer.close().await.unwrap();
+
+        let file_group = "tag1";
+        let file_id = "0000000010";
+        let mut writer = provider.create(file_group, file_id).await.unwrap();
+        writer.write_all(b"foo").await.unwrap();
+        writer.flush().await.unwrap();
+        writer.close().await.unwrap();
+
+        let readers = provider.read_all("tag0").await.unwrap();
+        assert_eq!(readers.len(), 2);
+        for (_, mut reader) in readers {
+            let mut buf = Vec::new();
+            reader.read_to_end(&mut buf).await.unwrap();
+            assert!(matches!(buf.as_slice(), b"hello" | b"world"));
+        }
+        let readers = provider.read_all("tag1").await.unwrap();
+        assert_eq!(readers.len(), 1);
+        let mut reader = readers.into_iter().map(|x| x.1).next().unwrap();
+        let mut buf = Vec::new();
+        reader.read_to_end(&mut buf).await.unwrap();
+        assert_eq!(buf, b"foo");
+
+        provider.cleanup().await.unwrap();
+
+        assert!(provider
+            .manager
+            .store()
+            .list(location.dir_to_cleanup())
+            .await
+            .unwrap()
+            .is_empty());
+    }
 }

src/mito2/src/sst/index/inverted_index.rs

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 pub(crate) mod applier;
-mod codec;
 pub(crate) mod creator;
 
 const INDEX_BLOB_TYPE: &str = "greptime-inverted-index-v1";

src/mito2/src/sst/index/inverted_index/applier/builder.rs

@@ -37,8 +37,8 @@ use crate::cache::file_cache::FileCacheRef;
 use crate::cache::index::inverted_index::InvertedIndexCacheRef;
 use crate::error::{BuildIndexApplierSnafu, ColumnNotFoundSnafu, ConvertValueSnafu, Result};
 use crate::row_converter::SortField;
+use crate::sst::index::codec::IndexValueCodec;
 use crate::sst::index::inverted_index::applier::InvertedIndexApplier;
-use crate::sst::index::inverted_index::codec::IndexValueCodec;
 use crate::sst::index::puffin_manager::PuffinManagerFactory;
 
 /// Constructs an [`InvertedIndexApplier`] which applies predicates to SST files during scan.

src/mito2/src/sst/index/inverted_index/creator.rs

@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub(crate) mod temp_provider;
-
 use std::collections::HashSet;
 use std::num::NonZeroUsize;
 use std::sync::atomic::AtomicUsize;
@@ -38,9 +36,10 @@ use crate::error::{
 use crate::read::Batch;
 use crate::row_converter::SortField;
 use crate::sst::file::FileId;
-use crate::sst::index::intermediate::{IntermediateLocation, IntermediateManager};
-use crate::sst::index::inverted_index::codec::{IndexValueCodec, IndexValuesCodec};
-use crate::sst::index::inverted_index::creator::temp_provider::TempFileProvider;
+use crate::sst::index::codec::{IndexValueCodec, IndexValuesCodec};
+use crate::sst::index::intermediate::{
+    IntermediateLocation, IntermediateManager, TempFileProvider,
+};
 use crate::sst::index::inverted_index::INDEX_BLOB_TYPE;
 use crate::sst::index::puffin_manager::SstPuffinWriter;
 use crate::sst::index::statistics::{ByteCount, RowCount, Statistics};

src/mito2/src/sst/index/inverted_index/creator/temp_provider.rs

@@ -1,182 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use async_trait::async_trait;
-use common_error::ext::BoxedError;
-use common_telemetry::warn;
-use futures::{AsyncRead, AsyncWrite};
-use index::error as index_error;
-use index::error::Result as IndexResult;
-use index::external_provider::ExternalTempFileProvider;
-use snafu::ResultExt;
-
-use crate::error::Result;
-use crate::metrics::{
-    INDEX_INTERMEDIATE_FLUSH_OP_TOTAL, INDEX_INTERMEDIATE_READ_BYTES_TOTAL,
-    INDEX_INTERMEDIATE_READ_OP_TOTAL, INDEX_INTERMEDIATE_SEEK_OP_TOTAL,
-    INDEX_INTERMEDIATE_WRITE_BYTES_TOTAL, INDEX_INTERMEDIATE_WRITE_OP_TOTAL,
-};
-use crate::sst::index::intermediate::{IntermediateLocation, IntermediateManager};
-
-/// `TempFileProvider` implements `ExternalTempFileProvider`.
-/// It uses `InstrumentedStore` to create and read intermediate files.
-pub(crate) struct TempFileProvider {
-    /// Provides the location of intermediate files.
-    location: IntermediateLocation,
-    /// Provides store to access to intermediate files.
-    manager: IntermediateManager,
-}
-
-#[async_trait]
-impl ExternalTempFileProvider for TempFileProvider {
-    async fn create(
-        &self,
-        file_group: &str,
-        file_id: &str,
-    ) -> IndexResult<Box<dyn AsyncWrite + Unpin + Send>> {
-        let path = self.location.file_path(file_group, file_id);
-        let writer = self
-            .manager
-            .store()
-            .writer(
-                &path,
-                &INDEX_INTERMEDIATE_WRITE_BYTES_TOTAL,
-                &INDEX_INTERMEDIATE_WRITE_OP_TOTAL,
-                &INDEX_INTERMEDIATE_FLUSH_OP_TOTAL,
-            )
-            .await
-            .map_err(BoxedError::new)
-            .context(index_error::ExternalSnafu)?;
-        Ok(Box::new(writer))
-    }
-
-    async fn read_all(
-        &self,
-        file_group: &str,
-    ) -> IndexResult<Vec<(String, Box<dyn AsyncRead + Unpin + Send>)>> {
-        let file_group_path = self.location.file_group_path(file_group);
-        let entries = self
-            .manager
-            .store()
-            .list(&file_group_path)
-            .await
-            .map_err(BoxedError::new)
-            .context(index_error::ExternalSnafu)?;
-        let mut readers = Vec::with_capacity(entries.len());
-
-        for entry in entries {
-            if entry.metadata().is_dir() {
-                warn!("Unexpected entry in index creation dir: {:?}", entry.path());
-                continue;
-            }
-
-            let im_file_id = self.location.im_file_id_from_path(entry.path());
-
-            let reader = self
-                .manager
-                .store()
-                .reader(
-                    entry.path(),
-                    &INDEX_INTERMEDIATE_READ_BYTES_TOTAL,
-                    &INDEX_INTERMEDIATE_READ_OP_TOTAL,
-                    &INDEX_INTERMEDIATE_SEEK_OP_TOTAL,
-                )
-                .await
-                .map_err(BoxedError::new)
-                .context(index_error::ExternalSnafu)?;
-            readers.push((im_file_id, Box::new(reader) as _));
-        }
-
-        Ok(readers)
-    }
-}
-
-impl TempFileProvider {
-    /// Creates a new `TempFileProvider`.
-    pub fn new(location: IntermediateLocation, manager: IntermediateManager) -> Self {
-        Self { location, manager }
-    }
-
-    /// Removes all intermediate files.
-    pub async fn cleanup(&self) -> Result<()> {
-        self.manager
-            .store()
-            .remove_all(self.location.dir_to_cleanup())
-            .await
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use common_test_util::temp_dir;
-    use futures::{AsyncReadExt, AsyncWriteExt};
-    use store_api::storage::RegionId;
-
-    use super::*;
-    use crate::sst::file::FileId;
-
-    #[tokio::test]
-    async fn test_temp_file_provider_basic() {
-        let temp_dir = temp_dir::create_temp_dir("intermediate");
-        let path = temp_dir.path().display().to_string();
-
-        let location = IntermediateLocation::new(&RegionId::new(0, 0), &FileId::random());
-        let store = IntermediateManager::init_fs(path).await.unwrap();
-        let provider = TempFileProvider::new(location.clone(), store);
-
-        let file_group = "tag0";
-        let file_id = "0000000010";
-        let mut writer = provider.create(file_group, file_id).await.unwrap();
-        writer.write_all(b"hello").await.unwrap();
-        writer.flush().await.unwrap();
-        writer.close().await.unwrap();
-
-        let file_id = "0000000100";
-        let mut writer = provider.create(file_group, file_id).await.unwrap();
-        writer.write_all(b"world").await.unwrap();
-        writer.flush().await.unwrap();
-        writer.close().await.unwrap();
-
-        let file_group = "tag1";
-        let file_id = "0000000010";
-        let mut writer = provider.create(file_group, file_id).await.unwrap();
-        writer.write_all(b"foo").await.unwrap();
-        writer.flush().await.unwrap();
-        writer.close().await.unwrap();
-
-        let readers = provider.read_all("tag0").await.unwrap();
-        assert_eq!(readers.len(), 2);
-        for (_, mut reader) in readers {
-            let mut buf = Vec::new();
-            reader.read_to_end(&mut buf).await.unwrap();
-            assert!(matches!(buf.as_slice(), b"hello" | b"world"));
-        }
-        let readers = provider.read_all("tag1").await.unwrap();
-        assert_eq!(readers.len(), 1);
-        let mut reader = readers.into_iter().map(|x| x.1).next().unwrap();
-        let mut buf = Vec::new();
-        reader.read_to_end(&mut buf).await.unwrap();
-        assert_eq!(buf, b"foo");
-
-        provider.cleanup().await.unwrap();
-
-        assert!(provider
-            .manager
-            .store()
-            .list(location.dir_to_cleanup())
-            .await
-            .unwrap()
-            .is_empty());
-    }
-}

src/mito2/src/sst/parquet/reader.rs

@@ -51,6 +51,7 @@ use crate::read::prune::{PruneReader, Source};
 use crate::read::{Batch, BatchReader};
 use crate::row_converter::{McmpRowCodec, SortField};
 use crate::sst::file::FileHandle;
+use crate::sst::index::bloom_filter::applier::BloomFilterIndexApplierRef;
 use crate::sst::index::fulltext_index::applier::FulltextIndexApplierRef;
 use crate::sst::index::inverted_index::applier::InvertedIndexApplierRef;
 use crate::sst::parquet::file_range::{FileRangeContext, FileRangeContextRef};
@@ -80,6 +81,7 @@ pub struct ParquetReaderBuilder {
     cache_manager: Option<CacheManagerRef>,
     /// Index appliers.
     inverted_index_applier: Option<InvertedIndexApplierRef>,
+    bloom_filter_index_applier: Option<BloomFilterIndexApplierRef>,
     fulltext_index_applier: Option<FulltextIndexApplierRef>,
     /// Expected metadata of the region while reading the SST.
     /// This is usually the latest metadata of the region. The reader use
@@ -102,6 +104,7 @@ impl ParquetReaderBuilder {
             projection: None,
             cache_manager: None,
             inverted_index_applier: None,
+            bloom_filter_index_applier: None,
             fulltext_index_applier: None,
             expected_metadata: None,
         }
@@ -140,6 +143,16 @@ impl ParquetReaderBuilder {
         self
     }
 
+    /// Attaches the bloom filter index applier to the builder.
+    #[must_use]
+    pub(crate) fn bloom_filter_index_applier(
+        mut self,
+        index_applier: Option<BloomFilterIndexApplierRef>,
+    ) -> Self {
+        self.bloom_filter_index_applier = index_applier;
+        self
+    }
+
     /// Attaches the fulltext index applier to the builder.
     #[must_use]
     pub(crate) fn fulltext_index_applier(
@@ -337,8 +350,8 @@ impl ParquetReaderBuilder {
             return BTreeMap::default();
         }
 
-        metrics.num_row_groups_before_filtering += num_row_groups;
-        metrics.num_rows_in_row_group_before_filtering += num_rows as usize;
+        metrics.rg_total += num_row_groups;
+        metrics.rows_total += num_rows as usize;
 
         let mut output = (0..num_row_groups).map(|i| (i, None)).collect();
 
@@ -359,6 +372,9 @@ impl ParquetReaderBuilder {
             self.prune_row_groups_by_minmax(read_format, parquet_meta, &mut output, metrics);
         }
 
+        self.prune_row_groups_by_bloom_filter(parquet_meta, &mut output, metrics)
+            .await;
+
         output
     }
 
@@ -382,7 +398,7 @@ impl ParquetReaderBuilder {
             Err(err) => {
                 if cfg!(any(test, feature = "test")) {
                     panic!(
-                        "Failed to apply full-text index, region_id: {}, file_id: {}, err: {}",
+                        "Failed to apply full-text index, region_id: {}, file_id: {}, err: {:?}",
                         self.file_handle.region_id(),
                         self.file_handle.file_id(),
                         err
@@ -404,8 +420,8 @@ impl ParquetReaderBuilder {
             parquet_meta,
             row_group_to_row_ids,
             output,
-            &mut metrics.num_row_groups_fulltext_index_filtered,
-            &mut metrics.num_rows_in_row_group_fulltext_index_filtered,
+            &mut metrics.rg_fulltext_filtered,
+            &mut metrics.rows_fulltext_filtered,
         );
 
         true
@@ -466,7 +482,7 @@ impl ParquetReaderBuilder {
             Err(err) => {
                 if cfg!(any(test, feature = "test")) {
                     panic!(
-                        "Failed to apply inverted index, region_id: {}, file_id: {}, err: {}",
+                        "Failed to apply inverted index, region_id: {}, file_id: {}, err: {:?}",
                         self.file_handle.region_id(),
                         self.file_handle.file_id(),
                         err
@@ -505,8 +521,8 @@ impl ParquetReaderBuilder {
             parquet_meta,
             ranges_in_row_groups,
             output,
-            &mut metrics.num_row_groups_inverted_index_filtered,
-            &mut metrics.num_rows_in_row_group_inverted_index_filtered,
+            &mut metrics.rg_inverted_filtered,
+            &mut metrics.rows_inverted_filtered,
         );
 
         true
@@ -548,7 +564,7 @@ impl ParquetReaderBuilder {
             .collect::<BTreeMap<_, _>>();
 
         let row_groups_after = res.len();
-        metrics.num_row_groups_min_max_filtered += row_groups_before - row_groups_after;
+        metrics.rg_minmax_filtered += row_groups_before - row_groups_after;
 
         *output = res;
         true
@@ -607,6 +623,56 @@ impl ParquetReaderBuilder {
         *output = res;
     }
 
+    async fn prune_row_groups_by_bloom_filter(
+        &self,
+        parquet_meta: &ParquetMetaData,
+        output: &mut BTreeMap<usize, Option<RowSelection>>,
+        metrics: &mut ReaderFilterMetrics,
+    ) -> bool {
+        let Some(index_applier) = &self.bloom_filter_index_applier else {
+            return false;
+        };
+
+        if !self.file_handle.meta_ref().bloom_filter_index_available() {
+            return false;
+        }
+
+        let before_rg = output.len();
+
+        let file_size_hint = self.file_handle.meta_ref().bloom_filter_index_size();
+        if let Err(err) = index_applier
+            .apply(
+                self.file_handle.file_id(),
+                file_size_hint,
+                parquet_meta.row_groups(),
+                output,
+            )
+            .await
+        {
+            if cfg!(any(test, feature = "test")) {
+                panic!(
+                    "Failed to apply bloom filter index, region_id: {}, file_id: {}, err: {:?}",
+                    self.file_handle.region_id(),
+                    self.file_handle.file_id(),
+                    err
+                );
+            } else {
+                warn!(
+                    err; "Failed to apply bloom filter index, region_id: {}, file_id: {}",
+                    self.file_handle.region_id(), self.file_handle.file_id()
+                );
+            }
+
+            return false;
+        };
+
+        let after_rg = output.len();
+        // Update metrics.
+        metrics.rg_bloom_filtered += before_rg - after_rg;
+
+        true
+    }
+
     /// Prunes row groups by ranges. The `ranges_in_row_groups` is like a map from row group to
     /// a list of row ranges to keep.
     fn prune_row_groups_by_ranges(
@@ -664,64 +730,77 @@ impl ParquetReaderBuilder {
 #[derive(Debug, Default, Clone, Copy)]
 pub(crate) struct ReaderFilterMetrics {
     /// Number of row groups before filtering.
-    pub(crate) num_row_groups_before_filtering: usize,
+    pub(crate) rg_total: usize,
     /// Number of row groups filtered by fulltext index.
-    pub(crate) num_row_groups_fulltext_index_filtered: usize,
+    pub(crate) rg_fulltext_filtered: usize,
     /// Number of row groups filtered by inverted index.
-    pub(crate) num_row_groups_inverted_index_filtered: usize,
+    pub(crate) rg_inverted_filtered: usize,
     /// Number of row groups filtered by min-max index.
-    pub(crate) num_row_groups_min_max_filtered: usize,
-    /// Number of rows filtered by precise filter.
-    pub(crate) num_rows_precise_filtered: usize,
+    pub(crate) rg_minmax_filtered: usize,
+    /// Number of row groups filtered by bloom filter index.
+    pub(crate) rg_bloom_filtered: usize,
+
     /// Number of rows in row group before filtering.
-    pub(crate) num_rows_in_row_group_before_filtering: usize,
+    pub(crate) rows_total: usize,
     /// Number of rows in row group filtered by fulltext index.
-    pub(crate) num_rows_in_row_group_fulltext_index_filtered: usize,
+    pub(crate) rows_fulltext_filtered: usize,
     /// Number of rows in row group filtered by inverted index.
-    pub(crate) num_rows_in_row_group_inverted_index_filtered: usize,
+    pub(crate) rows_inverted_filtered: usize,
+    /// Number of rows in row group filtered by bloom filter index.
+    pub(crate) rows_bloom_filtered: usize,
+    /// Number of rows filtered by precise filter.
+    pub(crate) rows_precise_filtered: usize,
 }
 
 impl ReaderFilterMetrics {
     /// Adds `other` metrics to this metrics.
     pub(crate) fn merge_from(&mut self, other: &ReaderFilterMetrics) {
-        self.num_row_groups_before_filtering += other.num_row_groups_before_filtering;
-        self.num_row_groups_fulltext_index_filtered += other.num_row_groups_fulltext_index_filtered;
-        self.num_row_groups_inverted_index_filtered += other.num_row_groups_inverted_index_filtered;
-        self.num_row_groups_min_max_filtered += other.num_row_groups_min_max_filtered;
-        self.num_rows_precise_filtered += other.num_rows_precise_filtered;
-        self.num_rows_in_row_group_before_filtering += other.num_rows_in_row_group_before_filtering;
-        self.num_rows_in_row_group_fulltext_index_filtered +=
-            other.num_rows_in_row_group_fulltext_index_filtered;
-        self.num_rows_in_row_group_inverted_index_filtered +=
-            other.num_rows_in_row_group_inverted_index_filtered;
+        self.rg_total += other.rg_total;
+        self.rg_fulltext_filtered += other.rg_fulltext_filtered;
+        self.rg_inverted_filtered += other.rg_inverted_filtered;
+        self.rg_minmax_filtered += other.rg_minmax_filtered;
+        self.rg_bloom_filtered += other.rg_bloom_filtered;
+
+        self.rows_total += other.rows_total;
+        self.rows_fulltext_filtered += other.rows_fulltext_filtered;
+        self.rows_inverted_filtered += other.rows_inverted_filtered;
+        self.rows_bloom_filtered += other.rows_bloom_filtered;
+        self.rows_precise_filtered += other.rows_precise_filtered;
     }
 
     /// Reports metrics.
     pub(crate) fn observe(&self) {
         READ_ROW_GROUPS_TOTAL
             .with_label_values(&["before_filtering"])
-            .inc_by(self.num_row_groups_before_filtering as u64);
+            .inc_by(self.rg_total as u64);
         READ_ROW_GROUPS_TOTAL
             .with_label_values(&["fulltext_index_filtered"])
-            .inc_by(self.num_row_groups_fulltext_index_filtered as u64);
+            .inc_by(self.rg_fulltext_filtered as u64);
         READ_ROW_GROUPS_TOTAL
             .with_label_values(&["inverted_index_filtered"])
-            .inc_by(self.num_row_groups_inverted_index_filtered as u64);
+            .inc_by(self.rg_inverted_filtered as u64);
         READ_ROW_GROUPS_TOTAL
             .with_label_values(&["minmax_index_filtered"])
-            .inc_by(self.num_row_groups_min_max_filtered as u64);
+            .inc_by(self.rg_minmax_filtered as u64);
+        READ_ROW_GROUPS_TOTAL
+            .with_label_values(&["bloom_filter_index_filtered"])
+            .inc_by(self.rg_bloom_filtered as u64);
+
         PRECISE_FILTER_ROWS_TOTAL
             .with_label_values(&["parquet"])
-            .inc_by(self.num_rows_precise_filtered as u64);
+            .inc_by(self.rows_precise_filtered as u64);
         READ_ROWS_IN_ROW_GROUP_TOTAL
             .with_label_values(&["before_filtering"])
-            .inc_by(self.num_rows_in_row_group_before_filtering as u64);
+            .inc_by(self.rows_total as u64);
         READ_ROWS_IN_ROW_GROUP_TOTAL
             .with_label_values(&["fulltext_index_filtered"])
-            .inc_by(self.num_rows_in_row_group_fulltext_index_filtered as u64);
+            .inc_by(self.rows_fulltext_filtered as u64);
         READ_ROWS_IN_ROW_GROUP_TOTAL
             .with_label_values(&["inverted_index_filtered"])
-            .inc_by(self.num_rows_in_row_group_inverted_index_filtered as u64);
+            .inc_by(self.rows_inverted_filtered as u64);
+        READ_ROWS_IN_ROW_GROUP_TOTAL
+            .with_label_values(&["bloom_filter_index_filtered"])
+            .inc_by(self.rows_bloom_filtered as u64);
     }
 }
 
@@ -977,12 +1056,12 @@ impl Drop for ParquetReader {
             self.context.reader_builder().file_handle.region_id(),
             self.context.reader_builder().file_handle.file_id(),
             self.context.reader_builder().file_handle.time_range(),
-            metrics.filter_metrics.num_row_groups_before_filtering
-                - metrics
-                    .filter_metrics
-                    .num_row_groups_inverted_index_filtered
-                - metrics.filter_metrics.num_row_groups_min_max_filtered,
-            metrics.filter_metrics.num_row_groups_before_filtering,
+            metrics.filter_metrics.rg_total
+                - metrics.filter_metrics.rg_inverted_filtered
+                - metrics.filter_metrics.rg_minmax_filtered
+                - metrics.filter_metrics.rg_fulltext_filtered
+                - metrics.filter_metrics.rg_bloom_filtered,
+            metrics.filter_metrics.rg_total,
             metrics
         );
 

src/mito2/src/test_util.rs

@@ -643,17 +643,9 @@ impl TestEnv {
             .await
             .unwrap();
 
-        let object_store_manager = self.get_object_store_manager().unwrap();
-        let write_cache = WriteCache::new(
-            local_store,
-            object_store_manager,
-            capacity,
-            None,
-            puffin_mgr,
-            intm_mgr,
-        )
-        .await
-        .unwrap();
+        let write_cache = WriteCache::new(local_store, capacity, None, puffin_mgr, intm_mgr)
+            .await
+            .unwrap();
 
         Arc::new(write_cache)
     }

src/mito2/src/worker.rs

@@ -157,7 +157,6 @@ impl WorkerGroup {
         let purge_scheduler = Arc::new(LocalScheduler::new(config.max_background_purges));
         let write_cache = write_cache_from_config(
             &config,
-            object_store_manager.clone(),
             puffin_manager_factory.clone(),
             intermediate_manager.clone(),
         )
@@ -303,7 +302,6 @@ impl WorkerGroup {
             .with_buffer_size(Some(config.index.write_buffer_size.as_bytes() as _));
         let write_cache = write_cache_from_config(
             &config,
-            object_store_manager.clone(),
             puffin_manager_factory.clone(),
             intermediate_manager.clone(),
         )
@@ -364,7 +362,6 @@ fn region_id_to_index(id: RegionId, num_workers: usize) -> usize {
 
 async fn write_cache_from_config(
     config: &MitoConfig,
-    object_store_manager: ObjectStoreManagerRef,
     puffin_manager_factory: PuffinManagerFactory,
     intermediate_manager: IntermediateManager,
 ) -> Result<Option<WriteCacheRef>> {
@@ -383,7 +380,6 @@ async fn write_cache_from_config(
 
     let cache = WriteCache::new_fs(
         &config.experimental_write_cache_path,
-        object_store_manager,
         config.experimental_write_cache_size,
         config.experimental_write_cache_ttl,
         puffin_manager_factory,

src/object-store/src/layers.rs

@@ -25,18 +25,23 @@ mod prometheus {
 
     static PROMETHEUS_LAYER: OnceLock<Mutex<PrometheusLayer>> = OnceLock::new();
 
-    pub fn build_prometheus_metrics_layer(with_path_label: bool) -> PrometheusLayer {
+    /// This logical tries to extract parent path from the object storage operation
+    /// the function also relies on assumption that the region path is built from
+    /// pattern `<data|index>/catalog/schema/table_id/...` OR `greptimedb/object_cache/<read|write>/...`
+    ///
+    /// We'll get the data/catalog/schema from path.
+    pub fn build_prometheus_metrics_layer(_with_path_label: bool) -> PrometheusLayer {
         PROMETHEUS_LAYER
             .get_or_init(|| {
-                // This logical tries to extract parent path from the object storage operation
-                // the function also relies on assumption that the region path is built from
-                // pattern `<data|index>/catalog/schema/table_id/....`
-                //
-                // We'll get the data/catalog/schema from path.
-                let path_level = if with_path_label { 3 } else { 0 };
+                // let path_level = if with_path_label { 3 } else { 0 };
+
+                // opendal doesn't support dynamic path label trim
+                // we have uuid in index path, which causes the label size to explode
+                // remove path label first, waiting for later fix
+                // TODO(shuiyisong): add dynamic path label trim for opendal
 
                 let layer = PrometheusLayer::builder()
-                    .path_label(path_level)
+                    .path_label(0)
                     .register_default()
                     .unwrap();
 

src/object-store/src/layers/lru_cache.rs

@@ -117,9 +117,7 @@ impl<I: Access, C: Access> LayeredAccess for LruCacheAccess<I, C> {
     async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
         let result = self.inner.write(path, args).await;
 
-        self.read_cache
-            .invalidate_entries_with_prefix(format!("{:x}", md5::compute(path)))
-            .await;
+        self.read_cache.invalidate_entries_with_prefix(path);
 
         result
     }
@@ -127,9 +125,7 @@ impl<I: Access, C: Access> LayeredAccess for LruCacheAccess<I, C> {
     async fn delete(&self, path: &str, args: OpDelete) -> Result<RpDelete> {
         let result = self.inner.delete(path, args).await;
 
-        self.read_cache
-            .invalidate_entries_with_prefix(format!("{:x}", md5::compute(path)))
-            .await;
+        self.read_cache.invalidate_entries_with_prefix(path);
 
         result
     }
@@ -146,8 +142,7 @@ impl<I: Access, C: Access> LayeredAccess for LruCacheAccess<I, C> {
     fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
         let result = self.inner.blocking_write(path, args);
 
-        self.read_cache
-            .blocking_invalidate_entries_with_prefix(format!("{:x}", md5::compute(path)));
+        self.read_cache.invalidate_entries_with_prefix(path);
 
         result
     }

src/object-store/src/layers/lru_cache/read_cache.rs

@@ -20,7 +20,7 @@ use moka::future::Cache;
 use moka::notification::ListenerFuture;
 use opendal::raw::oio::{Read, Reader, Write};
 use opendal::raw::{Access, OpDelete, OpRead, OpStat, OpWrite, RpRead};
-use opendal::{Error as OpendalError, ErrorKind, Metakey, OperatorBuilder, Result};
+use opendal::{EntryMode, Error as OpendalError, ErrorKind, Metakey, OperatorBuilder, Result};
 
 use crate::metrics::{
     OBJECT_STORE_LRU_CACHE_BYTES, OBJECT_STORE_LRU_CACHE_ENTRIES, OBJECT_STORE_LRU_CACHE_HIT,
@@ -28,6 +28,10 @@ use crate::metrics::{
 };
 
 const RECOVER_CACHE_LIST_CONCURRENT: usize = 8;
+/// Subdirectory of cached files for read.
+///
+/// This must contain three layers, corresponding to [`build_prometheus_metrics_layer`](object_store::layers::build_prometheus_metrics_layer).
+const READ_CACHE_DIR: &str = "cache/object/read";
 
 /// Cache value for read file
 #[derive(Debug, Clone, PartialEq, Eq, Copy)]
@@ -56,12 +60,20 @@ fn can_cache(path: &str) -> bool {
 /// Generate a unique cache key for the read path and range.
 fn read_cache_key(path: &str, args: &OpRead) -> String {
     format!(
-        "{:x}.cache-{}",
+        "{READ_CACHE_DIR}/{:x}.cache-{}",
         md5::compute(path),
         args.range().to_header()
     )
 }
 
+fn read_cache_root() -> String {
+    format!("/{READ_CACHE_DIR}/")
+}
+
+fn read_cache_key_prefix(path: &str) -> String {
+    format!("{READ_CACHE_DIR}/{:x}", md5::compute(path))
+}
+
 /// Local read cache for files in object storage
 #[derive(Debug)]
 pub(crate) struct ReadCache<C> {
@@ -125,16 +137,9 @@ impl<C: Access> ReadCache<C> {
         (self.mem_cache.entry_count(), self.mem_cache.weighted_size())
     }
 
-    /// Invalidate all cache items which key starts with `prefix`.
-    pub(crate) async fn invalidate_entries_with_prefix(&self, prefix: String) {
-        // Safety: always ok when building cache with `support_invalidation_closures`.
-        self.mem_cache
-            .invalidate_entries_if(move |k: &String, &_v| k.starts_with(&prefix))
-            .ok();
-    }
-
-    /// Blocking version of `invalidate_entries_with_prefix`.
-    pub(crate) fn blocking_invalidate_entries_with_prefix(&self, prefix: String) {
+    /// Invalidate all cache items belong to the specific path.
+    pub(crate) fn invalidate_entries_with_prefix(&self, path: &str) {
+        let prefix = read_cache_key_prefix(path);
         // Safety: always ok when building cache with `support_invalidation_closures`.
         self.mem_cache
             .invalidate_entries_if(move |k: &String, &_v| k.starts_with(&prefix))
@@ -145,8 +150,9 @@ impl<C: Access> ReadCache<C> {
     /// Return entry count and total approximate entry size in bytes.
     pub(crate) async fn recover_cache(&self) -> Result<(u64, u64)> {
         let op = OperatorBuilder::new(self.file_cache.clone()).finish();
+        let root = read_cache_root();
         let mut entries = op
-            .list_with("/")
+            .list_with(&root)
             .metakey(Metakey::ContentLength | Metakey::ContentType)
             .concurrent(RECOVER_CACHE_LIST_CONCURRENT)
             .await?;
@@ -157,7 +163,7 @@ impl<C: Access> ReadCache<C> {
             OBJECT_STORE_LRU_CACHE_ENTRIES.inc();
             OBJECT_STORE_LRU_CACHE_BYTES.add(size as i64);
             // ignore root path
-            if entry.path() != "/" {
+            if entry.metadata().mode() == EntryMode::FILE {
                 self.mem_cache
                     .insert(read_key.to_string(), ReadResult::Success(size as u32))
                     .await;

src/object-store/tests/object_store_test.rs

@@ -27,6 +27,9 @@ use opendal::raw::{Access, OpList, OpRead};
 use opendal::services::{Azblob, Gcs, Oss};
 use opendal::{EntryMode, OperatorBuilder};
 
+/// Duplicate of the constant in `src/layers/lru_cache/read_cache.rs`
+const READ_CACHE_DIR: &str = "cache/object/read";
+
 async fn test_object_crud(store: &ObjectStore) -> Result<()> {
     // Create object handler.
     // Write data info object;
@@ -267,7 +270,8 @@ async fn test_file_backend_with_lru_cache() -> Result<()> {
 
 async fn assert_lru_cache<C: Access>(cache_layer: &LruCacheLayer<C>, file_names: &[&str]) {
     for file_name in file_names {
-        assert!(cache_layer.contains_file(file_name).await, "{file_name}");
+        let file_path = format!("{READ_CACHE_DIR}/{file_name}");
+        assert!(cache_layer.contains_file(&file_path).await, "{file_path:?}");
     }
 }
 

src/operator/src/expr_factory.rs

@@ -68,6 +68,7 @@ impl CreateExprFactory {
         table_name: &TableReference<'_>,
         column_schemas: &[api::v1::ColumnSchema],
         engine: &str,
+        desc: Option<&str>,
     ) -> Result<CreateTableExpr> {
         let column_exprs = ColumnExpr::from_column_schemas(column_schemas);
         let create_expr = common_grpc_expr::util::build_create_table_expr(
@@ -75,7 +76,7 @@ impl CreateExprFactory {
             table_name,
             column_exprs,
             engine,
-            "Created on insertion",
+            desc.unwrap_or("Created on insertion"),
         )
         .context(BuildCreateExprOnInsertionSnafu)?;
 

src/operator/src/insert.rs

@@ -870,5 +870,5 @@ fn build_create_table_expr(
     request_schema: &[ColumnSchema],
     engine: &str,
 ) -> Result<CreateTableExpr> {
-    CreateExprFactory.create_table_expr_by_column_schemas(table, request_schema, engine)
+    CreateExprFactory.create_table_expr_by_column_schemas(table, request_schema, engine, None)
 }

src/query/Cargo.toml

@@ -70,9 +70,11 @@ uuid.workspace = true
 [dev-dependencies]
 arrow.workspace = true
 catalog = { workspace = true, features = ["testing"] }
+common-function.workspace = true
 common-macro.workspace = true
 common-query = { workspace = true, features = ["testing"] }
 fastrand = "2.0"
+nalgebra.workspace = true
 num = "0.4"
 num-traits = "0.2"
 paste = "1.0"

src/query/src/tests.rs

@@ -33,6 +33,7 @@ mod time_range_filter_test;
 
 mod function;
 mod pow;
+mod vec_sum_test;
 
 async fn exec_selection(engine: QueryEngineRef, sql: &str) -> Vec<RecordBatch> {
     let query_ctx = QueryContext::arc();

src/query/src/tests/function.rs

@@ -14,12 +14,13 @@
 
 use std::sync::Arc;
 
+use common_function::scalars::vector::impl_conv::veclit_to_binlit;
 use common_recordbatch::RecordBatch;
 use datatypes::for_all_primitive_types;
 use datatypes::prelude::*;
 use datatypes::schema::{ColumnSchema, Schema};
 use datatypes::types::WrapperType;
-use datatypes::vectors::Helper;
+use datatypes::vectors::{BinaryVector, Helper};
 use rand::Rng;
 use table::test_util::MemTable;
 
@@ -52,6 +53,34 @@ pub fn create_query_engine() -> QueryEngineRef {
     new_query_engine_with_table(number_table)
 }
 
+pub fn create_query_engine_for_vector10x3() -> QueryEngineRef {
+    let mut column_schemas = vec![];
+    let mut columns = vec![];
+    let mut rng = rand::thread_rng();
+
+    let column_name = "vector";
+    let column_schema = ColumnSchema::new(column_name, ConcreteDataType::binary_datatype(), true);
+    column_schemas.push(column_schema);
+
+    let vectors = (0..10)
+        .map(|_| {
+            let veclit = [
+                rng.gen_range(-100f32..100.0),
+                rng.gen_range(-100f32..100.0),
+                rng.gen_range(-100f32..100.0),
+            ];
+            veclit_to_binlit(&veclit)
+        })
+        .collect::<Vec<_>>();
+    let column: VectorRef = Arc::new(BinaryVector::from(vectors));
+    columns.push(column);
+
+    let schema = Arc::new(Schema::new(column_schemas.clone()));
+    let recordbatch = RecordBatch::new(schema, columns).unwrap();
+    let vector_table = MemTable::table("vectors", recordbatch);
+    new_query_engine_with_table(vector_table)
+}
+
 pub async fn get_numbers_from_table<'s, T>(
     column_name: &'s str,
     table_name: &'s str,