diff --git a/config/config.md b/config/config.md index 14f22f5b6c..7f48597737 100644 --- a/config/config.md +++ b/config/config.md @@ -129,6 +129,8 @@ | `region_engine.mito.inverted_index.apply_on_query` | String | `auto` | Whether to apply the index on query
- `auto`: automatically (default)
- `disable`: never | | `region_engine.mito.inverted_index.mem_threshold_on_create` | String | `auto` | Memory threshold for performing an external sort during index creation.
- `auto`: automatically determine the threshold based on the system memory size (default)
- `unlimited`: no memory limit
- `[size]` e.g. `64MB`: fixed memory threshold | | `region_engine.mito.inverted_index.intermediate_path` | String | `""` | Deprecated, use `region_engine.mito.index.aux_path` instead. | +| `region_engine.mito.inverted_index.metadata_cache_size` | String | `64MiB` | Cache size for inverted index metadata. | +| `region_engine.mito.inverted_index.content_cache_size` | String | `128MiB` | Cache size for inverted index content. | | `region_engine.mito.fulltext_index` | -- | -- | The options for full-text index in Mito engine. | | `region_engine.mito.fulltext_index.create_on_flush` | String | `auto` | Whether to create the index on flush.
- `auto`: automatically (default)
- `disable`: never | | `region_engine.mito.fulltext_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.
- `auto`: automatically (default)
- `disable`: never | diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 0944d5985e..7d40927703 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -459,6 +459,12 @@ mem_threshold_on_create = "auto" ## Deprecated, use `region_engine.mito.index.aux_path` instead. intermediate_path = "" +## Cache size for inverted index metadata. +metadata_cache_size = "64MiB" + +## Cache size for inverted index content. +content_cache_size = "128MiB" + ## The options for full-text index in Mito engine. [region_engine.mito.fulltext_index] diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs index 012c31aaad..7fd87d774f 100644 --- a/src/mito2/src/config.rs +++ b/src/mito2/src/config.rs @@ -39,6 +39,8 @@ const DEFAULT_SCAN_CHANNEL_SIZE: usize = 32; const GLOBAL_WRITE_BUFFER_SIZE_FACTOR: u64 = 8; /// Use `1/SST_META_CACHE_SIZE_FACTOR` of OS memory size as SST meta cache size in default mode const SST_META_CACHE_SIZE_FACTOR: u64 = 32; +/// Use `1/INDEX_CONTENT_CACHE_SIZE_FACTOR` of OS memory size for inverted index file content cache by default. +const INDEX_CONTENT_CACHE_SIZE_FACTOR: u64 = 32; /// Use `1/MEM_CACHE_SIZE_FACTOR` of OS memory size as mem cache size in default mode const MEM_CACHE_SIZE_FACTOR: u64 = 16; /// Use `1/INDEX_CREATE_MEM_THRESHOLD_FACTOR` of OS memory size as mem threshold for creating index @@ -389,19 +391,35 @@ pub struct InvertedIndexConfig { pub content_cache_size: ReadableSize, } +impl InvertedIndexConfig { + /// Adjusts the cache size of [InvertedIndexConfig] according to system memory size. + fn adjust_cache_size(&mut self, sys_memory: ReadableSize) { + let content_cache_size = cmp::min( + sys_memory / INDEX_CONTENT_CACHE_SIZE_FACTOR, + ReadableSize::mb(128), + ); + self.content_cache_size = content_cache_size; + } +} + impl Default for InvertedIndexConfig { #[allow(deprecated)] fn default() -> Self { - Self { + let mut index_config = Self { create_on_flush: Mode::Auto, create_on_compaction: Mode::Auto, apply_on_query: Mode::Auto, mem_threshold_on_create: MemoryThreshold::Auto, write_buffer_size: ReadableSize::mb(8), intermediate_path: String::new(), - metadata_cache_size: ReadableSize::mb(32), - content_cache_size: ReadableSize::mb(32), + metadata_cache_size: ReadableSize::mb(64), + content_cache_size: ReadableSize::mb(128), + }; + + if let Some(sys_memory) = common_config::utils::get_sys_total_memory() { + index_config.adjust_cache_size(sys_memory); } + index_config } } diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 75939b7c94..38c87d865d 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -839,8 +839,6 @@ create_on_flush = "auto" create_on_compaction = "auto" apply_on_query = "auto" mem_threshold_on_create = "auto" -metadata_cache_size = "32MiB" -content_cache_size = "32MiB" [region_engine.mito.fulltext_index] create_on_flush = "auto" @@ -889,6 +887,8 @@ fn drop_lines_with_inconsistent_results(input: String) -> String { "vector_cache_size =", "page_cache_size =", "selector_result_cache_size =", + "metadata_cache_size =", + "content_cache_size =", ]; input