From ba3c5a939e3319b1973bfc57db77c011f1b511f0 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Fri, 28 Aug 2026 05:56:44 +0000 Subject: [PATCH] chore(mito2): reduce default auto flush interval (#8971) Signed-off-by: evenyag --- config/config.md | 4 ++-- config/datanode.example.toml | 2 +- config/standalone.example.toml | 2 +- src/cmd/tests/load_config_test.rs | 4 ++-- src/mito2/src/config.rs | 12 ++++++++++-- tests-integration/tests/http.rs | 2 +- 6 files changed, 17 insertions(+), 9 deletions(-) diff --git a/config/config.md b/config/config.md index 992296d482..1343f60dbd 100644 --- a/config/config.md +++ b/config/config.md @@ -161,7 +161,7 @@ | `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). | | `region_engine.mito.experimental_compaction_memory_limit` | String | 0 | Memory budget for compaction tasks.
Supports absolute size (e.g., "2GiB", "512MB") or percentage of system memory (e.g., "50%").
Setting it to 0 or "unlimited" disables the limit. | | `region_engine.mito.experimental_compaction_on_exhausted` | String | wait | Behavior when compaction cannot acquire memory from the budget.
Options: "wait" (default, 10s), "wait()", "fail" | -| `region_engine.mito.auto_flush_interval` | String | `1h` | Interval to auto flush a region if it has not flushed yet. | +| `region_engine.mito.auto_flush_interval` | String | `10m` | Interval to auto flush a region if it has not flushed yet. | | `region_engine.mito.global_write_buffer_size` | String | Auto | Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. | | `region_engine.mito.global_write_buffer_reject_size` | String | Auto | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size`. | | `region_engine.mito.default_region_write_buffer_size` | String | `0` | Default write buffer size for each region. Regions stall at this size and reject writes at twice this size. Setting it to 0 disables both limits unless the table specifies `write_buffer_size`. | @@ -585,7 +585,7 @@ | `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). | | `region_engine.mito.experimental_compaction_memory_limit` | String | 0 | Memory budget for compaction tasks.
Supports absolute size (e.g., "2GiB", "512MB") or percentage of system memory (e.g., "50%").
Setting it to 0 or "unlimited" disables the limit. | | `region_engine.mito.experimental_compaction_on_exhausted` | String | wait | Behavior when compaction cannot acquire memory from the budget.
Options: "wait" (default, 10s), "wait()", "fail" | -| `region_engine.mito.auto_flush_interval` | String | `1h` | Interval to auto flush a region if it has not flushed yet. | +| `region_engine.mito.auto_flush_interval` | String | `10m` | Interval to auto flush a region if it has not flushed yet. | | `region_engine.mito.global_write_buffer_size` | String | Auto | Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. | | `region_engine.mito.global_write_buffer_reject_size` | String | Auto | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size` | | `region_engine.mito.default_region_write_buffer_size` | String | `0` | Default write buffer size for each region. Regions stall at this size and reject writes at twice this size. Setting it to 0 disables both limits unless the table specifies `write_buffer_size`. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 5966405452..20c84fd9ee 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -462,7 +462,7 @@ compress_manifest = false #+ experimental_compaction_on_exhausted = "wait" ## Interval to auto flush a region if it has not flushed yet. -auto_flush_interval = "1h" +auto_flush_interval = "10m" ## Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. ## @toml2docs:none-default="Auto" diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 70f74c7f94..6ca13566d7 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -625,7 +625,7 @@ compress_manifest = false #+ experimental_compaction_on_exhausted = "wait" ## Interval to auto flush a region if it has not flushed yet. -auto_flush_interval = "1h" +auto_flush_interval = "10m" ## Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. ## @toml2docs:none-default="Auto" diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs index 70594d0653..8af216256f 100644 --- a/src/cmd/tests/load_config_test.rs +++ b/src/cmd/tests/load_config_test.rs @@ -113,7 +113,7 @@ fn test_load_datanode_example_config() { }, region_engine: vec![ RegionEngineConfig::Mito(MitoConfig { - auto_flush_interval: Duration::from_secs(3600), + auto_flush_interval: Duration::from_secs(10 * 60), default_region_write_buffer_size: ReadableSize::mb(0), write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)), scan_memory_limit: MemoryLimit::Unlimited, @@ -354,7 +354,7 @@ fn test_load_standalone_example_config() { }), region_engine: vec![ RegionEngineConfig::Mito(MitoConfig { - auto_flush_interval: Duration::from_secs(3600), + auto_flush_interval: Duration::from_secs(10 * 60), default_region_write_buffer_size: ReadableSize::mb(0), write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)), scan_memory_limit: MemoryLimit::Unlimited, diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs index 54445549a0..be74fa67c6 100644 --- a/src/mito2/src/config.rs +++ b/src/mito2/src/config.rs @@ -102,7 +102,7 @@ pub struct MitoConfig { pub experimental_compaction_on_exhausted: OnExhaustedPolicy, // Flush configs: - /// Interval to auto flush a region if it has not flushed yet (default 30 min). + /// Interval to auto flush a region if it has not flushed yet (default 10 min). #[serde(with = "humantime_serde")] pub auto_flush_interval: Duration, /// Global write buffer size threshold to trigger flush. @@ -210,7 +210,7 @@ impl Default for MitoConfig { max_background_purges: get_total_cpu_cores(), experimental_compaction_memory_limit: MemoryLimit::Unlimited, experimental_compaction_on_exhausted: OnExhaustedPolicy::default(), - auto_flush_interval: Duration::from_secs(30 * 60), + auto_flush_interval: Duration::from_secs(10 * 60), global_write_buffer_size: ReadableSize::gb(1), global_write_buffer_reject_size: ReadableSize::gb(2), default_region_write_buffer_size: ReadableSize::mb(0), @@ -381,6 +381,14 @@ impl MitoConfig { mod tests { use super::*; + #[test] + fn test_default_auto_flush_interval() { + assert_eq!( + Duration::from_secs(10 * 60), + MitoConfig::default().auto_flush_interval + ); + } + #[test] fn test_adjust_sst_metadata_and_prefilter_cache_caps_independently() { let mut config = MitoConfig::default(); diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 43ba8701d7..f4cfda24dc 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -2278,7 +2278,7 @@ experimental_manifest_keep_removed_file_ttl = "1h" compress_manifest = false experimental_compaction_memory_limit = "unlimited" experimental_compaction_on_exhausted = "wait" -auto_flush_interval = "30m" +auto_flush_interval = "10m" default_region_write_buffer_size = "0KiB" enable_write_cache = false write_cache_path = ""