From ab96703d8ff53658a37474b6353680c6dc8a8d90 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Mon, 1 Sep 2025 17:26:20 +0800 Subject: [PATCH] chore: enlarge max file limit to 384 (#6868) chore: enlarge max concurrent files to 384 Signed-off-by: evenyag --- config/config.md | 4 ++-- config/datanode.example.toml | 2 +- config/standalone.example.toml | 2 +- src/mito2/src/config.rs | 4 ++-- tests-integration/tests/http.rs | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/config/config.md b/config/config.md index 847c18cf08..46d7b440c1 100644 --- a/config/config.md +++ b/config/config.md @@ -148,7 +148,7 @@ | `region_engine.mito.write_cache_ttl` | String | Unset | TTL for write cache. | | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. | | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. | -| `region_engine.mito.max_concurrent_scan_files` | Integer | `128` | Maximum number of SST files to scan concurrently. | +| `region_engine.mito.max_concurrent_scan_files` | Integer | `384` | Maximum number of SST files to scan concurrently. | | `region_engine.mito.allow_stale_entries` | Bool | `false` | Whether to allow stale WAL entries read during replay. | | `region_engine.mito.min_compaction_interval` | String | `0m` | Minimum time interval between two compactions.
To align with the old behavior, the default value is 0 (no restrictions). | | `region_engine.mito.index` | -- | -- | The options for index in Mito engine. | @@ -540,7 +540,7 @@ | `region_engine.mito.write_cache_ttl` | String | Unset | TTL for write cache. | | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. | | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. | -| `region_engine.mito.max_concurrent_scan_files` | Integer | `128` | Maximum number of SST files to scan concurrently. | +| `region_engine.mito.max_concurrent_scan_files` | Integer | `384` | Maximum number of SST files to scan concurrently. | | `region_engine.mito.allow_stale_entries` | Bool | `false` | Whether to allow stale WAL entries read during replay. | | `region_engine.mito.min_compaction_interval` | String | `0m` | Minimum time interval between two compactions.
To align with the old behavior, the default value is 0 (no restrictions). | | `region_engine.mito.index` | -- | -- | The options for index in Mito engine. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 5bde0947e4..7e04748059 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -488,7 +488,7 @@ sst_write_buffer_size = "8MB" parallel_scan_channel_size = 32 ## Maximum number of SST files to scan concurrently. -max_concurrent_scan_files = 128 +max_concurrent_scan_files = 384 ## Whether to allow stale WAL entries read during replay. allow_stale_entries = false diff --git a/config/standalone.example.toml b/config/standalone.example.toml index e93d66a135..22dd8105a9 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -567,7 +567,7 @@ sst_write_buffer_size = "8MB" parallel_scan_channel_size = 32 ## Maximum number of SST files to scan concurrently. -max_concurrent_scan_files = 128 +max_concurrent_scan_files = 384 ## Whether to allow stale WAL entries read during replay. allow_stale_entries = false diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs index 942a2c8dcd..9a88ba96f8 100644 --- a/src/mito2/src/config.rs +++ b/src/mito2/src/config.rs @@ -31,7 +31,7 @@ const MULTIPART_UPLOAD_MINIMUM_SIZE: ReadableSize = ReadableSize::mb(5); /// Default channel size for parallel scan task. pub(crate) const DEFAULT_SCAN_CHANNEL_SIZE: usize = 32; /// Default maximum number of SST files to scan concurrently. -pub(crate) const DEFAULT_MAX_CONCURRENT_SCAN_FILES: usize = 128; +pub(crate) const DEFAULT_MAX_CONCURRENT_SCAN_FILES: usize = 384; // Use `1/GLOBAL_WRITE_BUFFER_SIZE_FACTOR` of OS memory as global write buffer size in default mode const GLOBAL_WRITE_BUFFER_SIZE_FACTOR: u64 = 8; @@ -120,7 +120,7 @@ pub struct MitoConfig { pub sst_write_buffer_size: ReadableSize, /// Capacity of the channel to send data from parallel scan tasks to the main task (default 32). pub parallel_scan_channel_size: usize, - /// Maximum number of SST files to scan concurrently (default 128). + /// Maximum number of SST files to scan concurrently (default 384). pub max_concurrent_scan_files: usize, /// Whether to allow stale entries read during replay. pub allow_stale_entries: bool, diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index a0a9bd5d3f..4decd7f5f3 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -1430,7 +1430,7 @@ write_cache_path = "" write_cache_size = "5GiB" sst_write_buffer_size = "8MiB" parallel_scan_channel_size = 32 -max_concurrent_scan_files = 128 +max_concurrent_scan_files = 384 allow_stale_entries = false min_compaction_interval = "0s"