From 2bf4b08a6bbd94f0beba011e17a07245622906b9 Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:03:29 +0800 Subject: [PATCH] chore: change default factor to compute memory size (#3211) * chore: change default factor to compute memory size * chore: update doc * chore: update comment in example config * chore: extract factor to const and update comments * chore: update comment by cr suggestion Co-authored-by: dennis zhuang --------- Co-authored-by: dennis zhuang --- config/datanode.example.toml | 13 ++++++++----- config/standalone.example.toml | 13 ++++++++----- src/mito2/src/config.rs | 25 +++++++++++++++++++------ 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 62502cabe5..73e50b21e3 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -94,15 +94,18 @@ compress_manifest = false max_background_jobs = 4 # Interval to auto flush a region if it has not flushed yet. auto_flush_interval = "1h" -# Global write buffer size for all regions. +# Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. global_write_buffer_size = "1GB" -# Global write buffer size threshold to reject write requests (default 2G). +# Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size` global_write_buffer_reject_size = "2GB" -# Cache size for SST metadata (default 128MB). Setting it to 0 to disable the cache. +# Cache size for SST metadata. Setting it to 0 to disable the cache. +# If not set, it's default to 1/32 of OS memory with a max limitation of 128MB. sst_meta_cache_size = "128MB" -# Cache size for vectors and arrow arrays (default 512MB). Setting it to 0 to disable the cache. +# Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache. +# If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. vector_cache_size = "512MB" -# Cache size for pages of SST row groups (default 512MB). Setting it to 0 to disable the cache. +# Cache size for pages of SST row groups. Setting it to 0 to disable the cache. +# If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. page_cache_size = "512MB" # Buffer size for SST writing. sst_write_buffer_size = "8MB" diff --git a/config/standalone.example.toml b/config/standalone.example.toml index f3bb54aa2a..8b1644b9e2 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -193,15 +193,18 @@ compress_manifest = false max_background_jobs = 4 # Interval to auto flush a region if it has not flushed yet. auto_flush_interval = "1h" -# Global write buffer size for all regions. +# Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. global_write_buffer_size = "1GB" -# Global write buffer size threshold to reject write requests (default 2G). +# Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size` global_write_buffer_reject_size = "2GB" -# Cache size for SST metadata (default 128MB). Setting it to 0 to disable the cache. +# Cache size for SST metadata. Setting it to 0 to disable the cache. +# If not set, it's default to 1/32 of OS memory with a max limitation of 128MB. sst_meta_cache_size = "128MB" -# Cache size for vectors and arrow arrays (default 512MB). Setting it to 0 to disable the cache. +# Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache. +# If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. vector_cache_size = "512MB" -# Cache size for pages of SST row groups (default 512MB). Setting it to 0 to disable the cache. +# Cache size for pages of SST row groups. Setting it to 0 to disable the cache. +# If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. page_cache_size = "512MB" # Buffer size for SST writing. sst_write_buffer_size = "8MB" diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs index 093b8c06d0..cc25be263b 100644 --- a/src/mito2/src/config.rs +++ b/src/mito2/src/config.rs @@ -33,6 +33,13 @@ const MULTIPART_UPLOAD_MINIMUM_SIZE: ReadableSize = ReadableSize::mb(5); /// Default channel size for parallel scan task. const DEFAULT_SCAN_CHANNEL_SIZE: usize = 32; +// Use `1/GLOBAL_WRITE_BUFFER_SIZE_FACTOR` of OS memory as global write buffer size in default mode +const GLOBAL_WRITE_BUFFER_SIZE_FACTOR: u64 = 8; +/// Use `1/SST_META_CACHE_SIZE_FACTOR` of OS memory size as SST meta cache size in default mode +const SST_META_CACHE_SIZE_FACTOR: u64 = 32; +/// Use `1/MEM_CACHE_SIZE_FACTOR` of OS memory size as mem cache size in default mode +const MEM_CACHE_SIZE_FACTOR: u64 = 16; + /// Configuration for [MitoEngine](crate::engine::MitoEngine). #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] @@ -192,14 +199,20 @@ impl MitoConfig { } fn adjust_buffer_and_cache_size(&mut self, sys_memory: ReadableSize) { - // Use 1/64 of OS memory as global write buffer size, it shouldn't be greater than 1G in default mode. - let global_write_buffer_size = cmp::min(sys_memory / 64, ReadableSize::gb(1)); + // shouldn't be greater than 1G in default mode. + let global_write_buffer_size = cmp::min( + sys_memory / GLOBAL_WRITE_BUFFER_SIZE_FACTOR, + ReadableSize::gb(1), + ); // Use 2x of global write buffer size as global write buffer reject size. let global_write_buffer_reject_size = global_write_buffer_size * 2; - // Use 1/256 of OS memory size as SST meta cache size, it shouldn't be greater than 128MB in default mode. - let sst_meta_cache_size = cmp::min(sys_memory / 256, ReadableSize::mb(128)); - // Use 1/128 of OS memory size as mem cache size, it shouldn't be greater than 512MB in default mode. - let mem_cache_size = cmp::min(sys_memory / 128, ReadableSize::mb(512)); + // shouldn't be greater than 128MB in default mode. + let sst_meta_cache_size = cmp::min( + sys_memory / SST_META_CACHE_SIZE_FACTOR, + ReadableSize::mb(128), + ); + // shouldn't be greater than 512MB in default mode. + let mem_cache_size = cmp::min(sys_memory / MEM_CACHE_SIZE_FACTOR, ReadableSize::mb(512)); self.global_write_buffer_size = global_write_buffer_size; self.global_write_buffer_reject_size = global_write_buffer_reject_size;