rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-01-04 20:32:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: change default factor to compute memory size (#3211)

Browse Source 
* chore: change default factor to compute memory size

* chore: update doc

* chore: update comment in example config

* chore: extract factor to const and update comments

* chore: update comment by cr suggestion

Co-authored-by: dennis zhuang <killme2008@gmail.com>

---------

Co-authored-by: dennis zhuang <killme2008@gmail.com>
This commit is contained in:
shuiyisong
2024-01-22 17:03:29 +08:00
committed by GitHub
parent 8cc7129397
commit 2bf4b08a6b
3 changed files with 35 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
config/datanode.example.toml
View File

@@ -94,15 +94,18 @@ compress_manifest = false
max_background_jobs = 4
# Interval to auto flush a region if it has not flushed yet.
auto_flush_interval = "1h"
# Global write buffer size for all regions.
# Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB.
global_write_buffer_size = "1GB"
# Global write buffer size threshold to reject write requests (default 2G).
# Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size`
global_write_buffer_reject_size = "2GB"
# Cache size for SST metadata (default 128MB). Setting it to 0 to disable the cache.
# Cache size for SST metadata. Setting it to 0 to disable the cache.
# If not set, it's default to 1/32 of OS memory with a max limitation of 128MB.
sst_meta_cache_size = "128MB"
# Cache size for vectors and arrow arrays (default 512MB). Setting it to 0 to disable the cache.
# Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.
# If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
vector_cache_size = "512MB"
# Cache size for pages of SST row groups (default 512MB). Setting it to 0 to disable the cache.
# Cache size for pages of SST row groups. Setting it to 0 to disable the cache.
# If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
page_cache_size = "512MB"
# Buffer size for SST writing.
sst_write_buffer_size = "8MB"

13
config/standalone.example.toml
View File

@@ -193,15 +193,18 @@ compress_manifest = false
max_background_jobs = 4
# Interval to auto flush a region if it has not flushed yet.
auto_flush_interval = "1h"
# Global write buffer size for all regions.
# Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB.
global_write_buffer_size = "1GB"
# Global write buffer size threshold to reject write requests (default 2G).
# Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size`
global_write_buffer_reject_size = "2GB"
# Cache size for SST metadata (default 128MB). Setting it to 0 to disable the cache.
# Cache size for SST metadata. Setting it to 0 to disable the cache.
# If not set, it's default to 1/32 of OS memory with a max limitation of 128MB.
sst_meta_cache_size = "128MB"
# Cache size for vectors and arrow arrays (default 512MB). Setting it to 0 to disable the cache.
# Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.
# If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
vector_cache_size = "512MB"
# Cache size for pages of SST row groups (default 512MB). Setting it to 0 to disable the cache.
# Cache size for pages of SST row groups. Setting it to 0 to disable the cache.
# If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
page_cache_size = "512MB"
# Buffer size for SST writing.
sst_write_buffer_size = "8MB"

25
src/mito2/src/config.rs
View File

@@ -33,6 +33,13 @@ const MULTIPART_UPLOAD_MINIMUM_SIZE: ReadableSize = ReadableSize::mb(5);
/// Default channel size for parallel scan task.
const DEFAULT_SCAN_CHANNEL_SIZE: usize = 32;
// Use `1/GLOBAL_WRITE_BUFFER_SIZE_FACTOR` of OS memory as global write buffer size in default mode
const GLOBAL_WRITE_BUFFER_SIZE_FACTOR: u64 = 8;
/// Use `1/SST_META_CACHE_SIZE_FACTOR` of OS memory size as SST meta cache size in default mode
const SST_META_CACHE_SIZE_FACTOR: u64 = 32;
/// Use `1/MEM_CACHE_SIZE_FACTOR` of OS memory size as mem cache size in default mode
const MEM_CACHE_SIZE_FACTOR: u64 = 16;
/// Configuration for [MitoEngine](crate::engine::MitoEngine).
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
#[serde(default)]
@@ -192,14 +199,20 @@ impl MitoConfig {
}
fn adjust_buffer_and_cache_size(&mut self, sys_memory: ReadableSize) {
// Use 1/64 of OS memory as global write buffer size, it shouldn't be greater than 1G in default mode.
let global_write_buffer_size = cmp::min(sys_memory / 64, ReadableSize::gb(1));
// shouldn't be greater than 1G in default mode.
let global_write_buffer_size = cmp::min(
sys_memory / GLOBAL_WRITE_BUFFER_SIZE_FACTOR,
ReadableSize::gb(1),
);
// Use 2x of global write buffer size as global write buffer reject size.
let global_write_buffer_reject_size = global_write_buffer_size * 2;
// Use 1/256 of OS memory size as SST meta cache size, it shouldn't be greater than 128MB in default mode.
let sst_meta_cache_size = cmp::min(sys_memory / 256, ReadableSize::mb(128));
// Use 1/128 of OS memory size as mem cache size, it shouldn't be greater than 512MB in default mode.
let mem_cache_size = cmp::min(sys_memory / 128, ReadableSize::mb(512));
// shouldn't be greater than 128MB in default mode.
let sst_meta_cache_size = cmp::min(
sys_memory / SST_META_CACHE_SIZE_FACTOR,
ReadableSize::mb(128),
);
// shouldn't be greater than 512MB in default mode.
let mem_cache_size = cmp::min(sys_memory / MEM_CACHE_SIZE_FACTOR, ReadableSize::mb(512));
self.global_write_buffer_size = global_write_buffer_size;
self.global_write_buffer_reject_size = global_write_buffer_reject_size;