feat: auto config cache size according to memory size (#3165)

* feat: auto config cache and buffer size according to mem size

* feat: utils

* refactor: add util function to common config

* refactor: check cgroups

* refactor: code

* fix: test

* fix: test

* chore: cr comment

Co-authored-by: Yingwen <realevenyag@gmail.com>
Co-authored-by: Dennis Zhuang <killme2008@gmail.com>

* chore: remove default comment

---------

Co-authored-by: Yingwen <realevenyag@gmail.com>
Co-authored-by: Dennis Zhuang <killme2008@gmail.com>

src/common/base/src/readable_size.rs

@@ -33,7 +33,7 @@ pub const GIB: u64 = MIB * BINARY_DATA_MAGNITUDE;
 pub const TIB: u64 = GIB * BINARY_DATA_MAGNITUDE;
 pub const PIB: u64 = TIB * BINARY_DATA_MAGNITUDE;
 
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd)]
+#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]
 pub struct ReadableSize(pub u64);
 
 impl ReadableSize {

src/common/config/Cargo.toml

@@ -7,8 +7,10 @@ license.workspace = true
 [dependencies]
 common-base.workspace = true
 humantime-serde.workspace = true
+num_cpus.workspace = true
 rskafka.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_with = "3"
+sysinfo.workspace = true
 toml.workspace = true

src/common/config/src/lib.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod utils;
 pub mod wal;
 
 use common_base::readable_size::ReadableSize;

src/common/config/src/utils.rs

@@ -0,0 +1,55 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_base::readable_size::ReadableSize;
+use sysinfo::System;
+
+/// Get the CPU core number of system, aware of cgroups.
+pub fn get_cpus() -> usize {
+    // This function will check cgroups
+    num_cpus::get()
+}
+
+/// Get the total memory of the system.
+/// If `cgroup_limits` is enabled, it will also check it.
+pub fn get_sys_total_memory() -> Option<ReadableSize> {
+    if sysinfo::IS_SUPPORTED_SYSTEM {
+        let mut sys_info = System::new();
+        sys_info.refresh_memory();
+        let mut total_memory = sys_info.total_memory();
+        // Compare with cgroups memory limit, use smaller values
+        // This method is only implemented for Linux. It always returns None for all other systems.
+        if let Some(cgroup_limits) = sys_info.cgroup_limits() {
+            total_memory = total_memory.min(cgroup_limits.total_memory)
+        }
+        Some(ReadableSize(total_memory))
+    } else {
+        None
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_cpus() {
+        assert!(get_cpus() > 0);
+    }
+
+    #[test]
+    fn test_get_sys_total_memory() {
+        assert!(get_sys_total_memory().unwrap() > ReadableSize::mb(0));
+    }
+}