feat: mem manager on compaction (#7305)

* feat: mem manager on compaction Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: by copilot review comment Signed-off-by: jeremyhi <fengjiachun@gmail.com> * feat: experimental_ Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: refine estimate_compaction_bytes Signed-off-by: jeremyhi <fengjiachun@gmail.com> * feat: make them into config example Signed-off-by: jeremyhi <fengjiachun@gmail.com> * chore: by copilot comment Signed-off-by: jeremyhi <fengjiachun@gmail.com> * Update src/mito2/src/compaction.rs Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com> * fix: dedup the regions waiting Signed-off-by: jeremyhi <fengjiachun@gmail.com> * chore: by comment Signed-off-by: jeremyhi <fengjiachun@gmail.com> * chore: minor change Signed-off-by: jeremyhi <fengjiachun@gmail.com> * feat: add AdditionalMemoryGuard for the running compaction task Signed-off-by: jeremyhi <fengjiachun@gmail.com> * refactor: do OnExhaustedPolicy before running task Signed-off-by: jeremyhi <fengjiachun@gmail.com> * refactor: use OwnedSemaphorePermit to impl guard Signed-off-by: jeremyhi <fengjiachun@gmail.com> * feat: add early_release_partial method to release a portion of memory Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: 0 bytes make request_additional unlimited Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: fail-fast on acquire Signed-off-by: jeremyhi <fengjiachun@gmail.com> --------- Signed-off-by: jeremyhi <fengjiachun@gmail.com> Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com>
2026-08-12 09:19:44 +00:00 · 2025-12-12 14:49:58 +08:00
parent 11a5e1618d
commit baffed8c6a
29 changed files with 1024 additions and 11 deletions
@@ -145,6 +145,17 @@ impl ObjbenchCommand {
        let region_meta = extract_region_metadata(&self.source, &parquet_meta)?;
        let num_rows = parquet_meta.file_metadata().num_rows() as u64;
        let num_row_groups = parquet_meta.num_row_groups() as u64;
+        let max_row_group_uncompressed_size: u64 = parquet_meta
+            .row_groups()
+            .iter()
+            .map(|rg| {
+                rg.columns()
+                    .iter()
+                    .map(|c| c.uncompressed_size() as u64)
+                    .sum::<u64>()
+            })
+            .max()
+            .unwrap_or(0);

        println!(
            "{} Metadata loaded - rows: {}, size: {} bytes",
@@ -160,6 +171,7 @@ impl ObjbenchCommand {
            time_range: Default::default(),
            level: 0,
            file_size,
+            max_row_group_uncompressed_size,
            available_indexes: Default::default(),
            indexes: Default::default(),
            index_file_size: 0,