From 29f215531af154ebf6ce3f525006d54265dd8b38 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Tue, 10 Sep 2024 15:12:23 +0800 Subject: [PATCH] feat: parallel in row group level under append mode (#4704) feat: append in row group level under append mode Signed-off-by: Ruihang Xia --- src/mito2/src/read/scan_region.rs | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs index ec45c9b934..716d372945 100644 --- a/src/mito2/src/read/scan_region.rs +++ b/src/mito2/src/read/scan_region.rs @@ -731,14 +731,28 @@ impl ScanInput { } for file in &self.files { - let range = PartitionRange { - start: file.meta_ref().time_range.0, - end: file.meta_ref().time_range.1, - num_rows: file.meta_ref().num_rows as usize, - identifier: id, - }; - id += 1; - container.push(range); + if self.append_mode { + // For append mode, we can parallelize reading row groups. + for _ in 0..file.meta_ref().num_row_groups { + let range = PartitionRange { + start: file.time_range().0, + end: file.time_range().1, + num_rows: file.num_rows(), + identifier: id, + }; + id += 1; + container.push(range); + } + } else { + let range = PartitionRange { + start: file.meta_ref().time_range.0, + end: file.meta_ref().time_range.1, + num_rows: file.meta_ref().num_rows as usize, + identifier: id, + }; + id += 1; + container.push(range); + } } container