fix!: fix regression caused by unbalanced partitions and splitting ranges (#5090)

* feat: assign partition ranges by rows

* feat: balance partition rows

* feat: get uppoer bound for part nums

* feat: only split in non-compaction seq scan

* fix: parallel scan on multiple sources

* fix: can split check

* feat: scanner prepare by request

* feat: remove scan_parallelism

* docs: upate docs

* chore: update comment

* style: fix clippy

* feat: skip merge and dedup if there is only one source

* chore: Revert "feat: skip merge and dedup if there is only one source"

Since memtable won't do dedup jobs

This reverts commit 2fc7a54b11.

* test: avoid compaction in sqlness window sort test

* chore: do not create semaphore if num partitions is enough

* chore: more assertions

* chore: fix typo

* fix: compaction flag not set

* chore: address review comments

src/table/src/table/scan.rs

@@ -35,7 +35,7 @@ use datafusion_common::{ColumnStatistics, DataFusionError, Statistics};
 use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalSortExpr};
 use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef;
 use futures::{Stream, StreamExt};
-use store_api::region_engine::{PartitionRange, RegionScannerRef};
+use store_api::region_engine::{PartitionRange, PrepareRequest, RegionScannerRef};
 
 use crate::table::metrics::StreamMetrics;
 
@@ -112,6 +112,7 @@ impl RegionScanExec {
     pub fn with_new_partitions(
         &self,
         partitions: Vec<Vec<PartitionRange>>,
+        target_partitions: usize,
     ) -> Result<Self, BoxedError> {
         if self.is_partition_set {
             warn!("Setting partition ranges more than once for RegionScanExec");
@@ -123,8 +124,11 @@ impl RegionScanExec {
 
         {
             let mut scanner = self.scanner.lock().unwrap();
-            let distinguish_partition_range = scanner.properties().distinguish_partition_range();
-            scanner.prepare(partitions, distinguish_partition_range)?;
+            scanner.prepare(
+                PrepareRequest::default()
+                    .with_ranges(partitions)
+                    .with_target_partitions(target_partitions),
+            )?;
         }
 
         Ok(Self {
@@ -141,9 +145,10 @@ impl RegionScanExec {
 
     pub fn with_distinguish_partition_range(&self, distinguish_partition_range: bool) {
         let mut scanner = self.scanner.lock().unwrap();
-        let partition_ranges = scanner.properties().partitions.clone();
         // set distinguish_partition_range won't fail
-        let _ = scanner.prepare(partition_ranges, distinguish_partition_range);
+        let _ = scanner.prepare(
+            PrepareRequest::default().with_distinguish_partition_range(distinguish_partition_range),
+        );
     }
 
     pub fn time_index(&self) -> String {