fix: backport scan statistics and partitioning fixes to release/5e02efa9 (#9783)

* fix: preserve count correctness after repartition (#9154) * fix: preserve count correctness after repartition Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: restore full predicate guard for count statistics Signed-off-by: WenyXu <wenymedia@gmail.com> * test: update count plans for full predicate guard Signed-off-by: WenyXu <wenymedia@gmail.com> * fix: preserve count statistics for safe partition scans Signed-off-by: WenyXu <wenymedia@gmail.com> * test: retain repartition home guard and cover staging flush Signed-off-by: WenyXu <wenymedia@gmail.com> --------- Signed-off-by: WenyXu <wenymedia@gmail.com> (cherry picked from commit d9b97796a6) Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> * fix: only push down aggregates grouped by the partition columns themselves (#9337) * fix: only push down aggregates grouped by the partition columns themselves Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * fix: keep grouping sets on the frontend for partitioned tables Signed-off-by: Dennis Zhuang <killme2008@gmail.com> --------- Signed-off-by: Dennis Zhuang <killme2008@gmail.com> (cherry picked from commit def0c2ec5c) Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> * fix(table): require complete primary keys for hash partitioning (#9507) Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> (cherry picked from commit 28ec3ff5aa) Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> * fix(table): stop claiming exact null and distinct counts for append scans (#9593) RegionScanExec reported null_count = 0 and distinct_count = total_rows for every column of an append-only table. DataFusion answers count(col) and count(DISTINCT col) from these statistics, so both returned the row count. Keep the exact row count and an exact zero null count for non-nullable columns, so count(*) is still answered from statistics. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> (cherry picked from commit 5cfdc30a69) Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> * fix(table): stop claiming tag order for logical metric scans (#9522) * test: cover metric logical scan tag ordering Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix(table): only claim projected primary key prefix ordering Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix(table): keep physical key ordering for metric projections Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test(table): compare unknown partition counts explicitly Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test(table): model scoped internal TSID scan properties Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> --------- Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> (cherry picked from commit 75bc4ce46d) Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> * test: regenerate append_count_nulls result for release plan rendering Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> --------- Signed-off-by: WenyXu <wenymedia@gmail.com> Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> Signed-off-by: Dennis Zhuang <killme2008@gmail.com> Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> Co-authored-by: Weny Xu <wenymedia@gmail.com> Co-authored-by: dennis zhuang <killme2008@gmail.com> Co-authored-by: discord9 <discord9@163.com>
2026-10-27 22:26:15 +00:00 · 2026-10-20 08:32:32 -07:00
parent 1e5fef5809
commit fd3faeb7ed
16 changed files with 1442 additions and 45 deletions
@@ -1688,6 +1688,27 @@ impl ScanInput {
        );
    }

+    /// Whether physical source row counts equal the rows visible to this region,
+    /// before applying query predicates.
+    pub(crate) fn total_rows_is_exact(&self) -> bool {
+        if self.region_partition_expr.is_none() {
+            return true;
+        }
+
+        // Extension ranges do not expose their partition expressions.
+        #[cfg(feature = "enterprise")]
+        if !self.extension_ranges.is_empty() {
+            return false;
+        }
+
+        // Repartition flushes existing memtables before entering staging. New
+        // writes are routed under the target partition rules, whereas historical
+        // SSTs can be shared by regions and still need partition filtering.
+        self.files
+            .iter()
+            .all(|file| self.should_skip_region_partition(file))
+    }
+
    pub(crate) fn total_rows(&self) -> usize {
        let rows_in_files: usize = self.files.iter().map(|f| f.num_rows()).sum();
        let rows_in_memtables: usize = self.memtables.iter().map(|m| m.stats().num_rows()).sum();
@@ -2325,6 +2346,43 @@ mod tests {
            .with_files(vec![file])
    }

+    #[tokio::test]
+    async fn test_total_rows_is_exact_after_partition_filter() {
+        let expr = partition_col("k0").gt_eq(Value::String("foo".into()));
+        let other = partition_col("k0").gt_eq(Value::String("bar".into()));
+        for (region_expr, file_exprs, exact) in [
+            (None, vec![None], true),
+            (None, vec![Some(expr.clone())], true),
+            (Some(expr.clone()), vec![], true),
+            (Some(expr.clone()), vec![Some(expr.clone())], true),
+            (Some(expr.clone()), vec![None], false),
+            (Some(expr.clone()), vec![Some(other)], false),
+            (Some(expr.clone()), vec![Some(expr), None], false),
+        ] {
+            let mut builder =
+                RegionMetadataBuilder::from_existing(metadata_with_primary_key(vec![0, 1], false));
+            builder.partition_expr_json(region_expr.map(|expr| expr.as_json_str().unwrap()));
+            let metadata = Arc::new(builder.build_without_validation().unwrap());
+            let files = file_exprs
+                .into_iter()
+                .map(|partition_expr| {
+                    FileHandle::new(
+                        FileMeta {
+                            partition_expr,
+                            ..Default::default()
+                        },
+                        Arc::new(crate::sst::file_purger::NoopFilePurger),
+                    )
+                })
+                .collect();
+            let input = new_scan_input(metadata, vec![])
+                .await
+                .with_files(files)
+                .build();
+            assert_eq!(input.total_rows_is_exact(), exact);
+        }
+    }
+
    /// Helper to create a timestamp millisecond literal.
    fn ts_lit(val: i64) -> datafusion_expr::Expr {
        lit(ScalarValue::TimestampMillisecond(Some(val), None))
@@ -76,7 +76,8 @@ impl SeqScan {
    pub(crate) fn new(input: ScanInput) -> Self {
        let mut properties = ScannerProperties::default()
            .with_append_mode(input.append_mode)
-            .with_total_rows(input.total_rows());
+            .with_total_rows(input.total_rows())
+            .with_total_rows_is_exact(input.append_mode && input.total_rows_is_exact());
        if let Some(counters) = input.query_stat_counters.clone() {
            properties.set_query_stat_counters(counters);
        }
@@ -147,7 +147,8 @@ impl SeriesScan {
        };
        let mut properties = ScannerProperties::default()
            .with_append_mode(input.append_mode)
-            .with_total_rows(input.total_rows());
+            .with_total_rows(input.total_rows())
+            .with_total_rows_is_exact(input.append_mode && input.total_rows_is_exact());
        if let Some(counters) = input.query_stat_counters.clone() {
            properties.set_query_stat_counters(counters);
        }
@@ -61,7 +61,8 @@ impl UnorderedScan {
    pub(crate) fn new(input: ScanInput) -> Self {
        let mut properties = ScannerProperties::default()
            .with_append_mode(input.append_mode)
-            .with_total_rows(input.total_rows());
+            .with_total_rows(input.total_rows())
+            .with_total_rows_is_exact(input.append_mode && input.total_rows_is_exact());
        if let Some(counters) = input.query_stat_counters.clone() {
            properties.set_query_stat_counters(counters);
        }