rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-21 19:32:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(table): stop claiming exact null and distinct counts for append scans (#9593)

Browse Source 
RegionScanExec reported null_count = 0 and distinct_count = total_rows for
every column of an append-only table. DataFusion answers count(col) and
count(DISTINCT col) from these statistics, so both returned the row count.
Keep the exact row count and an exact zero null count for non-nullable
columns, so count(*) is still answered from statistics.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
This commit is contained in:
dennis zhuang
2026-10-15 03:45:48 +00:00
committed by GitHub
parent ef1298ce02
commit 5cfdc30a69
3 changed files with 258 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/table/src/table/scan.rs
+26 -11
View File
@@ -522,13 +522,21 @@ impl ExecutionPlan for RegionScanExec {
let statistics = if scanner.properties().total_rows_is_exact()
&& !scanner.has_predicate_without_region()
{
// Only the row count is known. DataFusion answers `count(col)` as
// `num_rows - null_count` and `count(DISTINCT col)` as `distinct_count`
// without scanning, so these must stay absent unless the schema
// guarantees them. Non-nullable columns (including the time index that
// `count(*)` is rewritten to) keep an exact zero null count.
let column_statistics = self
.arrow_schema
.fields
.iter()
.map(|_| ColumnStatistics {
distinct_count: Precision::Exact(self.total_rows),
null_count: Precision::Exact(0), // all null rows are counted for append-only table
.map(|field| ColumnStatistics {
null_count: if field.is_nullable() {
Precision::Absent
} else {
Precision::Exact(0)
},
..Default::default()
})
.collect();
@@ -1218,7 +1226,8 @@ mod test {
(true, true, Precision::Exact(3)),
(false, true, Precision::Absent),
] {
let (schema, batch, metadata) = dynamic_filter_test_data(5685, false);
// Column `a` is nullable and `ts` is not.
let (schema, batch, metadata) = dynamic_filter_test_data(5685, true);
let scanner = Box::new(RepeatableScanner {
batches: RecordBatches::try_new(schema, vec![batch]).unwrap(),
properties: ScannerProperties::default()
@@ -1229,13 +1238,19 @@ mod test {
dynamic_filters: Arc::new(Mutex::new(Vec::new())),
});
let plan = RegionScanExec::new(scanner, ScanRequest::default(), None).unwrap();
assert_eq!(
StatisticsContext::new()
.compute(&plan, &StatisticsArgs::new())
.unwrap()
.num_rows,
expected,
);
let stats = StatisticsContext::new()
.compute(&plan, &StatisticsArgs::new())
.unwrap();
assert_eq!(stats.num_rows, expected);
if expected != Precision::Absent {
let [a, ts] = stats.column_statistics.as_slice() else {
panic!("unexpected column statistics: {stats:?}");
};
assert_eq!(a.null_count, Precision::Absent);
assert_eq!(ts.null_count, Precision::Exact(0));
assert_eq!(a.distinct_count, Precision::Absent);
assert_eq!(ts.distinct_count, Precision::Absent);
}
assert_eq!(
StatisticsContext::new()
.compute(&plan, &StatisticsArgs::new().with_partition(Some(0)))