From da14ee21cf32a0aa6f13bcc86b30f48369465d48 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com> Date: Mon, 14 Sep 2026 03:44:51 +0000 Subject: [PATCH] fix(mito2): prevent JSON2 SWCS data loss from misaligned Parquet statistics due to projection (#9129) * fix(mito2): look up row group stats by parquet leaf index for nested columns On flat-format tables a logical column can expand to multiple parquet leaf columns (e.g. a JSON2 struct stores the remainder and one leaf per promoted path). ParquetFlat used the logical column index in the SST schema directly as the leaf index when reading row group statistics, so min/max/null stats of every column after a nested column were read from wrong leaves. When the misplaced leaf held order-compatible statistics (e.g. a small Int64 JSON path vs. the timestamp window predicate), min-max pruning dropped whole row groups by mistake. SWCS compaction reads inputs with a time window predicate, so it silently lost all rows of such files; plain queries with time-range predicates were affected as well. Map each column to its first parquet leaf column and report NoStats for columns with multiple leaves, which makes pruning conservative for them. Add a unit test and a sqlness regression case that reproduces the data loss on the unfixed binary. Signed-off-by: Lei, HUANG * fix(mito2): skip nested root stats and correct SWCS regression baseline Signed-off-by: Lei, HUANG * fix(mito2): resolve statistics leaves for primary-key SST readers Resolve scalar roots against the actual Parquet schema in shared statistics helpers, covering both flat and primary-key readers. Remove flat-side translation to avoid mapping twice and align encoded primary-key statistics as well. Cover dense flat, legacy dense and sparse layouts with statistics and time-pruning regressions. Signed-off-by: Lei, HUANG * test(mito2): distinguish known null counts from unknown statistics Assert validity before reading timestamp null counts and add a nullable scalar after the nested root with a known nonzero count. Exercise the assertions for flat and primary-key SST layouts. Signed-off-by: Lei, HUANG * test(mito2): check JSON2 time pruning before SWCS compaction Query the first time window immediately after FLUSH to cover predicate reads on flush-written SSTs independently of compaction outputs. Regenerate the sqlness expectation and retain the post-compaction checks. Signed-off-by: Lei, HUANG * test(mito2): validate all business columns after JSON2 SWCS Expand the final regression query to all eight business columns so the generated expectation verifies complete rows, including tags and scalar fields, after repeated compaction. Signed-off-by: Lei, HUANG --------- Signed-off-by: Lei, HUANG (cherry picked from commit 743261f05e28ccdb96fdd13d64977abcdf2dc95c) Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> (cherry picked from commit 113db823e810e65c3baad7559758f0b77b5e136e) Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> --- src/mito2/src/sst/parquet/flat_format.rs | 349 +++++++++++++++++- src/mito2/src/sst/parquet/format.rs | 56 ++- .../types/json/json2_swcs_minmax_stats.result | 142 +++++++ .../types/json/json2_swcs_minmax_stats.sql | 72 ++++ 4 files changed, 600 insertions(+), 19 deletions(-) create mode 100644 tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.result create mode 100644 tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.sql diff --git a/src/mito2/src/sst/parquet/flat_format.rs b/src/mito2/src/sst/parquet/flat_format.rs index 5432c112f4..b6dfdf0ec4 100644 --- a/src/mito2/src/sst/parquet/flat_format.rs +++ b/src/mito2/src/sst/parquet/flat_format.rs @@ -560,7 +560,8 @@ struct ParquetFlat { arrow_schema: SchemaRef, /// Projection computed for the flat format. format_projection: FormatProjection, - /// Column id to index in SST. + /// Column id to top-level SST index. Shared statistics helpers resolve + /// physical leaves from each file's actual Parquet schema. column_id_to_sst_index: HashMap, } @@ -618,7 +619,6 @@ impl ParquetFlat { // No such column in the SST. return StatValues::NoColumn; }; - let stats = column_null_counts(row_groups, *index); StatValues::from_stats_opt(stats) } @@ -633,8 +633,9 @@ impl ParquetFlat { // No such column in the SST. return StatValues::NoColumn; }; - // Safety: `column_id_to_sst_index` is built from `metadata`. - let index = self.column_id_to_sst_index.get(&column_id).unwrap(); + let Some(index) = self.column_id_to_sst_index.get(&column_id) else { + return StatValues::NoStats; + }; let stats = column_values(row_groups, column, *index, is_min); StatValues::from_stats_opt(stats) @@ -895,16 +896,25 @@ mod tests { use api::v1::SemanticType; use datatypes::arrow::array::{ - ArrayRef, BinaryArray, TimestampMillisecondArray, UInt8Array, UInt32Array, UInt64Array, + ArrayRef, BinaryArray, Int64Array, TimestampMillisecondArray, UInt8Array, UInt32Array, + UInt64Array, }; - use datatypes::arrow::datatypes::DataType as ArrowDataType; + use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field, TimeUnit}; use datatypes::arrow::record_batch::RecordBatch; use datatypes::prelude::ConcreteDataType; use datatypes::schema::ColumnSchema; + use datatypes::types::json_type::{JsonNativeType, JsonObjectType}; + use parquet::arrow::ArrowSchemaConverter; + use parquet::basic::{Repetition, Type as PhysicalType}; + use parquet::file::metadata::{ColumnChunkMetaData, RowGroupMetaData}; + use parquet::file::statistics::Statistics; + use parquet::schema::types::{SchemaDescriptor, Type}; use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder}; use store_api::storage::RegionId; - use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME; + use store_api::storage::consts::{ + OP_TYPE_COLUMN_NAME, PRIMARY_KEY_COLUMN_NAME, SEQUENCE_COLUMN_NAME, + }; use super::*; use crate::read::read_columns::ReadColumns; @@ -964,6 +974,331 @@ mod tests { builder.build().unwrap() } + /// Builds the metadata of a table with a JSON2 struct field column: + /// `[tag_0, field_0, payload, nullable_after_payload, ts]` with primary key `tag_0`. + fn metadata_with_struct_field() -> RegionMetadata { + let mut builder = RegionMetadataBuilder::new(RegionId::new(0, 0)); + builder + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "tag_0".to_string(), + ConcreteDataType::string_datatype(), + true, + ), + semantic_type: SemanticType::Tag, + column_id: 0, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "field_0".to_string(), + ConcreteDataType::int64_datatype(), + true, + ), + semantic_type: SemanticType::Field, + column_id: 1, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "payload".to_string(), + ConcreteDataType::json2(JsonNativeType::Object(JsonObjectType::new())), + false, + ), + semantic_type: SemanticType::Field, + column_id: 2, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "nullable_after_payload".to_string(), + ConcreteDataType::int64_datatype(), + true, + ), + semantic_type: SemanticType::Field, + column_id: 4, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "ts".to_string(), + ConcreteDataType::timestamp_nanosecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: 3, + }); + builder.primary_key(vec![0]); + builder.primary_key_encoding(PrimaryKeyEncoding::Dense); + builder.build().unwrap() + } + + /// Builds a file schema and a row group in which the `payload` struct + /// column expands to three leaf columns. The `ns_edge` leaf carries small + /// Int64 statistics that must not be mistaken for the statistics of `ts`. + fn struct_column_file_and_row_group(raw_pk_columns: bool) -> (SchemaRef, RowGroupMetaData) { + let mut arrow_fields = vec![ + Field::new("tag_0", ArrowDataType::Utf8, true), + Field::new("field_0", ArrowDataType::Int64, true), + Field::new( + "payload", + ArrowDataType::Struct( + vec![ + Field::new("metadata", ArrowDataType::Binary, false), + Field::new("value", ArrowDataType::Binary, false), + Field::new("ns_edge", ArrowDataType::Int64, true), + ] + .into(), + ), + false, + ), + Field::new("nullable_after_payload", ArrowDataType::Int64, true), + Field::new( + "ts", + ArrowDataType::Timestamp(TimeUnit::Nanosecond, None), + false, + ), + Field::new(PRIMARY_KEY_COLUMN_NAME, ArrowDataType::Binary, false), + Field::new(SEQUENCE_COLUMN_NAME, ArrowDataType::UInt64, false), + Field::new(OP_TYPE_COLUMN_NAME, ArrowDataType::UInt8, false), + ]; + if !raw_pk_columns { + arrow_fields.remove(0); + } + let file_schema = Arc::new(Schema::new(arrow_fields)); + + let leaf = |name: &str, physical: PhysicalType| { + Arc::new( + Type::primitive_type_builder(name, physical) + .with_repetition(Repetition::OPTIONAL) + .build() + .unwrap(), + ) + }; + let payload = Arc::new( + Type::group_type_builder("payload") + .with_repetition(Repetition::OPTIONAL) + .with_fields(vec![ + leaf("metadata", PhysicalType::BYTE_ARRAY), + leaf("value", PhysicalType::BYTE_ARRAY), + leaf("ns_edge", PhysicalType::INT64), + ]) + .build() + .unwrap(), + ); + let mut parquet_fields = vec![ + leaf("tag_0", PhysicalType::BYTE_ARRAY), + leaf("field_0", PhysicalType::INT64), + payload, + leaf("nullable_after_payload", PhysicalType::INT64), + leaf("ts", PhysicalType::INT64), + leaf(PRIMARY_KEY_COLUMN_NAME, PhysicalType::BYTE_ARRAY), + leaf(SEQUENCE_COLUMN_NAME, PhysicalType::INT64), + leaf(OP_TYPE_COLUMN_NAME, PhysicalType::INT32), + ]; + if !raw_pk_columns { + parquet_fields.remove(0); + } + let schema_descr = Arc::new(SchemaDescriptor::new(Arc::new( + Type::group_type_builder("schema") + .with_fields(parquet_fields) + .build() + .unwrap(), + ))); + + // Omitted raw tags shift all subsequent leaves in primary-key SSTs. + let ns_edge_leaf = if raw_pk_columns { 4 } else { 3 }; + let nullable_leaf = ns_edge_leaf + 1; + let ts_leaf = nullable_leaf + 1; + let chunks: Vec<_> = (0..schema_descr.num_columns()) + .map(|i| { + let mut builder = ColumnChunkMetaData::builder(schema_descr.column(i)); + if i == ns_edge_leaf { + // Small values from the JSON payload, not timestamps. + builder = builder.set_statistics(Statistics::int64( + Some(0), + Some(86_400_000_000_000), + None, + Some(65), + true, + )); + } else if i == nullable_leaf { + builder = builder.set_statistics(Statistics::int64( + Some(100), + Some(200), + None, + Some(7), + true, + )); + } else if i == ts_leaf { + builder = builder.set_statistics(Statistics::int64( + Some(1_788_998_400_000_000_000), + Some(1_789_084_800_000_000_000), + None, + Some(0), + true, + )); + } + builder.build().unwrap() + }) + .collect(); + let row_group = RowGroupMetaData::builder(schema_descr) + .set_num_rows(69) + .set_total_byte_size(0) + .set_column_metadata(chunks) + .build() + .unwrap(); + + (file_schema, row_group) + } + + /// Regression test: row group statistics must be looked up by parquet leaf + /// column index. A struct field column (e.g. JSON2) expands to multiple + /// leaf columns, so statistics of columns after it must not be read from + /// the struct's leaves. Otherwise min-max pruning can drop a whole row + /// group by mistake (e.g. pruning `ts` with the small `ns_edge` stats), + /// which caused data loss during SWCS compaction. + #[test] + fn test_stats_with_struct_field_column() { + for (encoding, raw_pk_columns) in [ + (PrimaryKeyEncoding::Dense, true), + (PrimaryKeyEncoding::Dense, false), + (PrimaryKeyEncoding::Sparse, false), + ] { + let mut metadata = metadata_with_struct_field(); + metadata.primary_key_encoding = encoding; + let metadata = Arc::new(metadata); + let (file_schema, row_group) = struct_column_file_and_row_group(raw_pk_columns); + let read_format = FlatReadFormat::new( + metadata, + ReadColumns::new([0, 1, 2, 3, 4]), + Some(file_schema), + "test", + false, + ) + .unwrap(); + let row_groups = [&row_group]; + + // Statistics of `ts` come from the `ts` leaf column, not the leaves of + // the payload struct. + let StatValues::Values(min) = read_format.min_values(&row_groups, 3) else { + panic!("expected ts min values") + }; + let min = min.as_any().downcast_ref::().unwrap(); + assert_eq!(1_788_998_400_000_000_000, min.value(0)); + let StatValues::Values(max) = read_format.max_values(&row_groups, 3) else { + panic!("expected ts max values") + }; + let max = max.as_any().downcast_ref::().unwrap(); + assert_eq!(1_789_084_800_000_000_000, max.value(0)); + + let stats = crate::sst::parquet::stats::RowGroupPruningStats::new( + &row_groups, + &read_format, + None, + false, + ); + for (start, end, keep) in [ + (1_788_998_400_000_000_000, 1_789_084_800_000_000_000, true), + (1_789_084_800_000_000_001, 1_789_171_200_000_000_000, false), + ] { + let predicate = table::predicate::Predicate::new(vec![ + datafusion_expr::col("ts").gt_eq(datafusion_expr::lit( + datafusion_common::ScalarValue::TimestampNanosecond(Some(start), None), + )), + datafusion_expr::col("ts").lt(datafusion_expr::lit( + datafusion_common::ScalarValue::TimestampNanosecond(Some(end), None), + )), + ]); + assert_eq!( + vec![keep], + predicate + .prune_with_stats(&stats, read_format.metadata().schema.arrow_schema(),) + ); + } + + // Null counts of `ts` also read the correct leaf column. + let StatValues::Values(nulls) = read_format.null_counts(&row_groups, 3) else { + panic!("expected ts null counts") + }; + let nulls = nulls.as_any().downcast_ref::().unwrap(); + assert!(nulls.is_valid(0)); + assert_eq!(0, nulls.value(0)); + + // A null slot may contain an underlying zero. Check validity and + // a nonzero count to distinguish unknown or wrong-leaf statistics. + let StatValues::Values(nulls) = read_format.null_counts(&row_groups, 4) else { + panic!("expected nullable field null counts") + }; + let nulls = nulls.as_any().downcast_ref::().unwrap(); + assert!(nulls.is_valid(0)); + assert_eq!(7, nulls.value(0)); + + // A column that expands to multiple leaf columns has no single column + // statistics. + assert!(matches!( + read_format.min_values(&row_groups, 2), + StatValues::NoStats + )); + assert!(matches!( + read_format.max_values(&row_groups, 2), + StatValues::NoStats + )); + assert!(matches!( + read_format.null_counts(&row_groups, 2), + StatValues::NoStats + )); + } + } + + /// Even one leaf cannot supply the null count of a nested parent: + /// {"a": null} and [null] are non-null parents with null children. + #[test] + fn test_single_leaf_nested_columns_have_no_root_stats() { + let child = Arc::new(Field::new("a", ArrowDataType::Int64, true)); + for nested_type in [ + ArrowDataType::Struct(vec![child.clone()].into()), + ArrowDataType::List(child.clone()), + ArrowDataType::LargeList(child.clone()), + ArrowDataType::FixedSizeList(child, 1), + ] { + let metadata = Arc::new(metadata_with_struct_field()); + let (schema, _) = struct_column_file_and_row_group(true); + let mut fields = schema.fields().to_vec(); + fields[2] = Arc::new(Field::new("payload", nested_type.clone(), true)); + let schema = Arc::new(Schema::new(fields)); + let descriptor = Arc::new(ArrowSchemaConverter::new().convert(&schema).unwrap()); + let chunks = descriptor + .columns() + .iter() + .map(|column| { + ColumnChunkMetaData::builder(column.clone()) + .build() + .unwrap() + }) + .collect(); + let row_group = RowGroupMetaData::builder(descriptor) + .set_num_rows(1) + .set_total_byte_size(0) + .set_column_metadata(chunks) + .build() + .unwrap(); + let format = ParquetFlat::new(metadata, ReadColumns::new([0, 1, 2, 3]), schema); + + // The nested root has no usable statistics, independently of the + // values stored in any row group. Its leaf still occupies a slot. + let groups = &[row_group]; + assert!( + matches!(format.min_values(groups, 2), StatValues::NoStats), + "{nested_type:?}" + ); + assert!( + matches!(format.max_values(groups, 2), StatValues::NoStats), + "{nested_type:?}" + ); + assert!( + matches!(format.null_counts(groups, 2), StatValues::NoStats), + "{nested_type:?}" + ); + } + } + #[test] fn test_field_column_start() { // (num_tags, num_fields, encoding, expected) diff --git a/src/mito2/src/sst/parquet/format.rs b/src/mito2/src/sst/parquet/format.rs index c0cfc7b981..7dc39b4c27 100644 --- a/src/mito2/src/sst/parquet/format.rs +++ b/src/mito2/src/sst/parquet/format.rs @@ -36,7 +36,7 @@ use datafusion_common::ScalarValue; use datatypes::arrow::array::{ ArrayRef, BinaryArray, BinaryDictionaryBuilder, DictionaryArray, UInt64Array, }; -use datatypes::arrow::datatypes::{SchemaRef, UInt32Type}; +use datatypes::arrow::datatypes::{DataType as ArrowDataType, SchemaRef, UInt32Type}; use datatypes::arrow::record_batch::RecordBatch; use datatypes::prelude::DataType; use datatypes::types::json_type::JsonNativeType; @@ -46,6 +46,7 @@ use mito_codec::row_converter::{ }; use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData}; use parquet::file::statistics::Statistics; +use parquet::schema::types::SchemaDescriptor; use snafu::{OptionExt, ResultExt, ensure}; use store_api::metadata::{ColumnMetadata, RegionMetadataRef}; use store_api::storage::{ColumnId, NestedPath, SequenceNumber}; @@ -136,12 +137,25 @@ pub(crate) fn column_values( column_index: usize, is_min: bool, ) -> Option { - let null_scalar: ScalarValue = column - .column_schema - .data_type - .as_arrow_type() - .try_into() - .ok()?; + column_values_by_type( + row_groups, + &column.column_schema.data_type.as_arrow_type(), + column_index, + is_min, + ) +} + +/// Returns min/max values for a top-level column with the given Arrow data type. +/// Resolves its leaf from the actual Parquet schema, not an inferred Arrow layout. +pub(crate) fn column_values_by_type( + row_groups: &[impl Borrow], + data_type: &ArrowDataType, + column_index: usize, + is_min: bool, +) -> Option { + let column_index = + scalar_leaf_index(row_groups.first()?.borrow().schema_descr(), column_index)?; + let null_scalar: ScalarValue = data_type.try_into().ok()?; let scalar_values = row_groups .iter() .map(|meta| { @@ -191,12 +205,14 @@ pub(crate) fn column_values( ScalarValue::iter_to_array(scalar_values).ok() } -/// Returns null counts of specific columns. +/// Returns null counts of a top-level column. /// The column should not be encoded as a part of a primary key. pub(crate) fn column_null_counts( row_groups: &[impl Borrow], column_index: usize, ) -> Option { + let column_index = + scalar_leaf_index(row_groups.first()?.borrow().schema_descr(), column_index)?; let values = row_groups.iter().map(|meta| { let col = meta.borrow().column(column_index); let stat = col.statistics()?; @@ -205,6 +221,21 @@ pub(crate) fn column_null_counts( Some(Arc::new(UInt64Array::from_iter(values))) } +/// Maps a scalar root to its physical leaf. Nested roots have no root-level +/// statistics, even with one leaf: child null counts do not describe parents. +/// All row groups of a file share the same schema. +fn scalar_leaf_index(schema: &SchemaDescriptor, root_index: usize) -> Option { + if !schema + .root_schema() + .get_fields() + .get(root_index)? + .is_primitive() + { + return None; + } + (0..schema.num_columns()).find(|&leaf| schema.get_column_root_idx(leaf) == root_index) +} + /// Helper for reading the SST format. pub struct PrimaryKeyReadFormat { /// The metadata stored in the SST. @@ -523,11 +554,12 @@ impl PrimaryKeyReadFormat { .into_iter(), ); + let primary_key_leaf = scalar_leaf_index( + row_groups.first()?.borrow().schema_descr(), + self.primary_key_position(), + )?; let values = row_groups.iter().map(|meta| { - let stats = meta - .borrow() - .column(self.primary_key_position()) - .statistics()?; + let stats = meta.borrow().column(primary_key_leaf).statistics()?; match stats { Statistics::Boolean(_) => None, Statistics::Int32(_) => None, diff --git a/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.result b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.result new file mode 100644 index 0000000000..77d4822503 --- /dev/null +++ b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.result @@ -0,0 +1,142 @@ +-- Regression test: min-max pruning must look up row group statistics by +-- parquet leaf column index. On flat-format tables with a JSON2 column, the +-- JSON2 struct expands to multiple leaf columns, so columns after it were +-- previously misaligned with leaves of the JSON2 struct when reading row +-- group statistics. Here `event_time` is the 8th logical column (2 tags + 5 +-- fields), so its statistics were read from the 8th parquet leaf, which is +-- the small-integer `ns_edge` path of the payload. SWCS compaction reads +-- inputs with a time window predicate, so min-max pruning compared the window +-- against the tiny `ns_edge` statistics, dropped every row group and silently +-- lost all rows. The same misaligned statistics also affected plain queries +-- with time-range predicates. +create table json2_swcs_minmax_stats ( + workspace_id string not null, + session_id string not null, + seq bigint not null, + event_time timestamp(9) not null time index, + entry_kind string not null, + payload json2 not null, + schema_version int not null, + is_error boolean not null, + primary key (workspace_id, session_id) +) with ( + 'append_mode' = 'true', + 'sst_format' = 'flat', + 'ttl' = 'forever' +); + +Affected Rows: 0 + +-- 9 rows in the 2026-09-10 window and 2 rows in the 2026-09-11 window. +insert into json2_swcs_minmax_stats values + ('ws', 's1', 1, 1788998400000000001, 'synthetic', '{"case":"swcs-repro","nested":[1,true],"ns_edge":0,"ordinal":1,"synthetic":true}', 1, false), + ('ws', 's2', 2, 1788998400000000002, 'synthetic', '{"case":"swcs-repro","nested":[2,false],"ns_edge":1,"ordinal":2,"synthetic":true}', 1, false), + ('ws', 's3', 3, 1788998400000000003, 'synthetic', '{"case":"swcs-repro","nested":[3,null],"ns_edge":2,"ordinal":3,"synthetic":true}', 1, false), + ('ws', 's1', 4, 1788998400000000004, 'synthetic', '{"case":"swcs-repro","nested":[4,true],"ns_edge":3,"ordinal":4,"synthetic":true}', 1, false), + ('ws', 's2', 5, 1788998400000000005, 'synthetic', '{"case":"swcs-repro","nested":[5,false],"ns_edge":4,"ordinal":5,"synthetic":true}', 1, false), + ('ws', 's3', 6, 1788998400000000006, 'synthetic', '{"case":"swcs-repro","nested":[6,null],"ns_edge":5,"ordinal":6,"synthetic":true}', 1, false), + ('ws', 's1', 7, 1788998400000000007, 'synthetic', '{"case":"swcs-repro","nested":[7,true],"ns_edge":6,"ordinal":7,"synthetic":true}', 1, false), + ('ws', 's2', 8, 1788998400000000008, 'synthetic', '{"case":"swcs-repro","nested":[8,false],"ns_edge":7,"ordinal":8,"synthetic":true}', 1, false), + ('ws', 's3', 9, 1788998400000000009, 'synthetic', '{"case":"swcs-repro","nested":[9,null],"ns_edge":8,"ordinal":9,"synthetic":true}', 1, false), + ('ws', 's1', 10, 1789084800000000001, 'synthetic', '{"case":"swcs-repro","nested":[10,true],"ns_edge":9,"ordinal":10,"synthetic":true}', 1, false), + ('ws', 's2', 11, 1789084800000000002, 'synthetic', '{"case":"swcs-repro","nested":[11,false],"ns_edge":10,"ordinal":11,"synthetic":true}', 1, false); + +Affected Rows: 11 + +select count(*) from json2_swcs_minmax_stats; + ++----------+ +| count(*) | ++----------+ +| 11 | ++----------+ + +admin flush_table('json2_swcs_minmax_stats'); + ++----------------------------------------------+ +| ADMIN flush_table('json2_swcs_minmax_stats') | ++----------------------------------------------+ +| 0 | ++----------------------------------------------+ + +-- Check pruning on flush-written SSTs before any manual compaction. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + ++----------+ +| count(*) | ++----------+ +| 9 | ++----------+ + +-- SWCS compaction reads the SST with a time window predicate. Before the +-- fix, min-max pruning compared the window against the small `ns_edge` +-- statistics and dropped every row group. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + ++-----------------------------------------------------------------+ +| ADMIN compact_table('json2_swcs_minmax_stats', 'swcs', '86400') | ++-----------------------------------------------------------------+ +| 0 | ++-----------------------------------------------------------------+ + +select count(*) from json2_swcs_minmax_stats; + ++----------+ +| count(*) | ++----------+ +| 11 | ++----------+ + +-- Compact once more to also exercise reading compaction-written SSTs. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + ++-----------------------------------------------------------------+ +| ADMIN compact_table('json2_swcs_minmax_stats', 'swcs', '86400') | ++-----------------------------------------------------------------+ +| 0 | ++-----------------------------------------------------------------+ + +select count(*) from json2_swcs_minmax_stats; + ++----------+ +| count(*) | ++----------+ +| 11 | ++----------+ + +-- Time-range predicates use the same statistics and must not prune the rows +-- either. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + ++----------+ +| count(*) | ++----------+ +| 9 | ++----------+ + +select workspace_id, session_id, seq, event_time, entry_kind, payload, schema_version, is_error +from json2_swcs_minmax_stats +order by seq; + ++--------------+------------+-----+-------------------------------+------------+--------------------------------------------------------------------------------------+----------------+----------+ +| workspace_id | session_id | seq | event_time | entry_kind | payload | schema_version | is_error | ++--------------+------------+-----+-------------------------------+------------+--------------------------------------------------------------------------------------+----------------+----------+ +| ws | s1 | 1 | 2026-09-10T00:00:00.000000001 | synthetic | {"case":"swcs-repro","nested":[1,true],"ns_edge":0,"ordinal":1,"synthetic":true} | 1 | false | +| ws | s2 | 2 | 2026-09-10T00:00:00.000000002 | synthetic | {"case":"swcs-repro","nested":[2,false],"ns_edge":1,"ordinal":2,"synthetic":true} | 1 | false | +| ws | s3 | 3 | 2026-09-10T00:00:00.000000003 | synthetic | {"case":"swcs-repro","nested":[3,null],"ns_edge":2,"ordinal":3,"synthetic":true} | 1 | false | +| ws | s1 | 4 | 2026-09-10T00:00:00.000000004 | synthetic | {"case":"swcs-repro","nested":[4,true],"ns_edge":3,"ordinal":4,"synthetic":true} | 1 | false | +| ws | s2 | 5 | 2026-09-10T00:00:00.000000005 | synthetic | {"case":"swcs-repro","nested":[5,false],"ns_edge":4,"ordinal":5,"synthetic":true} | 1 | false | +| ws | s3 | 6 | 2026-09-10T00:00:00.000000006 | synthetic | {"case":"swcs-repro","nested":[6,null],"ns_edge":5,"ordinal":6,"synthetic":true} | 1 | false | +| ws | s1 | 7 | 2026-09-10T00:00:00.000000007 | synthetic | {"case":"swcs-repro","nested":[7,true],"ns_edge":6,"ordinal":7,"synthetic":true} | 1 | false | +| ws | s2 | 8 | 2026-09-10T00:00:00.000000008 | synthetic | {"case":"swcs-repro","nested":[8,false],"ns_edge":7,"ordinal":8,"synthetic":true} | 1 | false | +| ws | s3 | 9 | 2026-09-10T00:00:00.000000009 | synthetic | {"case":"swcs-repro","nested":[9,null],"ns_edge":8,"ordinal":9,"synthetic":true} | 1 | false | +| ws | s1 | 10 | 2026-09-11T00:00:00.000000001 | synthetic | {"case":"swcs-repro","nested":[10,true],"ns_edge":9,"ordinal":10,"synthetic":true} | 1 | false | +| ws | s2 | 11 | 2026-09-11T00:00:00.000000002 | synthetic | {"case":"swcs-repro","nested":[11,false],"ns_edge":10,"ordinal":11,"synthetic":true} | 1 | false | ++--------------+------------+-----+-------------------------------+------------+--------------------------------------------------------------------------------------+----------------+----------+ + +drop table json2_swcs_minmax_stats; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.sql b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.sql new file mode 100644 index 0000000000..7bfea33442 --- /dev/null +++ b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.sql @@ -0,0 +1,72 @@ +-- Regression test: min-max pruning must look up row group statistics by +-- parquet leaf column index. On flat-format tables with a JSON2 column, the +-- JSON2 struct expands to multiple leaf columns, so columns after it were +-- previously misaligned with leaves of the JSON2 struct when reading row +-- group statistics. Here `event_time` is the 8th logical column (2 tags + 5 +-- fields), so its statistics were read from the 8th parquet leaf, which is +-- the small-integer `ns_edge` path of the payload. SWCS compaction reads +-- inputs with a time window predicate, so min-max pruning compared the window +-- against the tiny `ns_edge` statistics, dropped every row group and silently +-- lost all rows. The same misaligned statistics also affected plain queries +-- with time-range predicates. + +create table json2_swcs_minmax_stats ( + workspace_id string not null, + session_id string not null, + seq bigint not null, + event_time timestamp(9) not null time index, + entry_kind string not null, + payload json2 not null, + schema_version int not null, + is_error boolean not null, + primary key (workspace_id, session_id) +) with ( + 'append_mode' = 'true', + 'sst_format' = 'flat', + 'ttl' = 'forever' +); + +-- 9 rows in the 2026-09-10 window and 2 rows in the 2026-09-11 window. +insert into json2_swcs_minmax_stats values + ('ws', 's1', 1, 1788998400000000001, 'synthetic', '{"case":"swcs-repro","nested":[1,true],"ns_edge":0,"ordinal":1,"synthetic":true}', 1, false), + ('ws', 's2', 2, 1788998400000000002, 'synthetic', '{"case":"swcs-repro","nested":[2,false],"ns_edge":1,"ordinal":2,"synthetic":true}', 1, false), + ('ws', 's3', 3, 1788998400000000003, 'synthetic', '{"case":"swcs-repro","nested":[3,null],"ns_edge":2,"ordinal":3,"synthetic":true}', 1, false), + ('ws', 's1', 4, 1788998400000000004, 'synthetic', '{"case":"swcs-repro","nested":[4,true],"ns_edge":3,"ordinal":4,"synthetic":true}', 1, false), + ('ws', 's2', 5, 1788998400000000005, 'synthetic', '{"case":"swcs-repro","nested":[5,false],"ns_edge":4,"ordinal":5,"synthetic":true}', 1, false), + ('ws', 's3', 6, 1788998400000000006, 'synthetic', '{"case":"swcs-repro","nested":[6,null],"ns_edge":5,"ordinal":6,"synthetic":true}', 1, false), + ('ws', 's1', 7, 1788998400000000007, 'synthetic', '{"case":"swcs-repro","nested":[7,true],"ns_edge":6,"ordinal":7,"synthetic":true}', 1, false), + ('ws', 's2', 8, 1788998400000000008, 'synthetic', '{"case":"swcs-repro","nested":[8,false],"ns_edge":7,"ordinal":8,"synthetic":true}', 1, false), + ('ws', 's3', 9, 1788998400000000009, 'synthetic', '{"case":"swcs-repro","nested":[9,null],"ns_edge":8,"ordinal":9,"synthetic":true}', 1, false), + ('ws', 's1', 10, 1789084800000000001, 'synthetic', '{"case":"swcs-repro","nested":[10,true],"ns_edge":9,"ordinal":10,"synthetic":true}', 1, false), + ('ws', 's2', 11, 1789084800000000002, 'synthetic', '{"case":"swcs-repro","nested":[11,false],"ns_edge":10,"ordinal":11,"synthetic":true}', 1, false); + +select count(*) from json2_swcs_minmax_stats; + +admin flush_table('json2_swcs_minmax_stats'); + +-- Check pruning on flush-written SSTs before any manual compaction. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + +-- SWCS compaction reads the SST with a time window predicate. Before the +-- fix, min-max pruning compared the window against the small `ns_edge` +-- statistics and dropped every row group. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + +select count(*) from json2_swcs_minmax_stats; + +-- Compact once more to also exercise reading compaction-written SSTs. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + +select count(*) from json2_swcs_minmax_stats; + +-- Time-range predicates use the same statistics and must not prune the rows +-- either. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + +select workspace_id, session_id, seq, event_time, entry_kind, payload, schema_version, is_error +from json2_swcs_minmax_stats +order by seq; + +drop table json2_swcs_minmax_stats;