diff --git a/src/mito2/src/sst/parquet/flat_format.rs b/src/mito2/src/sst/parquet/flat_format.rs index 5432c112f4..b6dfdf0ec4 100644 --- a/src/mito2/src/sst/parquet/flat_format.rs +++ b/src/mito2/src/sst/parquet/flat_format.rs @@ -560,7 +560,8 @@ struct ParquetFlat { arrow_schema: SchemaRef, /// Projection computed for the flat format. format_projection: FormatProjection, - /// Column id to index in SST. + /// Column id to top-level SST index. Shared statistics helpers resolve + /// physical leaves from each file's actual Parquet schema. column_id_to_sst_index: HashMap, } @@ -618,7 +619,6 @@ impl ParquetFlat { // No such column in the SST. return StatValues::NoColumn; }; - let stats = column_null_counts(row_groups, *index); StatValues::from_stats_opt(stats) } @@ -633,8 +633,9 @@ impl ParquetFlat { // No such column in the SST. return StatValues::NoColumn; }; - // Safety: `column_id_to_sst_index` is built from `metadata`. - let index = self.column_id_to_sst_index.get(&column_id).unwrap(); + let Some(index) = self.column_id_to_sst_index.get(&column_id) else { + return StatValues::NoStats; + }; let stats = column_values(row_groups, column, *index, is_min); StatValues::from_stats_opt(stats) @@ -895,16 +896,25 @@ mod tests { use api::v1::SemanticType; use datatypes::arrow::array::{ - ArrayRef, BinaryArray, TimestampMillisecondArray, UInt8Array, UInt32Array, UInt64Array, + ArrayRef, BinaryArray, Int64Array, TimestampMillisecondArray, UInt8Array, UInt32Array, + UInt64Array, }; - use datatypes::arrow::datatypes::DataType as ArrowDataType; + use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field, TimeUnit}; use datatypes::arrow::record_batch::RecordBatch; use datatypes::prelude::ConcreteDataType; use datatypes::schema::ColumnSchema; + use datatypes::types::json_type::{JsonNativeType, JsonObjectType}; + use parquet::arrow::ArrowSchemaConverter; + use parquet::basic::{Repetition, Type as PhysicalType}; + use parquet::file::metadata::{ColumnChunkMetaData, RowGroupMetaData}; + use parquet::file::statistics::Statistics; + use parquet::schema::types::{SchemaDescriptor, Type}; use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder}; use store_api::storage::RegionId; - use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME; + use store_api::storage::consts::{ + OP_TYPE_COLUMN_NAME, PRIMARY_KEY_COLUMN_NAME, SEQUENCE_COLUMN_NAME, + }; use super::*; use crate::read::read_columns::ReadColumns; @@ -964,6 +974,331 @@ mod tests { builder.build().unwrap() } + /// Builds the metadata of a table with a JSON2 struct field column: + /// `[tag_0, field_0, payload, nullable_after_payload, ts]` with primary key `tag_0`. + fn metadata_with_struct_field() -> RegionMetadata { + let mut builder = RegionMetadataBuilder::new(RegionId::new(0, 0)); + builder + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "tag_0".to_string(), + ConcreteDataType::string_datatype(), + true, + ), + semantic_type: SemanticType::Tag, + column_id: 0, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "field_0".to_string(), + ConcreteDataType::int64_datatype(), + true, + ), + semantic_type: SemanticType::Field, + column_id: 1, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "payload".to_string(), + ConcreteDataType::json2(JsonNativeType::Object(JsonObjectType::new())), + false, + ), + semantic_type: SemanticType::Field, + column_id: 2, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "nullable_after_payload".to_string(), + ConcreteDataType::int64_datatype(), + true, + ), + semantic_type: SemanticType::Field, + column_id: 4, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "ts".to_string(), + ConcreteDataType::timestamp_nanosecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: 3, + }); + builder.primary_key(vec![0]); + builder.primary_key_encoding(PrimaryKeyEncoding::Dense); + builder.build().unwrap() + } + + /// Builds a file schema and a row group in which the `payload` struct + /// column expands to three leaf columns. The `ns_edge` leaf carries small + /// Int64 statistics that must not be mistaken for the statistics of `ts`. + fn struct_column_file_and_row_group(raw_pk_columns: bool) -> (SchemaRef, RowGroupMetaData) { + let mut arrow_fields = vec![ + Field::new("tag_0", ArrowDataType::Utf8, true), + Field::new("field_0", ArrowDataType::Int64, true), + Field::new( + "payload", + ArrowDataType::Struct( + vec![ + Field::new("metadata", ArrowDataType::Binary, false), + Field::new("value", ArrowDataType::Binary, false), + Field::new("ns_edge", ArrowDataType::Int64, true), + ] + .into(), + ), + false, + ), + Field::new("nullable_after_payload", ArrowDataType::Int64, true), + Field::new( + "ts", + ArrowDataType::Timestamp(TimeUnit::Nanosecond, None), + false, + ), + Field::new(PRIMARY_KEY_COLUMN_NAME, ArrowDataType::Binary, false), + Field::new(SEQUENCE_COLUMN_NAME, ArrowDataType::UInt64, false), + Field::new(OP_TYPE_COLUMN_NAME, ArrowDataType::UInt8, false), + ]; + if !raw_pk_columns { + arrow_fields.remove(0); + } + let file_schema = Arc::new(Schema::new(arrow_fields)); + + let leaf = |name: &str, physical: PhysicalType| { + Arc::new( + Type::primitive_type_builder(name, physical) + .with_repetition(Repetition::OPTIONAL) + .build() + .unwrap(), + ) + }; + let payload = Arc::new( + Type::group_type_builder("payload") + .with_repetition(Repetition::OPTIONAL) + .with_fields(vec![ + leaf("metadata", PhysicalType::BYTE_ARRAY), + leaf("value", PhysicalType::BYTE_ARRAY), + leaf("ns_edge", PhysicalType::INT64), + ]) + .build() + .unwrap(), + ); + let mut parquet_fields = vec![ + leaf("tag_0", PhysicalType::BYTE_ARRAY), + leaf("field_0", PhysicalType::INT64), + payload, + leaf("nullable_after_payload", PhysicalType::INT64), + leaf("ts", PhysicalType::INT64), + leaf(PRIMARY_KEY_COLUMN_NAME, PhysicalType::BYTE_ARRAY), + leaf(SEQUENCE_COLUMN_NAME, PhysicalType::INT64), + leaf(OP_TYPE_COLUMN_NAME, PhysicalType::INT32), + ]; + if !raw_pk_columns { + parquet_fields.remove(0); + } + let schema_descr = Arc::new(SchemaDescriptor::new(Arc::new( + Type::group_type_builder("schema") + .with_fields(parquet_fields) + .build() + .unwrap(), + ))); + + // Omitted raw tags shift all subsequent leaves in primary-key SSTs. + let ns_edge_leaf = if raw_pk_columns { 4 } else { 3 }; + let nullable_leaf = ns_edge_leaf + 1; + let ts_leaf = nullable_leaf + 1; + let chunks: Vec<_> = (0..schema_descr.num_columns()) + .map(|i| { + let mut builder = ColumnChunkMetaData::builder(schema_descr.column(i)); + if i == ns_edge_leaf { + // Small values from the JSON payload, not timestamps. + builder = builder.set_statistics(Statistics::int64( + Some(0), + Some(86_400_000_000_000), + None, + Some(65), + true, + )); + } else if i == nullable_leaf { + builder = builder.set_statistics(Statistics::int64( + Some(100), + Some(200), + None, + Some(7), + true, + )); + } else if i == ts_leaf { + builder = builder.set_statistics(Statistics::int64( + Some(1_788_998_400_000_000_000), + Some(1_789_084_800_000_000_000), + None, + Some(0), + true, + )); + } + builder.build().unwrap() + }) + .collect(); + let row_group = RowGroupMetaData::builder(schema_descr) + .set_num_rows(69) + .set_total_byte_size(0) + .set_column_metadata(chunks) + .build() + .unwrap(); + + (file_schema, row_group) + } + + /// Regression test: row group statistics must be looked up by parquet leaf + /// column index. A struct field column (e.g. JSON2) expands to multiple + /// leaf columns, so statistics of columns after it must not be read from + /// the struct's leaves. Otherwise min-max pruning can drop a whole row + /// group by mistake (e.g. pruning `ts` with the small `ns_edge` stats), + /// which caused data loss during SWCS compaction. + #[test] + fn test_stats_with_struct_field_column() { + for (encoding, raw_pk_columns) in [ + (PrimaryKeyEncoding::Dense, true), + (PrimaryKeyEncoding::Dense, false), + (PrimaryKeyEncoding::Sparse, false), + ] { + let mut metadata = metadata_with_struct_field(); + metadata.primary_key_encoding = encoding; + let metadata = Arc::new(metadata); + let (file_schema, row_group) = struct_column_file_and_row_group(raw_pk_columns); + let read_format = FlatReadFormat::new( + metadata, + ReadColumns::new([0, 1, 2, 3, 4]), + Some(file_schema), + "test", + false, + ) + .unwrap(); + let row_groups = [&row_group]; + + // Statistics of `ts` come from the `ts` leaf column, not the leaves of + // the payload struct. + let StatValues::Values(min) = read_format.min_values(&row_groups, 3) else { + panic!("expected ts min values") + }; + let min = min.as_any().downcast_ref::().unwrap(); + assert_eq!(1_788_998_400_000_000_000, min.value(0)); + let StatValues::Values(max) = read_format.max_values(&row_groups, 3) else { + panic!("expected ts max values") + }; + let max = max.as_any().downcast_ref::().unwrap(); + assert_eq!(1_789_084_800_000_000_000, max.value(0)); + + let stats = crate::sst::parquet::stats::RowGroupPruningStats::new( + &row_groups, + &read_format, + None, + false, + ); + for (start, end, keep) in [ + (1_788_998_400_000_000_000, 1_789_084_800_000_000_000, true), + (1_789_084_800_000_000_001, 1_789_171_200_000_000_000, false), + ] { + let predicate = table::predicate::Predicate::new(vec![ + datafusion_expr::col("ts").gt_eq(datafusion_expr::lit( + datafusion_common::ScalarValue::TimestampNanosecond(Some(start), None), + )), + datafusion_expr::col("ts").lt(datafusion_expr::lit( + datafusion_common::ScalarValue::TimestampNanosecond(Some(end), None), + )), + ]); + assert_eq!( + vec![keep], + predicate + .prune_with_stats(&stats, read_format.metadata().schema.arrow_schema(),) + ); + } + + // Null counts of `ts` also read the correct leaf column. + let StatValues::Values(nulls) = read_format.null_counts(&row_groups, 3) else { + panic!("expected ts null counts") + }; + let nulls = nulls.as_any().downcast_ref::().unwrap(); + assert!(nulls.is_valid(0)); + assert_eq!(0, nulls.value(0)); + + // A null slot may contain an underlying zero. Check validity and + // a nonzero count to distinguish unknown or wrong-leaf statistics. + let StatValues::Values(nulls) = read_format.null_counts(&row_groups, 4) else { + panic!("expected nullable field null counts") + }; + let nulls = nulls.as_any().downcast_ref::().unwrap(); + assert!(nulls.is_valid(0)); + assert_eq!(7, nulls.value(0)); + + // A column that expands to multiple leaf columns has no single column + // statistics. + assert!(matches!( + read_format.min_values(&row_groups, 2), + StatValues::NoStats + )); + assert!(matches!( + read_format.max_values(&row_groups, 2), + StatValues::NoStats + )); + assert!(matches!( + read_format.null_counts(&row_groups, 2), + StatValues::NoStats + )); + } + } + + /// Even one leaf cannot supply the null count of a nested parent: + /// {"a": null} and [null] are non-null parents with null children. + #[test] + fn test_single_leaf_nested_columns_have_no_root_stats() { + let child = Arc::new(Field::new("a", ArrowDataType::Int64, true)); + for nested_type in [ + ArrowDataType::Struct(vec![child.clone()].into()), + ArrowDataType::List(child.clone()), + ArrowDataType::LargeList(child.clone()), + ArrowDataType::FixedSizeList(child, 1), + ] { + let metadata = Arc::new(metadata_with_struct_field()); + let (schema, _) = struct_column_file_and_row_group(true); + let mut fields = schema.fields().to_vec(); + fields[2] = Arc::new(Field::new("payload", nested_type.clone(), true)); + let schema = Arc::new(Schema::new(fields)); + let descriptor = Arc::new(ArrowSchemaConverter::new().convert(&schema).unwrap()); + let chunks = descriptor + .columns() + .iter() + .map(|column| { + ColumnChunkMetaData::builder(column.clone()) + .build() + .unwrap() + }) + .collect(); + let row_group = RowGroupMetaData::builder(descriptor) + .set_num_rows(1) + .set_total_byte_size(0) + .set_column_metadata(chunks) + .build() + .unwrap(); + let format = ParquetFlat::new(metadata, ReadColumns::new([0, 1, 2, 3]), schema); + + // The nested root has no usable statistics, independently of the + // values stored in any row group. Its leaf still occupies a slot. + let groups = &[row_group]; + assert!( + matches!(format.min_values(groups, 2), StatValues::NoStats), + "{nested_type:?}" + ); + assert!( + matches!(format.max_values(groups, 2), StatValues::NoStats), + "{nested_type:?}" + ); + assert!( + matches!(format.null_counts(groups, 2), StatValues::NoStats), + "{nested_type:?}" + ); + } + } + #[test] fn test_field_column_start() { // (num_tags, num_fields, encoding, expected) diff --git a/src/mito2/src/sst/parquet/format.rs b/src/mito2/src/sst/parquet/format.rs index 1a4325b7a7..6ca7a48662 100644 --- a/src/mito2/src/sst/parquet/format.rs +++ b/src/mito2/src/sst/parquet/format.rs @@ -46,6 +46,7 @@ use mito_codec::row_converter::{ }; use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData}; use parquet::file::statistics::Statistics; +use parquet::schema::types::SchemaDescriptor; use snafu::{OptionExt, ResultExt, ensure}; use store_api::metadata::{ColumnMetadata, RegionMetadataRef}; use store_api::storage::{ColumnId, NestedPath, SequenceNumber}; @@ -144,13 +145,16 @@ pub(crate) fn column_values( ) } -/// Returns min/max values of a parquet column with the given Arrow data type. +/// Returns min/max values for a top-level column with the given Arrow data type. +/// Resolves its leaf from the actual Parquet schema, not an inferred Arrow layout. pub(crate) fn column_values_by_type( row_groups: &[impl Borrow], data_type: &ArrowDataType, column_index: usize, is_min: bool, ) -> Option { + let column_index = + scalar_leaf_index(row_groups.first()?.borrow().schema_descr(), column_index)?; let null_scalar: ScalarValue = data_type.try_into().ok()?; let scalar_values = row_groups .iter() @@ -207,12 +211,14 @@ pub(crate) fn column_values_by_type( ScalarValue::iter_to_array(scalar_values).ok() } -/// Returns null counts of specific columns. +/// Returns null counts of a top-level column. /// The column should not be encoded as a part of a primary key. pub(crate) fn column_null_counts( row_groups: &[impl Borrow], column_index: usize, ) -> Option { + let column_index = + scalar_leaf_index(row_groups.first()?.borrow().schema_descr(), column_index)?; let values = row_groups.iter().map(|meta| { let col = meta.borrow().column(column_index); let stat = col.statistics()?; @@ -221,6 +227,21 @@ pub(crate) fn column_null_counts( Some(Arc::new(UInt64Array::from_iter(values))) } +/// Maps a scalar root to its physical leaf. Nested roots have no root-level +/// statistics, even with one leaf: child null counts do not describe parents. +/// All row groups of a file share the same schema. +fn scalar_leaf_index(schema: &SchemaDescriptor, root_index: usize) -> Option { + if !schema + .root_schema() + .get_fields() + .get(root_index)? + .is_primitive() + { + return None; + } + (0..schema.num_columns()).find(|&leaf| schema.get_column_root_idx(leaf) == root_index) +} + /// Helper for reading the SST format. pub struct PrimaryKeyReadFormat { /// The metadata stored in the SST. @@ -539,11 +560,12 @@ impl PrimaryKeyReadFormat { .into_iter(), ); + let primary_key_leaf = scalar_leaf_index( + row_groups.first()?.borrow().schema_descr(), + self.primary_key_position(), + )?; let values = row_groups.iter().map(|meta| { - let stats = meta - .borrow() - .column(self.primary_key_position()) - .statistics()?; + let stats = meta.borrow().column(primary_key_leaf).statistics()?; match stats { Statistics::Boolean(_) => None, Statistics::Int32(_) => None, diff --git a/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.result b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.result new file mode 100644 index 0000000000..77d4822503 --- /dev/null +++ b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.result @@ -0,0 +1,142 @@ +-- Regression test: min-max pruning must look up row group statistics by +-- parquet leaf column index. On flat-format tables with a JSON2 column, the +-- JSON2 struct expands to multiple leaf columns, so columns after it were +-- previously misaligned with leaves of the JSON2 struct when reading row +-- group statistics. Here `event_time` is the 8th logical column (2 tags + 5 +-- fields), so its statistics were read from the 8th parquet leaf, which is +-- the small-integer `ns_edge` path of the payload. SWCS compaction reads +-- inputs with a time window predicate, so min-max pruning compared the window +-- against the tiny `ns_edge` statistics, dropped every row group and silently +-- lost all rows. The same misaligned statistics also affected plain queries +-- with time-range predicates. +create table json2_swcs_minmax_stats ( + workspace_id string not null, + session_id string not null, + seq bigint not null, + event_time timestamp(9) not null time index, + entry_kind string not null, + payload json2 not null, + schema_version int not null, + is_error boolean not null, + primary key (workspace_id, session_id) +) with ( + 'append_mode' = 'true', + 'sst_format' = 'flat', + 'ttl' = 'forever' +); + +Affected Rows: 0 + +-- 9 rows in the 2026-09-10 window and 2 rows in the 2026-09-11 window. +insert into json2_swcs_minmax_stats values + ('ws', 's1', 1, 1788998400000000001, 'synthetic', '{"case":"swcs-repro","nested":[1,true],"ns_edge":0,"ordinal":1,"synthetic":true}', 1, false), + ('ws', 's2', 2, 1788998400000000002, 'synthetic', '{"case":"swcs-repro","nested":[2,false],"ns_edge":1,"ordinal":2,"synthetic":true}', 1, false), + ('ws', 's3', 3, 1788998400000000003, 'synthetic', '{"case":"swcs-repro","nested":[3,null],"ns_edge":2,"ordinal":3,"synthetic":true}', 1, false), + ('ws', 's1', 4, 1788998400000000004, 'synthetic', '{"case":"swcs-repro","nested":[4,true],"ns_edge":3,"ordinal":4,"synthetic":true}', 1, false), + ('ws', 's2', 5, 1788998400000000005, 'synthetic', '{"case":"swcs-repro","nested":[5,false],"ns_edge":4,"ordinal":5,"synthetic":true}', 1, false), + ('ws', 's3', 6, 1788998400000000006, 'synthetic', '{"case":"swcs-repro","nested":[6,null],"ns_edge":5,"ordinal":6,"synthetic":true}', 1, false), + ('ws', 's1', 7, 1788998400000000007, 'synthetic', '{"case":"swcs-repro","nested":[7,true],"ns_edge":6,"ordinal":7,"synthetic":true}', 1, false), + ('ws', 's2', 8, 1788998400000000008, 'synthetic', '{"case":"swcs-repro","nested":[8,false],"ns_edge":7,"ordinal":8,"synthetic":true}', 1, false), + ('ws', 's3', 9, 1788998400000000009, 'synthetic', '{"case":"swcs-repro","nested":[9,null],"ns_edge":8,"ordinal":9,"synthetic":true}', 1, false), + ('ws', 's1', 10, 1789084800000000001, 'synthetic', '{"case":"swcs-repro","nested":[10,true],"ns_edge":9,"ordinal":10,"synthetic":true}', 1, false), + ('ws', 's2', 11, 1789084800000000002, 'synthetic', '{"case":"swcs-repro","nested":[11,false],"ns_edge":10,"ordinal":11,"synthetic":true}', 1, false); + +Affected Rows: 11 + +select count(*) from json2_swcs_minmax_stats; + ++----------+ +| count(*) | ++----------+ +| 11 | ++----------+ + +admin flush_table('json2_swcs_minmax_stats'); + ++----------------------------------------------+ +| ADMIN flush_table('json2_swcs_minmax_stats') | ++----------------------------------------------+ +| 0 | ++----------------------------------------------+ + +-- Check pruning on flush-written SSTs before any manual compaction. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + ++----------+ +| count(*) | ++----------+ +| 9 | ++----------+ + +-- SWCS compaction reads the SST with a time window predicate. Before the +-- fix, min-max pruning compared the window against the small `ns_edge` +-- statistics and dropped every row group. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + ++-----------------------------------------------------------------+ +| ADMIN compact_table('json2_swcs_minmax_stats', 'swcs', '86400') | ++-----------------------------------------------------------------+ +| 0 | ++-----------------------------------------------------------------+ + +select count(*) from json2_swcs_minmax_stats; + ++----------+ +| count(*) | ++----------+ +| 11 | ++----------+ + +-- Compact once more to also exercise reading compaction-written SSTs. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + ++-----------------------------------------------------------------+ +| ADMIN compact_table('json2_swcs_minmax_stats', 'swcs', '86400') | ++-----------------------------------------------------------------+ +| 0 | ++-----------------------------------------------------------------+ + +select count(*) from json2_swcs_minmax_stats; + ++----------+ +| count(*) | ++----------+ +| 11 | ++----------+ + +-- Time-range predicates use the same statistics and must not prune the rows +-- either. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + ++----------+ +| count(*) | ++----------+ +| 9 | ++----------+ + +select workspace_id, session_id, seq, event_time, entry_kind, payload, schema_version, is_error +from json2_swcs_minmax_stats +order by seq; + ++--------------+------------+-----+-------------------------------+------------+--------------------------------------------------------------------------------------+----------------+----------+ +| workspace_id | session_id | seq | event_time | entry_kind | payload | schema_version | is_error | ++--------------+------------+-----+-------------------------------+------------+--------------------------------------------------------------------------------------+----------------+----------+ +| ws | s1 | 1 | 2026-09-10T00:00:00.000000001 | synthetic | {"case":"swcs-repro","nested":[1,true],"ns_edge":0,"ordinal":1,"synthetic":true} | 1 | false | +| ws | s2 | 2 | 2026-09-10T00:00:00.000000002 | synthetic | {"case":"swcs-repro","nested":[2,false],"ns_edge":1,"ordinal":2,"synthetic":true} | 1 | false | +| ws | s3 | 3 | 2026-09-10T00:00:00.000000003 | synthetic | {"case":"swcs-repro","nested":[3,null],"ns_edge":2,"ordinal":3,"synthetic":true} | 1 | false | +| ws | s1 | 4 | 2026-09-10T00:00:00.000000004 | synthetic | {"case":"swcs-repro","nested":[4,true],"ns_edge":3,"ordinal":4,"synthetic":true} | 1 | false | +| ws | s2 | 5 | 2026-09-10T00:00:00.000000005 | synthetic | {"case":"swcs-repro","nested":[5,false],"ns_edge":4,"ordinal":5,"synthetic":true} | 1 | false | +| ws | s3 | 6 | 2026-09-10T00:00:00.000000006 | synthetic | {"case":"swcs-repro","nested":[6,null],"ns_edge":5,"ordinal":6,"synthetic":true} | 1 | false | +| ws | s1 | 7 | 2026-09-10T00:00:00.000000007 | synthetic | {"case":"swcs-repro","nested":[7,true],"ns_edge":6,"ordinal":7,"synthetic":true} | 1 | false | +| ws | s2 | 8 | 2026-09-10T00:00:00.000000008 | synthetic | {"case":"swcs-repro","nested":[8,false],"ns_edge":7,"ordinal":8,"synthetic":true} | 1 | false | +| ws | s3 | 9 | 2026-09-10T00:00:00.000000009 | synthetic | {"case":"swcs-repro","nested":[9,null],"ns_edge":8,"ordinal":9,"synthetic":true} | 1 | false | +| ws | s1 | 10 | 2026-09-11T00:00:00.000000001 | synthetic | {"case":"swcs-repro","nested":[10,true],"ns_edge":9,"ordinal":10,"synthetic":true} | 1 | false | +| ws | s2 | 11 | 2026-09-11T00:00:00.000000002 | synthetic | {"case":"swcs-repro","nested":[11,false],"ns_edge":10,"ordinal":11,"synthetic":true} | 1 | false | ++--------------+------------+-----+-------------------------------+------------+--------------------------------------------------------------------------------------+----------------+----------+ + +drop table json2_swcs_minmax_stats; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.sql b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.sql new file mode 100644 index 0000000000..7bfea33442 --- /dev/null +++ b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.sql @@ -0,0 +1,72 @@ +-- Regression test: min-max pruning must look up row group statistics by +-- parquet leaf column index. On flat-format tables with a JSON2 column, the +-- JSON2 struct expands to multiple leaf columns, so columns after it were +-- previously misaligned with leaves of the JSON2 struct when reading row +-- group statistics. Here `event_time` is the 8th logical column (2 tags + 5 +-- fields), so its statistics were read from the 8th parquet leaf, which is +-- the small-integer `ns_edge` path of the payload. SWCS compaction reads +-- inputs with a time window predicate, so min-max pruning compared the window +-- against the tiny `ns_edge` statistics, dropped every row group and silently +-- lost all rows. The same misaligned statistics also affected plain queries +-- with time-range predicates. + +create table json2_swcs_minmax_stats ( + workspace_id string not null, + session_id string not null, + seq bigint not null, + event_time timestamp(9) not null time index, + entry_kind string not null, + payload json2 not null, + schema_version int not null, + is_error boolean not null, + primary key (workspace_id, session_id) +) with ( + 'append_mode' = 'true', + 'sst_format' = 'flat', + 'ttl' = 'forever' +); + +-- 9 rows in the 2026-09-10 window and 2 rows in the 2026-09-11 window. +insert into json2_swcs_minmax_stats values + ('ws', 's1', 1, 1788998400000000001, 'synthetic', '{"case":"swcs-repro","nested":[1,true],"ns_edge":0,"ordinal":1,"synthetic":true}', 1, false), + ('ws', 's2', 2, 1788998400000000002, 'synthetic', '{"case":"swcs-repro","nested":[2,false],"ns_edge":1,"ordinal":2,"synthetic":true}', 1, false), + ('ws', 's3', 3, 1788998400000000003, 'synthetic', '{"case":"swcs-repro","nested":[3,null],"ns_edge":2,"ordinal":3,"synthetic":true}', 1, false), + ('ws', 's1', 4, 1788998400000000004, 'synthetic', '{"case":"swcs-repro","nested":[4,true],"ns_edge":3,"ordinal":4,"synthetic":true}', 1, false), + ('ws', 's2', 5, 1788998400000000005, 'synthetic', '{"case":"swcs-repro","nested":[5,false],"ns_edge":4,"ordinal":5,"synthetic":true}', 1, false), + ('ws', 's3', 6, 1788998400000000006, 'synthetic', '{"case":"swcs-repro","nested":[6,null],"ns_edge":5,"ordinal":6,"synthetic":true}', 1, false), + ('ws', 's1', 7, 1788998400000000007, 'synthetic', '{"case":"swcs-repro","nested":[7,true],"ns_edge":6,"ordinal":7,"synthetic":true}', 1, false), + ('ws', 's2', 8, 1788998400000000008, 'synthetic', '{"case":"swcs-repro","nested":[8,false],"ns_edge":7,"ordinal":8,"synthetic":true}', 1, false), + ('ws', 's3', 9, 1788998400000000009, 'synthetic', '{"case":"swcs-repro","nested":[9,null],"ns_edge":8,"ordinal":9,"synthetic":true}', 1, false), + ('ws', 's1', 10, 1789084800000000001, 'synthetic', '{"case":"swcs-repro","nested":[10,true],"ns_edge":9,"ordinal":10,"synthetic":true}', 1, false), + ('ws', 's2', 11, 1789084800000000002, 'synthetic', '{"case":"swcs-repro","nested":[11,false],"ns_edge":10,"ordinal":11,"synthetic":true}', 1, false); + +select count(*) from json2_swcs_minmax_stats; + +admin flush_table('json2_swcs_minmax_stats'); + +-- Check pruning on flush-written SSTs before any manual compaction. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + +-- SWCS compaction reads the SST with a time window predicate. Before the +-- fix, min-max pruning compared the window against the small `ns_edge` +-- statistics and dropped every row group. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + +select count(*) from json2_swcs_minmax_stats; + +-- Compact once more to also exercise reading compaction-written SSTs. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + +select count(*) from json2_swcs_minmax_stats; + +-- Time-range predicates use the same statistics and must not prune the rows +-- either. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + +select workspace_id, session_id, seq, event_time, entry_kind, payload, schema_version, is_error +from json2_swcs_minmax_stats +order by seq; + +drop table json2_swcs_minmax_stats;