diff --git a/src/metric-engine/src/engine/create.rs b/src/metric-engine/src/engine/create.rs index c18b947c4d0..156edf3cd86 100644 --- a/src/metric-engine/src/engine/create.rs +++ b/src/metric-engine/src/engine/create.rs @@ -313,6 +313,8 @@ impl MetricEngineInner { .unwrap()) .clone(); if metadata.semantic_type == SemanticType::Field { + // Physical split-value fields are nullable because each row uses only one + // half. Preserve the user-facing field schema in logical metadata. column_metadata.column_schema = metadata.column_schema.clone(); } column_metadata @@ -619,6 +621,8 @@ fn configure_metric_value_int_columns( || (metadata.semantic_type == SemanticType::Field && metadata.column_schema.data_type == ConcreteDataType::float64_datatype()) { + // Every split row leaves either the float or integer half null. Logical metadata + // retains the original nullability constraint. metadata.column_schema.set_nullable(); } value_split_enabled || !is_int_column diff --git a/src/mito2/src/metric_value.rs b/src/mito2/src/metric_value.rs index 62feb169ed4..a0e87fea8b6 100644 --- a/src/mito2/src/metric_value.rs +++ b/src/mito2/src/metric_value.rs @@ -29,7 +29,9 @@ use crate::error::{InvalidMetadataSnafu, Result}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct MetricValueColumn { + /// Index of the logical Float64 value in `RegionMetadata::column_metadatas`. pub(crate) value_index: usize, + /// Index of its Int64 companion in `RegionMetadata::column_metadatas`. pub(crate) int_index: usize, } diff --git a/src/mito2/src/sst/parquet/metric_value_split.rs b/src/mito2/src/sst/parquet/metric_value_split.rs index 80276ccda62..6b1ffea623e 100644 --- a/src/mito2/src/sst/parquet/metric_value_split.rs +++ b/src/mito2/src/sst/parquet/metric_value_split.rs @@ -24,11 +24,16 @@ use snafu::{OptionExt, ResultExt}; use store_api::metadata::RegionMetadata; use crate::error::{InvalidRecordBatchSnafu, NewRecordBatchSnafu, Result}; -use crate::metric_value::metric_value_columns; +use crate::metric_value::{MetricValueColumn, metric_value_columns}; +/// Maps a split metric value pair in region metadata to an Arrow batch schema. #[derive(Debug, Clone)] pub(crate) struct MetricValueSplitColumn { + /// The value pair's indices in region metadata. + pub(crate) region_column: MetricValueColumn, + /// Index of the Float64 value in the Arrow batch schema. pub(crate) float_index: usize, + /// Index of the Int64 companion in the Arrow batch schema. pub(crate) int_index: usize, } @@ -46,6 +51,7 @@ pub(crate) fn metric_value_split_columns( .ok()?; let int_index = arrow_schema.index_of(&int_column.column_schema.name).ok()?; Some(MetricValueSplitColumn { + region_column: column, float_index, int_index, }) @@ -293,6 +299,10 @@ mod tests { let batch = split_metric_value_columns( &batch, &[MetricValueSplitColumn { + region_column: MetricValueColumn { + value_index: 0, + int_index: 1, + }, float_index: 0, int_index: 1, }], diff --git a/src/mito2/src/sst/parquet/writer.rs b/src/mito2/src/sst/parquet/writer.rs index 6a743749332..de932166dba 100644 --- a/src/mito2/src/sst/parquet/writer.rs +++ b/src/mito2/src/sst/parquet/writer.rs @@ -408,10 +408,18 @@ where metric_value_split_columns(region_metadata, schema) .into_iter() .fold(builder, |builder, split_column| { - let float_col = - ColumnPath::new(vec![schema.field(split_column.float_index).name().clone()]); - let int_col = - ColumnPath::new(vec![schema.field(split_column.int_index).name().clone()]); + let float_col = ColumnPath::new(vec![ + region_metadata.column_metadatas[split_column.region_column.value_index] + .column_schema + .name + .clone(), + ]); + let int_col = ColumnPath::new(vec![ + region_metadata.column_metadatas[split_column.region_column.int_index] + .column_schema + .name + .clone(), + ]); builder .set_column_encoding(float_col.clone(), Encoding::PLAIN) .set_column_dictionary_enabled(float_col, false)