fix(mito2): avoid unrelated sparse tag expansion during compaction (#10414)

* fix(mito2): avoid unrelated sparse tag expansion during compaction

Signed-off-by: evenyag <realevenyag@gmail.com>

* test(mito2): cover sparse compaction filter and compatibility boundaries

Signed-off-by: evenyag <realevenyag@gmail.com>

* fix(mito2): handle projected fields in sparse compaction mapper

Signed-off-by: evenyag <realevenyag@gmail.com>

* refactor(mito2): remove redundant compaction projection mapper

Signed-off-by: evenyag <realevenyag@gmail.com>

---------

Signed-off-by: evenyag <realevenyag@gmail.com>
This commit is contained in:
Yingwen
2026-11-15 02:34:57 +00:00
committed by GitHub
parent 1e39eab06b
commit 1dbc0cd80f
10 changed files with 257 additions and 119 deletions
-1
View File
@@ -106,7 +106,6 @@ impl BulkIterContext {
codec,
// we don't need to compat batch since all batch in memtable have the same schema.
compat_batch: None,
compaction_projection_mapper: None,
pre_filter_mode,
partition_filter: None,
},
+8 -1
View File
@@ -1552,7 +1552,7 @@ mod tests {
}
#[test]
fn test_flat_compat_batch_compact_sparse() {
fn test_flat_compat_batch_compact_sparse_added_and_dropped_fields() {
let mut actual_metadata = new_metadata(
&[
(
@@ -1561,6 +1561,7 @@ mod tests {
ConcreteDataType::timestamp_millisecond_datatype(),
),
(2, SemanticType::Field, ConcreteDataType::int64_datatype()),
(4, SemanticType::Field, ConcreteDataType::int64_datatype()),
],
&[],
);
@@ -1599,6 +1600,7 @@ mod tests {
let sparse_k1 = encode_sparse_key(&[]);
let input_columns: Vec<ArrayRef> = vec![
Arc::new(Int64Array::from(vec![100, 200])),
Arc::new(Int64Array::from(vec![300, 400])), // dropped field
Arc::new(TimestampMillisecondArray::from_iter_values([1000, 2000])),
build_flat_test_pk_array(&[&sparse_k1, &sparse_k1]),
Arc::new(UInt64Array::from_iter_values([1, 2])),
@@ -1613,6 +1615,11 @@ mod tests {
);
let input_batch = RecordBatch::try_new(input_schema, input_columns).unwrap();
// The physical read projection omits the dropped source field. Sparse
// conversion preserves this encoded layout, which goes directly to compat.
assert!(read_format.projected_index_by_id(4).is_none());
let input_batch = input_batch.project(&[0, 2, 3, 4, 5]).unwrap();
let input_batch = read_format.convert_batch(input_batch, None).unwrap();
let result = compat_batch.compat(input_batch).unwrap();
let expected_columns: Vec<ArrayRef> = vec![
+6 -42
View File
@@ -31,7 +31,7 @@ use store_api::metadata::{RegionMetadata, RegionMetadataRef};
use store_api::storage::ColumnId;
use crate::cache::CacheStrategy;
use crate::error::{InvalidRequestSnafu, NewRecordBatchSnafu, Result};
use crate::error::{InvalidRequestSnafu, Result};
use crate::read::projection::{read_column_ids_from_projection, repeated_vector_with_cache};
use crate::read::read_columns::{JsonTargetTypes, ReadColumns};
use crate::sst::parquet::Json2RewriteTargets;
@@ -46,6 +46,11 @@ use crate::sst::{
///
/// This mapper support duplicate and unsorted projection indices.
/// The output schema is determined by the projection indices.
/// Input indices address the compatible merge schema, not physical Parquet
/// columns or temporary predicate projections. Sparse compaction uses
/// `input_arrow_schema(true)` without expanded tags and bypasses query output
/// conversion. Sparse compaction retains encoded keys through filtering and
/// passes batches directly to compatibility; predicate tags stay temporary.
pub struct FlatProjectionMapper {
/// Metadata of the region.
metadata: RegionMetadataRef,
@@ -477,47 +482,6 @@ pub(crate) fn compute_input_arrow_schema(
Arc::new(datatypes::arrow::datatypes::Schema::new(new_fields))
}
/// Helper to project compaction batches into flat format columns
/// (fields + time index + __primary_key + __sequence + __op_type).
pub(crate) struct CompactionProjectionMapper {
projection: Vec<usize>,
}
impl CompactionProjectionMapper {
pub(crate) fn try_new(metadata: &RegionMetadataRef) -> Result<Self> {
let projection = metadata
.column_metadatas
.iter()
.enumerate()
.filter_map(|(idx, col)| {
if matches!(col.semantic_type, SemanticType::Field) {
Some(idx)
} else {
None
}
})
.chain([metadata.time_index_column_pos()])
.collect::<Vec<_>>();
let read_col_ids = metadata.column_metadatas.iter().map(|col| col.column_id);
let read_cols = ReadColumns::new(read_col_ids);
let mapper = FlatProjectionMapper::new_with_read_columns(metadata, projection, read_cols)?;
Ok(Self {
projection: mapper.batch_indices,
})
}
/// Projects columns and appends internal columns for compaction output.
///
/// The input batch is expected to be in flat format with internal columns appended.
pub(crate) fn project(&self, batch: DfRecordBatch) -> Result<DfRecordBatch> {
let mut projection = self.projection.clone();
projection.extend(batch.num_columns() - 3..batch.num_columns());
// Projection must preserve the Arrow layouts already produced by compaction Rewrite.
batch.project(&projection).context(NewRecordBatchSnafu)
}
}
#[cfg(test)]
mod tests {
use datatypes::schema::ColumnSchema;
-6
View File
@@ -1616,13 +1616,7 @@ pub fn build_flat_file_range_scan_stream(
.file_handle()
.is_effective_target_sequence_trusted(stream_ctx.input.region_metadata().region_id);
let mapper = range.compaction_projection_mapper();
while let Some(record_batch) = reader.next_batch().await? {
let record_batch = if let Some(mapper) = mapper {
mapper.project(record_batch)?
} else {
record_batch
};
let Some(record_batch) = filter_flat_batch_by_sequence(
record_batch,
stream_ctx.input.sequence_range,
-5
View File
@@ -571,11 +571,6 @@ fn scan_series_file_ranges(
reader_metrics.filter_metrics.rows_precise_filtered +=
num_rows_before_filter - record_batch.num_rows();
let record_batch = if let Some(mapper) = range.compaction_projection_mapper() {
mapper.project(record_batch)?
} else {
record_batch
};
if let Some(compat) = range.compat_batch() {
yield compat.compat(record_batch)?;
} else {
+113 -26
View File
@@ -46,7 +46,6 @@ use crate::error::{
NewRecordBatchSnafu, RecordBatchSnafu, Result, StatsNotPresentSnafu, UnexpectedSnafu,
};
use crate::read::compat::FlatCompatBatch;
use crate::read::flat_projection::CompactionProjectionMapper;
use crate::read::last_row::FlatRowGroupLastRowCachedReader;
use crate::read::prune::FlatPruneReader;
use crate::sst::file::FileHandle;
@@ -388,11 +387,6 @@ impl FileRange {
self.context.compat_batch()
}
/// Returns the helper to project batches.
pub(crate) fn compaction_projection_mapper(&self) -> Option<&CompactionProjectionMapper> {
self.context.compaction_projection_mapper()
}
/// Filters a full-projection batch using this range's precise filters.
pub(crate) fn precise_filter_flat(
&self,
@@ -535,19 +529,12 @@ impl FileRangeContext {
self.base.compat_batch.as_ref()
}
/// Returns the helper to project batches.
pub(crate) fn compaction_projection_mapper(&self) -> Option<&CompactionProjectionMapper> {
self.base.compaction_projection_mapper.as_ref()
}
/// Sets the compat helper to the context.
pub(crate) fn set_compat_batch(&mut self, compat: Option<FlatCompatBatch>) {
self.base.compat_batch = compat;
}
/// Filters the input RecordBatch by the pushed down predicate and returns RecordBatch.
/// If a partition expr filter is configured, it is also applied.
/// Physical filter exprs are not evaluated here; they are only applied during prefiltering.
/// Applies source-batch filtering; see [RangeBase::precise_filter_flat] for layout invariants.
pub(crate) fn precise_filter_flat(
&self,
input: RecordBatch,
@@ -626,8 +613,6 @@ pub(crate) struct RangeBase {
pub(crate) codec: Arc<dyn PrimaryKeyCodec>,
/// Optional helper to compat batches.
pub(crate) compat_batch: Option<FlatCompatBatch>,
/// Optional helper to project batches.
pub(crate) compaction_projection_mapper: Option<CompactionProjectionMapper>,
/// Mode to pre-filter columns.
pub(crate) pre_filter_mode: PreFilterMode,
/// Partition filter.
@@ -649,9 +634,11 @@ impl TagDecodeState {
}
impl RangeBase {
/// Filters the input RecordBatch by the pushed down predicate and returns RecordBatch.
/// Filters converted source batches before compatibility and merge.
/// Predicate projections are temporary: only their row mask is applied to the
/// original batch, preserving its schema and complete encoded primary keys.
///
/// It assumes all necessary tags are already decoded from the primary key.
/// Missing predicate tags are decoded from the primary key on demand.
///
/// # Arguments
/// * `input` - The RecordBatch to filter
@@ -1041,7 +1028,6 @@ mod tests {
prune_schema: metadata.schema.clone(),
codec: mito_codec::row_converter::build_primary_key_codec(metadata.as_ref()),
compat_batch: None,
compaction_projection_mapper: None,
pre_filter_mode: PreFilterMode::All,
partition_filter: None,
}
@@ -1087,11 +1073,28 @@ mod tests {
assert_eq!(batch, filtered);
}
#[test]
fn test_sparse_filter_and_partition_tags_preserve_skip_modes() {
let metadata = Arc::new(sst_region_metadata_with_encoding(
PrimaryKeyEncoding::Sparse,
));
#[rstest::rstest]
#[case(2)]
#[case(40)]
#[case(3600)]
fn test_sparse_filter_and_partition_tags_preserve_skip_modes(#[case] tag_count: u32) {
let source = sst_region_metadata_with_encoding(PrimaryKeyEncoding::Sparse);
let mut keys = source.primary_key.clone();
let mut builder = RegionMetadataBuilder::from_existing(source);
for id in 4..tag_count + 2 {
builder.push_column_metadata(store_api::metadata::ColumnMetadata {
column_schema: ColumnSchema::new(
format!("tag_{id}"),
ConcreteDataType::string_datatype(),
true,
),
semantic_type: SemanticType::Tag,
column_id: id,
});
keys.push(id);
}
builder.primary_key(keys);
let metadata = Arc::new(builder.build().unwrap());
let codec = SparsePrimaryKeyCodec::schemaless();
let long_tag = "x".repeat(80);
let mut pk_builder = BinaryDictionaryBuilder::<UInt32Type>::new();
@@ -1159,7 +1162,10 @@ mod tests {
)
.unwrap();
let batch = read_format.convert_batch(raw.clone(), None).unwrap();
let base = RangeBase {
if !materialized {
assert_eq!(batch, raw);
}
let mut base = RangeBase {
filters: filters.clone(),
dyn_filters: vec![],
read_format,
@@ -1167,7 +1173,6 @@ mod tests {
prune_schema: metadata.schema.clone(),
codec: Arc::new(codec.clone()),
compat_batch: None,
compaction_projection_mapper: None,
pre_filter_mode: PreFilterMode::All,
partition_filter: Some(PartitionFilterContext {
partition_schema: partition_schema.clone(),
@@ -1180,6 +1185,19 @@ mod tests {
(false, true, vec![3, 4, 5]),
(true, true, vec![0, 1, 3, 4, 5]),
] {
let mut state = TagDecodeState::new();
base.compute_filter_mask_flat(&batch, skip_fields, skip_tags, &mut state)
.unwrap();
let mut decoded = state.decoded_tag_cache.keys().copied().collect::<Vec<_>>();
decoded.sort_unstable();
assert_eq!(
decoded,
if !materialized && !skip_tags {
vec![0, 1]
} else {
vec![1]
}
);
let output = base
.precise_filter_flat(batch.clone(), skip_fields, skip_tags)
.unwrap()
@@ -1194,7 +1212,76 @@ mod tests {
expected.as_slice(),
"materialized={materialized}, skip_fields={skip_fields}, skip_tags={skip_tags}"
);
let mask = BooleanArray::from(
(0..6)
.map(|row| expected.contains(&row))
.collect::<Vec<_>>(),
);
assert_eq!(
output,
datatypes::arrow::compute::filter_record_batch(&batch, &mask).unwrap()
);
// Query materialization stays separate from compaction. Both
// compaction consumers pass encoded batches directly to compat.
if materialized {
continue;
}
let compacted = output;
assert_eq!(
compacted,
datatypes::arrow::compute::filter_record_batch(&raw, &mask).unwrap()
);
// Exercise that boundary with an added nullable field.
let mut target = RegionMetadataBuilder::from_existing(metadata.as_ref().clone());
target.push_column_metadata(store_api::metadata::ColumnMetadata {
column_schema: ColumnSchema::new(
"added",
ConcreteDataType::uint64_datatype(),
true,
),
semantic_type: SemanticType::Field,
column_id: tag_count + 2,
});
let target = Arc::new(target.build().unwrap());
let projection =
crate::read::flat_projection::FlatProjectionMapper::all(&target).unwrap();
let compat = FlatCompatBatch::try_new(&projection, &base.read_format, true)
.unwrap()
.unwrap();
let compatible = compat.compat(compacted.clone()).unwrap();
assert_eq!(compatible.schema(), projection.input_arrow_schema(true));
assert_eq!(compatible.column(1).null_count(), compatible.num_rows());
assert_eq!(compatible.column(0), compacted.column(0));
assert_eq!(&compatible.columns()[2..], &compacted.columns()[1..]);
}
// Missing partition tags produce NULL, which must reject the row
// even when ordinary tag and field predicates are skipped.
base.filters.clear();
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
"tag_0",
ConcreteDataType::dictionary_datatype(
ConcreteDataType::uint32_datatype(),
ConcreteDataType::string_datatype(),
),
true,
)]));
base.partition_filter = Some(PartitionFilterContext {
region_partition_physical_expr: partition_col("tag_0")
.eq(Value::String("".into()))
.try_as_physical_expr(schema.arrow_schema())
.unwrap(),
partition_schema: schema.clone(),
});
let temporary = base
.project_record_batch_for_pruning_flat(&batch, &schema, &mut TagDecodeState::new())
.unwrap();
// Dictionary keys can be valid while the referenced value is NULL.
assert_eq!(temporary.column(0).logical_null_count(), 1);
let filtered = base
.precise_filter_flat(batch.clone(), true, true)
.unwrap()
.unwrap();
assert_eq!(filtered, batch.slice(1, 1));
}
}
+6 -2
View File
@@ -183,7 +183,11 @@ pub(crate) fn field_column_start(metadata: &RegionMetadata, num_columns: usize)
// sparse encoding).
/// Helper for reading the flat SST format with projection.
///
/// It only supports flat format that stores primary keys additionally.
/// Physical Parquet projections describe stored columns and drive pruning and
/// prefiltering. Converted projections describe batches returned by `convert_batch`.
/// Sparse compaction can retain encoded-only tags; query reads may expand them.
/// Neither layout is the final user projection. Compatibility aligns converted
/// batches before merge; temporary predicate columns never change their layout.
pub struct FlatReadFormat {
/// Sequence number to override the sequence read from the SST.
override_sequence: Option<SequenceNumber>,
@@ -252,7 +256,7 @@ impl FlatReadFormat {
self.pk_dict_wrap_schema = Some(output_schema);
}
/// Index of a column in the projected batch by its column id.
/// Index in the converted batch, never in a separately projected prefilter batch.
pub fn projected_index_by_id(&self, column_id: ColumnId) -> Option<usize> {
self.format_projection()
.column_id_to_projected_index
+5 -3
View File
@@ -1651,8 +1651,10 @@ mod tests {
tag1: String,
}
#[test]
fn test_flat_read_format_convert_format_with_dense_encoding() {
#[rstest::rstest]
#[case(false)]
#[case(true)]
fn test_flat_read_format_convert_format_with_dense_encoding(#[case] skip_auto_convert: bool) {
let metadata = build_test_region_metadata();
let column_ids: Vec<_> = metadata
@@ -1665,7 +1667,7 @@ mod tests {
ReadColumns::new(column_ids),
Some(build_test_arrow_schema()),
"test",
false,
skip_auto_convert,
)
.unwrap();
+119 -32
View File
@@ -47,7 +47,6 @@ use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData};
use parquet::file::properties::DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT;
use partition::expr::PartitionExpr;
use snafu::{OptionExt, ResultExt};
use store_api::codec::PrimaryKeyEncoding;
use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef};
use store_api::region_request::PathType;
use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME;
@@ -64,7 +63,6 @@ use crate::error::{
UnexpectedSnafu,
};
use crate::metrics::{READ_ROWS_TOTAL, READ_STAGE_ELAPSED};
use crate::read::flat_projection::CompactionProjectionMapper;
use crate::read::prune::FlatPruneReader;
use crate::read::read_columns::ReadColumns;
use crate::series_index::SeriesIndexReadContext;
@@ -469,35 +467,9 @@ impl ParquetReaderBuilder {
let mut parquet_meta = sst_meta.parquet_metadata();
let mut parquet_metadata_size = sst_meta.parquet_metadata_size();
let region_meta = sst_meta.region_metadata();
let region_partition_expr_str = self
.expected_metadata
.as_ref()
.and_then(|meta| meta.partition_expr.as_deref());
let (_, is_same_region_partition) = Self::is_same_region_partition(
region_partition_expr_str,
self.file_handle.meta_ref().partition_expr.as_deref(),
)?;
// Skip auto convert when:
// - compaction is enabled
// - region partition expr is same with file partition expr (no need to auto convert)
let skip_auto_convert = self.compaction && is_same_region_partition;
// Build a compaction projection helper when:
// - compaction is enabled
// - region partition expr differs from file partition expr
// - flat format is enabled
// - primary key encoding is sparse
//
// This is applied after row-group filtering to align batches with flat output schema
// before compat handling.
let compaction_projection_mapper = if self.compaction
&& !is_same_region_partition
&& region_meta.primary_key_encoding == PrimaryKeyEncoding::Sparse
{
Some(CompactionProjectionMapper::try_new(&region_meta)?)
} else {
None
};
// Sparse compaction preserves encoded keys. Partition predicates decode only
// their dependencies into temporary arrays during precise filtering.
let skip_auto_convert = self.compaction;
let read_cols = if let Some(read_cols) = &self.read_cols {
read_cols.clone()
@@ -716,7 +688,6 @@ impl ParquetReaderBuilder {
prune_schema,
codec,
compat_batch: None,
compaction_projection_mapper,
pre_filter_mode: self.pre_filter_mode,
partition_filter,
},
@@ -741,6 +712,9 @@ impl ParquetReaderBuilder {
Ok((region_partition_expr, is_same))
}
/// Builds the dependency-only schema for row-level partition evaluation.
/// Its indices are independent of both Parquet and converted batch projections;
/// precise filtering gathers these columns temporarily before compatibility.
/// Compare partition expressions from expected metadata and file metadata,
/// and build a partition filter if they differ.
fn build_partition_filter(
@@ -927,6 +901,8 @@ impl ParquetReaderBuilder {
}
/// Computes row groups to read, along with their respective row selections.
/// Statistics and index lookups use source-file column mappings, independently
/// of whether conversion will later expand encoded tags.
#[tracing::instrument(
skip_all,
fields(
@@ -2974,6 +2950,117 @@ mod tests {
}
}
#[rstest::rstest]
#[case(None)]
#[case(Some(false))]
#[case(Some(true))]
#[tokio::test]
async fn test_sparse_compaction_partition_mismatch_keeps_encoded_layout(
#[case] prefilter: Option<bool>,
) {
let source = crate::test_util::sst_util::sst_region_metadata_with_encoding(
PrimaryKeyEncoding::Sparse,
);
let props = WriterProperties::builder()
.set_key_value_metadata(Some(vec![parquet::file::metadata::KeyValue::new(
crate::sst::parquet::PARQUET_METADATA_KEY.to_string(),
source.to_json().unwrap(),
)]))
.build();
let store = ObjectStore::new(Memory::default()).unwrap();
let (fixture, metadata) = prefilter_test_builder_with(
store.clone(),
3,
2,
PageIndexPolicy::Skip,
Some(props),
Predicate::new(vec![col("tag_0").eq(lit("a"))]),
CacheStrategy::Disabled,
)
.await;
for partition in [None, Some("a"), Some("b"), Some("missing")] {
let mut expected = metadata.as_ref().clone();
expected.set_partition_expr(partition.map(|value| {
partition::expr::col("tag_0")
.eq(datatypes::value::Value::String(value.into()))
.as_json_str()
.unwrap()
}));
for compaction in [false, true] {
let builder = ParquetReaderBuilder::new(
"prefilter_test".into(),
PathType::Bare,
fixture.file_handle.clone(),
store.clone(),
)
.expected_metadata(Some(Arc::new(expected.clone())))
.compaction(compaction);
let builder = if let Some(enabled) = prefilter {
builder
.predicate(Some(Predicate::new(vec![
col("tag_1").eq(lit("x")),
col("field_0").gt(lit(1_u64)),
])))
.enable_predicate_prefilter(enabled)
} else {
builder
};
let mut reader = builder.build().await.unwrap().unwrap();
assert_eq!(
reader.context.reader_builder().prefilter_builder.is_some(),
prefilter == Some(true),
);
let mut count = 0;
while let Some(batch) = reader.next_record_batch().await.unwrap() {
assert_eq!(
batch.num_columns(),
if compaction {
5
} else {
metadata.column_metadatas.len() + 3
}
);
if compaction {
assert!(
reader
.context
.read_format()
.projected_index_by_id(0)
.is_none()
);
assert!(
reader
.context
.read_format()
.projected_index_by_id(1)
.is_none()
);
}
count += batch.num_rows();
}
assert_eq!(
count,
match partition {
None =>
if prefilter.is_some() {
4
} else {
6
},
Some("a") =>
if prefilter.is_some() {
3
} else {
5
},
Some("b") => 1,
_ => 0,
}
);
}
}
}
async fn prefilter_test_builder(
object_store: ObjectStore,
predicate: Predicate,
-1
View File
@@ -84,7 +84,6 @@ pub fn tag_filter_for_bench(
prune_schema: metadata.schema.clone(),
codec: mito_codec::row_converter::build_primary_key_codec(&metadata),
compat_batch: None,
compaction_projection_mapper: None,
pre_filter_mode: PreFilterMode::All,
partition_filter: None,
};