mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 16:54:09 +00:00
fix(mito2): avoid unrelated sparse tag expansion during compaction (#10414)
* fix(mito2): avoid unrelated sparse tag expansion during compaction Signed-off-by: evenyag <realevenyag@gmail.com> * test(mito2): cover sparse compaction filter and compatibility boundaries Signed-off-by: evenyag <realevenyag@gmail.com> * fix(mito2): handle projected fields in sparse compaction mapper Signed-off-by: evenyag <realevenyag@gmail.com> * refactor(mito2): remove redundant compaction projection mapper Signed-off-by: evenyag <realevenyag@gmail.com> --------- Signed-off-by: evenyag <realevenyag@gmail.com>
This commit is contained in:
@@ -106,7 +106,6 @@ impl BulkIterContext {
|
||||
codec,
|
||||
// we don't need to compat batch since all batch in memtable have the same schema.
|
||||
compat_batch: None,
|
||||
compaction_projection_mapper: None,
|
||||
pre_filter_mode,
|
||||
partition_filter: None,
|
||||
},
|
||||
|
||||
@@ -1552,7 +1552,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flat_compat_batch_compact_sparse() {
|
||||
fn test_flat_compat_batch_compact_sparse_added_and_dropped_fields() {
|
||||
let mut actual_metadata = new_metadata(
|
||||
&[
|
||||
(
|
||||
@@ -1561,6 +1561,7 @@ mod tests {
|
||||
ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
),
|
||||
(2, SemanticType::Field, ConcreteDataType::int64_datatype()),
|
||||
(4, SemanticType::Field, ConcreteDataType::int64_datatype()),
|
||||
],
|
||||
&[],
|
||||
);
|
||||
@@ -1599,6 +1600,7 @@ mod tests {
|
||||
let sparse_k1 = encode_sparse_key(&[]);
|
||||
let input_columns: Vec<ArrayRef> = vec![
|
||||
Arc::new(Int64Array::from(vec![100, 200])),
|
||||
Arc::new(Int64Array::from(vec![300, 400])), // dropped field
|
||||
Arc::new(TimestampMillisecondArray::from_iter_values([1000, 2000])),
|
||||
build_flat_test_pk_array(&[&sparse_k1, &sparse_k1]),
|
||||
Arc::new(UInt64Array::from_iter_values([1, 2])),
|
||||
@@ -1613,6 +1615,11 @@ mod tests {
|
||||
);
|
||||
let input_batch = RecordBatch::try_new(input_schema, input_columns).unwrap();
|
||||
|
||||
// The physical read projection omits the dropped source field. Sparse
|
||||
// conversion preserves this encoded layout, which goes directly to compat.
|
||||
assert!(read_format.projected_index_by_id(4).is_none());
|
||||
let input_batch = input_batch.project(&[0, 2, 3, 4, 5]).unwrap();
|
||||
let input_batch = read_format.convert_batch(input_batch, None).unwrap();
|
||||
let result = compat_batch.compat(input_batch).unwrap();
|
||||
|
||||
let expected_columns: Vec<ArrayRef> = vec![
|
||||
|
||||
@@ -31,7 +31,7 @@ use store_api::metadata::{RegionMetadata, RegionMetadataRef};
|
||||
use store_api::storage::ColumnId;
|
||||
|
||||
use crate::cache::CacheStrategy;
|
||||
use crate::error::{InvalidRequestSnafu, NewRecordBatchSnafu, Result};
|
||||
use crate::error::{InvalidRequestSnafu, Result};
|
||||
use crate::read::projection::{read_column_ids_from_projection, repeated_vector_with_cache};
|
||||
use crate::read::read_columns::{JsonTargetTypes, ReadColumns};
|
||||
use crate::sst::parquet::Json2RewriteTargets;
|
||||
@@ -46,6 +46,11 @@ use crate::sst::{
|
||||
///
|
||||
/// This mapper support duplicate and unsorted projection indices.
|
||||
/// The output schema is determined by the projection indices.
|
||||
/// Input indices address the compatible merge schema, not physical Parquet
|
||||
/// columns or temporary predicate projections. Sparse compaction uses
|
||||
/// `input_arrow_schema(true)` without expanded tags and bypasses query output
|
||||
/// conversion. Sparse compaction retains encoded keys through filtering and
|
||||
/// passes batches directly to compatibility; predicate tags stay temporary.
|
||||
pub struct FlatProjectionMapper {
|
||||
/// Metadata of the region.
|
||||
metadata: RegionMetadataRef,
|
||||
@@ -477,47 +482,6 @@ pub(crate) fn compute_input_arrow_schema(
|
||||
Arc::new(datatypes::arrow::datatypes::Schema::new(new_fields))
|
||||
}
|
||||
|
||||
/// Helper to project compaction batches into flat format columns
|
||||
/// (fields + time index + __primary_key + __sequence + __op_type).
|
||||
pub(crate) struct CompactionProjectionMapper {
|
||||
projection: Vec<usize>,
|
||||
}
|
||||
|
||||
impl CompactionProjectionMapper {
|
||||
pub(crate) fn try_new(metadata: &RegionMetadataRef) -> Result<Self> {
|
||||
let projection = metadata
|
||||
.column_metadatas
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(idx, col)| {
|
||||
if matches!(col.semantic_type, SemanticType::Field) {
|
||||
Some(idx)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.chain([metadata.time_index_column_pos()])
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let read_col_ids = metadata.column_metadatas.iter().map(|col| col.column_id);
|
||||
let read_cols = ReadColumns::new(read_col_ids);
|
||||
let mapper = FlatProjectionMapper::new_with_read_columns(metadata, projection, read_cols)?;
|
||||
Ok(Self {
|
||||
projection: mapper.batch_indices,
|
||||
})
|
||||
}
|
||||
|
||||
/// Projects columns and appends internal columns for compaction output.
|
||||
///
|
||||
/// The input batch is expected to be in flat format with internal columns appended.
|
||||
pub(crate) fn project(&self, batch: DfRecordBatch) -> Result<DfRecordBatch> {
|
||||
let mut projection = self.projection.clone();
|
||||
projection.extend(batch.num_columns() - 3..batch.num_columns());
|
||||
// Projection must preserve the Arrow layouts already produced by compaction Rewrite.
|
||||
batch.project(&projection).context(NewRecordBatchSnafu)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use datatypes::schema::ColumnSchema;
|
||||
|
||||
@@ -1616,13 +1616,7 @@ pub fn build_flat_file_range_scan_stream(
|
||||
.file_handle()
|
||||
.is_effective_target_sequence_trusted(stream_ctx.input.region_metadata().region_id);
|
||||
|
||||
let mapper = range.compaction_projection_mapper();
|
||||
while let Some(record_batch) = reader.next_batch().await? {
|
||||
let record_batch = if let Some(mapper) = mapper {
|
||||
mapper.project(record_batch)?
|
||||
} else {
|
||||
record_batch
|
||||
};
|
||||
let Some(record_batch) = filter_flat_batch_by_sequence(
|
||||
record_batch,
|
||||
stream_ctx.input.sequence_range,
|
||||
|
||||
@@ -571,11 +571,6 @@ fn scan_series_file_ranges(
|
||||
reader_metrics.filter_metrics.rows_precise_filtered +=
|
||||
num_rows_before_filter - record_batch.num_rows();
|
||||
|
||||
let record_batch = if let Some(mapper) = range.compaction_projection_mapper() {
|
||||
mapper.project(record_batch)?
|
||||
} else {
|
||||
record_batch
|
||||
};
|
||||
if let Some(compat) = range.compat_batch() {
|
||||
yield compat.compat(record_batch)?;
|
||||
} else {
|
||||
|
||||
@@ -46,7 +46,6 @@ use crate::error::{
|
||||
NewRecordBatchSnafu, RecordBatchSnafu, Result, StatsNotPresentSnafu, UnexpectedSnafu,
|
||||
};
|
||||
use crate::read::compat::FlatCompatBatch;
|
||||
use crate::read::flat_projection::CompactionProjectionMapper;
|
||||
use crate::read::last_row::FlatRowGroupLastRowCachedReader;
|
||||
use crate::read::prune::FlatPruneReader;
|
||||
use crate::sst::file::FileHandle;
|
||||
@@ -388,11 +387,6 @@ impl FileRange {
|
||||
self.context.compat_batch()
|
||||
}
|
||||
|
||||
/// Returns the helper to project batches.
|
||||
pub(crate) fn compaction_projection_mapper(&self) -> Option<&CompactionProjectionMapper> {
|
||||
self.context.compaction_projection_mapper()
|
||||
}
|
||||
|
||||
/// Filters a full-projection batch using this range's precise filters.
|
||||
pub(crate) fn precise_filter_flat(
|
||||
&self,
|
||||
@@ -535,19 +529,12 @@ impl FileRangeContext {
|
||||
self.base.compat_batch.as_ref()
|
||||
}
|
||||
|
||||
/// Returns the helper to project batches.
|
||||
pub(crate) fn compaction_projection_mapper(&self) -> Option<&CompactionProjectionMapper> {
|
||||
self.base.compaction_projection_mapper.as_ref()
|
||||
}
|
||||
|
||||
/// Sets the compat helper to the context.
|
||||
pub(crate) fn set_compat_batch(&mut self, compat: Option<FlatCompatBatch>) {
|
||||
self.base.compat_batch = compat;
|
||||
}
|
||||
|
||||
/// Filters the input RecordBatch by the pushed down predicate and returns RecordBatch.
|
||||
/// If a partition expr filter is configured, it is also applied.
|
||||
/// Physical filter exprs are not evaluated here; they are only applied during prefiltering.
|
||||
/// Applies source-batch filtering; see [RangeBase::precise_filter_flat] for layout invariants.
|
||||
pub(crate) fn precise_filter_flat(
|
||||
&self,
|
||||
input: RecordBatch,
|
||||
@@ -626,8 +613,6 @@ pub(crate) struct RangeBase {
|
||||
pub(crate) codec: Arc<dyn PrimaryKeyCodec>,
|
||||
/// Optional helper to compat batches.
|
||||
pub(crate) compat_batch: Option<FlatCompatBatch>,
|
||||
/// Optional helper to project batches.
|
||||
pub(crate) compaction_projection_mapper: Option<CompactionProjectionMapper>,
|
||||
/// Mode to pre-filter columns.
|
||||
pub(crate) pre_filter_mode: PreFilterMode,
|
||||
/// Partition filter.
|
||||
@@ -649,9 +634,11 @@ impl TagDecodeState {
|
||||
}
|
||||
|
||||
impl RangeBase {
|
||||
/// Filters the input RecordBatch by the pushed down predicate and returns RecordBatch.
|
||||
/// Filters converted source batches before compatibility and merge.
|
||||
/// Predicate projections are temporary: only their row mask is applied to the
|
||||
/// original batch, preserving its schema and complete encoded primary keys.
|
||||
///
|
||||
/// It assumes all necessary tags are already decoded from the primary key.
|
||||
/// Missing predicate tags are decoded from the primary key on demand.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `input` - The RecordBatch to filter
|
||||
@@ -1041,7 +1028,6 @@ mod tests {
|
||||
prune_schema: metadata.schema.clone(),
|
||||
codec: mito_codec::row_converter::build_primary_key_codec(metadata.as_ref()),
|
||||
compat_batch: None,
|
||||
compaction_projection_mapper: None,
|
||||
pre_filter_mode: PreFilterMode::All,
|
||||
partition_filter: None,
|
||||
}
|
||||
@@ -1087,11 +1073,28 @@ mod tests {
|
||||
assert_eq!(batch, filtered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sparse_filter_and_partition_tags_preserve_skip_modes() {
|
||||
let metadata = Arc::new(sst_region_metadata_with_encoding(
|
||||
PrimaryKeyEncoding::Sparse,
|
||||
));
|
||||
#[rstest::rstest]
|
||||
#[case(2)]
|
||||
#[case(40)]
|
||||
#[case(3600)]
|
||||
fn test_sparse_filter_and_partition_tags_preserve_skip_modes(#[case] tag_count: u32) {
|
||||
let source = sst_region_metadata_with_encoding(PrimaryKeyEncoding::Sparse);
|
||||
let mut keys = source.primary_key.clone();
|
||||
let mut builder = RegionMetadataBuilder::from_existing(source);
|
||||
for id in 4..tag_count + 2 {
|
||||
builder.push_column_metadata(store_api::metadata::ColumnMetadata {
|
||||
column_schema: ColumnSchema::new(
|
||||
format!("tag_{id}"),
|
||||
ConcreteDataType::string_datatype(),
|
||||
true,
|
||||
),
|
||||
semantic_type: SemanticType::Tag,
|
||||
column_id: id,
|
||||
});
|
||||
keys.push(id);
|
||||
}
|
||||
builder.primary_key(keys);
|
||||
let metadata = Arc::new(builder.build().unwrap());
|
||||
let codec = SparsePrimaryKeyCodec::schemaless();
|
||||
let long_tag = "x".repeat(80);
|
||||
let mut pk_builder = BinaryDictionaryBuilder::<UInt32Type>::new();
|
||||
@@ -1159,7 +1162,10 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
let batch = read_format.convert_batch(raw.clone(), None).unwrap();
|
||||
let base = RangeBase {
|
||||
if !materialized {
|
||||
assert_eq!(batch, raw);
|
||||
}
|
||||
let mut base = RangeBase {
|
||||
filters: filters.clone(),
|
||||
dyn_filters: vec![],
|
||||
read_format,
|
||||
@@ -1167,7 +1173,6 @@ mod tests {
|
||||
prune_schema: metadata.schema.clone(),
|
||||
codec: Arc::new(codec.clone()),
|
||||
compat_batch: None,
|
||||
compaction_projection_mapper: None,
|
||||
pre_filter_mode: PreFilterMode::All,
|
||||
partition_filter: Some(PartitionFilterContext {
|
||||
partition_schema: partition_schema.clone(),
|
||||
@@ -1180,6 +1185,19 @@ mod tests {
|
||||
(false, true, vec![3, 4, 5]),
|
||||
(true, true, vec![0, 1, 3, 4, 5]),
|
||||
] {
|
||||
let mut state = TagDecodeState::new();
|
||||
base.compute_filter_mask_flat(&batch, skip_fields, skip_tags, &mut state)
|
||||
.unwrap();
|
||||
let mut decoded = state.decoded_tag_cache.keys().copied().collect::<Vec<_>>();
|
||||
decoded.sort_unstable();
|
||||
assert_eq!(
|
||||
decoded,
|
||||
if !materialized && !skip_tags {
|
||||
vec![0, 1]
|
||||
} else {
|
||||
vec![1]
|
||||
}
|
||||
);
|
||||
let output = base
|
||||
.precise_filter_flat(batch.clone(), skip_fields, skip_tags)
|
||||
.unwrap()
|
||||
@@ -1194,7 +1212,76 @@ mod tests {
|
||||
expected.as_slice(),
|
||||
"materialized={materialized}, skip_fields={skip_fields}, skip_tags={skip_tags}"
|
||||
);
|
||||
let mask = BooleanArray::from(
|
||||
(0..6)
|
||||
.map(|row| expected.contains(&row))
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
assert_eq!(
|
||||
output,
|
||||
datatypes::arrow::compute::filter_record_batch(&batch, &mask).unwrap()
|
||||
);
|
||||
// Query materialization stays separate from compaction. Both
|
||||
// compaction consumers pass encoded batches directly to compat.
|
||||
if materialized {
|
||||
continue;
|
||||
}
|
||||
let compacted = output;
|
||||
assert_eq!(
|
||||
compacted,
|
||||
datatypes::arrow::compute::filter_record_batch(&raw, &mask).unwrap()
|
||||
);
|
||||
// Exercise that boundary with an added nullable field.
|
||||
let mut target = RegionMetadataBuilder::from_existing(metadata.as_ref().clone());
|
||||
target.push_column_metadata(store_api::metadata::ColumnMetadata {
|
||||
column_schema: ColumnSchema::new(
|
||||
"added",
|
||||
ConcreteDataType::uint64_datatype(),
|
||||
true,
|
||||
),
|
||||
semantic_type: SemanticType::Field,
|
||||
column_id: tag_count + 2,
|
||||
});
|
||||
let target = Arc::new(target.build().unwrap());
|
||||
let projection =
|
||||
crate::read::flat_projection::FlatProjectionMapper::all(&target).unwrap();
|
||||
let compat = FlatCompatBatch::try_new(&projection, &base.read_format, true)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let compatible = compat.compat(compacted.clone()).unwrap();
|
||||
assert_eq!(compatible.schema(), projection.input_arrow_schema(true));
|
||||
assert_eq!(compatible.column(1).null_count(), compatible.num_rows());
|
||||
assert_eq!(compatible.column(0), compacted.column(0));
|
||||
assert_eq!(&compatible.columns()[2..], &compacted.columns()[1..]);
|
||||
}
|
||||
// Missing partition tags produce NULL, which must reject the row
|
||||
// even when ordinary tag and field predicates are skipped.
|
||||
base.filters.clear();
|
||||
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
|
||||
"tag_0",
|
||||
ConcreteDataType::dictionary_datatype(
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
),
|
||||
true,
|
||||
)]));
|
||||
base.partition_filter = Some(PartitionFilterContext {
|
||||
region_partition_physical_expr: partition_col("tag_0")
|
||||
.eq(Value::String("".into()))
|
||||
.try_as_physical_expr(schema.arrow_schema())
|
||||
.unwrap(),
|
||||
partition_schema: schema.clone(),
|
||||
});
|
||||
let temporary = base
|
||||
.project_record_batch_for_pruning_flat(&batch, &schema, &mut TagDecodeState::new())
|
||||
.unwrap();
|
||||
// Dictionary keys can be valid while the referenced value is NULL.
|
||||
assert_eq!(temporary.column(0).logical_null_count(), 1);
|
||||
let filtered = base
|
||||
.precise_filter_flat(batch.clone(), true, true)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(filtered, batch.slice(1, 1));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -183,7 +183,11 @@ pub(crate) fn field_column_start(metadata: &RegionMetadata, num_columns: usize)
|
||||
// sparse encoding).
|
||||
/// Helper for reading the flat SST format with projection.
|
||||
///
|
||||
/// It only supports flat format that stores primary keys additionally.
|
||||
/// Physical Parquet projections describe stored columns and drive pruning and
|
||||
/// prefiltering. Converted projections describe batches returned by `convert_batch`.
|
||||
/// Sparse compaction can retain encoded-only tags; query reads may expand them.
|
||||
/// Neither layout is the final user projection. Compatibility aligns converted
|
||||
/// batches before merge; temporary predicate columns never change their layout.
|
||||
pub struct FlatReadFormat {
|
||||
/// Sequence number to override the sequence read from the SST.
|
||||
override_sequence: Option<SequenceNumber>,
|
||||
@@ -252,7 +256,7 @@ impl FlatReadFormat {
|
||||
self.pk_dict_wrap_schema = Some(output_schema);
|
||||
}
|
||||
|
||||
/// Index of a column in the projected batch by its column id.
|
||||
/// Index in the converted batch, never in a separately projected prefilter batch.
|
||||
pub fn projected_index_by_id(&self, column_id: ColumnId) -> Option<usize> {
|
||||
self.format_projection()
|
||||
.column_id_to_projected_index
|
||||
|
||||
@@ -1651,8 +1651,10 @@ mod tests {
|
||||
tag1: String,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flat_read_format_convert_format_with_dense_encoding() {
|
||||
#[rstest::rstest]
|
||||
#[case(false)]
|
||||
#[case(true)]
|
||||
fn test_flat_read_format_convert_format_with_dense_encoding(#[case] skip_auto_convert: bool) {
|
||||
let metadata = build_test_region_metadata();
|
||||
|
||||
let column_ids: Vec<_> = metadata
|
||||
@@ -1665,7 +1667,7 @@ mod tests {
|
||||
ReadColumns::new(column_ids),
|
||||
Some(build_test_arrow_schema()),
|
||||
"test",
|
||||
false,
|
||||
skip_auto_convert,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
||||
@@ -47,7 +47,6 @@ use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData};
|
||||
use parquet::file::properties::DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT;
|
||||
use partition::expr::PartitionExpr;
|
||||
use snafu::{OptionExt, ResultExt};
|
||||
use store_api::codec::PrimaryKeyEncoding;
|
||||
use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef};
|
||||
use store_api::region_request::PathType;
|
||||
use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME;
|
||||
@@ -64,7 +63,6 @@ use crate::error::{
|
||||
UnexpectedSnafu,
|
||||
};
|
||||
use crate::metrics::{READ_ROWS_TOTAL, READ_STAGE_ELAPSED};
|
||||
use crate::read::flat_projection::CompactionProjectionMapper;
|
||||
use crate::read::prune::FlatPruneReader;
|
||||
use crate::read::read_columns::ReadColumns;
|
||||
use crate::series_index::SeriesIndexReadContext;
|
||||
@@ -469,35 +467,9 @@ impl ParquetReaderBuilder {
|
||||
let mut parquet_meta = sst_meta.parquet_metadata();
|
||||
let mut parquet_metadata_size = sst_meta.parquet_metadata_size();
|
||||
let region_meta = sst_meta.region_metadata();
|
||||
let region_partition_expr_str = self
|
||||
.expected_metadata
|
||||
.as_ref()
|
||||
.and_then(|meta| meta.partition_expr.as_deref());
|
||||
let (_, is_same_region_partition) = Self::is_same_region_partition(
|
||||
region_partition_expr_str,
|
||||
self.file_handle.meta_ref().partition_expr.as_deref(),
|
||||
)?;
|
||||
// Skip auto convert when:
|
||||
// - compaction is enabled
|
||||
// - region partition expr is same with file partition expr (no need to auto convert)
|
||||
let skip_auto_convert = self.compaction && is_same_region_partition;
|
||||
|
||||
// Build a compaction projection helper when:
|
||||
// - compaction is enabled
|
||||
// - region partition expr differs from file partition expr
|
||||
// - flat format is enabled
|
||||
// - primary key encoding is sparse
|
||||
//
|
||||
// This is applied after row-group filtering to align batches with flat output schema
|
||||
// before compat handling.
|
||||
let compaction_projection_mapper = if self.compaction
|
||||
&& !is_same_region_partition
|
||||
&& region_meta.primary_key_encoding == PrimaryKeyEncoding::Sparse
|
||||
{
|
||||
Some(CompactionProjectionMapper::try_new(®ion_meta)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Sparse compaction preserves encoded keys. Partition predicates decode only
|
||||
// their dependencies into temporary arrays during precise filtering.
|
||||
let skip_auto_convert = self.compaction;
|
||||
|
||||
let read_cols = if let Some(read_cols) = &self.read_cols {
|
||||
read_cols.clone()
|
||||
@@ -716,7 +688,6 @@ impl ParquetReaderBuilder {
|
||||
prune_schema,
|
||||
codec,
|
||||
compat_batch: None,
|
||||
compaction_projection_mapper,
|
||||
pre_filter_mode: self.pre_filter_mode,
|
||||
partition_filter,
|
||||
},
|
||||
@@ -741,6 +712,9 @@ impl ParquetReaderBuilder {
|
||||
Ok((region_partition_expr, is_same))
|
||||
}
|
||||
|
||||
/// Builds the dependency-only schema for row-level partition evaluation.
|
||||
/// Its indices are independent of both Parquet and converted batch projections;
|
||||
/// precise filtering gathers these columns temporarily before compatibility.
|
||||
/// Compare partition expressions from expected metadata and file metadata,
|
||||
/// and build a partition filter if they differ.
|
||||
fn build_partition_filter(
|
||||
@@ -927,6 +901,8 @@ impl ParquetReaderBuilder {
|
||||
}
|
||||
|
||||
/// Computes row groups to read, along with their respective row selections.
|
||||
/// Statistics and index lookups use source-file column mappings, independently
|
||||
/// of whether conversion will later expand encoded tags.
|
||||
#[tracing::instrument(
|
||||
skip_all,
|
||||
fields(
|
||||
@@ -2974,6 +2950,117 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case(None)]
|
||||
#[case(Some(false))]
|
||||
#[case(Some(true))]
|
||||
#[tokio::test]
|
||||
async fn test_sparse_compaction_partition_mismatch_keeps_encoded_layout(
|
||||
#[case] prefilter: Option<bool>,
|
||||
) {
|
||||
let source = crate::test_util::sst_util::sst_region_metadata_with_encoding(
|
||||
PrimaryKeyEncoding::Sparse,
|
||||
);
|
||||
let props = WriterProperties::builder()
|
||||
.set_key_value_metadata(Some(vec![parquet::file::metadata::KeyValue::new(
|
||||
crate::sst::parquet::PARQUET_METADATA_KEY.to_string(),
|
||||
source.to_json().unwrap(),
|
||||
)]))
|
||||
.build();
|
||||
let store = ObjectStore::new(Memory::default()).unwrap();
|
||||
let (fixture, metadata) = prefilter_test_builder_with(
|
||||
store.clone(),
|
||||
3,
|
||||
2,
|
||||
PageIndexPolicy::Skip,
|
||||
Some(props),
|
||||
Predicate::new(vec![col("tag_0").eq(lit("a"))]),
|
||||
CacheStrategy::Disabled,
|
||||
)
|
||||
.await;
|
||||
for partition in [None, Some("a"), Some("b"), Some("missing")] {
|
||||
let mut expected = metadata.as_ref().clone();
|
||||
expected.set_partition_expr(partition.map(|value| {
|
||||
partition::expr::col("tag_0")
|
||||
.eq(datatypes::value::Value::String(value.into()))
|
||||
.as_json_str()
|
||||
.unwrap()
|
||||
}));
|
||||
for compaction in [false, true] {
|
||||
let builder = ParquetReaderBuilder::new(
|
||||
"prefilter_test".into(),
|
||||
PathType::Bare,
|
||||
fixture.file_handle.clone(),
|
||||
store.clone(),
|
||||
)
|
||||
.expected_metadata(Some(Arc::new(expected.clone())))
|
||||
.compaction(compaction);
|
||||
let builder = if let Some(enabled) = prefilter {
|
||||
builder
|
||||
.predicate(Some(Predicate::new(vec![
|
||||
col("tag_1").eq(lit("x")),
|
||||
col("field_0").gt(lit(1_u64)),
|
||||
])))
|
||||
.enable_predicate_prefilter(enabled)
|
||||
} else {
|
||||
builder
|
||||
};
|
||||
let mut reader = builder.build().await.unwrap().unwrap();
|
||||
assert_eq!(
|
||||
reader.context.reader_builder().prefilter_builder.is_some(),
|
||||
prefilter == Some(true),
|
||||
);
|
||||
let mut count = 0;
|
||||
while let Some(batch) = reader.next_record_batch().await.unwrap() {
|
||||
assert_eq!(
|
||||
batch.num_columns(),
|
||||
if compaction {
|
||||
5
|
||||
} else {
|
||||
metadata.column_metadatas.len() + 3
|
||||
}
|
||||
);
|
||||
if compaction {
|
||||
assert!(
|
||||
reader
|
||||
.context
|
||||
.read_format()
|
||||
.projected_index_by_id(0)
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
reader
|
||||
.context
|
||||
.read_format()
|
||||
.projected_index_by_id(1)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
count += batch.num_rows();
|
||||
}
|
||||
assert_eq!(
|
||||
count,
|
||||
match partition {
|
||||
None =>
|
||||
if prefilter.is_some() {
|
||||
4
|
||||
} else {
|
||||
6
|
||||
},
|
||||
Some("a") =>
|
||||
if prefilter.is_some() {
|
||||
3
|
||||
} else {
|
||||
5
|
||||
},
|
||||
Some("b") => 1,
|
||||
_ => 0,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn prefilter_test_builder(
|
||||
object_store: ObjectStore,
|
||||
predicate: Predicate,
|
||||
|
||||
@@ -84,7 +84,6 @@ pub fn tag_filter_for_bench(
|
||||
prune_schema: metadata.schema.clone(),
|
||||
codec: mito_codec::row_converter::build_primary_key_codec(&metadata),
|
||||
compat_batch: None,
|
||||
compaction_projection_mapper: None,
|
||||
pre_filter_mode: PreFilterMode::All,
|
||||
partition_filter: None,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user