fix(mito2): track window coverage and separate index builds

Signed-off-by: evenyag <realevenyag@gmail.com>
This commit is contained in:
evenyag
2026-09-11 14:58:44 +08:00
parent de1a33e15e
commit fc8fa70b04
5 changed files with 758 additions and 207 deletions
+2
View File
@@ -62,6 +62,8 @@ filtered `RecordBatch` stream.
- **Manifest format** (`manifest/action.rs`): affects crash recovery and
follower replay. Keep it backward compatible.
- **SST/Parquet layout** (`sst/`): readers must stay compatible with existing files.
- **Series-index coverage** (`series_index/catalog.rs`): `SeriesIndexEntry` stores
compaction-window width and per-window sequences in both catalogs and Parquet footers.
- **Request types** (`request.rs`): usually tied to proto definitions consumed by `datanode`.
- **WAL/memtable encoding** (`wal/`, `memtable/`): breaks replay if changed incompatibly.
+297 -67
View File
@@ -32,7 +32,11 @@ pub(crate) struct IndexBucket {
pub(crate) start: Timestamp,
pub(crate) end: Timestamp,
pub(crate) index_ids: SmallVec<[FileId; 2]>,
pub(crate) max_file_sequence: u64,
/// Zero means that merged indexes use incompatible window widths.
pub(crate) compaction_window_secs: i64,
/// Indexed SST sequence maxima per compaction window. Merging takes maxima only
/// for matching windows. See [`SeriesIndexEntry::window_sequences`] for the layout.
pub(crate) window_sequences: BTreeMap<i64, u64>,
}
impl IndexBucket {
@@ -41,15 +45,38 @@ impl IndexBucket {
start,
end,
index_ids: SmallVec::new(),
max_file_sequence: 0,
compaction_window_secs: 0,
window_sequences: BTreeMap::new(),
}
}
pub(crate) fn from_entry(entry: &SeriesIndexEntry) -> Self {
Self {
start: entry.bucket_start,
end: entry.bucket_end,
index_ids: smallvec![entry.index_uuid],
compaction_window_secs: entry.compaction_window_secs,
window_sequences: entry.window_sequences.clone(),
}
}
fn merge(&mut self, mut other: Self) {
self.start = self.start.min(other.start);
self.end = self.end.max(other.end);
if self.index_ids.is_empty() {
self.compaction_window_secs = other.compaction_window_secs;
self.window_sequences = std::mem::take(&mut other.window_sequences);
} else if !other.index_ids.is_empty() {
if self.compaction_window_secs == other.compaction_window_secs {
merge_window_sequences(&mut self.window_sequences, other.window_sequences);
} else {
// Do not compare maps with different window boundaries, including
// a previous incompatible merge, against current SST coverage.
self.compaction_window_secs = 0;
self.window_sequences.clear();
}
}
self.index_ids.append(&mut other.index_ids);
self.max_file_sequence = self.max_file_sequence.max(other.max_file_sequence);
}
/// Inserts a bucket, consuming overlapping entries and expanding their interval.
@@ -72,9 +99,9 @@ impl IndexBucket {
/// SSTs grouped into a half-open time interval for an aggregate series-index build.
///
/// Reconciliation may expand the interval through existing index coverage and retain
/// only files above its sequence watermark. Unknown source sequences prevent builds
/// even after filtering, since their coverage cannot be determined safely.
/// Reconciliation may expand the interval through existing index coverage. A changed
/// bucket is rebuilt from all its SSTs. Unknown source sequences prevent builds,
/// since their coverage cannot be determined safely.
#[derive(Debug, Clone)]
pub(crate) struct SeriesBucket {
pub(crate) start: Timestamp,
@@ -83,6 +110,10 @@ pub(crate) struct SeriesBucket {
pub(crate) has_unknown_sequence: bool,
/// Maximum known source sequence, or zero when all sequences are unknown.
pub(crate) max_file_sequence: u64,
pub(crate) compaction_window_secs: i64,
/// Current SST sequence maxima per window, compared with indexed coverage.
/// Unknown sequences contribute zero and prevent building an index.
pub(crate) window_sequences: BTreeMap<i64, u64>,
}
/// Next bucket coverage and the work required to publish it, computed without I/O.
@@ -90,11 +121,13 @@ pub(crate) struct SeriesIndexPlan {
pub(crate) index_buckets: BTreeMap<Timestamp, IndexBucket>,
pub(crate) builds: Vec<(SeriesBucket, SeriesIndexEntry)>,
pub(crate) expired_index_ids: Vec<FileId>,
/// Retire only after replacement builds and catalog publication succeed.
pub(crate) superseded_index_ids: Vec<FileId>,
pub(crate) computed_buckets: usize,
pub(crate) skipped_buckets: usize,
}
/// Plans complete sequence suffixes after merging time coverage. The returned bucket
/// Plans whole-bucket replacements when per-window coverage changes. The returned bucket
/// map is publishable only after every planned build and the catalog writes succeed.
pub(crate) fn plan_series_indexes(
buckets: Vec<SeriesBucket>,
@@ -102,7 +135,9 @@ pub(crate) fn plan_series_indexes(
ttl: Option<TimeToLive>,
now_ms: i64,
) -> SeriesIndexPlan {
let buckets = reconcile_series_buckets(buckets, &mut index_buckets);
// Reconciliation changes geometry, not established coverage. In particular,
// a deferred bridge must not change the indexed snapshot.
let buckets = reconcile_series_buckets(buckets, &mut index_buckets.clone());
let computed_buckets = buckets.len();
let expired = |end| {
ttl.is_some_and(|ttl| {
@@ -120,28 +155,31 @@ pub(crate) fn plan_series_indexes(
}
});
let mut builds = Vec::new();
for mut bucket in buckets {
let mut superseded_index_ids = Vec::new();
for bucket in buckets {
if expired(bucket.end) {
continue;
}
let index_bucket = index_buckets
.entry(bucket.start)
.or_insert_with(|| IndexBucket::new(bucket.start, bucket.end));
if bucket.has_unknown_sequence || bucket.max_file_sequence <= index_bucket.max_file_sequence
{
if bucket.has_unknown_sequence {
continue;
}
if index_buckets.get(&bucket.start).is_some_and(|indexed| {
indexed.end == bucket.end
&& indexed.compaction_window_secs == bucket.compaction_window_secs
&& indexed.window_sequences == bucket.window_sequences
}) {
continue;
}
// Select all SSTs above the watermark, including every file sharing a sequence.
// File IDs may change under compaction without advancing indexed coverage.
// The maximum remains valid because it exceeds the watermark and is retained.
bucket.files.retain(|file| {
file.meta_ref()
.sequence
.is_some_and(|sequence| sequence.get() > index_bucket.max_file_sequence)
});
if let Some(entry) = bucket.to_series_entry() {
index_bucket.index_ids.push(entry.index_uuid);
index_bucket.max_file_sequence = entry.max_file_sequence;
index_buckets.retain(|_, indexed| {
if indexed.start < bucket.end && bucket.start < indexed.end {
superseded_index_ids.extend_from_slice(&indexed.index_ids);
false
} else {
true
}
});
IndexBucket::from_entry(&entry).insert_into(&mut index_buckets);
builds.push((bucket, entry));
}
}
@@ -151,6 +189,7 @@ pub(crate) fn plan_series_indexes(
skipped_buckets: computed_buckets - builds.len(),
builds,
expired_index_ids,
superseded_index_ids,
computed_buckets,
}
}
@@ -169,18 +208,28 @@ pub(crate) fn rounded_bucket_width(
/// Groups SSTs across levels into sorted, disjoint time buckets without planning builds.
///
/// `width_secs` must be positive. Inclusive SST ranges are rounded outward to aligned,
/// Both widths must be positive, and `width_secs` must be a multiple of the compaction
/// window width. Inclusive SST ranges are rounded outward to aligned,
/// half-open intervals in seconds. Overlapping intervals merge; adjacent ones stay
/// separate. Each merged bucket tracks its maximum sequence and any unknown sequence.
pub(crate) fn group_files_into_series_buckets(
files: &[FileHandle],
width_secs: i64,
compaction_window_secs: i64,
) -> Vec<SeriesBucket> {
let mut spans = files
.iter()
.map(|file| {
let start = file.time_range().0.split().0;
let end = file.time_range().1.split().0;
let sequence = file
.meta_ref()
.sequence
.map_or(0, |sequence| sequence.get());
let window_sequences = (start.div_euclid(compaction_window_secs)
..=end.div_euclid(compaction_window_secs))
.map(|window| (window.saturating_mul(compaction_window_secs), sequence))
.collect();
SeriesBucket {
start: Timestamp::new_second(
start.div_euclid(width_secs).saturating_mul(width_secs),
@@ -192,10 +241,9 @@ pub(crate) fn group_files_into_series_buckets(
),
files: smallvec![file.clone()],
has_unknown_sequence: file.meta_ref().sequence.is_none(),
max_file_sequence: file
.meta_ref()
.sequence
.map_or(0, |sequence| sequence.get()),
max_file_sequence: sequence,
compaction_window_secs,
window_sequences,
}
})
.collect::<Vec<_>>();
@@ -231,6 +279,7 @@ fn group_series_buckets(spans: Vec<SeriesBucket>) -> Vec<SeriesBucket> {
last.files.append(&mut span.files);
last.has_unknown_sequence |= span.has_unknown_sequence;
last.max_file_sequence = last.max_file_sequence.max(span.max_file_sequence);
merge_window_sequences(&mut last.window_sequences, span.window_sequences);
} else {
buckets.push(span);
}
@@ -238,6 +287,15 @@ fn group_series_buckets(spans: Vec<SeriesBucket>) -> Vec<SeriesBucket> {
buckets
}
fn merge_window_sequences(target: &mut BTreeMap<i64, u64>, source: BTreeMap<i64, u64>) {
for (window, sequence) in source {
target
.entry(window)
.and_modify(|max| *max = (*max).max(sequence))
.or_insert(sequence);
}
}
impl SeriesBucket {
/// Creates entry metadata with a fresh UUID and sorted source file IDs.
/// Returns `None` if a source sequence is unknown or too few files remain to build.
@@ -263,6 +321,8 @@ impl SeriesBucket {
source_file_ids,
min_file_sequence,
max_file_sequence: self.max_file_sequence,
compaction_window_secs: self.compaction_window_secs,
window_sequences: self.window_sequences.clone(),
})
}
}
@@ -320,7 +380,7 @@ mod tests {
Timestamp::new_second(61),
),
];
let buckets = group_files_into_series_buckets(&files, width);
let buckets = group_files_into_series_buckets(&files, width, 10);
let spans = buckets
.iter()
.map(|b| (b.start, b.end, b.files.len()))
@@ -344,7 +404,7 @@ mod tests {
Timestamp::new_second(1),
));
assert!(
group_files_into_series_buckets(&files, width)[0]
group_files_into_series_buckets(&files, width, 10)[0]
.to_series_entry()
.is_none()
);
@@ -354,7 +414,7 @@ mod tests {
fn test_seconds_do_not_require_millisecond_conversion() {
let start = Timestamp::new_second(i64::MAX / 1000 + 100);
assert!(start.convert_to(TimeUnit::Millisecond).is_none());
let buckets = group_files_into_series_buckets(&[file(Some(1), 0, start, start)], 1);
let buckets = group_files_into_series_buckets(&[file(Some(1), 0, start, start)], 1, 1);
assert_eq!(start, buckets[0].start);
assert_eq!(Timestamp::new_second(start.value() + 1), buckets[0].end);
@@ -367,6 +427,7 @@ mod tests {
Timestamp::new_microsecond(1),
)],
width,
1,
);
assert_eq!(
(Timestamp::new_second(-1), Timestamp::new_second(1)),
@@ -388,7 +449,11 @@ mod tests {
start: ts(start),
end: ts(end),
index_ids: smallvec![id],
max_file_sequence: max_sequence,
compaction_window_secs: 10,
window_sequences: (start..end)
.step_by(10)
.map(|w| (w, max_sequence))
.collect(),
}
.insert_into(&mut indexes);
}
@@ -400,17 +465,16 @@ mod tests {
file(Some(33), 0, ts(90), ts(95)),
file(Some(34), 0, ts(100), ts(105)),
];
let planned = group_files_into_series_buckets(&files, 10);
let planned = group_files_into_series_buckets(&files, 10, 10);
assert_eq!(4, planned.len());
let plan = plan_series_indexes(planned, indexes, None, 0);
assert_eq!((2, 1), (plan.computed_buckets, plan.skipped_buckets));
let [(bucket, entry)] = plan.builds.as_slice() else {
panic!("expected one incremental build");
panic!("expected one replacement build");
};
assert_eq!((ts(0), ts(100)), (bucket.start, bucket.end));
// Ignore the gap at sequence 15, include both files at 32, and exclude the
// adjacent time bucket even though its sequence exceeds the merged watermark.
let expected = files[1..5]
// Include sequence 15 and both files at 32, but exclude the adjacent bucket.
let expected = files[..5]
.iter()
.map(|file| file.file_id().file_id())
.collect::<HashSet<_>>();
@@ -423,26 +487,24 @@ mod tests {
.collect()
);
assert_eq!(expected, entry.source_file_ids.iter().copied().collect());
assert_eq!((31, 33), (entry.min_file_sequence, entry.max_file_sequence));
assert_eq!((15, 33), (entry.min_file_sequence, entry.max_file_sequence));
assert!(plan.expired_index_ids.is_empty());
assert_eq!(1, plan.index_buckets.len());
let merged = &plan.index_buckets[&ts(0)];
assert_eq!((ts(0), ts(100)), (merged.start, merged.end));
assert_eq!(33, bucket.max_file_sequence);
assert_eq!(33, merged.max_file_sequence);
assert_eq!(
[ids[0], ids[1], ids[2], entry.index_uuid].as_slice(),
merged.index_ids.as_slice()
);
assert_eq!(bucket.window_sequences, merged.window_sequences);
assert_eq!(ids.as_slice(), plan.superseded_index_ids.as_slice());
assert_eq!([entry.index_uuid].as_slice(), merged.index_ids.as_slice());
}
#[test]
fn test_plan_reuses_replaced_sources_and_builds_complete_sequence_suffix() {
fn test_plan_reuses_replaced_sources_and_rebuilds_changed_bucket() {
let ts = Timestamp::new_second;
let make_file = |sequence| file(Some(sequence), 0, ts(1), ts(2));
let original = (1..=4).map(make_file).collect::<Vec<_>>();
let initial = plan_series_indexes(
group_files_into_series_buckets(&original, 10),
group_files_into_series_buckets(&original, 10, 10),
BTreeMap::new(),
None,
0,
@@ -452,7 +514,7 @@ mod tests {
// New file IDs with already indexed sequences do not invalidate the index.
let mut files = (1..=4).map(make_file).collect::<Vec<_>>();
let replaced = plan_series_indexes(
group_files_into_series_buckets(&files, 10),
group_files_into_series_buckets(&files, 10, 10),
initial.index_buckets.clone(),
None,
0,
@@ -461,27 +523,17 @@ mod tests {
assert!(replaced.expired_index_ids.is_empty());
assert_eq!(initial.index_buckets, replaced.index_buckets);
files.extend((5..=7).map(make_file));
let deferred = plan_series_indexes(
group_files_into_series_buckets(&files, 10),
files.push(make_file(5));
let ready = plan_series_indexes(
group_files_into_series_buckets(&files, 10, 10),
replaced.index_buckets,
None,
0,
);
assert!(deferred.builds.is_empty());
assert_eq!(initial.index_buckets, deferred.index_buckets);
files.push(make_file(8));
let ready = plan_series_indexes(
group_files_into_series_buckets(&files, 10),
deferred.index_buckets,
None,
0,
);
let [(bucket, entry)] = ready.builds.as_slice() else {
panic!("the fourth new SST must trigger one build");
panic!("a changed window must rebuild the whole bucket");
};
let expected = files[4..]
let expected = files[..]
.iter()
.map(|file| file.file_id().file_id())
.collect::<HashSet<_>>();
@@ -494,13 +546,14 @@ mod tests {
.collect()
);
assert_eq!(expected, entry.source_file_ids.iter().copied().collect());
assert_eq!((5, 8), (entry.min_file_sequence, entry.max_file_sequence));
assert_eq!((1, 5), (entry.min_file_sequence, entry.max_file_sequence));
assert_eq!(
[first_id, entry.index_uuid].as_slice(),
[entry.index_uuid].as_slice(),
ready.index_buckets[&ts(0)].index_ids.as_slice()
);
assert_eq!(vec![first_id], ready.superseded_index_ids);
let repeated = plan_series_indexes(
group_files_into_series_buckets(&files, 10),
group_files_into_series_buckets(&files, 10, 10),
ready.index_buckets.clone(),
None,
0,
@@ -508,7 +561,7 @@ mod tests {
assert!(repeated.builds.is_empty());
assert_eq!(ready.index_buckets, repeated.index_buckets);
// Retiring the whole bucket must retire both historical and incremental indexes.
// TTL retires the replacement; the previous index is already superseded.
let expired = plan_series_indexes(
Vec::new(),
ready.index_buckets,
@@ -518,8 +571,185 @@ mod tests {
assert!(expired.builds.is_empty());
assert!(expired.index_buckets.is_empty());
assert_eq!(
[first_id, entry.index_uuid].as_slice(),
[entry.index_uuid].as_slice(),
expired.expired_index_ids.as_slice()
);
}
#[test]
fn test_window_coverage_includes_every_intersected_window() {
let ts = Timestamp::new_millisecond;
let files = [
file(Some(10), 0, ts(-1), ts(19_999)),
file(Some(30), 1, ts(10_000), ts(20_000)),
];
let buckets = group_files_into_series_buckets(&files, 100, 10);
assert_eq!(1, buckets.len());
assert_eq!(
BTreeMap::from([(-10, 10), (0, 10), (10, 30), (20, 30)]),
buckets[0].window_sequences
);
}
#[test]
fn test_lower_window_changes_with_unchanged_bucket_maximum() {
let ts = Timestamp::new_second;
let mut files = [
file(Some(1), 0, ts(1), ts(2)),
file(Some(10), 0, ts(1), ts(2)),
file(Some(20), 0, ts(11), ts(12)),
file(Some(30), 0, ts(11), ts(12)),
]
.to_vec();
let initial = plan_series_indexes(
group_files_into_series_buckets(&files, 100, 10),
BTreeMap::new(),
None,
0,
);
files.extend((11..=13).map(|seq| file(Some(seq), 0, ts(1), ts(2))));
let plan = plan_series_indexes(
group_files_into_series_buckets(&files, 100, 10),
initial.index_buckets,
None,
0,
);
let [(bucket, entry)] = plan.builds.as_slice() else {
panic!("a changed lower watermark must rebuild the bucket");
};
assert_eq!(files.len(), bucket.files.len());
assert_eq!(30, entry.max_file_sequence);
assert_eq!(BTreeMap::from([(0, 13), (10, 30)]), entry.window_sequences);
}
#[test]
fn test_deferred_bridge_preserves_established_coverage() {
let ts = Timestamp::new_second;
let mut indexes = BTreeMap::new();
for (start, seq) in [(0, 10), (20, 30)] {
IndexBucket {
start: ts(start),
end: ts(start + 10),
index_ids: smallvec![FileId::random()],
compaction_window_secs: 10,
window_sequences: BTreeMap::from([(start, seq)]),
}
.insert_into(&mut indexes);
}
let mut files = vec![
file(Some(13), 0, ts(1), ts(21)),
file(Some(30), 0, ts(21), ts(22)),
];
let deferred = plan_series_indexes(
group_files_into_series_buckets(&files, 10, 10),
indexes.clone(),
None,
0,
);
assert!(deferred.builds.is_empty());
assert!(deferred.superseded_index_ids.is_empty());
assert_eq!(indexes, deferred.index_buckets);
files.extend((11..=12).map(|seq| file(Some(seq), 0, ts(1), ts(2))));
let plan = plan_series_indexes(
group_files_into_series_buckets(&files, 10, 10),
deferred.index_buckets,
None,
0,
);
assert_eq!(1, plan.builds.len());
assert_eq!(4, plan.builds[0].0.files.len());
assert_eq!(2, plan.superseded_index_ids.len());
assert_eq!(
BTreeMap::from([(0, 13), (10, 13), (20, 30)]),
plan.builds[0].1.window_sequences
);
files.push(file(None, 0, ts(1), ts(2)));
let unknown = plan_series_indexes(
group_files_into_series_buckets(&files, 10, 10),
indexes.clone(),
None,
0,
);
assert!(unknown.builds.is_empty());
assert_eq!(indexes, unknown.index_buckets);
}
#[rstest::rstest]
#[case::removed_window(100, 10, false)]
#[case::added_window(100, 10, true)]
#[case::window_width(100, 20, false)]
#[case::bucket_width(200, 10, false)]
fn test_coverage_shape_changes_rebuild(
#[case] bucket_width: i64,
#[case] window_width: i64,
#[case] add_window: bool,
) {
let ts = Timestamp::new_second;
let mut files = (27..=30)
.map(|seq| file(Some(seq), 0, ts(11), ts(12)))
.collect::<Vec<_>>();
let mut original = files.clone();
original.push(file(Some(10), 0, ts(1), ts(2)));
let initial = plan_series_indexes(
group_files_into_series_buckets(&original, 100, 10),
BTreeMap::new(),
None,
0,
);
if add_window {
files = original;
files.push(file(Some(15), 0, ts(21), ts(22)));
} else if bucket_width != 100 || window_width != 10 {
files = original;
}
let plan = plan_series_indexes(
group_files_into_series_buckets(&files, bucket_width, window_width),
initial.index_buckets,
None,
0,
);
assert_eq!(1, plan.builds.len());
assert_eq!(1, plan.superseded_index_ids.len());
let repeated = plan_series_indexes(
group_files_into_series_buckets(&files, bucket_width, window_width),
plan.index_buckets,
None,
0,
);
assert!(repeated.builds.is_empty());
}
#[test]
fn test_index_map_merge_is_order_independent() {
let ts = Timestamp::new_second;
let make_index = |start, end, width, windows: &[(i64, u64)]| IndexBucket {
start: ts(start),
end: ts(end),
index_ids: smallvec![FileId::random()],
compaction_window_secs: width,
window_sequences: windows.iter().copied().collect(),
};
let indexes = [
make_index(0, 20, 10, &[(0, 10), (10, 20)]),
make_index(10, 30, 10, &[(10, 30), (20, 15)]),
make_index(20, 40, 10, &[(20, 25), (30, 5)]),
];
for order in [[0, 1, 2], [2, 1, 0], [1, 0, 2], [0, 2, 1]] {
let mut map = BTreeMap::new();
for i in order {
indexes[i].clone().insert_into(&mut map);
}
assert_eq!(1, map.len());
assert_eq!(10, map[&ts(0)].compaction_window_secs);
assert_eq!(
BTreeMap::from([(0, 10), (10, 30), (20, 25), (30, 5)]),
map[&ts(0)].window_sequences
);
make_index(10, 30, 20, &[(0, 30)]).insert_into(&mut map);
indexes[0].clone().insert_into(&mut map);
assert_eq!(0, map[&ts(0)].compaction_window_secs);
assert!(map[&ts(0)].window_sequences.is_empty());
}
}
}
+444 -130
View File
@@ -12,12 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Builds range and aggregate series indexes from SST readers.
//! Independently builds range and aggregate series indexes from SST readers.
use std::collections::HashSet;
use std::sync::{Arc, Mutex};
use std::sync::Arc;
use async_stream::try_stream;
use common_telemetry::warn;
use futures::TryStreamExt;
use object_store::ObjectStore;
use snafu::OptionExt;
@@ -60,6 +60,7 @@ async fn reader_input(
.map(|(context, selection)| (Arc::new(context), selection)))
}
/// Builds one range index, or returns `None` when the SST has no readable input.
pub(crate) async fn build_range_index(
store: &ObjectStore,
region: &MitoRegionRef,
@@ -67,6 +68,9 @@ pub(crate) async fn build_range_index(
file: FileHandle,
) -> Result<Option<FileId>> {
let file_id = file.file_id().file_id();
let Some((context, mut selection)) = reader_input(region, file).await? else {
return Ok(None);
};
let path = range_index_path(region.region_id, file_id);
let mut writer = SstRangeIndexWriter::try_new(
version.metadata.clone(),
@@ -75,68 +79,58 @@ pub(crate) async fn build_range_index(
SstRangeIndexWriterOptions::default(),
)
.await?;
let Some((context, mut selection)) = reader_input(region, file).await? else {
writer.abort().await?;
return Ok(None);
};
let fetch_metrics = ParquetFetchMetrics::default();
while let Some((row_group_id, row_selection)) = selection.pop_first() {
let parquet_reader = context
.reader_builder()
.build(context.build_context(row_group_id, Some(row_selection), Some(&fetch_metrics)))
.await?;
let mut reader = FlatPruneReader::new_with_row_group_reader(
context.clone(),
FlatRowGroupReader::new(context.clone(), parquet_reader),
context.pre_filter_mode().skip_fields(),
);
while let Some(batch) = reader.next_batch().await? {
writer.write(row_group_id as u32, &batch).await?;
let result: Result<()> = async {
let fetch_metrics = ParquetFetchMetrics::default();
while let Some((row_group_id, row_selection)) = selection.pop_first() {
let parquet_reader = context
.reader_builder()
.build(context.build_context(
row_group_id,
Some(row_selection),
Some(&fetch_metrics),
))
.await?;
let mut reader = FlatPruneReader::new_with_row_group_reader(
context.clone(),
FlatRowGroupReader::new(context.clone(), parquet_reader),
context.pre_filter_mode().skip_fields(),
);
while let Some(batch) = reader.next_batch().await? {
writer.write(row_group_id as u32, &batch).await?;
}
}
Ok(())
}
.await;
if let Err(error) = result {
if let Err(cleanup_error) = writer.abort().await {
warn!(cleanup_error; "Failed to abort range-index build");
}
return Err(error);
}
writer.finish().await?;
file_operation(IndexFileType::Range, "build", "success");
Ok(Some(file_id))
}
/// Builds only the series index. Callers build needed range indexes separately.
pub(crate) async fn build_series_index(
store: &ObjectStore,
region: &MitoRegionRef,
version: &VersionRef,
bucket: &SeriesBucket,
entry: &SeriesIndexEntry,
missing_range_ids: &HashSet<FileId>,
purger: &IndexFilePurger,
) -> Result<(SeriesIndexFileHandle, HashSet<FileId>)> {
let completed_ranges = Arc::new(Mutex::new(HashSet::new()));
) -> Result<SeriesIndexFileHandle> {
let mut sources = Vec::<BoxedRecordBatchStream>::new();
let mut schema = None;
let mut expected_ranges = 0;
for file in &bucket.files {
let file_id = file.file_id().file_id();
let Some((context, mut selection)) = reader_input(region, file.clone()).await? else {
continue;
};
schema.get_or_insert(context.read_format().output_arrow_schema()?);
let range_writer = if missing_range_ids.contains(&file_id) {
expected_ranges += 1;
let path = range_index_path(region.region_id, file_id);
Some(
SstRangeIndexWriter::try_new(
version.metadata.clone(),
store.clone(),
&path,
SstRangeIndexWriterOptions::default(),
)
.await?,
)
} else {
None
};
let completed_ranges = completed_ranges.clone();
sources.push(Box::pin(try_stream! {
let fetch_metrics = ParquetFetchMetrics::default();
let mut range_writer = range_writer;
while let Some((row_group_id, row_selection)) = selection.pop_first() {
let parquet_reader = context.reader_builder().build(context.build_context(
row_group_id,
@@ -149,23 +143,15 @@ pub(crate) async fn build_series_index(
context.pre_filter_mode().skip_fields(),
);
while let Some(batch) = reader.next_batch().await? {
if let Some(writer) = &mut range_writer {
writer.write(row_group_id as u32, &batch).await?;
}
yield batch;
}
}
if let Some(writer) = range_writer {
writer.finish().await?;
file_operation(IndexFileType::Range, "build", "success");
completed_ranges.lock().unwrap().insert(file_id);
}
}));
}
let schema = schema.context(UnexpectedSnafu {
reason: "series-index bucket has no readable SST",
})?;
let merged: BoxedRecordBatchStream = if sources.len() == 1 {
let mut visible: BoxedRecordBatchStream = if sources.len() == 1 {
sources.pop().context(UnexpectedSnafu {
reason: "series-index source disappeared",
})?
@@ -177,73 +163,58 @@ pub(crate) async fn build_series_index(
)
};
// TODO: Deduplicate update-mode rows before series indexes are used by queries.
let mut visible = merged;
let path = series_index_path(region.region_id, entry.index_uuid);
let mut writer = SeriesIndexWriter::try_new(
region.metadata(),
version.metadata.clone(),
store.clone(),
&path,
SeriesIndexWriterOptions::default(),
Some(series_metadata(entry)?),
)
.await?;
while let Some(batch) = visible.try_next().await? {
writer.write(&batch).await?;
}
let range_indexes = std::mem::take(&mut *completed_ranges.lock().unwrap());
if range_indexes.len() != expected_ranges {
return UnexpectedSnafu {
reason: "not all combined range-index builds completed",
let result: Result<()> = async {
while let Some(batch) = visible.try_next().await? {
writer.write(&batch).await?;
}
.fail();
Ok(())
}
.await;
if let Err(error) = result {
if let Err(cleanup_error) = writer.abort().await {
warn!(cleanup_error; "Failed to abort series-index build");
}
return Err(error);
}
writer.finish().await?;
file_operation(IndexFileType::Series, "build", "success");
Ok((
SeriesIndexFileHandle::new(region.region_id, entry.clone(), purger.clone()),
range_indexes,
Ok(SeriesIndexFileHandle::new(
region.region_id,
entry.clone(),
purger.clone(),
))
}
#[cfg(test)]
mod tests {
use std::collections::{BTreeMap, HashMap};
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};
use datatypes::data_type::ConcreteDataType;
use object_store::layers::mock::{self, MockLayerBuilder, oio};
use object_store::services::Memory;
use store_api::region_engine::RegionEngine;
use super::*;
use crate::series_index::bucket::{group_files_into_series_buckets, plan_series_indexes};
use crate::series_index::catalog::{
SeriesIndexCatalog, load_version_control, series_catalog_path, store_catalog,
};
use crate::series_index::purger::series_index_channel;
use crate::series_index::tests::prepare_region;
use crate::test_util::TestEnv;
struct FailingSeriesWriter(oio::Writer);
impl mock::Write for FailingSeriesWriter {
async fn write(&mut self, buffer: mock::Buffer) -> mock::Result<()> {
self.0.write(buffer).await
}
async fn close(&mut self) -> mock::Result<mock::Metadata> {
Err(mock::Error::new(
mock::ErrorKind::Unexpected,
"injected series write failure",
))
}
async fn abort(&mut self) -> mock::Result<()> {
self.0.abort().await
}
}
#[rstest::rstest]
#[case::success(false)]
#[case::series_failure(true)]
#[tokio::test]
async fn test_build_indexes_without_publication(#[case] fail_series: bool) {
let mut env = TestEnv::with_prefix("series-builder").await;
let (engine, region) = prepare_region(&mut env).await;
let version = region.version();
fn build_input(version: &VersionRef) -> (SeriesBucket, SeriesIndexEntry) {
let files = version
.ssts
.levels()
@@ -251,53 +222,83 @@ mod tests {
.flat_map(|level| level.files())
.cloned()
.collect::<Vec<_>>();
let store = ObjectStore::new(Memory::default()).unwrap();
let (purger, mut receiver) = series_index_channel(store.clone());
// Build one range separately, then reuse it during the combined build.
let existing = build_range_index(&store, &region, &version, files[0].clone())
.await
.unwrap()
.unwrap();
let existing_path = range_index_path(region.region_id, existing);
let existing_bytes = store.read(&existing_path).await.unwrap().to_bytes();
let missing = files
.iter()
.map(|file| file.file_id().file_id())
.filter(|id| *id != existing)
.collect::<HashSet<_>>();
let mut plan = plan_series_indexes(
group_files_into_series_buckets(&files, 100),
Default::default(),
group_files_into_series_buckets(&files, 100, 10),
BTreeMap::new(),
None,
0,
);
assert_eq!(plan.builds.len(), 1);
let (bucket, entry) = plan.builds.pop().unwrap();
let layer = MockLayerBuilder::default()
.writer_factory(Arc::new(move |path, _, inner| -> oio::Writer {
if fail_series && path.contains("/series/") {
Box::new(FailingSeriesWriter(inner))
} else {
inner
}
}))
.build()
.unwrap();
assert_eq!(1, plan.builds.len());
plan.builds.pop().unwrap()
}
fn metadata_with_seconds(version: &VersionRef) -> store_api::metadata::RegionMetadataRef {
let mut metadata = (*version.metadata).clone();
let time_index = metadata.time_index_column_pos();
metadata.column_metadatas[time_index]
.column_schema
.data_type = ConcreteDataType::timestamp_second_datatype();
Arc::new(metadata)
}
#[rstest::rstest]
#[case::range_then_series(true, false)]
#[case::series_only(false, false)]
#[case::series_failure(true, true)]
#[tokio::test]
async fn test_build_indexes_without_publication(
#[case] build_ranges: bool,
#[case] fail_series: bool,
) {
let mut env = TestEnv::with_prefix("series-builder").await;
let (engine, region) = prepare_region(&mut env).await;
let version = region.version();
let (bucket, entry) = build_input(&version);
let files = &bucket.files;
let store = ObjectStore::new(Memory::default()).unwrap();
let (purger, mut receiver) = series_index_channel(store.clone());
let mut range_bytes = HashMap::new();
if build_ranges {
for file in files {
let id = build_range_index(&store, &region, &version, file.clone())
.await
.unwrap()
.unwrap();
range_bytes.insert(
id,
store
.read(&range_index_path(region.region_id, id))
.await
.unwrap()
.to_bytes(),
);
}
}
// Both builders use the captured version, even if live metadata changes.
region
.version_control
.alter_metadata(metadata_with_seconds(&version));
let layer = writer_layer(&WriterStates::default(), move |path| {
assert!(path.contains("/series/"), "series stage opened {path}");
if fail_series {
WriterFailure::Finish
} else {
WriterFailure::None
}
});
let result = build_series_index(
&store.clone().layer(layer),
&region,
&version,
&bucket,
&entry,
&missing,
&purger,
)
.await;
if fail_series {
assert!(result.is_err());
} else {
let (handle, built_ranges) = result.unwrap();
assert_eq!(built_ranges, missing);
let handle = result.unwrap();
assert_eq!(handle.entry(), &entry);
assert!(
store
@@ -305,10 +306,94 @@ mod tests {
.await
.unwrap()
);
store_catalog(
&store,
&series_catalog_path(region.region_id),
&SeriesIndexCatalog {
indexes: vec![handle.entry().clone()],
},
)
.await
.unwrap();
let recovered = load_version_control(&store, region.region_id, &purger)
.await
.current();
let repeated = plan_series_indexes(
group_files_into_series_buckets(files, 100, 10),
recovered.index_buckets.clone(),
None,
0,
);
assert!(repeated.builds.is_empty());
assert_eq!(recovered.index_buckets, repeated.index_buckets);
}
// A failed series build must retain completed companion range indexes.
for file in &files {
assert!(
// Series success or failure leaves completed range indexes unchanged.
for file in files {
let id = file.file_id().file_id();
let path = range_index_path(region.region_id, id);
if let Some(bytes) = range_bytes.get(&id) {
assert_eq!(*bytes, store.read(&path).await.unwrap().to_bytes());
} else {
assert!(!store.exists(&path).await.unwrap());
}
}
assert!(region.series_index_version().range_indexes.is_empty());
assert!(region.series_index_version().series_indexes.is_empty());
assert!(receiver.try_recv().is_err());
engine.stop().await.unwrap();
}
#[tokio::test]
async fn test_range_failure_preserves_completed_indexes_and_stops_series_stage() {
let mut env = TestEnv::with_prefix("range-builder-stage-failure").await;
let (engine, region) = prepare_region(&mut env).await;
let version = region.version();
let (bucket, entry) = build_input(&version);
let files = &bucket.files;
let failed_path = range_index_path(region.region_id, files[1].file_id().file_id());
let states = WriterStates::default();
let layer = writer_layer(&states, move |path| {
if path == failed_path {
WriterFailure::Finish
} else {
WriterFailure::None
}
});
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
let (purger, _receiver) = series_index_channel(store.clone());
let mut completed = Vec::new();
let result: Result<Option<SeriesIndexFileHandle>> = async {
for file in files {
let Some(id) = build_range_index(&store, &region, &version, file.clone()).await?
else {
// Defer series construction if a needed range is not ready.
return Ok(None);
};
completed.push(id);
}
build_series_index(&store, &region, &version, &bucket, &entry, &purger)
.await
.map(Some)
}
.await;
assert!(format!("{:?}", result.unwrap_err()).contains("injected index finish failure"));
assert_eq!(vec![files[0].file_id().file_id()], completed);
{
let states = states.lock().unwrap();
assert_eq!(2, states.len());
assert!(states.keys().all(|path| !path.contains("/series/")));
assert_eq!(
1,
states[&range_index_path(region.region_id, completed[0])].closed
);
assert_eq!(
1,
states[&range_index_path(region.region_id, files[1].file_id().file_id())].aborted
);
}
for (i, file) in files.iter().enumerate() {
assert_eq!(
i == 0,
store
.exists(&range_index_path(
region.region_id,
@@ -318,13 +403,242 @@ mod tests {
.unwrap()
);
}
assert_eq!(
store.read(&existing_path).await.unwrap().to_bytes(),
existing_bytes
engine.stop().await.unwrap();
}
#[derive(Default)]
struct WriterState {
closed: usize,
aborted: usize,
}
type WriterStates = Arc<Mutex<BTreeMap<String, WriterState>>>;
enum WriterFailure {
None,
Finish,
Abort,
}
fn writer_layer(
states: &WriterStates,
failure: impl Fn(&str) -> WriterFailure + Send + Sync + 'static,
) -> mock::MockLayer {
let states = states.clone();
MockLayerBuilder::default()
.writer_factory(Arc::new(move |path, _, inner| -> oio::Writer {
states
.lock()
.unwrap()
.insert(path.to_string(), WriterState::default());
Box::new(RecordingWriter {
inner,
path: path.to_string(),
states: states.clone(),
failure: failure(path),
})
}))
.build()
.unwrap()
}
struct RecordingWriter {
inner: oio::Writer,
path: String,
states: WriterStates,
failure: WriterFailure,
}
impl mock::Write for RecordingWriter {
async fn write(&mut self, buffer: mock::Buffer) -> mock::Result<()> {
self.inner.write(buffer).await
}
async fn close(&mut self) -> mock::Result<mock::Metadata> {
if matches!(self.failure, WriterFailure::Finish) {
return Err(mock::Error::new(
mock::ErrorKind::Unexpected,
"injected index finish failure",
));
}
let metadata = self.inner.close().await?;
self.states
.lock()
.unwrap()
.get_mut(&self.path)
.unwrap()
.closed += 1;
Ok(metadata)
}
async fn abort(&mut self) -> mock::Result<()> {
self.states
.lock()
.unwrap()
.get_mut(&self.path)
.unwrap()
.aborted += 1;
if matches!(self.failure, WriterFailure::Abort) {
return Err(mock::Error::new(
mock::ErrorKind::Unexpected,
"injected abort failure",
));
}
self.inner.abort().await
}
}
struct FailingSstReader {
inner: oio::Reader,
fail: Arc<AtomicBool>,
}
impl mock::Read for FailingSstReader {
async fn read(
&self,
range: mock::BytesRange,
) -> mock::Result<(mock::RpRead, mock::Buffer)> {
if self.fail.load(Ordering::Relaxed) {
return Err(mock::Error::new(
mock::ErrorKind::Unexpected,
"injected SST read failure",
));
}
self.inner.read(range).await
}
async fn open(
&self,
range: mock::BytesRange,
) -> mock::Result<(mock::RpRead, Box<dyn mock::ReadStreamDyn>)> {
if self.fail.load(Ordering::Relaxed) {
return Err(mock::Error::new(
mock::ErrorKind::Unexpected,
"injected SST read failure",
));
}
self.inner.open(range).await
}
}
#[rstest::rstest]
#[case::range(false, false, false)]
#[case::range_abort_failure(false, false, true)]
#[case::series_setup(true, true, false)]
#[case::series_read(true, false, false)]
#[case::series_abort_failure(true, false, true)]
#[tokio::test]
async fn test_read_failure_aborts_unfinished_outputs(
#[case] series: bool,
#[case] fail_before_open: bool,
#[case] fail_abort: bool,
) {
let fail = Arc::new(AtomicBool::new(false));
let read_fail = fail.clone();
let read_layer = MockLayerBuilder::default()
.reader_factory(Arc::new(move |path, _, inner| -> oio::Reader {
if path.ends_with(".parquet") {
Box::new(FailingSstReader {
inner,
fail: read_fail.clone(),
})
} else {
inner
}
}))
.build()
.unwrap();
let mut env = TestEnv::with_prefix("series-builder-read-failure")
.await
.with_mock_layer(read_layer);
let (engine, region) = prepare_region(&mut env).await;
let version = region.version();
let (mut bucket, mut entry) = build_input(&version);
// A single source is first polled after opening the series writer.
bucket.files.truncate(1);
entry.source_file_ids = vec![bucket.files[0].file_id().file_id()];
let states = WriterStates::default();
let write_fail = fail.clone();
let write_layer = writer_layer(&states, move |_| {
write_fail.store(true, Ordering::Relaxed);
if fail_abort {
WriterFailure::Abort
} else {
WriterFailure::None
}
});
let store = ObjectStore::new(Memory::default())
.unwrap()
.layer(write_layer);
fail.store(fail_before_open, Ordering::Relaxed);
let result = if !series {
build_range_index(&store, &region, &version, bucket.files[0].clone())
.await
.map(|_| ())
} else {
let (purger, _receiver) = series_index_channel(store.clone());
build_series_index(&store, &region, &version, &bucket, &entry, &purger)
.await
.map(|_| ())
};
fail.store(false, Ordering::Relaxed);
let error = result.unwrap_err();
assert!(
format!("{error:?}").contains("injected SST read failure"),
"{error:?}"
);
assert!(region.series_index_version().range_indexes.is_empty());
{
let states = states.lock().unwrap();
assert_eq!(usize::from(!fail_before_open), states.len());
for (path, state) in states.iter() {
assert_eq!(0, state.closed, "{path}");
assert_eq!(1, state.aborted, "{path}");
}
}
assert!(store.list("/").await.unwrap().is_empty());
assert!(region.series_index_version().series_indexes.is_empty());
engine.stop().await.unwrap();
}
#[tokio::test]
async fn test_series_batch_error_aborts_writer() {
let mut env = TestEnv::with_prefix("series-builder-batch-failure").await;
let (engine, region) = prepare_region(&mut env).await;
let version = region.version();
let (bucket, entry) = build_input(&version);
let files = &bucket.files;
// Give the series writer a different timestamp unit from the SST batches.
let mut bad_version = (*version).clone();
bad_version.metadata = metadata_with_seconds(&version);
let bad_version = Arc::new(bad_version);
let states = WriterStates::default();
let layer = writer_layer(&states, |_| WriterFailure::None);
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
let (purger, _receiver) = series_index_channel(store.clone());
for file in files {
build_range_index(&store, &region, &version, file.clone())
.await
.unwrap()
.unwrap();
}
let error = build_series_index(&store, &region, &bad_version, &bucket, &entry, &purger)
.await
.unwrap_err();
assert!(
format!("{error:?}").contains("does not match the index unit"),
"{error:?}"
);
let path = series_index_path(region.region_id, entry.index_uuid);
{
let states = states.lock().unwrap();
assert_eq!(1, states[&path].aborted);
assert_eq!(0, states[&path].closed);
for state in states.values() {
assert_eq!(1, state.aborted + state.closed);
}
}
assert!(!store.exists(&path).await.unwrap());
assert!(region.series_index_version().series_indexes.is_empty());
assert!(receiver.try_recv().is_err());
engine.stop().await.unwrap();
}
}
+14 -1
View File
@@ -14,6 +14,8 @@
//! Index catalog persistence, coverage metadata, and file paths.
use std::collections::BTreeMap;
use common_telemetry::warn;
use common_time::Timestamp;
use object_store::{ErrorKind, ObjectStore};
@@ -46,6 +48,13 @@ pub(crate) struct SeriesIndexEntry {
pub(crate) source_file_ids: Vec<FileId>,
pub(crate) min_file_sequence: u64,
pub(crate) max_file_sequence: u64,
/// Width used to align the half-open compaction windows, in seconds.
pub(crate) compaction_window_secs: i64,
/// Maximum source SST sequence per compaction window, persisted for recovery.
/// Keys are window starts in epoch seconds; each window spans
/// `[start, start + compaction_window_secs)`. An SST contributes to every window
/// its time range intersects. Windows without source SSTs are omitted.
pub(crate) window_sequences: BTreeMap<i64, u64>,
}
#[derive(Debug, Default, Serialize, Deserialize)]
@@ -157,6 +166,7 @@ pub(crate) async fn load_version_control(
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use std::sync::Arc;
use common_time::Timestamp;
@@ -258,6 +268,8 @@ mod tests {
source_file_ids: vec![FileId::random()],
min_file_sequence: 1,
max_file_sequence: 2,
compaction_window_secs: 10,
window_sequences: BTreeMap::from([(0, 1), (10, 2)]),
};
store_catalog(
&store,
@@ -279,7 +291,8 @@ mod tests {
assert_eq!(bucket.start, entry.bucket_start);
assert_eq!(bucket.end, entry.bucket_end);
assert_eq!(bucket.index_ids.as_slice(), &[entry.index_uuid]);
assert_eq!(bucket.max_file_sequence, entry.max_file_sequence);
assert_eq!(bucket.compaction_window_secs, entry.compaction_window_secs);
assert_eq!(bucket.window_sequences, entry.window_sequences);
let metadata = series_metadata(&entry).unwrap();
let decoded: SeriesIndexEntry =
+1 -9
View File
@@ -20,7 +20,6 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, RwLock};
use common_time::Timestamp;
use smallvec::smallvec;
use store_api::storage::{FileId, RegionId};
use crate::series_index::bucket::IndexBucket;
@@ -103,14 +102,7 @@ impl SeriesIndexVersion {
) -> Self {
let mut index_buckets = BTreeMap::new();
for handle in series_indexes.values() {
let entry = handle.entry();
IndexBucket {
start: entry.bucket_start,
end: entry.bucket_end,
index_ids: smallvec![entry.index_uuid],
max_file_sequence: entry.max_file_sequence,
}
.insert_into(&mut index_buckets);
IndexBucket::from_entry(handle.entry()).insert_into(&mut index_buckets);
}
Self {
range_indexes,