From de1a33e15ee0089f1e59dae44af44c5b24abe3fc Mon Sep 17 00:00:00 2001 From: evenyag Date: Wed, 9 Sep 2026 21:42:59 +0800 Subject: [PATCH] feat(mito2): add series index planning and builders Signed-off-by: evenyag --- AGENTS.md | 5 + src/mito2/AGENTS.md | 2 +- src/mito2/src/series_index.rs | 8 + src/mito2/src/series_index/bucket.rs | 525 ++++++++++++++++++++++++++ src/mito2/src/series_index/builder.rs | 330 ++++++++++++++++ src/mito2/src/series_index/catalog.rs | 68 +++- src/mito2/src/series_index/tests.rs | 113 ++++++ src/mito2/src/series_index/version.rs | 31 +- src/mito2/src/sst/range_index.rs | 2 - 9 files changed, 1062 insertions(+), 22 deletions(-) create mode 100644 src/mito2/src/series_index/bucket.rs create mode 100644 src/mito2/src/series_index/builder.rs create mode 100644 src/mito2/src/series_index/tests.rs diff --git a/AGENTS.md b/AGENTS.md index 819679f3048..e8feb4abe4b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,6 +69,11 @@ a module map, read/write paths, change-coupling points, and gotchas: - [`tests/compatibility/AGENTS.md`](tests/compatibility/AGENTS.md) - [`tests/perf/AGENTS.md`](tests/perf/AGENTS.md) +## Rust imports + +Prefer crate-rooted imports (`use crate::...`) over `use super::...` in +production code. Relative imports using `super` are allowed in tests. + ## Read before changing code - [`.agents/architecture-invariants.md`](.agents/architecture-invariants.md) — diff --git a/src/mito2/AGENTS.md b/src/mito2/AGENTS.md index b22ee758bb9..623f7df18e7 100644 --- a/src/mito2/AGENTS.md +++ b/src/mito2/AGENTS.md @@ -26,7 +26,7 @@ snapshot isolation). It implements the `RegionEngine` trait from `store-api`. | `compaction` | `src/mito2/src/compaction/` | Compaction scheduler (`scheduler.rs` + `scheduler/`), TWCS picker, strict-window manual picker, compactor, memory control | | `access_layer` | `src/mito2/src/access_layer.rs` | SST read/write over the object store | | `sst` | `src/mito2/src/sst/` | Parquet format, file metadata, index layout | -| `series_index` | `src/mito2/src/series_index/` | Series index writer/searcher, catalogs, immutable snapshots, file lifecycle, and background maintenance | +| `series_index` | `src/mito2/src/series_index/` | Series index writer/searcher, bucket planning and SST builders, catalogs, immutable snapshots, file lifecycle, and background maintenance | | `read` | `src/mito2/src/read/` | `ScanRegion`, merge, dedup, projection, streaming | | `manifest` | `src/mito2/src/manifest/` | `RegionManifestManager`, manifest actions/edits | | `cache` | `src/mito2/src/cache.rs` | Write/file/page caches | diff --git a/src/mito2/src/series_index.rs b/src/mito2/src/series_index.rs index f08178456a5..8173050f2ab 100644 --- a/src/mito2/src/series_index.rs +++ b/src/mito2/src/series_index.rs @@ -17,10 +17,18 @@ // These components are consumed by the upcoming query and maintenance integration. #[allow(dead_code)] mod catalog; +// Consumed by the follow-up background-maintenance integration. +#[allow(dead_code)] +mod bucket; +// Consumed by the follow-up background-maintenance integration. +#[allow(dead_code)] +mod builder; #[allow(dead_code)] mod purger; mod searcher; mod task; +#[cfg(test)] +mod tests; #[allow(dead_code)] mod version; mod writer; diff --git a/src/mito2/src/series_index/bucket.rs b/src/mito2/src/series_index/bucket.rs new file mode 100644 index 00000000000..72f043fd79d --- /dev/null +++ b/src/mito2/src/series_index/bucket.rs @@ -0,0 +1,525 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Event-time bucket planning and source coverage. + +use std::collections::BTreeMap; +use std::time::Duration; + +use common_time::{TimeToLive, Timestamp}; +use smallvec::{SmallVec, smallvec}; +use store_api::storage::FileId; + +use crate::series_index::catalog::SeriesIndexEntry; +use crate::sst::file::FileHandle; + +const SERIES_INDEX_TRIGGER_FILES: usize = 4; + +/// Index files sharing a non-overlapping, half-open time bucket. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct IndexBucket { + pub(crate) start: Timestamp, + pub(crate) end: Timestamp, + pub(crate) index_ids: SmallVec<[FileId; 2]>, + pub(crate) max_file_sequence: u64, +} + +impl IndexBucket { + pub(crate) fn new(start: Timestamp, end: Timestamp) -> Self { + Self { + start, + end, + index_ids: SmallVec::new(), + max_file_sequence: 0, + } + } + + fn merge(&mut self, mut other: Self) { + self.start = self.start.min(other.start); + self.end = self.end.max(other.end); + self.index_ids.append(&mut other.index_ids); + self.max_file_sequence = self.max_file_sequence.max(other.max_file_sequence); + } + + /// Inserts a bucket, consuming overlapping entries and expanding their interval. + /// Each map key equals the stored bucket start; adjacent intervals remain separate. + pub(crate) fn insert_into(mut self, buckets: &mut BTreeMap) { + if let Some((&previous_start, previous)) = buckets.range(..self.start).next_back() + && previous.end > self.start + { + self.start = previous_start; + } + // Expanding the end may expose more overlaps, so look up the next entry again. + while let Some((&next_start, _)) = buckets.range(self.start..self.end).next() { + if let Some(other) = buckets.remove(&next_start) { + self.merge(other); + } + } + buckets.insert(self.start, self); + } +} + +/// SSTs grouped into a half-open time interval for an aggregate series-index build. +/// +/// Reconciliation may expand the interval through existing index coverage and retain +/// only files above its sequence watermark. Unknown source sequences prevent builds +/// even after filtering, since their coverage cannot be determined safely. +#[derive(Debug, Clone)] +pub(crate) struct SeriesBucket { + pub(crate) start: Timestamp, + pub(crate) end: Timestamp, + pub(crate) files: SmallVec<[FileHandle; 2]>, + pub(crate) has_unknown_sequence: bool, + /// Maximum known source sequence, or zero when all sequences are unknown. + pub(crate) max_file_sequence: u64, +} + +/// Next bucket coverage and the work required to publish it, computed without I/O. +pub(crate) struct SeriesIndexPlan { + pub(crate) index_buckets: BTreeMap, + pub(crate) builds: Vec<(SeriesBucket, SeriesIndexEntry)>, + pub(crate) expired_index_ids: Vec, + pub(crate) computed_buckets: usize, + pub(crate) skipped_buckets: usize, +} + +/// Plans complete sequence suffixes after merging time coverage. The returned bucket +/// map is publishable only after every planned build and the catalog writes succeed. +pub(crate) fn plan_series_indexes( + buckets: Vec, + mut index_buckets: BTreeMap, + ttl: Option, + now_ms: i64, +) -> SeriesIndexPlan { + let buckets = reconcile_series_buckets(buckets, &mut index_buckets); + let computed_buckets = buckets.len(); + let expired = |end| { + ttl.is_some_and(|ttl| { + ttl.is_expired(&end, &Timestamp::new_millisecond(now_ms)) + .unwrap_or(false) + }) + }; + let mut expired_index_ids = Vec::new(); + index_buckets.retain(|_, bucket| { + if expired(bucket.end) { + expired_index_ids.extend_from_slice(&bucket.index_ids); + false + } else { + true + } + }); + let mut builds = Vec::new(); + for mut bucket in buckets { + if expired(bucket.end) { + continue; + } + let index_bucket = index_buckets + .entry(bucket.start) + .or_insert_with(|| IndexBucket::new(bucket.start, bucket.end)); + if bucket.has_unknown_sequence || bucket.max_file_sequence <= index_bucket.max_file_sequence + { + continue; + } + // Select all SSTs above the watermark, including every file sharing a sequence. + // File IDs may change under compaction without advancing indexed coverage. + // The maximum remains valid because it exceeds the watermark and is retained. + bucket.files.retain(|file| { + file.meta_ref() + .sequence + .is_some_and(|sequence| sequence.get() > index_bucket.max_file_sequence) + }); + if let Some(entry) = bucket.to_series_entry() { + index_bucket.index_ids.push(entry.index_uuid); + index_bucket.max_file_sequence = entry.max_file_sequence; + builds.push((bucket, entry)); + } + } + index_buckets.retain(|_, bucket| !bucket.index_ids.is_empty()); + SeriesIndexPlan { + index_buckets, + skipped_buckets: computed_buckets - builds.len(), + builds, + expired_index_ids, + computed_buckets, + } +} + +pub(crate) fn rounded_bucket_width( + requested: Duration, + compaction_window: Duration, +) -> Option { + let window_secs = i64::try_from(compaction_window.as_secs()).ok()?.max(1); + let requested_secs = i64::try_from(requested.as_secs()) + .unwrap_or(i64::MAX) + .max(1); + let multiples = requested_secs / window_secs + i64::from(requested_secs % window_secs != 0); + multiples.checked_mul(window_secs) +} + +/// Groups SSTs across levels into sorted, disjoint time buckets without planning builds. +/// +/// `width_secs` must be positive. Inclusive SST ranges are rounded outward to aligned, +/// half-open intervals in seconds. Overlapping intervals merge; adjacent ones stay +/// separate. Each merged bucket tracks its maximum sequence and any unknown sequence. +pub(crate) fn group_files_into_series_buckets( + files: &[FileHandle], + width_secs: i64, +) -> Vec { + let mut spans = files + .iter() + .map(|file| { + let start = file.time_range().0.split().0; + let end = file.time_range().1.split().0; + SeriesBucket { + start: Timestamp::new_second( + start.div_euclid(width_secs).saturating_mul(width_secs), + ), + end: Timestamp::new_second( + end.div_euclid(width_secs) + .saturating_add(1) + .saturating_mul(width_secs), + ), + files: smallvec![file.clone()], + has_unknown_sequence: file.meta_ref().sequence.is_none(), + max_file_sequence: file + .meta_ref() + .sequence + .map_or(0, |sequence| sequence.get()), + } + }) + .collect::>(); + spans.sort_unstable_by(|a, b| a.start.cmp(&b.start).then_with(|| b.end.cmp(&a.end))); + group_series_buckets(spans) +} + +/// Expands SST buckets through existing index coverage before grouping build inputs. +fn reconcile_series_buckets( + mut buckets: Vec, + index_buckets: &mut BTreeMap, +) -> Vec { + for bucket in &buckets { + IndexBucket::new(bucket.start, bucket.end).insert_into(index_buckets); + } + for bucket in &mut buckets { + if let Some((&start, index_bucket)) = index_buckets.range(..=bucket.start).next_back() { + bucket.start = start; + bucket.end = index_bucket.end; + } + } + // Expanding sorted, disjoint spans preserves their order, but may join several of them. + group_series_buckets(buckets) +} + +fn group_series_buckets(spans: Vec) -> Vec { + let mut buckets: Vec = Vec::new(); + for mut span in spans { + if let Some(last) = buckets.last_mut() + && span.start < last.end + { + last.end = last.end.max(span.end); + last.files.append(&mut span.files); + last.has_unknown_sequence |= span.has_unknown_sequence; + last.max_file_sequence = last.max_file_sequence.max(span.max_file_sequence); + } else { + buckets.push(span); + } + } + buckets +} + +impl SeriesBucket { + /// Creates entry metadata with a fresh UUID and sorted source file IDs. + /// Returns `None` if a source sequence is unknown or too few files remain to build. + fn to_series_entry(&self) -> Option { + if self.has_unknown_sequence || self.files.len() < SERIES_INDEX_TRIGGER_FILES { + return None; + } + let mut source_file_ids = self + .files + .iter() + .map(|file| file.file_id().file_id()) + .collect::>(); + source_file_ids.sort_unstable_by(|left, right| left.as_bytes().cmp(right.as_bytes())); + let min_file_sequence = self + .files + .iter() + .filter_map(|file| file.meta_ref().sequence.map(|sequence| sequence.get())) + .min()?; + Some(SeriesIndexEntry { + index_uuid: FileId::random(), + bucket_start: self.start, + bucket_end: self.end, + source_file_ids, + min_file_sequence, + max_file_sequence: self.max_file_sequence, + }) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::num::NonZeroU64; + + use common_time::timestamp::TimeUnit; + + use super::*; + use crate::sst::file::FileMeta; + use crate::test_util::new_noop_file_purger; + + fn file(sequence: Option, level: u8, start: Timestamp, end: Timestamp) -> FileHandle { + FileHandle::new( + FileMeta { + file_id: FileId::random(), + sequence: sequence.and_then(NonZeroU64::new), + level, + time_range: (start, end), + ..Default::default() + }, + new_noop_file_purger(), + ) + } + + #[test] + fn test_second_resolution_buckets_merge_spans_across_levels() { + let width = rounded_bucket_width(Duration::from_secs(11), Duration::from_secs(10)).unwrap(); + let files = [ + file( + Some(1), + 0, + Timestamp::new_millisecond(-1), + Timestamp::new_millisecond(19999), + ), + file( + Some(2), + 1, + Timestamp::new_microsecond(19000000), + Timestamp::new_microsecond(39000000), + ), + file( + Some(3), + 2, + Timestamp::new_nanosecond(39000000000), + Timestamp::new_nanosecond(40000000000), + ), + file( + Some(4), + 1, + Timestamp::new_second(60), + Timestamp::new_second(61), + ), + ]; + let buckets = group_files_into_series_buckets(&files, width); + let spans = buckets + .iter() + .map(|b| (b.start, b.end, b.files.len())) + .collect::>(); + assert_eq!( + vec![ + (Timestamp::new_second(-20), Timestamp::new_second(60), 3), + (Timestamp::new_second(60), Timestamp::new_second(80), 1), + ], + spans + ); + assert_eq!(3, buckets[0].max_file_sequence); + assert_eq!(4, buckets[1].max_file_sequence); + assert!(buckets[0].to_series_entry().is_none()); + assert!(buckets[1].to_series_entry().is_none()); + let mut files = files.to_vec(); + files.push(file( + None, + 0, + Timestamp::new_second(0), + Timestamp::new_second(1), + )); + assert!( + group_files_into_series_buckets(&files, width)[0] + .to_series_entry() + .is_none() + ); + } + + #[test] + fn test_seconds_do_not_require_millisecond_conversion() { + let start = Timestamp::new_second(i64::MAX / 1000 + 100); + assert!(start.convert_to(TimeUnit::Millisecond).is_none()); + let buckets = group_files_into_series_buckets(&[file(Some(1), 0, start, start)], 1); + assert_eq!(start, buckets[0].start); + assert_eq!(Timestamp::new_second(start.value() + 1), buckets[0].end); + + let width = rounded_bucket_width(Duration::ZERO, Duration::from_millis(100)).unwrap(); + let buckets = group_files_into_series_buckets( + &[file( + Some(1), + 0, + Timestamp::new_nanosecond(-1), + Timestamp::new_microsecond(1), + )], + width, + ); + assert_eq!( + (Timestamp::new_second(-1), Timestamp::new_second(1)), + (buckets[0].start, buckets[0].end) + ); + } + + #[test] + fn test_reconcile_bridge_expands_through_index_and_sst_buckets() { + let ts = Timestamp::new_second; + let mut indexes = BTreeMap::new(); + let ids = [FileId::random(), FileId::random(), FileId::random()]; + for (start, end, max_sequence, id) in [ + (0, 20, 10, ids[0]), + (30, 60, 30, ids[1]), + (70, 100, 20, ids[2]), + ] { + IndexBucket { + start: ts(start), + end: ts(end), + index_ids: smallvec![id], + max_file_sequence: max_sequence, + } + .insert_into(&mut indexes); + } + let files = [ + file(Some(15), 1, ts(10), ts(30)), + file(Some(31), 0, ts(50), ts(70)), + file(Some(32), 0, ts(90), ts(95)), + file(Some(32), 1, ts(90), ts(95)), + file(Some(33), 0, ts(90), ts(95)), + file(Some(34), 0, ts(100), ts(105)), + ]; + let planned = group_files_into_series_buckets(&files, 10); + assert_eq!(4, planned.len()); + let plan = plan_series_indexes(planned, indexes, None, 0); + assert_eq!((2, 1), (plan.computed_buckets, plan.skipped_buckets)); + let [(bucket, entry)] = plan.builds.as_slice() else { + panic!("expected one incremental build"); + }; + assert_eq!((ts(0), ts(100)), (bucket.start, bucket.end)); + // Ignore the gap at sequence 15, include both files at 32, and exclude the + // adjacent time bucket even though its sequence exceeds the merged watermark. + let expected = files[1..5] + .iter() + .map(|file| file.file_id().file_id()) + .collect::>(); + assert_eq!( + expected, + bucket + .files + .iter() + .map(|file| file.file_id().file_id()) + .collect() + ); + assert_eq!(expected, entry.source_file_ids.iter().copied().collect()); + assert_eq!((31, 33), (entry.min_file_sequence, entry.max_file_sequence)); + assert!(plan.expired_index_ids.is_empty()); + assert_eq!(1, plan.index_buckets.len()); + let merged = &plan.index_buckets[&ts(0)]; + assert_eq!((ts(0), ts(100)), (merged.start, merged.end)); + assert_eq!(33, bucket.max_file_sequence); + assert_eq!(33, merged.max_file_sequence); + assert_eq!( + [ids[0], ids[1], ids[2], entry.index_uuid].as_slice(), + merged.index_ids.as_slice() + ); + } + + #[test] + fn test_plan_reuses_replaced_sources_and_builds_complete_sequence_suffix() { + let ts = Timestamp::new_second; + let make_file = |sequence| file(Some(sequence), 0, ts(1), ts(2)); + let original = (1..=4).map(make_file).collect::>(); + let initial = plan_series_indexes( + group_files_into_series_buckets(&original, 10), + BTreeMap::new(), + None, + 0, + ); + let first_id = initial.builds[0].1.index_uuid; + + // New file IDs with already indexed sequences do not invalidate the index. + let mut files = (1..=4).map(make_file).collect::>(); + let replaced = plan_series_indexes( + group_files_into_series_buckets(&files, 10), + initial.index_buckets.clone(), + None, + 0, + ); + assert!(replaced.builds.is_empty()); + assert!(replaced.expired_index_ids.is_empty()); + assert_eq!(initial.index_buckets, replaced.index_buckets); + + files.extend((5..=7).map(make_file)); + let deferred = plan_series_indexes( + group_files_into_series_buckets(&files, 10), + replaced.index_buckets, + None, + 0, + ); + assert!(deferred.builds.is_empty()); + assert_eq!(initial.index_buckets, deferred.index_buckets); + + files.push(make_file(8)); + let ready = plan_series_indexes( + group_files_into_series_buckets(&files, 10), + deferred.index_buckets, + None, + 0, + ); + let [(bucket, entry)] = ready.builds.as_slice() else { + panic!("the fourth new SST must trigger one build"); + }; + let expected = files[4..] + .iter() + .map(|file| file.file_id().file_id()) + .collect::>(); + assert_eq!( + expected, + bucket + .files + .iter() + .map(|file| file.file_id().file_id()) + .collect() + ); + assert_eq!(expected, entry.source_file_ids.iter().copied().collect()); + assert_eq!((5, 8), (entry.min_file_sequence, entry.max_file_sequence)); + assert_eq!( + [first_id, entry.index_uuid].as_slice(), + ready.index_buckets[&ts(0)].index_ids.as_slice() + ); + let repeated = plan_series_indexes( + group_files_into_series_buckets(&files, 10), + ready.index_buckets.clone(), + None, + 0, + ); + assert!(repeated.builds.is_empty()); + assert_eq!(ready.index_buckets, repeated.index_buckets); + + // Retiring the whole bucket must retire both historical and incremental indexes. + let expired = plan_series_indexes( + Vec::new(), + ready.index_buckets, + Some(TimeToLive::Duration(Duration::from_secs(10))), + 21_000, + ); + assert!(expired.builds.is_empty()); + assert!(expired.index_buckets.is_empty()); + assert_eq!( + [first_id, entry.index_uuid].as_slice(), + expired.expired_index_ids.as_slice() + ); + } +} diff --git a/src/mito2/src/series_index/builder.rs b/src/mito2/src/series_index/builder.rs new file mode 100644 index 00000000000..60321286ec8 --- /dev/null +++ b/src/mito2/src/series_index/builder.rs @@ -0,0 +1,330 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Builds range and aggregate series indexes from SST readers. + +use std::collections::HashSet; +use std::sync::{Arc, Mutex}; + +use async_stream::try_stream; +use futures::TryStreamExt; +use object_store::ObjectStore; +use snafu::OptionExt; +use store_api::storage::FileId; + +use crate::error::{Result, UnexpectedSnafu}; +use crate::read::BoxedRecordBatchStream; +use crate::read::flat_merge::FlatMergeReader; +use crate::read::prune::FlatPruneReader; +use crate::read::read_columns::ReadColumns; +use crate::region::MitoRegionRef; +use crate::region::version::VersionRef; +use crate::series_index::bucket::SeriesBucket; +use crate::series_index::catalog::{ + SeriesIndexEntry, range_index_path, series_index_path, series_metadata, +}; +use crate::series_index::purger::{IndexFilePurger, IndexFileType, file_operation}; +use crate::series_index::version::SeriesIndexFileHandle; +use crate::series_index::{SeriesIndexWriter, SeriesIndexWriterOptions}; +use crate::sst::file::FileHandle; +use crate::sst::parquet::reader::{FlatRowGroupReader, ReaderMetrics}; +use crate::sst::parquet::row_group::ParquetFetchMetrics; +use crate::sst::range_index::{SstRangeIndexWriter, SstRangeIndexWriterOptions}; + +async fn reader_input( + region: &MitoRegionRef, + file: FileHandle, +) -> Result< + Option<( + Arc, + crate::sst::parquet::row_selection::RowGroupSelection, + )>, +> { + Ok(region + .access_layer + .read_sst(file) + .projection(Some(ReadColumns::new([]))) + .build_reader_input(&mut ReaderMetrics::default()) + .await? + .map(|(context, selection)| (Arc::new(context), selection))) +} + +pub(crate) async fn build_range_index( + store: &ObjectStore, + region: &MitoRegionRef, + version: &VersionRef, + file: FileHandle, +) -> Result> { + let file_id = file.file_id().file_id(); + let path = range_index_path(region.region_id, file_id); + let mut writer = SstRangeIndexWriter::try_new( + version.metadata.clone(), + store.clone(), + &path, + SstRangeIndexWriterOptions::default(), + ) + .await?; + let Some((context, mut selection)) = reader_input(region, file).await? else { + writer.abort().await?; + return Ok(None); + }; + let fetch_metrics = ParquetFetchMetrics::default(); + while let Some((row_group_id, row_selection)) = selection.pop_first() { + let parquet_reader = context + .reader_builder() + .build(context.build_context(row_group_id, Some(row_selection), Some(&fetch_metrics))) + .await?; + let mut reader = FlatPruneReader::new_with_row_group_reader( + context.clone(), + FlatRowGroupReader::new(context.clone(), parquet_reader), + context.pre_filter_mode().skip_fields(), + ); + while let Some(batch) = reader.next_batch().await? { + writer.write(row_group_id as u32, &batch).await?; + } + } + writer.finish().await?; + file_operation(IndexFileType::Range, "build", "success"); + Ok(Some(file_id)) +} + +pub(crate) async fn build_series_index( + store: &ObjectStore, + region: &MitoRegionRef, + version: &VersionRef, + bucket: &SeriesBucket, + entry: &SeriesIndexEntry, + missing_range_ids: &HashSet, + purger: &IndexFilePurger, +) -> Result<(SeriesIndexFileHandle, HashSet)> { + let completed_ranges = Arc::new(Mutex::new(HashSet::new())); + let mut sources = Vec::::new(); + let mut schema = None; + let mut expected_ranges = 0; + for file in &bucket.files { + let file_id = file.file_id().file_id(); + let Some((context, mut selection)) = reader_input(region, file.clone()).await? else { + continue; + }; + schema.get_or_insert(context.read_format().output_arrow_schema()?); + let range_writer = if missing_range_ids.contains(&file_id) { + expected_ranges += 1; + let path = range_index_path(region.region_id, file_id); + Some( + SstRangeIndexWriter::try_new( + version.metadata.clone(), + store.clone(), + &path, + SstRangeIndexWriterOptions::default(), + ) + .await?, + ) + } else { + None + }; + let completed_ranges = completed_ranges.clone(); + sources.push(Box::pin(try_stream! { + let fetch_metrics = ParquetFetchMetrics::default(); + let mut range_writer = range_writer; + while let Some((row_group_id, row_selection)) = selection.pop_first() { + let parquet_reader = context.reader_builder().build(context.build_context( + row_group_id, + Some(row_selection), + Some(&fetch_metrics), + )).await?; + let mut reader = FlatPruneReader::new_with_row_group_reader( + context.clone(), + FlatRowGroupReader::new(context.clone(), parquet_reader), + context.pre_filter_mode().skip_fields(), + ); + while let Some(batch) = reader.next_batch().await? { + if let Some(writer) = &mut range_writer { + writer.write(row_group_id as u32, &batch).await?; + } + yield batch; + } + } + if let Some(writer) = range_writer { + writer.finish().await?; + file_operation(IndexFileType::Range, "build", "success"); + completed_ranges.lock().unwrap().insert(file_id); + } + })); + } + let schema = schema.context(UnexpectedSnafu { + reason: "series-index bucket has no readable SST", + })?; + let merged: BoxedRecordBatchStream = if sources.len() == 1 { + sources.pop().context(UnexpectedSnafu { + reason: "series-index source disappeared", + })? + } else { + Box::pin( + FlatMergeReader::new(schema, sources, 8192, None) + .await? + .into_stream(), + ) + }; + // TODO: Deduplicate update-mode rows before series indexes are used by queries. + let mut visible = merged; + let path = series_index_path(region.region_id, entry.index_uuid); + let mut writer = SeriesIndexWriter::try_new( + region.metadata(), + store.clone(), + &path, + SeriesIndexWriterOptions::default(), + Some(series_metadata(entry)?), + ) + .await?; + while let Some(batch) = visible.try_next().await? { + writer.write(&batch).await?; + } + let range_indexes = std::mem::take(&mut *completed_ranges.lock().unwrap()); + if range_indexes.len() != expected_ranges { + return UnexpectedSnafu { + reason: "not all combined range-index builds completed", + } + .fail(); + } + writer.finish().await?; + file_operation(IndexFileType::Series, "build", "success"); + Ok(( + SeriesIndexFileHandle::new(region.region_id, entry.clone(), purger.clone()), + range_indexes, + )) +} + +#[cfg(test)] +mod tests { + use object_store::layers::mock::{self, MockLayerBuilder, oio}; + use object_store::services::Memory; + use store_api::region_engine::RegionEngine; + + use super::*; + use crate::series_index::bucket::{group_files_into_series_buckets, plan_series_indexes}; + use crate::series_index::purger::series_index_channel; + use crate::series_index::tests::prepare_region; + use crate::test_util::TestEnv; + + struct FailingSeriesWriter(oio::Writer); + + impl mock::Write for FailingSeriesWriter { + async fn write(&mut self, buffer: mock::Buffer) -> mock::Result<()> { + self.0.write(buffer).await + } + + async fn close(&mut self) -> mock::Result { + Err(mock::Error::new( + mock::ErrorKind::Unexpected, + "injected series write failure", + )) + } + + async fn abort(&mut self) -> mock::Result<()> { + self.0.abort().await + } + } + + #[rstest::rstest] + #[case::success(false)] + #[case::series_failure(true)] + #[tokio::test] + async fn test_build_indexes_without_publication(#[case] fail_series: bool) { + let mut env = TestEnv::with_prefix("series-builder").await; + let (engine, region) = prepare_region(&mut env).await; + let version = region.version(); + let files = version + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .cloned() + .collect::>(); + let store = ObjectStore::new(Memory::default()).unwrap(); + let (purger, mut receiver) = series_index_channel(store.clone()); + // Build one range separately, then reuse it during the combined build. + let existing = build_range_index(&store, ®ion, &version, files[0].clone()) + .await + .unwrap() + .unwrap(); + let existing_path = range_index_path(region.region_id, existing); + let existing_bytes = store.read(&existing_path).await.unwrap().to_bytes(); + let missing = files + .iter() + .map(|file| file.file_id().file_id()) + .filter(|id| *id != existing) + .collect::>(); + let mut plan = plan_series_indexes( + group_files_into_series_buckets(&files, 100), + Default::default(), + None, + 0, + ); + assert_eq!(plan.builds.len(), 1); + let (bucket, entry) = plan.builds.pop().unwrap(); + let layer = MockLayerBuilder::default() + .writer_factory(Arc::new(move |path, _, inner| -> oio::Writer { + if fail_series && path.contains("/series/") { + Box::new(FailingSeriesWriter(inner)) + } else { + inner + } + })) + .build() + .unwrap(); + let result = build_series_index( + &store.clone().layer(layer), + ®ion, + &version, + &bucket, + &entry, + &missing, + &purger, + ) + .await; + if fail_series { + assert!(result.is_err()); + } else { + let (handle, built_ranges) = result.unwrap(); + assert_eq!(built_ranges, missing); + assert_eq!(handle.entry(), &entry); + assert!( + store + .exists(&series_index_path(region.region_id, entry.index_uuid)) + .await + .unwrap() + ); + } + // A failed series build must retain completed companion range indexes. + for file in &files { + assert!( + store + .exists(&range_index_path( + region.region_id, + file.file_id().file_id() + )) + .await + .unwrap() + ); + } + assert_eq!( + store.read(&existing_path).await.unwrap().to_bytes(), + existing_bytes + ); + assert!(region.series_index_version().range_indexes.is_empty()); + assert!(region.series_index_version().series_indexes.is_empty()); + assert!(receiver.try_recv().is_err()); + engine.stop().await.unwrap(); + } +} diff --git a/src/mito2/src/series_index/catalog.rs b/src/mito2/src/series_index/catalog.rs index a59c1926fc0..6bb37e9d1f2 100644 --- a/src/mito2/src/series_index/catalog.rs +++ b/src/mito2/src/series_index/catalog.rs @@ -28,6 +28,8 @@ use crate::series_index::purger::IndexFilePurger; use crate::series_index::version::{ SeriesIndexFileHandle, SeriesIndexVersion, SeriesIndexVersionControl, }; +pub(crate) use crate::sst::range_index::range_index_path; + const SERIES_DIR: &str = "series"; const RANGE_CATALOG: &str = "range-index.json"; const SERIES_CATALOG: &str = "series-index.json"; @@ -135,9 +137,9 @@ pub(crate) async fn load_version_control( .await .unwrap_or_default(); // TODO: Handle catalog entries whose index files are missing from storage. - let version = SeriesIndexVersion { - range_indexes: range.indexes.into_iter().collect(), - series_indexes: series + let version = SeriesIndexVersion::new( + range.indexes.into_iter().collect(), + series .indexes .into_iter() .map(|entry| { @@ -147,7 +149,7 @@ pub(crate) async fn load_version_control( ) }) .collect(), - }; + ); let control = SeriesIndexVersionControl::default(); control.publish(std::sync::Arc::new(version)); control @@ -164,8 +166,9 @@ mod tests { use store_api::storage::{FileId, RegionId}; use crate::series_index::catalog::{ - SeriesIndexCatalog, SeriesIndexEntry, load_catalog, load_version_control, - series_catalog_path, series_metadata, store_catalog, + RangeIndexCatalog, SeriesIndexCatalog, SeriesIndexEntry, load_catalog, + load_version_control, range_catalog_path, series_catalog_path, series_metadata, + store_catalog, }; use crate::series_index::purger::series_index_channel; @@ -194,31 +197,54 @@ mod tests { } #[tokio::test] - async fn test_load_catalog_returns_none_on_error() { + async fn test_load_catalog_defaults_on_missing_invalid_or_unreadable_catalog() { let store = ObjectStore::new(Memory::default()).unwrap(); - let path = series_catalog_path(RegionId::new(1, 1)); - // Missing catalog. + let region_id = RegionId::new(1, 1); + let (purger, _receiver) = series_index_channel(store.clone()); assert!( - load_catalog::(&store, &path) - .await - .is_none() - ); - store.write(&path, "invalid").await.unwrap(); - assert!( - load_catalog::(&store, &path) + load_catalog::(&store, &series_catalog_path(region_id)) .await .is_none() ); + let control = load_version_control(&store, region_id, &purger).await; + assert!(control.current().range_indexes.is_empty()); + assert!(control.current().series_indexes.is_empty()); + let file_id = FileId::random(); + store + .write( + &range_catalog_path(region_id), + serde_json::to_vec(&RangeIndexCatalog { + indexes: vec![file_id], + }) + .unwrap(), + ) + .await + .unwrap(); + store + .write(&series_catalog_path(region_id), "invalid") + .await + .unwrap(); + let control = load_version_control(&store, region_id, &purger).await; + assert!(control.current().range_indexes.contains(&file_id)); + assert!(control.current().series_indexes.is_empty()); let layer = MockLayerBuilder::default() .reader_factory(Arc::new(|_, _, _| Box::new(FailingCatalogReader))) .build() .unwrap(); - let store = store.layer(layer); assert!( - load_catalog::(&store, &path) + load_catalog::(&store, &series_catalog_path(region_id)) .await .is_none() ); + let store = store.layer(layer); + assert!( + load_catalog::(&store, &range_catalog_path(region_id)) + .await + .is_none() + ); + let control = load_version_control(&store, region_id, &purger).await; + assert!(control.current().range_indexes.is_empty()); + assert!(control.current().series_indexes.is_empty()); } #[tokio::test] @@ -248,6 +274,12 @@ mod tests { .current(); assert!(current.range_indexes.is_empty()); assert_eq!(&entry, current.series_indexes[&entry.index_uuid].entry()); + assert_eq!(current.index_buckets.len(), 1); + let bucket = ¤t.index_buckets[&entry.bucket_start]; + assert_eq!(bucket.start, entry.bucket_start); + assert_eq!(bucket.end, entry.bucket_end); + assert_eq!(bucket.index_ids.as_slice(), &[entry.index_uuid]); + assert_eq!(bucket.max_file_sequence, entry.max_file_sequence); let metadata = series_metadata(&entry).unwrap(); let decoded: SeriesIndexEntry = diff --git a/src/mito2/src/series_index/tests.rs b/src/mito2/src/series_index/tests.rs new file mode 100644 index 00000000000..4656208f596 --- /dev/null +++ b/src/mito2/src/series_index/tests.rs @@ -0,0 +1,113 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared fixtures for series-index construction tests. + +use std::sync::Arc; + +use api::v1::helper::row; +use api::v1::value::ValueData; +use api::v1::{ColumnDataType, Rows, SemanticType, WriteHint}; +use store_api::codec::PrimaryKeyEncoding; +use store_api::metric_engine_consts::PRIMARY_KEY_ENCODING; +use store_api::region_engine::RegionEngine; +use store_api::region_request::{RegionPutRequest, RegionRequest}; +use store_api::storage::RegionId; +use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME; + +use crate::config::MitoConfig; +use crate::engine::MitoEngine; +use crate::region::MitoRegionRef; +use crate::test_util::sst_util::{new_sparse_primary_key, sst_region_metadata_with_encoding}; +use crate::test_util::{CreateRequestBuilder, TestEnv, flush_region, rows_schema}; + +/// Builds real sparse SSTs; background maintenance is disabled so tests control publication. +pub(super) async fn prepare_region(env: &mut TestEnv) -> (MitoEngine, MitoRegionRef) { + prepare_region_with_timestamps(env, &[1000, 2000, 3000, 4000]).await +} + +async fn prepare_region_with_timestamps( + env: &mut TestEnv, + timestamps: &[i64], +) -> (MitoEngine, MitoRegionRef) { + let engine = env.create_engine(MitoConfig::default()).await; + let metadata = Arc::new(sst_region_metadata_with_encoding( + PrimaryKeyEncoding::Sparse, + )); + let region_id = RegionId::new(1, 1); + let mut request = CreateRequestBuilder::new().build(); + request.column_metadatas = metadata.column_metadatas.clone(); + request.primary_key = metadata.primary_key.clone(); + request + .options + .insert(PRIMARY_KEY_ENCODING.to_string(), "sparse".to_string()); + request + .options + .insert("memtable.type".to_string(), "bulk".to_string()); + request + .options + .insert("sst_format".to_string(), "flat".to_string()); + request + .options + .insert("compaction.type".to_string(), "twcs".to_string()); + request.options.insert( + "compaction.twcs.time_window".to_string(), + "100s".to_string(), + ); + // Keep the source SSTs stable while tests reconcile multiple buckets. + request.options.insert( + "compaction.twcs.trigger_file_num".to_string(), + "100".to_string(), + ); + let full_schema = rows_schema(&request); + let mut pk_column = full_schema[0].clone(); + pk_column.column_name = PRIMARY_KEY_COLUMN_NAME.to_string(); + pk_column.datatype = ColumnDataType::Binary.into(); + pk_column.semantic_type = SemanticType::Tag.into(); + let schema = vec![pk_column, full_schema[5].clone(), full_schema[4].clone()]; + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + for &ts in timestamps { + engine + .handle_request( + region_id, + RegionRequest::Put(RegionPutRequest { + rows: Rows { + schema: schema.clone(), + rows: vec![row(vec![ + ValueData::BinaryValue(new_sparse_primary_key( + &["a", "x"], + &metadata, + 10, + 0, + )), + ValueData::TimestampMillisecondValue(ts), + ValueData::U64Value(1), + ])], + }, + hint: Some(WriteHint { + primary_key_encoding: api::v1::PrimaryKeyEncoding::Sparse.into(), + }), + partition_expr_version: None, + }), + ) + .await + .unwrap(); + flush_region(&engine, region_id, None).await; + } + let region = engine.get_region(region_id).unwrap(); + (engine, region) +} diff --git a/src/mito2/src/series_index/version.rs b/src/mito2/src/series_index/version.rs index 7db23da2f93..1af4cabcd42 100644 --- a/src/mito2/src/series_index/version.rs +++ b/src/mito2/src/series_index/version.rs @@ -14,13 +14,16 @@ //! Immutable index snapshots and aggregate series-file handles. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt::{self, Debug, Formatter}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; +use common_time::Timestamp; +use smallvec::smallvec; use store_api::storage::{FileId, RegionId}; +use crate::series_index::bucket::IndexBucket; use crate::series_index::catalog::SeriesIndexEntry; use crate::series_index::purger::{IndexFilePurger, PurgeRequest}; use crate::sst::file::RegionFileId; @@ -85,11 +88,37 @@ impl Drop for SeriesIndexFileHandleInner { /// Immutable series-index snapshot for one region. #[derive(Debug, Default)] pub(crate) struct SeriesIndexVersion { + /// Range indexes for visible SSTs; reconciliation removes IDs absent from its SST snapshot. + /// Physical deletion is independently handled by the SST file purger. pub(crate) range_indexes: HashSet, pub(crate) series_indexes: HashMap, + pub(crate) index_buckets: BTreeMap, } impl SeriesIndexVersion { + /// Restores bucket lookup from immutable index coverage stored in the catalog. + pub(crate) fn new( + range_indexes: HashSet, + series_indexes: HashMap, + ) -> Self { + let mut index_buckets = BTreeMap::new(); + for handle in series_indexes.values() { + let entry = handle.entry(); + IndexBucket { + start: entry.bucket_start, + end: entry.bucket_end, + index_ids: smallvec![entry.index_uuid], + max_file_sequence: entry.max_file_sequence, + } + .insert_into(&mut index_buckets); + } + Self { + range_indexes, + series_indexes, + index_buckets, + } + } + fn mark_all_deleted(&self) { self.series_indexes .values() diff --git a/src/mito2/src/sst/range_index.rs b/src/mito2/src/sst/range_index.rs index c4e576b6cd9..6d9eadc5532 100644 --- a/src/mito2/src/sst/range_index.rs +++ b/src/mito2/src/sst/range_index.rs @@ -28,8 +28,6 @@ pub use writer::{ }; pub use crate::sst::range_index::deleter::RangeIndexDeleter; -// Used by the upcoming query and index-building integration. -#[allow(unused_imports)] pub(crate) use crate::sst::range_index::deleter::range_index_path; const ROW_GROUP_ID_COLUMN: &str = "row_group_id";