mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 08:14:56 +00:00
feat(mito2): add series index planning and builders
Signed-off-by: evenyag <realevenyag@gmail.com>
This commit is contained in:
@@ -69,6 +69,11 @@ a module map, read/write paths, change-coupling points, and gotchas:
|
||||
- [`tests/compatibility/AGENTS.md`](tests/compatibility/AGENTS.md)
|
||||
- [`tests/perf/AGENTS.md`](tests/perf/AGENTS.md)
|
||||
|
||||
## Rust imports
|
||||
|
||||
Prefer crate-rooted imports (`use crate::...`) over `use super::...` in
|
||||
production code. Relative imports using `super` are allowed in tests.
|
||||
|
||||
## Read before changing code
|
||||
|
||||
- [`.agents/architecture-invariants.md`](.agents/architecture-invariants.md) —
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@ snapshot isolation). It implements the `RegionEngine` trait from `store-api`.
|
||||
| `compaction` | `src/mito2/src/compaction/` | Compaction scheduler (`scheduler.rs` + `scheduler/`), TWCS picker, strict-window manual picker, compactor, memory control |
|
||||
| `access_layer` | `src/mito2/src/access_layer.rs` | SST read/write over the object store |
|
||||
| `sst` | `src/mito2/src/sst/` | Parquet format, file metadata, index layout |
|
||||
| `series_index` | `src/mito2/src/series_index/` | Series index writer/searcher, catalogs, immutable snapshots, file lifecycle, and background maintenance |
|
||||
| `series_index` | `src/mito2/src/series_index/` | Series index writer/searcher, bucket planning and SST builders, catalogs, immutable snapshots, file lifecycle, and background maintenance |
|
||||
| `read` | `src/mito2/src/read/` | `ScanRegion`, merge, dedup, projection, streaming |
|
||||
| `manifest` | `src/mito2/src/manifest/` | `RegionManifestManager`, manifest actions/edits |
|
||||
| `cache` | `src/mito2/src/cache.rs` | Write/file/page caches |
|
||||
|
||||
@@ -17,10 +17,18 @@
|
||||
// These components are consumed by the upcoming query and maintenance integration.
|
||||
#[allow(dead_code)]
|
||||
mod catalog;
|
||||
// Consumed by the follow-up background-maintenance integration.
|
||||
#[allow(dead_code)]
|
||||
mod bucket;
|
||||
// Consumed by the follow-up background-maintenance integration.
|
||||
#[allow(dead_code)]
|
||||
mod builder;
|
||||
#[allow(dead_code)]
|
||||
mod purger;
|
||||
mod searcher;
|
||||
mod task;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
#[allow(dead_code)]
|
||||
mod version;
|
||||
mod writer;
|
||||
|
||||
@@ -0,0 +1,525 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Event-time bucket planning and source coverage.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use common_time::{TimeToLive, Timestamp};
|
||||
use smallvec::{SmallVec, smallvec};
|
||||
use store_api::storage::FileId;
|
||||
|
||||
use crate::series_index::catalog::SeriesIndexEntry;
|
||||
use crate::sst::file::FileHandle;
|
||||
|
||||
const SERIES_INDEX_TRIGGER_FILES: usize = 4;
|
||||
|
||||
/// Index files sharing a non-overlapping, half-open time bucket.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct IndexBucket {
|
||||
pub(crate) start: Timestamp,
|
||||
pub(crate) end: Timestamp,
|
||||
pub(crate) index_ids: SmallVec<[FileId; 2]>,
|
||||
pub(crate) max_file_sequence: u64,
|
||||
}
|
||||
|
||||
impl IndexBucket {
|
||||
pub(crate) fn new(start: Timestamp, end: Timestamp) -> Self {
|
||||
Self {
|
||||
start,
|
||||
end,
|
||||
index_ids: SmallVec::new(),
|
||||
max_file_sequence: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn merge(&mut self, mut other: Self) {
|
||||
self.start = self.start.min(other.start);
|
||||
self.end = self.end.max(other.end);
|
||||
self.index_ids.append(&mut other.index_ids);
|
||||
self.max_file_sequence = self.max_file_sequence.max(other.max_file_sequence);
|
||||
}
|
||||
|
||||
/// Inserts a bucket, consuming overlapping entries and expanding their interval.
|
||||
/// Each map key equals the stored bucket start; adjacent intervals remain separate.
|
||||
pub(crate) fn insert_into(mut self, buckets: &mut BTreeMap<Timestamp, Self>) {
|
||||
if let Some((&previous_start, previous)) = buckets.range(..self.start).next_back()
|
||||
&& previous.end > self.start
|
||||
{
|
||||
self.start = previous_start;
|
||||
}
|
||||
// Expanding the end may expose more overlaps, so look up the next entry again.
|
||||
while let Some((&next_start, _)) = buckets.range(self.start..self.end).next() {
|
||||
if let Some(other) = buckets.remove(&next_start) {
|
||||
self.merge(other);
|
||||
}
|
||||
}
|
||||
buckets.insert(self.start, self);
|
||||
}
|
||||
}
|
||||
|
||||
/// SSTs grouped into a half-open time interval for an aggregate series-index build.
|
||||
///
|
||||
/// Reconciliation may expand the interval through existing index coverage and retain
|
||||
/// only files above its sequence watermark. Unknown source sequences prevent builds
|
||||
/// even after filtering, since their coverage cannot be determined safely.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct SeriesBucket {
|
||||
pub(crate) start: Timestamp,
|
||||
pub(crate) end: Timestamp,
|
||||
pub(crate) files: SmallVec<[FileHandle; 2]>,
|
||||
pub(crate) has_unknown_sequence: bool,
|
||||
/// Maximum known source sequence, or zero when all sequences are unknown.
|
||||
pub(crate) max_file_sequence: u64,
|
||||
}
|
||||
|
||||
/// Next bucket coverage and the work required to publish it, computed without I/O.
|
||||
pub(crate) struct SeriesIndexPlan {
|
||||
pub(crate) index_buckets: BTreeMap<Timestamp, IndexBucket>,
|
||||
pub(crate) builds: Vec<(SeriesBucket, SeriesIndexEntry)>,
|
||||
pub(crate) expired_index_ids: Vec<FileId>,
|
||||
pub(crate) computed_buckets: usize,
|
||||
pub(crate) skipped_buckets: usize,
|
||||
}
|
||||
|
||||
/// Plans complete sequence suffixes after merging time coverage. The returned bucket
|
||||
/// map is publishable only after every planned build and the catalog writes succeed.
|
||||
pub(crate) fn plan_series_indexes(
|
||||
buckets: Vec<SeriesBucket>,
|
||||
mut index_buckets: BTreeMap<Timestamp, IndexBucket>,
|
||||
ttl: Option<TimeToLive>,
|
||||
now_ms: i64,
|
||||
) -> SeriesIndexPlan {
|
||||
let buckets = reconcile_series_buckets(buckets, &mut index_buckets);
|
||||
let computed_buckets = buckets.len();
|
||||
let expired = |end| {
|
||||
ttl.is_some_and(|ttl| {
|
||||
ttl.is_expired(&end, &Timestamp::new_millisecond(now_ms))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
};
|
||||
let mut expired_index_ids = Vec::new();
|
||||
index_buckets.retain(|_, bucket| {
|
||||
if expired(bucket.end) {
|
||||
expired_index_ids.extend_from_slice(&bucket.index_ids);
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
let mut builds = Vec::new();
|
||||
for mut bucket in buckets {
|
||||
if expired(bucket.end) {
|
||||
continue;
|
||||
}
|
||||
let index_bucket = index_buckets
|
||||
.entry(bucket.start)
|
||||
.or_insert_with(|| IndexBucket::new(bucket.start, bucket.end));
|
||||
if bucket.has_unknown_sequence || bucket.max_file_sequence <= index_bucket.max_file_sequence
|
||||
{
|
||||
continue;
|
||||
}
|
||||
// Select all SSTs above the watermark, including every file sharing a sequence.
|
||||
// File IDs may change under compaction without advancing indexed coverage.
|
||||
// The maximum remains valid because it exceeds the watermark and is retained.
|
||||
bucket.files.retain(|file| {
|
||||
file.meta_ref()
|
||||
.sequence
|
||||
.is_some_and(|sequence| sequence.get() > index_bucket.max_file_sequence)
|
||||
});
|
||||
if let Some(entry) = bucket.to_series_entry() {
|
||||
index_bucket.index_ids.push(entry.index_uuid);
|
||||
index_bucket.max_file_sequence = entry.max_file_sequence;
|
||||
builds.push((bucket, entry));
|
||||
}
|
||||
}
|
||||
index_buckets.retain(|_, bucket| !bucket.index_ids.is_empty());
|
||||
SeriesIndexPlan {
|
||||
index_buckets,
|
||||
skipped_buckets: computed_buckets - builds.len(),
|
||||
builds,
|
||||
expired_index_ids,
|
||||
computed_buckets,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rounded_bucket_width(
|
||||
requested: Duration,
|
||||
compaction_window: Duration,
|
||||
) -> Option<i64> {
|
||||
let window_secs = i64::try_from(compaction_window.as_secs()).ok()?.max(1);
|
||||
let requested_secs = i64::try_from(requested.as_secs())
|
||||
.unwrap_or(i64::MAX)
|
||||
.max(1);
|
||||
let multiples = requested_secs / window_secs + i64::from(requested_secs % window_secs != 0);
|
||||
multiples.checked_mul(window_secs)
|
||||
}
|
||||
|
||||
/// Groups SSTs across levels into sorted, disjoint time buckets without planning builds.
|
||||
///
|
||||
/// `width_secs` must be positive. Inclusive SST ranges are rounded outward to aligned,
|
||||
/// half-open intervals in seconds. Overlapping intervals merge; adjacent ones stay
|
||||
/// separate. Each merged bucket tracks its maximum sequence and any unknown sequence.
|
||||
pub(crate) fn group_files_into_series_buckets(
|
||||
files: &[FileHandle],
|
||||
width_secs: i64,
|
||||
) -> Vec<SeriesBucket> {
|
||||
let mut spans = files
|
||||
.iter()
|
||||
.map(|file| {
|
||||
let start = file.time_range().0.split().0;
|
||||
let end = file.time_range().1.split().0;
|
||||
SeriesBucket {
|
||||
start: Timestamp::new_second(
|
||||
start.div_euclid(width_secs).saturating_mul(width_secs),
|
||||
),
|
||||
end: Timestamp::new_second(
|
||||
end.div_euclid(width_secs)
|
||||
.saturating_add(1)
|
||||
.saturating_mul(width_secs),
|
||||
),
|
||||
files: smallvec![file.clone()],
|
||||
has_unknown_sequence: file.meta_ref().sequence.is_none(),
|
||||
max_file_sequence: file
|
||||
.meta_ref()
|
||||
.sequence
|
||||
.map_or(0, |sequence| sequence.get()),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
spans.sort_unstable_by(|a, b| a.start.cmp(&b.start).then_with(|| b.end.cmp(&a.end)));
|
||||
group_series_buckets(spans)
|
||||
}
|
||||
|
||||
/// Expands SST buckets through existing index coverage before grouping build inputs.
|
||||
fn reconcile_series_buckets(
|
||||
mut buckets: Vec<SeriesBucket>,
|
||||
index_buckets: &mut BTreeMap<Timestamp, IndexBucket>,
|
||||
) -> Vec<SeriesBucket> {
|
||||
for bucket in &buckets {
|
||||
IndexBucket::new(bucket.start, bucket.end).insert_into(index_buckets);
|
||||
}
|
||||
for bucket in &mut buckets {
|
||||
if let Some((&start, index_bucket)) = index_buckets.range(..=bucket.start).next_back() {
|
||||
bucket.start = start;
|
||||
bucket.end = index_bucket.end;
|
||||
}
|
||||
}
|
||||
// Expanding sorted, disjoint spans preserves their order, but may join several of them.
|
||||
group_series_buckets(buckets)
|
||||
}
|
||||
|
||||
fn group_series_buckets(spans: Vec<SeriesBucket>) -> Vec<SeriesBucket> {
|
||||
let mut buckets: Vec<SeriesBucket> = Vec::new();
|
||||
for mut span in spans {
|
||||
if let Some(last) = buckets.last_mut()
|
||||
&& span.start < last.end
|
||||
{
|
||||
last.end = last.end.max(span.end);
|
||||
last.files.append(&mut span.files);
|
||||
last.has_unknown_sequence |= span.has_unknown_sequence;
|
||||
last.max_file_sequence = last.max_file_sequence.max(span.max_file_sequence);
|
||||
} else {
|
||||
buckets.push(span);
|
||||
}
|
||||
}
|
||||
buckets
|
||||
}
|
||||
|
||||
impl SeriesBucket {
|
||||
/// Creates entry metadata with a fresh UUID and sorted source file IDs.
|
||||
/// Returns `None` if a source sequence is unknown or too few files remain to build.
|
||||
fn to_series_entry(&self) -> Option<SeriesIndexEntry> {
|
||||
if self.has_unknown_sequence || self.files.len() < SERIES_INDEX_TRIGGER_FILES {
|
||||
return None;
|
||||
}
|
||||
let mut source_file_ids = self
|
||||
.files
|
||||
.iter()
|
||||
.map(|file| file.file_id().file_id())
|
||||
.collect::<Vec<_>>();
|
||||
source_file_ids.sort_unstable_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
|
||||
let min_file_sequence = self
|
||||
.files
|
||||
.iter()
|
||||
.filter_map(|file| file.meta_ref().sequence.map(|sequence| sequence.get()))
|
||||
.min()?;
|
||||
Some(SeriesIndexEntry {
|
||||
index_uuid: FileId::random(),
|
||||
bucket_start: self.start,
|
||||
bucket_end: self.end,
|
||||
source_file_ids,
|
||||
min_file_sequence,
|
||||
max_file_sequence: self.max_file_sequence,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashSet;
|
||||
use std::num::NonZeroU64;
|
||||
|
||||
use common_time::timestamp::TimeUnit;
|
||||
|
||||
use super::*;
|
||||
use crate::sst::file::FileMeta;
|
||||
use crate::test_util::new_noop_file_purger;
|
||||
|
||||
fn file(sequence: Option<u64>, level: u8, start: Timestamp, end: Timestamp) -> FileHandle {
|
||||
FileHandle::new(
|
||||
FileMeta {
|
||||
file_id: FileId::random(),
|
||||
sequence: sequence.and_then(NonZeroU64::new),
|
||||
level,
|
||||
time_range: (start, end),
|
||||
..Default::default()
|
||||
},
|
||||
new_noop_file_purger(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_second_resolution_buckets_merge_spans_across_levels() {
|
||||
let width = rounded_bucket_width(Duration::from_secs(11), Duration::from_secs(10)).unwrap();
|
||||
let files = [
|
||||
file(
|
||||
Some(1),
|
||||
0,
|
||||
Timestamp::new_millisecond(-1),
|
||||
Timestamp::new_millisecond(19999),
|
||||
),
|
||||
file(
|
||||
Some(2),
|
||||
1,
|
||||
Timestamp::new_microsecond(19000000),
|
||||
Timestamp::new_microsecond(39000000),
|
||||
),
|
||||
file(
|
||||
Some(3),
|
||||
2,
|
||||
Timestamp::new_nanosecond(39000000000),
|
||||
Timestamp::new_nanosecond(40000000000),
|
||||
),
|
||||
file(
|
||||
Some(4),
|
||||
1,
|
||||
Timestamp::new_second(60),
|
||||
Timestamp::new_second(61),
|
||||
),
|
||||
];
|
||||
let buckets = group_files_into_series_buckets(&files, width);
|
||||
let spans = buckets
|
||||
.iter()
|
||||
.map(|b| (b.start, b.end, b.files.len()))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
vec![
|
||||
(Timestamp::new_second(-20), Timestamp::new_second(60), 3),
|
||||
(Timestamp::new_second(60), Timestamp::new_second(80), 1),
|
||||
],
|
||||
spans
|
||||
);
|
||||
assert_eq!(3, buckets[0].max_file_sequence);
|
||||
assert_eq!(4, buckets[1].max_file_sequence);
|
||||
assert!(buckets[0].to_series_entry().is_none());
|
||||
assert!(buckets[1].to_series_entry().is_none());
|
||||
let mut files = files.to_vec();
|
||||
files.push(file(
|
||||
None,
|
||||
0,
|
||||
Timestamp::new_second(0),
|
||||
Timestamp::new_second(1),
|
||||
));
|
||||
assert!(
|
||||
group_files_into_series_buckets(&files, width)[0]
|
||||
.to_series_entry()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_seconds_do_not_require_millisecond_conversion() {
|
||||
let start = Timestamp::new_second(i64::MAX / 1000 + 100);
|
||||
assert!(start.convert_to(TimeUnit::Millisecond).is_none());
|
||||
let buckets = group_files_into_series_buckets(&[file(Some(1), 0, start, start)], 1);
|
||||
assert_eq!(start, buckets[0].start);
|
||||
assert_eq!(Timestamp::new_second(start.value() + 1), buckets[0].end);
|
||||
|
||||
let width = rounded_bucket_width(Duration::ZERO, Duration::from_millis(100)).unwrap();
|
||||
let buckets = group_files_into_series_buckets(
|
||||
&[file(
|
||||
Some(1),
|
||||
0,
|
||||
Timestamp::new_nanosecond(-1),
|
||||
Timestamp::new_microsecond(1),
|
||||
)],
|
||||
width,
|
||||
);
|
||||
assert_eq!(
|
||||
(Timestamp::new_second(-1), Timestamp::new_second(1)),
|
||||
(buckets[0].start, buckets[0].end)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reconcile_bridge_expands_through_index_and_sst_buckets() {
|
||||
let ts = Timestamp::new_second;
|
||||
let mut indexes = BTreeMap::new();
|
||||
let ids = [FileId::random(), FileId::random(), FileId::random()];
|
||||
for (start, end, max_sequence, id) in [
|
||||
(0, 20, 10, ids[0]),
|
||||
(30, 60, 30, ids[1]),
|
||||
(70, 100, 20, ids[2]),
|
||||
] {
|
||||
IndexBucket {
|
||||
start: ts(start),
|
||||
end: ts(end),
|
||||
index_ids: smallvec![id],
|
||||
max_file_sequence: max_sequence,
|
||||
}
|
||||
.insert_into(&mut indexes);
|
||||
}
|
||||
let files = [
|
||||
file(Some(15), 1, ts(10), ts(30)),
|
||||
file(Some(31), 0, ts(50), ts(70)),
|
||||
file(Some(32), 0, ts(90), ts(95)),
|
||||
file(Some(32), 1, ts(90), ts(95)),
|
||||
file(Some(33), 0, ts(90), ts(95)),
|
||||
file(Some(34), 0, ts(100), ts(105)),
|
||||
];
|
||||
let planned = group_files_into_series_buckets(&files, 10);
|
||||
assert_eq!(4, planned.len());
|
||||
let plan = plan_series_indexes(planned, indexes, None, 0);
|
||||
assert_eq!((2, 1), (plan.computed_buckets, plan.skipped_buckets));
|
||||
let [(bucket, entry)] = plan.builds.as_slice() else {
|
||||
panic!("expected one incremental build");
|
||||
};
|
||||
assert_eq!((ts(0), ts(100)), (bucket.start, bucket.end));
|
||||
// Ignore the gap at sequence 15, include both files at 32, and exclude the
|
||||
// adjacent time bucket even though its sequence exceeds the merged watermark.
|
||||
let expected = files[1..5]
|
||||
.iter()
|
||||
.map(|file| file.file_id().file_id())
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
expected,
|
||||
bucket
|
||||
.files
|
||||
.iter()
|
||||
.map(|file| file.file_id().file_id())
|
||||
.collect()
|
||||
);
|
||||
assert_eq!(expected, entry.source_file_ids.iter().copied().collect());
|
||||
assert_eq!((31, 33), (entry.min_file_sequence, entry.max_file_sequence));
|
||||
assert!(plan.expired_index_ids.is_empty());
|
||||
assert_eq!(1, plan.index_buckets.len());
|
||||
let merged = &plan.index_buckets[&ts(0)];
|
||||
assert_eq!((ts(0), ts(100)), (merged.start, merged.end));
|
||||
assert_eq!(33, bucket.max_file_sequence);
|
||||
assert_eq!(33, merged.max_file_sequence);
|
||||
assert_eq!(
|
||||
[ids[0], ids[1], ids[2], entry.index_uuid].as_slice(),
|
||||
merged.index_ids.as_slice()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_plan_reuses_replaced_sources_and_builds_complete_sequence_suffix() {
|
||||
let ts = Timestamp::new_second;
|
||||
let make_file = |sequence| file(Some(sequence), 0, ts(1), ts(2));
|
||||
let original = (1..=4).map(make_file).collect::<Vec<_>>();
|
||||
let initial = plan_series_indexes(
|
||||
group_files_into_series_buckets(&original, 10),
|
||||
BTreeMap::new(),
|
||||
None,
|
||||
0,
|
||||
);
|
||||
let first_id = initial.builds[0].1.index_uuid;
|
||||
|
||||
// New file IDs with already indexed sequences do not invalidate the index.
|
||||
let mut files = (1..=4).map(make_file).collect::<Vec<_>>();
|
||||
let replaced = plan_series_indexes(
|
||||
group_files_into_series_buckets(&files, 10),
|
||||
initial.index_buckets.clone(),
|
||||
None,
|
||||
0,
|
||||
);
|
||||
assert!(replaced.builds.is_empty());
|
||||
assert!(replaced.expired_index_ids.is_empty());
|
||||
assert_eq!(initial.index_buckets, replaced.index_buckets);
|
||||
|
||||
files.extend((5..=7).map(make_file));
|
||||
let deferred = plan_series_indexes(
|
||||
group_files_into_series_buckets(&files, 10),
|
||||
replaced.index_buckets,
|
||||
None,
|
||||
0,
|
||||
);
|
||||
assert!(deferred.builds.is_empty());
|
||||
assert_eq!(initial.index_buckets, deferred.index_buckets);
|
||||
|
||||
files.push(make_file(8));
|
||||
let ready = plan_series_indexes(
|
||||
group_files_into_series_buckets(&files, 10),
|
||||
deferred.index_buckets,
|
||||
None,
|
||||
0,
|
||||
);
|
||||
let [(bucket, entry)] = ready.builds.as_slice() else {
|
||||
panic!("the fourth new SST must trigger one build");
|
||||
};
|
||||
let expected = files[4..]
|
||||
.iter()
|
||||
.map(|file| file.file_id().file_id())
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
expected,
|
||||
bucket
|
||||
.files
|
||||
.iter()
|
||||
.map(|file| file.file_id().file_id())
|
||||
.collect()
|
||||
);
|
||||
assert_eq!(expected, entry.source_file_ids.iter().copied().collect());
|
||||
assert_eq!((5, 8), (entry.min_file_sequence, entry.max_file_sequence));
|
||||
assert_eq!(
|
||||
[first_id, entry.index_uuid].as_slice(),
|
||||
ready.index_buckets[&ts(0)].index_ids.as_slice()
|
||||
);
|
||||
let repeated = plan_series_indexes(
|
||||
group_files_into_series_buckets(&files, 10),
|
||||
ready.index_buckets.clone(),
|
||||
None,
|
||||
0,
|
||||
);
|
||||
assert!(repeated.builds.is_empty());
|
||||
assert_eq!(ready.index_buckets, repeated.index_buckets);
|
||||
|
||||
// Retiring the whole bucket must retire both historical and incremental indexes.
|
||||
let expired = plan_series_indexes(
|
||||
Vec::new(),
|
||||
ready.index_buckets,
|
||||
Some(TimeToLive::Duration(Duration::from_secs(10))),
|
||||
21_000,
|
||||
);
|
||||
assert!(expired.builds.is_empty());
|
||||
assert!(expired.index_buckets.is_empty());
|
||||
assert_eq!(
|
||||
[first_id, entry.index_uuid].as_slice(),
|
||||
expired.expired_index_ids.as_slice()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Builds range and aggregate series indexes from SST readers.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_stream::try_stream;
|
||||
use futures::TryStreamExt;
|
||||
use object_store::ObjectStore;
|
||||
use snafu::OptionExt;
|
||||
use store_api::storage::FileId;
|
||||
|
||||
use crate::error::{Result, UnexpectedSnafu};
|
||||
use crate::read::BoxedRecordBatchStream;
|
||||
use crate::read::flat_merge::FlatMergeReader;
|
||||
use crate::read::prune::FlatPruneReader;
|
||||
use crate::read::read_columns::ReadColumns;
|
||||
use crate::region::MitoRegionRef;
|
||||
use crate::region::version::VersionRef;
|
||||
use crate::series_index::bucket::SeriesBucket;
|
||||
use crate::series_index::catalog::{
|
||||
SeriesIndexEntry, range_index_path, series_index_path, series_metadata,
|
||||
};
|
||||
use crate::series_index::purger::{IndexFilePurger, IndexFileType, file_operation};
|
||||
use crate::series_index::version::SeriesIndexFileHandle;
|
||||
use crate::series_index::{SeriesIndexWriter, SeriesIndexWriterOptions};
|
||||
use crate::sst::file::FileHandle;
|
||||
use crate::sst::parquet::reader::{FlatRowGroupReader, ReaderMetrics};
|
||||
use crate::sst::parquet::row_group::ParquetFetchMetrics;
|
||||
use crate::sst::range_index::{SstRangeIndexWriter, SstRangeIndexWriterOptions};
|
||||
|
||||
async fn reader_input(
|
||||
region: &MitoRegionRef,
|
||||
file: FileHandle,
|
||||
) -> Result<
|
||||
Option<(
|
||||
Arc<crate::sst::parquet::file_range::FileRangeContext>,
|
||||
crate::sst::parquet::row_selection::RowGroupSelection,
|
||||
)>,
|
||||
> {
|
||||
Ok(region
|
||||
.access_layer
|
||||
.read_sst(file)
|
||||
.projection(Some(ReadColumns::new([])))
|
||||
.build_reader_input(&mut ReaderMetrics::default())
|
||||
.await?
|
||||
.map(|(context, selection)| (Arc::new(context), selection)))
|
||||
}
|
||||
|
||||
pub(crate) async fn build_range_index(
|
||||
store: &ObjectStore,
|
||||
region: &MitoRegionRef,
|
||||
version: &VersionRef,
|
||||
file: FileHandle,
|
||||
) -> Result<Option<FileId>> {
|
||||
let file_id = file.file_id().file_id();
|
||||
let path = range_index_path(region.region_id, file_id);
|
||||
let mut writer = SstRangeIndexWriter::try_new(
|
||||
version.metadata.clone(),
|
||||
store.clone(),
|
||||
&path,
|
||||
SstRangeIndexWriterOptions::default(),
|
||||
)
|
||||
.await?;
|
||||
let Some((context, mut selection)) = reader_input(region, file).await? else {
|
||||
writer.abort().await?;
|
||||
return Ok(None);
|
||||
};
|
||||
let fetch_metrics = ParquetFetchMetrics::default();
|
||||
while let Some((row_group_id, row_selection)) = selection.pop_first() {
|
||||
let parquet_reader = context
|
||||
.reader_builder()
|
||||
.build(context.build_context(row_group_id, Some(row_selection), Some(&fetch_metrics)))
|
||||
.await?;
|
||||
let mut reader = FlatPruneReader::new_with_row_group_reader(
|
||||
context.clone(),
|
||||
FlatRowGroupReader::new(context.clone(), parquet_reader),
|
||||
context.pre_filter_mode().skip_fields(),
|
||||
);
|
||||
while let Some(batch) = reader.next_batch().await? {
|
||||
writer.write(row_group_id as u32, &batch).await?;
|
||||
}
|
||||
}
|
||||
writer.finish().await?;
|
||||
file_operation(IndexFileType::Range, "build", "success");
|
||||
Ok(Some(file_id))
|
||||
}
|
||||
|
||||
pub(crate) async fn build_series_index(
|
||||
store: &ObjectStore,
|
||||
region: &MitoRegionRef,
|
||||
version: &VersionRef,
|
||||
bucket: &SeriesBucket,
|
||||
entry: &SeriesIndexEntry,
|
||||
missing_range_ids: &HashSet<FileId>,
|
||||
purger: &IndexFilePurger,
|
||||
) -> Result<(SeriesIndexFileHandle, HashSet<FileId>)> {
|
||||
let completed_ranges = Arc::new(Mutex::new(HashSet::new()));
|
||||
let mut sources = Vec::<BoxedRecordBatchStream>::new();
|
||||
let mut schema = None;
|
||||
let mut expected_ranges = 0;
|
||||
for file in &bucket.files {
|
||||
let file_id = file.file_id().file_id();
|
||||
let Some((context, mut selection)) = reader_input(region, file.clone()).await? else {
|
||||
continue;
|
||||
};
|
||||
schema.get_or_insert(context.read_format().output_arrow_schema()?);
|
||||
let range_writer = if missing_range_ids.contains(&file_id) {
|
||||
expected_ranges += 1;
|
||||
let path = range_index_path(region.region_id, file_id);
|
||||
Some(
|
||||
SstRangeIndexWriter::try_new(
|
||||
version.metadata.clone(),
|
||||
store.clone(),
|
||||
&path,
|
||||
SstRangeIndexWriterOptions::default(),
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let completed_ranges = completed_ranges.clone();
|
||||
sources.push(Box::pin(try_stream! {
|
||||
let fetch_metrics = ParquetFetchMetrics::default();
|
||||
let mut range_writer = range_writer;
|
||||
while let Some((row_group_id, row_selection)) = selection.pop_first() {
|
||||
let parquet_reader = context.reader_builder().build(context.build_context(
|
||||
row_group_id,
|
||||
Some(row_selection),
|
||||
Some(&fetch_metrics),
|
||||
)).await?;
|
||||
let mut reader = FlatPruneReader::new_with_row_group_reader(
|
||||
context.clone(),
|
||||
FlatRowGroupReader::new(context.clone(), parquet_reader),
|
||||
context.pre_filter_mode().skip_fields(),
|
||||
);
|
||||
while let Some(batch) = reader.next_batch().await? {
|
||||
if let Some(writer) = &mut range_writer {
|
||||
writer.write(row_group_id as u32, &batch).await?;
|
||||
}
|
||||
yield batch;
|
||||
}
|
||||
}
|
||||
if let Some(writer) = range_writer {
|
||||
writer.finish().await?;
|
||||
file_operation(IndexFileType::Range, "build", "success");
|
||||
completed_ranges.lock().unwrap().insert(file_id);
|
||||
}
|
||||
}));
|
||||
}
|
||||
let schema = schema.context(UnexpectedSnafu {
|
||||
reason: "series-index bucket has no readable SST",
|
||||
})?;
|
||||
let merged: BoxedRecordBatchStream = if sources.len() == 1 {
|
||||
sources.pop().context(UnexpectedSnafu {
|
||||
reason: "series-index source disappeared",
|
||||
})?
|
||||
} else {
|
||||
Box::pin(
|
||||
FlatMergeReader::new(schema, sources, 8192, None)
|
||||
.await?
|
||||
.into_stream(),
|
||||
)
|
||||
};
|
||||
// TODO: Deduplicate update-mode rows before series indexes are used by queries.
|
||||
let mut visible = merged;
|
||||
let path = series_index_path(region.region_id, entry.index_uuid);
|
||||
let mut writer = SeriesIndexWriter::try_new(
|
||||
region.metadata(),
|
||||
store.clone(),
|
||||
&path,
|
||||
SeriesIndexWriterOptions::default(),
|
||||
Some(series_metadata(entry)?),
|
||||
)
|
||||
.await?;
|
||||
while let Some(batch) = visible.try_next().await? {
|
||||
writer.write(&batch).await?;
|
||||
}
|
||||
let range_indexes = std::mem::take(&mut *completed_ranges.lock().unwrap());
|
||||
if range_indexes.len() != expected_ranges {
|
||||
return UnexpectedSnafu {
|
||||
reason: "not all combined range-index builds completed",
|
||||
}
|
||||
.fail();
|
||||
}
|
||||
writer.finish().await?;
|
||||
file_operation(IndexFileType::Series, "build", "success");
|
||||
Ok((
|
||||
SeriesIndexFileHandle::new(region.region_id, entry.clone(), purger.clone()),
|
||||
range_indexes,
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use object_store::layers::mock::{self, MockLayerBuilder, oio};
|
||||
use object_store::services::Memory;
|
||||
use store_api::region_engine::RegionEngine;
|
||||
|
||||
use super::*;
|
||||
use crate::series_index::bucket::{group_files_into_series_buckets, plan_series_indexes};
|
||||
use crate::series_index::purger::series_index_channel;
|
||||
use crate::series_index::tests::prepare_region;
|
||||
use crate::test_util::TestEnv;
|
||||
|
||||
struct FailingSeriesWriter(oio::Writer);
|
||||
|
||||
impl mock::Write for FailingSeriesWriter {
|
||||
async fn write(&mut self, buffer: mock::Buffer) -> mock::Result<()> {
|
||||
self.0.write(buffer).await
|
||||
}
|
||||
|
||||
async fn close(&mut self) -> mock::Result<mock::Metadata> {
|
||||
Err(mock::Error::new(
|
||||
mock::ErrorKind::Unexpected,
|
||||
"injected series write failure",
|
||||
))
|
||||
}
|
||||
|
||||
async fn abort(&mut self) -> mock::Result<()> {
|
||||
self.0.abort().await
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::success(false)]
|
||||
#[case::series_failure(true)]
|
||||
#[tokio::test]
|
||||
async fn test_build_indexes_without_publication(#[case] fail_series: bool) {
|
||||
let mut env = TestEnv::with_prefix("series-builder").await;
|
||||
let (engine, region) = prepare_region(&mut env).await;
|
||||
let version = region.version();
|
||||
let files = version
|
||||
.ssts
|
||||
.levels()
|
||||
.iter()
|
||||
.flat_map(|level| level.files())
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let store = ObjectStore::new(Memory::default()).unwrap();
|
||||
let (purger, mut receiver) = series_index_channel(store.clone());
|
||||
// Build one range separately, then reuse it during the combined build.
|
||||
let existing = build_range_index(&store, ®ion, &version, files[0].clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let existing_path = range_index_path(region.region_id, existing);
|
||||
let existing_bytes = store.read(&existing_path).await.unwrap().to_bytes();
|
||||
let missing = files
|
||||
.iter()
|
||||
.map(|file| file.file_id().file_id())
|
||||
.filter(|id| *id != existing)
|
||||
.collect::<HashSet<_>>();
|
||||
let mut plan = plan_series_indexes(
|
||||
group_files_into_series_buckets(&files, 100),
|
||||
Default::default(),
|
||||
None,
|
||||
0,
|
||||
);
|
||||
assert_eq!(plan.builds.len(), 1);
|
||||
let (bucket, entry) = plan.builds.pop().unwrap();
|
||||
let layer = MockLayerBuilder::default()
|
||||
.writer_factory(Arc::new(move |path, _, inner| -> oio::Writer {
|
||||
if fail_series && path.contains("/series/") {
|
||||
Box::new(FailingSeriesWriter(inner))
|
||||
} else {
|
||||
inner
|
||||
}
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let result = build_series_index(
|
||||
&store.clone().layer(layer),
|
||||
®ion,
|
||||
&version,
|
||||
&bucket,
|
||||
&entry,
|
||||
&missing,
|
||||
&purger,
|
||||
)
|
||||
.await;
|
||||
if fail_series {
|
||||
assert!(result.is_err());
|
||||
} else {
|
||||
let (handle, built_ranges) = result.unwrap();
|
||||
assert_eq!(built_ranges, missing);
|
||||
assert_eq!(handle.entry(), &entry);
|
||||
assert!(
|
||||
store
|
||||
.exists(&series_index_path(region.region_id, entry.index_uuid))
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
// A failed series build must retain completed companion range indexes.
|
||||
for file in &files {
|
||||
assert!(
|
||||
store
|
||||
.exists(&range_index_path(
|
||||
region.region_id,
|
||||
file.file_id().file_id()
|
||||
))
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
store.read(&existing_path).await.unwrap().to_bytes(),
|
||||
existing_bytes
|
||||
);
|
||||
assert!(region.series_index_version().range_indexes.is_empty());
|
||||
assert!(region.series_index_version().series_indexes.is_empty());
|
||||
assert!(receiver.try_recv().is_err());
|
||||
engine.stop().await.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,8 @@ use crate::series_index::purger::IndexFilePurger;
|
||||
use crate::series_index::version::{
|
||||
SeriesIndexFileHandle, SeriesIndexVersion, SeriesIndexVersionControl,
|
||||
};
|
||||
pub(crate) use crate::sst::range_index::range_index_path;
|
||||
|
||||
const SERIES_DIR: &str = "series";
|
||||
const RANGE_CATALOG: &str = "range-index.json";
|
||||
const SERIES_CATALOG: &str = "series-index.json";
|
||||
@@ -135,9 +137,9 @@ pub(crate) async fn load_version_control(
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
// TODO: Handle catalog entries whose index files are missing from storage.
|
||||
let version = SeriesIndexVersion {
|
||||
range_indexes: range.indexes.into_iter().collect(),
|
||||
series_indexes: series
|
||||
let version = SeriesIndexVersion::new(
|
||||
range.indexes.into_iter().collect(),
|
||||
series
|
||||
.indexes
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
@@ -147,7 +149,7 @@ pub(crate) async fn load_version_control(
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
);
|
||||
let control = SeriesIndexVersionControl::default();
|
||||
control.publish(std::sync::Arc::new(version));
|
||||
control
|
||||
@@ -164,8 +166,9 @@ mod tests {
|
||||
use store_api::storage::{FileId, RegionId};
|
||||
|
||||
use crate::series_index::catalog::{
|
||||
SeriesIndexCatalog, SeriesIndexEntry, load_catalog, load_version_control,
|
||||
series_catalog_path, series_metadata, store_catalog,
|
||||
RangeIndexCatalog, SeriesIndexCatalog, SeriesIndexEntry, load_catalog,
|
||||
load_version_control, range_catalog_path, series_catalog_path, series_metadata,
|
||||
store_catalog,
|
||||
};
|
||||
use crate::series_index::purger::series_index_channel;
|
||||
|
||||
@@ -194,31 +197,54 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_catalog_returns_none_on_error() {
|
||||
async fn test_load_catalog_defaults_on_missing_invalid_or_unreadable_catalog() {
|
||||
let store = ObjectStore::new(Memory::default()).unwrap();
|
||||
let path = series_catalog_path(RegionId::new(1, 1));
|
||||
// Missing catalog.
|
||||
let region_id = RegionId::new(1, 1);
|
||||
let (purger, _receiver) = series_index_channel(store.clone());
|
||||
assert!(
|
||||
load_catalog::<SeriesIndexCatalog>(&store, &path)
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
store.write(&path, "invalid").await.unwrap();
|
||||
assert!(
|
||||
load_catalog::<SeriesIndexCatalog>(&store, &path)
|
||||
load_catalog::<SeriesIndexCatalog>(&store, &series_catalog_path(region_id))
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
let control = load_version_control(&store, region_id, &purger).await;
|
||||
assert!(control.current().range_indexes.is_empty());
|
||||
assert!(control.current().series_indexes.is_empty());
|
||||
let file_id = FileId::random();
|
||||
store
|
||||
.write(
|
||||
&range_catalog_path(region_id),
|
||||
serde_json::to_vec(&RangeIndexCatalog {
|
||||
indexes: vec![file_id],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.write(&series_catalog_path(region_id), "invalid")
|
||||
.await
|
||||
.unwrap();
|
||||
let control = load_version_control(&store, region_id, &purger).await;
|
||||
assert!(control.current().range_indexes.contains(&file_id));
|
||||
assert!(control.current().series_indexes.is_empty());
|
||||
let layer = MockLayerBuilder::default()
|
||||
.reader_factory(Arc::new(|_, _, _| Box::new(FailingCatalogReader)))
|
||||
.build()
|
||||
.unwrap();
|
||||
let store = store.layer(layer);
|
||||
assert!(
|
||||
load_catalog::<SeriesIndexCatalog>(&store, &path)
|
||||
load_catalog::<SeriesIndexCatalog>(&store, &series_catalog_path(region_id))
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
let store = store.layer(layer);
|
||||
assert!(
|
||||
load_catalog::<RangeIndexCatalog>(&store, &range_catalog_path(region_id))
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
let control = load_version_control(&store, region_id, &purger).await;
|
||||
assert!(control.current().range_indexes.is_empty());
|
||||
assert!(control.current().series_indexes.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -248,6 +274,12 @@ mod tests {
|
||||
.current();
|
||||
assert!(current.range_indexes.is_empty());
|
||||
assert_eq!(&entry, current.series_indexes[&entry.index_uuid].entry());
|
||||
assert_eq!(current.index_buckets.len(), 1);
|
||||
let bucket = ¤t.index_buckets[&entry.bucket_start];
|
||||
assert_eq!(bucket.start, entry.bucket_start);
|
||||
assert_eq!(bucket.end, entry.bucket_end);
|
||||
assert_eq!(bucket.index_ids.as_slice(), &[entry.index_uuid]);
|
||||
assert_eq!(bucket.max_file_sequence, entry.max_file_sequence);
|
||||
|
||||
let metadata = series_metadata(&entry).unwrap();
|
||||
let decoded: SeriesIndexEntry =
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared fixtures for series-index construction tests.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use api::v1::helper::row;
|
||||
use api::v1::value::ValueData;
|
||||
use api::v1::{ColumnDataType, Rows, SemanticType, WriteHint};
|
||||
use store_api::codec::PrimaryKeyEncoding;
|
||||
use store_api::metric_engine_consts::PRIMARY_KEY_ENCODING;
|
||||
use store_api::region_engine::RegionEngine;
|
||||
use store_api::region_request::{RegionPutRequest, RegionRequest};
|
||||
use store_api::storage::RegionId;
|
||||
use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME;
|
||||
|
||||
use crate::config::MitoConfig;
|
||||
use crate::engine::MitoEngine;
|
||||
use crate::region::MitoRegionRef;
|
||||
use crate::test_util::sst_util::{new_sparse_primary_key, sst_region_metadata_with_encoding};
|
||||
use crate::test_util::{CreateRequestBuilder, TestEnv, flush_region, rows_schema};
|
||||
|
||||
/// Builds real sparse SSTs; background maintenance is disabled so tests control publication.
|
||||
pub(super) async fn prepare_region(env: &mut TestEnv) -> (MitoEngine, MitoRegionRef) {
|
||||
prepare_region_with_timestamps(env, &[1000, 2000, 3000, 4000]).await
|
||||
}
|
||||
|
||||
async fn prepare_region_with_timestamps(
|
||||
env: &mut TestEnv,
|
||||
timestamps: &[i64],
|
||||
) -> (MitoEngine, MitoRegionRef) {
|
||||
let engine = env.create_engine(MitoConfig::default()).await;
|
||||
let metadata = Arc::new(sst_region_metadata_with_encoding(
|
||||
PrimaryKeyEncoding::Sparse,
|
||||
));
|
||||
let region_id = RegionId::new(1, 1);
|
||||
let mut request = CreateRequestBuilder::new().build();
|
||||
request.column_metadatas = metadata.column_metadatas.clone();
|
||||
request.primary_key = metadata.primary_key.clone();
|
||||
request
|
||||
.options
|
||||
.insert(PRIMARY_KEY_ENCODING.to_string(), "sparse".to_string());
|
||||
request
|
||||
.options
|
||||
.insert("memtable.type".to_string(), "bulk".to_string());
|
||||
request
|
||||
.options
|
||||
.insert("sst_format".to_string(), "flat".to_string());
|
||||
request
|
||||
.options
|
||||
.insert("compaction.type".to_string(), "twcs".to_string());
|
||||
request.options.insert(
|
||||
"compaction.twcs.time_window".to_string(),
|
||||
"100s".to_string(),
|
||||
);
|
||||
// Keep the source SSTs stable while tests reconcile multiple buckets.
|
||||
request.options.insert(
|
||||
"compaction.twcs.trigger_file_num".to_string(),
|
||||
"100".to_string(),
|
||||
);
|
||||
let full_schema = rows_schema(&request);
|
||||
let mut pk_column = full_schema[0].clone();
|
||||
pk_column.column_name = PRIMARY_KEY_COLUMN_NAME.to_string();
|
||||
pk_column.datatype = ColumnDataType::Binary.into();
|
||||
pk_column.semantic_type = SemanticType::Tag.into();
|
||||
let schema = vec![pk_column, full_schema[5].clone(), full_schema[4].clone()];
|
||||
engine
|
||||
.handle_request(region_id, RegionRequest::Create(request))
|
||||
.await
|
||||
.unwrap();
|
||||
for &ts in timestamps {
|
||||
engine
|
||||
.handle_request(
|
||||
region_id,
|
||||
RegionRequest::Put(RegionPutRequest {
|
||||
rows: Rows {
|
||||
schema: schema.clone(),
|
||||
rows: vec![row(vec![
|
||||
ValueData::BinaryValue(new_sparse_primary_key(
|
||||
&["a", "x"],
|
||||
&metadata,
|
||||
10,
|
||||
0,
|
||||
)),
|
||||
ValueData::TimestampMillisecondValue(ts),
|
||||
ValueData::U64Value(1),
|
||||
])],
|
||||
},
|
||||
hint: Some(WriteHint {
|
||||
primary_key_encoding: api::v1::PrimaryKeyEncoding::Sparse.into(),
|
||||
}),
|
||||
partition_expr_version: None,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
flush_region(&engine, region_id, None).await;
|
||||
}
|
||||
let region = engine.get_region(region_id).unwrap();
|
||||
(engine, region)
|
||||
}
|
||||
@@ -14,13 +14,16 @@
|
||||
|
||||
//! Immutable index snapshots and aggregate series-file handles.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::fmt::{self, Debug, Formatter};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use common_time::Timestamp;
|
||||
use smallvec::smallvec;
|
||||
use store_api::storage::{FileId, RegionId};
|
||||
|
||||
use crate::series_index::bucket::IndexBucket;
|
||||
use crate::series_index::catalog::SeriesIndexEntry;
|
||||
use crate::series_index::purger::{IndexFilePurger, PurgeRequest};
|
||||
use crate::sst::file::RegionFileId;
|
||||
@@ -85,11 +88,37 @@ impl Drop for SeriesIndexFileHandleInner {
|
||||
/// Immutable series-index snapshot for one region.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct SeriesIndexVersion {
|
||||
/// Range indexes for visible SSTs; reconciliation removes IDs absent from its SST snapshot.
|
||||
/// Physical deletion is independently handled by the SST file purger.
|
||||
pub(crate) range_indexes: HashSet<FileId>,
|
||||
pub(crate) series_indexes: HashMap<FileId, SeriesIndexFileHandle>,
|
||||
pub(crate) index_buckets: BTreeMap<Timestamp, IndexBucket>,
|
||||
}
|
||||
|
||||
impl SeriesIndexVersion {
|
||||
/// Restores bucket lookup from immutable index coverage stored in the catalog.
|
||||
pub(crate) fn new(
|
||||
range_indexes: HashSet<FileId>,
|
||||
series_indexes: HashMap<FileId, SeriesIndexFileHandle>,
|
||||
) -> Self {
|
||||
let mut index_buckets = BTreeMap::new();
|
||||
for handle in series_indexes.values() {
|
||||
let entry = handle.entry();
|
||||
IndexBucket {
|
||||
start: entry.bucket_start,
|
||||
end: entry.bucket_end,
|
||||
index_ids: smallvec![entry.index_uuid],
|
||||
max_file_sequence: entry.max_file_sequence,
|
||||
}
|
||||
.insert_into(&mut index_buckets);
|
||||
}
|
||||
Self {
|
||||
range_indexes,
|
||||
series_indexes,
|
||||
index_buckets,
|
||||
}
|
||||
}
|
||||
|
||||
fn mark_all_deleted(&self) {
|
||||
self.series_indexes
|
||||
.values()
|
||||
|
||||
@@ -28,8 +28,6 @@ pub use writer::{
|
||||
};
|
||||
|
||||
pub use crate::sst::range_index::deleter::RangeIndexDeleter;
|
||||
// Used by the upcoming query and index-building integration.
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) use crate::sst::range_index::deleter::range_index_path;
|
||||
|
||||
const ROW_GROUP_ID_COLUMN: &str = "row_group_id";
|
||||
|
||||
Reference in New Issue
Block a user