From 77cb39b5d18d39410e2fcab89fb3954b994b7b83 Mon Sep 17 00:00:00 2001 From: jeremyhi Date: Tue, 10 Nov 2026 12:21:17 +0000 Subject: [PATCH] feat(mito2): record pending WAL tails in region manifests (#10308) * feat(mito2): record pending WAL tails in the region manifest Adds `pending_tails` and `handoff_floors` to the region manifest and its checkpoint, and an optional `pending_tails` / `handoff_floor` pair to `RegionEdit`. An edit without them leaves the records unchanged. A floor is dropped once the flushed entry id reaches it; a tail stays until a commit removes it. Neither is written when empty, so manifests without them encode as before. Opening a region replays its WAL after this node's tails and floor, and WAL releases at open and after a flush stay below this node's tails. Nothing writes these records yet, so every region behaves as before. Signed-off-by: jeremyhi * feat(mito2): hold back reads and region changes while WAL tails are pending A region with pending WAL tails is missing data that only those tails hold. Until they are recovered, scans and memtable and SST stats fail with the retryable `RegionPendingTails` error. Truncate, entering staging, a repartition flush, a close-time flush on request, and ALTERs other than adding nullable columns without a default or changing options are rejected before they change any region state. Compaction pauses. A full-listing GC pass skips the region while it has pending tails, or while a newer manifest may exist or the manifest listing fails. Signed-off-by: jeremyhi * docs(mito2): name the WAL releases that keep pending tails Signed-off-by: jeremyhi * fix(mito2): reject no-op ALTERs of restricted kinds while WAL tails are pending Signed-off-by: jeremyhi * test(mito2): check that reopening restores the pending-tail read gate Signed-off-by: jeremyhi * test(mito2): pin what the pending-tail fixtures and rejections promise Signed-off-by: jeremyhi * refactor(mito2): name the replay start in the open log and share the pending-tail check Signed-off-by: jeremyhi * docs(mito2): state the per-node order of pending tails and who keeps later writes out of them Signed-off-by: jeremyhi * test(mito2): check that a cancelled compaction starts nothing while WAL tails are pending When a compaction cancelled by task id finishes, the queued manual request goes through handle_pending_compaction_request, whose pending-tails check replies Ok(0). The reply is read with try_recv so that a dispatched request fails the test at once instead of hanging it. Signed-off-by: jeremyhi * fix(mito2): drop the pending compaction task state when WAL tails skip it Skipping a pending manual compaction while WAL tails are pending replied Ok(0) but left its task state in place, so task inspection kept listing it as queued and cancelling it by id still succeeded. Clear the pending task state and publish before replying. Signed-off-by: jeremyhi --------- Signed-off-by: jeremyhi --- .../src/bin/query_perf_fixture/direct_sst.rs | 2 + src/cmd/src/datanode/sst_replace.rs | 1 + .../src/region_server/catalog/inspect.rs | 1 + .../region_server/catalog/tests/manifest.rs | 2 + src/metric-engine/src/engine/sync/region.rs | 1 + src/mito2/AGENTS.md | 20 +- src/mito2/src/compaction/compactor.rs | 2 + src/mito2/src/compaction/scheduler.rs | 17 + src/mito2/src/compaction/scheduler/state.rs | 5 + src/mito2/src/compaction/scheduler_test.rs | 109 ++++- src/mito2/src/engine.rs | 28 +- .../src/engine/apply_staging_manifest_test.rs | 1 + src/mito2/src/engine/basic_test.rs | 1 + .../engine/bump_committed_sequence_test.rs | 1 + src/mito2/src/engine/catchup_test.rs | 1 + src/mito2/src/engine/edit_region_test.rs | 4 + src/mito2/src/engine/file_ref_test.rs | 2 + src/mito2/src/engine/flush_test.rs | 1 + src/mito2/src/engine/index_build_test.rs | 1 + src/mito2/src/engine/merge_mode_test.rs | 1 + src/mito2/src/engine/pending_tail_test.rs | 434 ++++++++++++++++++ src/mito2/src/engine/scan_test.rs | 3 + src/mito2/src/engine/staging_test.rs | 6 + src/mito2/src/engine/sync_test.rs | 1 + src/mito2/src/error.rs | 9 + src/mito2/src/flush.rs | 5 + src/mito2/src/gc.rs | 21 +- src/mito2/src/gc/worker_test.rs | 87 ++++ src/mito2/src/manifest/action.rs | 391 +++++++++++++++- src/mito2/src/manifest/manager.rs | 2 + src/mito2/src/manifest/tests/checkpoint.rs | 54 ++- src/mito2/src/region.rs | 3 + src/mito2/src/region/catchup.rs | 1 + src/mito2/src/region/opener.rs | 22 +- src/mito2/src/region/version.rs | 36 +- src/mito2/src/remap_manifest.rs | 4 + src/mito2/src/series_index/task.rs | 1 + src/mito2/src/series_index/tests.rs | 3 + src/mito2/src/sst/file_ref.rs | 3 +- src/mito2/src/sst/index.rs | 1 + src/mito2/src/sst/primary_key.rs | 1 + src/mito2/src/test_util.rs | 1 + src/mito2/src/test_util/version_util.rs | 1 + src/mito2/src/worker/handle_alter.rs | 20 + src/mito2/src/worker/handle_apply_staging.rs | 1 + src/mito2/src/worker/handle_close.rs | 7 + src/mito2/src/worker/handle_copy_region.rs | 1 + src/mito2/src/worker/handle_enter_staging.rs | 4 + src/mito2/src/worker/handle_flush.rs | 17 +- src/mito2/src/worker/handle_manifest.rs | 3 + src/mito2/src/worker/handle_rebuild_index.rs | 1 + src/mito2/src/worker/handle_truncate.rs | 5 + tests/compatibility/README.md | 3 + .../manifest/pending_tails/README.md | 27 ++ .../manifest/pending_tails/checkpoint.json | 1 + .../manifest/pending_tails/delta.json | 1 + 56 files changed, 1362 insertions(+), 20 deletions(-) create mode 100644 src/mito2/src/engine/pending_tail_test.rs create mode 100644 tests/compatibility/manifest/pending_tails/README.md create mode 100644 tests/compatibility/manifest/pending_tails/checkpoint.json create mode 100644 tests/compatibility/manifest/pending_tails/delta.json diff --git a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs index 0b09d1634ca..fc004c2af13 100644 --- a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs +++ b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs @@ -512,6 +512,8 @@ pub(super) async fn run_direct_sst(args: DirectArgs) { compaction_time_window: None, sst_format: format, append_mode: table.append_mode, + pending_tails: Default::default(), + handoff_floors: Default::default(), }; let checkpoint = RegionCheckpoint { last_version: args.checkpoint_version, diff --git a/src/cmd/src/datanode/sst_replace.rs b/src/cmd/src/datanode/sst_replace.rs index df13fcd6f25..d72a12a6002 100644 --- a/src/cmd/src/datanode/sst_replace.rs +++ b/src/cmd/src/datanode/sst_replace.rs @@ -208,6 +208,7 @@ impl SstReplaceCommand { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; let version = manifest_manager .update( diff --git a/src/datanode/src/region_server/catalog/inspect.rs b/src/datanode/src/region_server/catalog/inspect.rs index da3ebdb130b..875d7271b37 100644 --- a/src/datanode/src/region_server/catalog/inspect.rs +++ b/src/datanode/src/region_server/catalog/inspect.rs @@ -426,6 +426,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/datanode/src/region_server/catalog/tests/manifest.rs b/src/datanode/src/region_server/catalog/tests/manifest.rs index 91b9f757a3d..81288b506a6 100644 --- a/src/datanode/src/region_server/catalog/tests/manifest.rs +++ b/src/datanode/src/region_server/catalog/tests/manifest.rs @@ -74,6 +74,7 @@ async fn manifest_engine(regions: u32, files_per_region: usize) -> (TestEnv, Mit flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await @@ -189,6 +190,7 @@ async fn test_manifest_referring_region_and_origin_path() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/metric-engine/src/engine/sync/region.rs b/src/metric-engine/src/engine/sync/region.rs index c7b3ac8682c..dbf696db9ec 100644 --- a/src/metric-engine/src/engine/sync/region.rs +++ b/src/metric-engine/src/engine/sync/region.rs @@ -122,6 +122,7 @@ impl MetricEngineInner { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; self.mito .edit_region(target_metadata_region_id, edit) diff --git a/src/mito2/AGENTS.md b/src/mito2/AGENTS.md index 9f12cd750cd..ffd674917a2 100644 --- a/src/mito2/AGENTS.md +++ b/src/mito2/AGENTS.md @@ -82,7 +82,8 @@ pending. DDL cancellation retains its existing scheduling fence and priority. follower replay. Keep it backward compatible. Compaction task state does not belong in the manifest; publication uses ordinary file edits after locked input checks. Shared SSTs require Enterprise capability. Compatibility fixtures: - `tests/compatibility/manifest/table_compaction/`. + `tests/compatibility/manifest/table_compaction/` and + `tests/compatibility/manifest/pending_tails/`. - **SST/Parquet layout** (`sst/`): readers must stay compatible with existing files. - **Series-index disk limit**: tasks estimate usage from entry sizes in open-region snapshots and defer new builds when the shared estimate is full. Cleanup still prunes obsolete @@ -111,8 +112,21 @@ Tests live next to the code as `*_test.rs` (e.g. `src/mito2/src/engine/flush_tes depend on this. Do not change assignment lightly. - Manifest version is monotonic — never reset or skip it. - In regions whose writes require a lease and whose WAL is the object store WAL, - each manifest version has one fixed object name and only one writer can create - it (`manifest/storage.rs`). + each manifest version has one fixed object name, created conditionally. A late + create can succeed again once checkpoint cleanup frees the name, so a commit + checks its result (`ensure_version_not_checkpointed` in `manifest/storage.rs`). +- Pending tails (`RegionManifest::pending_tails`) are WAL ranges a region still has + to recover; the flushed entry id never prunes them. Opening a region replays its + WAL after `RegionManifest::wal_replay_start`. The WAL releases at open and after + a flush stay below this node's tails (`cap_wal_release`); truncate, drop, offline + cleanup and catchup release without that cap. Handoff floors are dropped once the + flushed entry id reaches them, in `RegionManifestBuilder::try_build`. While a + region has pending tails, scans and memtable/SST stats fail with + `RegionPendingTails`, and so do truncate, entering staging, a repartition flush, + a close with `flush_on_close`, and every ALTER other than adding nullable columns + without a default or changing options; compaction pauses. These checks run + before any region state changes. Full-listing GC skips a region that has pending + tails or whose manifest `has_update()` reports newer, or fails to check. - Lock ordering: take the manifest lock before updating `version_control`; the reverse deadlocks against concurrent flush/compaction. - All region I/O runs on tokio workers; never `block_on` inside a worker. diff --git a/src/mito2/src/compaction/compactor.rs b/src/mito2/src/compaction/compactor.rs index d0d2950018b..6cd51dae9a1 100644 --- a/src/mito2/src/compaction/compactor.rs +++ b/src/mito2/src/compaction/compactor.rs @@ -452,6 +452,7 @@ impl CompactionContext { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; let action_list = RegionMetaActionList::with_action(RegionMetaAction::Edit(edit.clone())); @@ -1144,6 +1145,7 @@ mod tests { compaction_time_window: None, flushed_entry_id: None, flushed_sequence: None, + wal_recovery: Default::default(), })), false, ) diff --git a/src/mito2/src/compaction/scheduler.rs b/src/mito2/src/compaction/scheduler.rs index 44886f6ccdb..c7a370df623 100644 --- a/src/mito2/src/compaction/scheduler.rs +++ b/src/mito2/src/compaction/scheduler.rs @@ -577,6 +577,14 @@ impl CompactionScheduler { waiter.send(Ok(0)); return Ok(false); } + if status.has_pending_tails() { + info!( + "Skipping compaction for region {} with pending WAL tails, options: {:?}", + region_id, compact_options + ); + waiter.send(Ok(0)); + return Ok(false); + } if status.is_compacting() { match trigger { @@ -644,6 +652,12 @@ impl CompactionScheduler { let Some(pending_request) = std::mem::take(&mut status.pending_request) else { return false; }; + if status.has_pending_tails() { + status.pending_task_state = None; + status.publish_state(); + pending_request.waiter.send(Ok(0)); + return false; + } let PendingCompaction { options, @@ -737,6 +751,9 @@ impl CompactionScheduler { let Some(status) = self.region_status.get_mut(®ion_id) else { return false; }; + if status.has_pending_tails() { + return false; + } // An external automatic trigger requires one unrestricted follow-up pick. let request = status.new_compaction_request( self.request_sender.clone(), diff --git a/src/mito2/src/compaction/scheduler/state.rs b/src/mito2/src/compaction/scheduler/state.rs index 3e232138610..f0afea33b7e 100644 --- a/src/mito2/src/compaction/scheduler/state.rs +++ b/src/mito2/src/compaction/scheduler/state.rs @@ -483,6 +483,11 @@ impl CompactionStatus { !self.pending_ddl_requests.is_empty() || self.executing_ddl.is_some() } + /// Compaction pauses while the region has WAL tails to recover. + pub(crate) fn has_pending_tails(&self) -> bool { + self.version_control.current().version.has_pending_tails() + } + /// Ends the current cycle without losing region or DDL execution state. pub(crate) fn become_idle(&mut self) { self.active = ActiveCompaction::idle(); diff --git a/src/mito2/src/compaction/scheduler_test.rs b/src/mito2/src/compaction/scheduler_test.rs index a14cf1afa88..a4d310a688f 100644 --- a/src/mito2/src/compaction/scheduler_test.rs +++ b/src/mito2/src/compaction/scheduler_test.rs @@ -41,6 +41,7 @@ use crate::compaction::test_util::new_file_handle; use crate::compaction::twcs::PickClass; use crate::compaction::{CompactionOutput, find_dynamic_options}; use crate::config::MitoConfig; +use crate::engine::pending_tail_test::{pending_tails_edit, tail}; use crate::error::InvalidSchedulerStateSnafu; use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions}; use crate::metrics::COMPACTION_MEMORY_REJECTED; @@ -51,9 +52,9 @@ use crate::schedule::remote_job_scheduler::{RemoteJob, RemoteJobSchedulerRef}; use crate::schedule::scheduler::{Job, Scheduler}; use crate::sst::FormatType; use crate::sst::file::FileHandle; -use crate::test_util::mock_schema_metadata_manager; use crate::test_util::scheduler_util::{SchedulerEnv, VecScheduler}; use crate::test_util::version_util::{VersionControlBuilder, apply_edit}; +use crate::test_util::{mock_schema_metadata_manager, new_noop_file_purger}; struct FailingScheduler; @@ -1193,6 +1194,7 @@ async fn test_pick_result_refreshes_replaced_selected_file() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], stale.file_purger(), @@ -3360,6 +3362,7 @@ async fn test_compaction_tasks_partial_memory_admission() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], builder.file_purger(), @@ -3664,6 +3667,110 @@ async fn test_compaction_tasks_linear_local_publication(#[case] reject_after_fir assert!(!scheduler.is_compacting(region_id)); } +#[tokio::test] +async fn test_finishing_compaction_with_pending_tails_starts_nothing() { + let env = SchedulerEnv::new().await; + let (tx, _rx) = mpsc::channel(4); + let mut scheduler = env.mock_compaction_scheduler(tx); + let builder = VersionControlBuilder::new(); + let version_control = Arc::new(builder.build()); + let region_id = builder.region_id(); + let manifest_ctx = env + .mock_manifest_context(version_control.current().version.metadata.clone()) + .await; + let (schema_metadata_manager, _kv_backend) = mock_schema_metadata_manager(); + let (waiter_tx, mut waiter_rx) = oneshot::channel(); + let mut status = + CompactionStatus::for_test(region_id, version_control.clone(), env.access_layer.clone()); + status.start_local_task(); + status.set_pending_request(PendingCompaction { + options: Options::Regular(Default::default()), + waiter: OptionOutputTx::from(waiter_tx), + max_parallelism: 1, + time_range: None, + }); + scheduler.region_status.insert(region_id, status); + let pending_id = scheduler.inspect_tasks(&MetadataFilter::default(), None)[0] + .task_id + .clone(); + version_control.apply_edit( + Some(pending_tails_edit(vec![tail(1, 0, 1)])), + &[], + builder.file_purger(), + ); + + scheduler + .on_compaction_finished(region_id, &manifest_ctx, schema_metadata_manager, true) + .await; + + assert_eq!(0, waiter_rx.try_recv().unwrap().unwrap()); + assert!(!scheduler.is_compacting(region_id)); + assert!( + scheduler + .inspect_tasks(&MetadataFilter::default(), None) + .is_empty() + ); + assert_eq!( + scheduler.cancel_task(region_id, &pending_id), + CancelCompactionResult::NotFound + ); +} + +#[tokio::test] +async fn test_cancelled_compaction_with_pending_tails_starts_nothing() { + let env = SchedulerEnv::new().await; + let (tx, mut rx) = mpsc::channel(8); + let mut scheduler = env.mock_compaction_scheduler(tx); + let version = compactable_version(); + let region_id = version.region_id(); + let (finished, manifest, schemas) = + begin_pick_result(&env, &mut scheduler, &mut rx, &version).await; + let active_id = scheduler.inspect_tasks(&MetadataFilter::default(), None)[0] + .task_id + .clone(); + let (sender, mut receiver) = oneshot::channel(); + scheduler + .schedule_manual_compaction( + Options::Regular(Default::default()), + &version, + &env.access_layer, + sender.into(), + &manifest, + schemas.clone(), + 1, + None, + ) + .unwrap(); + let pending_id = scheduler.inspect_tasks(&MetadataFilter::default(), None)[1] + .task_id + .clone(); + assert_eq!( + scheduler.cancel_task(region_id, &active_id), + CancelCompactionResult::CancelIssued + ); + version.apply_edit( + Some(pending_tails_edit(vec![tail(1, 0, 1)])), + &[], + new_noop_file_purger(), + ); + + scheduler + .handle_compaction_pick_finished(finished, &manifest, schemas) + .await; + + assert_eq!(0, receiver.try_recv().unwrap().unwrap()); + assert!(!scheduler.is_compacting(region_id)); + assert!( + scheduler + .inspect_tasks(&MetadataFilter::default(), None) + .is_empty() + ); + assert_eq!( + scheduler.cancel_task(region_id, &pending_id), + CancelCompactionResult::NotFound + ); +} + /// Pending task identity survives promotion, but replacement invalidates the old identity. #[tokio::test] async fn test_cancel_compaction_task_identity_and_pending() { diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs index 7c426c5e63b..8ff2b0f6ce0 100644 --- a/src/mito2/src/engine.rs +++ b/src/mito2/src/engine.rs @@ -63,6 +63,8 @@ mod open_test; #[cfg(test)] mod parallel_test; #[cfg(test)] +pub(crate) mod pending_tail_test; +#[cfg(test)] mod projection_test; #[cfg(test)] mod prune_test; @@ -159,7 +161,7 @@ use crate::error::{ #[cfg(feature = "enterprise")] use crate::extension::BoxedExtensionRangeProviderFactory; use crate::gc::GcLimiterRef; -use crate::manifest::action::RegionEdit; +use crate::manifest::action::{RegionEdit, WalRecoveryEdit}; use crate::memtable::MemtableStats; use crate::metrics::{ HANDLE_REQUEST_ELAPSED, SCAN_MEMORY_EXHAUSTED_TOTAL, SCAN_MEMORY_USAGE_BYTES, @@ -586,6 +588,7 @@ impl MitoEngine { .context(RegionNotFoundSnafu { region_id })?; let version = region.version(); + version.ensure_no_pending_tails()?; let memtable_stats = version .memtables .list_memtables() @@ -841,6 +844,7 @@ fn is_valid_region_edit(edit: &RegionEdit) -> bool { .. } ) + && edit.wal_recovery == WalRecoveryEdit::default() } /// Inner struct of [MitoEngine]. @@ -1172,6 +1176,7 @@ impl EngineInner { }); let version_data = region.version_control.current(); let version = version_data.version; + version.ensure_no_pending_tails()?; if request.snapshot_on_scan && request.memtable_max_sequence.is_none() { request.memtable_max_sequence = Some(version_data.committed_sequence); @@ -1889,6 +1894,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(is_valid_region_edit(&edit)); @@ -1901,6 +1907,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(!is_valid_region_edit(&edit)); @@ -1913,6 +1920,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(is_valid_region_edit(&edit)); @@ -1925,6 +1933,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(is_valid_region_edit(&edit)); @@ -1937,6 +1946,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(!is_valid_region_edit(&edit)); let edit = RegionEdit { @@ -1947,6 +1957,7 @@ mod tests { flushed_entry_id: Some(1), flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(!is_valid_region_edit(&edit)); let edit = RegionEdit { @@ -1957,6 +1968,21 @@ mod tests { flushed_entry_id: None, flushed_sequence: Some(1), committed_sequence: None, + wal_recovery: Default::default(), + }; + assert!(!is_valid_region_edit(&edit)); + let edit = RegionEdit { + files_to_add: vec![FileMeta::default()], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + wal_recovery: WalRecoveryEdit { + pending_tails: Some(vec![]), + handoff_floor: None, + }, }; assert!(!is_valid_region_edit(&edit)); } diff --git a/src/mito2/src/engine/apply_staging_manifest_test.rs b/src/mito2/src/engine/apply_staging_manifest_test.rs index e6ac2bc24b4..be6d651b48e 100644 --- a/src/mito2/src/engine/apply_staging_manifest_test.rs +++ b/src/mito2/src/engine/apply_staging_manifest_test.rs @@ -861,6 +861,7 @@ async fn test_apply_staging_manifest_change_edit_different_columns_fails_with_fo flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), ]), true, diff --git a/src/mito2/src/engine/basic_test.rs b/src/mito2/src/engine/basic_test.rs index ebc9647adc8..72f4949b12d 100644 --- a/src/mito2/src/engine/basic_test.rs +++ b/src/mito2/src/engine/basic_test.rs @@ -2216,6 +2216,7 @@ async fn add_manifest_scan_region( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; region .manifest_ctx diff --git a/src/mito2/src/engine/bump_committed_sequence_test.rs b/src/mito2/src/engine/bump_committed_sequence_test.rs index 23a5af88650..b5b7c23b8e4 100644 --- a/src/mito2/src/engine/bump_committed_sequence_test.rs +++ b/src/mito2/src/engine/bump_committed_sequence_test.rs @@ -84,6 +84,7 @@ async fn test_bump_committed_sequence_with_format(flat_format: bool) { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/catchup_test.rs b/src/mito2/src/engine/catchup_test.rs index b470db5e12a..2e2cdad014e 100644 --- a/src/mito2/src/engine/catchup_test.rs +++ b/src/mito2/src/engine/catchup_test.rs @@ -270,6 +270,7 @@ async fn test_catchup_preserves_region_edit_sequence( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/edit_region_test.rs b/src/mito2/src/engine/edit_region_test.rs index 68db3def4cf..f58daa55d34 100644 --- a/src/mito2/src/engine/edit_region_test.rs +++ b/src/mito2/src/engine/edit_region_test.rs @@ -137,6 +137,7 @@ async fn test_edit_region_schedule_compaction_with_format( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; engine .edit_region(region.region_id, new_edit(&[0, 10, 50, 80])) @@ -266,6 +267,7 @@ async fn test_edit_region_fill_cache( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; engine.edit_region(region.region_id, edit).await.unwrap(); @@ -775,6 +777,7 @@ fn test_region_edit(region_id: RegionId, file_id: FileId) -> RegionEdit { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), } } @@ -864,6 +867,7 @@ async fn test_edit_region_concurrently_with_format(flat_format: bool) { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; engine .edit_region(self.region.region_id, edit) diff --git a/src/mito2/src/engine/file_ref_test.rs b/src/mito2/src/engine/file_ref_test.rs index 3979c14553c..3bc1829c86b 100644 --- a/src/mito2/src/engine/file_ref_test.rs +++ b/src/mito2/src/engine/file_ref_test.rs @@ -114,6 +114,7 @@ async fn get_snapshot_preserves_related_region_old_index_readers() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })), false, ) @@ -219,6 +220,7 @@ async fn get_snapshot_preserves_old_index_readers_until_release( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })), false, ) diff --git a/src/mito2/src/engine/flush_test.rs b/src/mito2/src/engine/flush_test.rs index 399b5857700..8435a38c4a3 100644 --- a/src/mito2/src/engine/flush_test.rs +++ b/src/mito2/src/engine/flush_test.rs @@ -426,6 +426,7 @@ async fn test_ingested_parquet_bytes() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/index_build_test.rs b/src/mito2/src/engine/index_build_test.rs index 2877ac109d9..36ae38330ae 100644 --- a/src/mito2/src/engine/index_build_test.rs +++ b/src/mito2/src/engine/index_build_test.rs @@ -700,6 +700,7 @@ async fn test_index_build_uses_physical_file_region_and_logical_manifest_region( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/merge_mode_test.rs b/src/mito2/src/engine/merge_mode_test.rs index f5dd078103d..b628e2bab4e 100644 --- a/src/mito2/src/engine/merge_mode_test.rs +++ b/src/mito2/src/engine/merge_mode_test.rs @@ -127,6 +127,7 @@ async fn test_admitted_sst_compaction_preserves_intermediate_memtable( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/pending_tail_test.rs b/src/mito2/src/engine/pending_tail_test.rs new file mode 100644 index 00000000000..5f2627f0633 --- /dev/null +++ b/src/mito2/src/engine/pending_tail_test.rs @@ -0,0 +1,434 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use api::v1::region::StrictWindow; +use api::v1::region::compact_request::Options; +use api::v1::{Rows, SemanticType}; +use common_error::ext::{BoxedError, ErrorExt, RetryHint}; +use common_error::status_code::StatusCode; +use common_time::Timestamp; +use datatypes::prelude::ConcreteDataType; +use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema}; +use rstest::rstest; +use store_api::logstore::provider::RaftEngineProvider; +use store_api::metadata::ColumnMetadata; +use store_api::region_engine::RegionEngine; +use store_api::region_request::{ + AddColumn, AlterKind, EnterStagingRequest, RegionAlterRequest, RegionCloseRequest, + RegionCompactRequest, RegionCreateRequest, RegionDropRequest, RegionFlushReason, + RegionFlushRequest, RegionRequest, RegionTruncateRequest, SetRegionOption, + StagingPartitionDirective, +}; +use store_api::storage::{RegionId, ScanRequest}; + +use crate::config::MitoConfig; +use crate::engine::MitoEngine; +use crate::error::Error; +use crate::manifest::action::{ + HandoffFloor, PendingTail, RegionEdit, RegionMetaAction, RegionMetaActionList, UptoKind, + WalRecoveryEdit, +}; +use crate::region::RegionLeaderState; +use crate::test_util::{ + CreateRequestBuilder, LogStoreImpl, TestEnv, build_rows, flush_region, put_rows, reopen_region, + rows_schema, +}; +use crate::wal::EntryId; + +const NODE_ID: u64 = 1; + +pub(crate) fn tail(node_id: u64, after: EntryId, upto: EntryId) -> PendingTail { + PendingTail { + node_id, + after, + upto, + upto_kind: UptoKind::Reported, + } +} + +/// Creates a region and writes one row in each of WAL entries `1..=num_entries`. +async fn create_region_with_entries( + engine: &MitoEngine, + region_id: RegionId, + num_entries: usize, +) -> RegionCreateRequest { + let request = CreateRequestBuilder::new().build(); + let schema = rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request.clone())) + .await + .unwrap(); + for i in 0..num_entries { + let rows = Rows { + schema: schema.clone(), + rows: build_rows(i, i + 1), + }; + put_rows(engine, region_id, rows).await; + } + request +} + +pub(crate) fn pending_tails_edit(pending_tails: Vec) -> RegionEdit { + RegionEdit { + files_to_add: vec![], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + wal_recovery: WalRecoveryEdit { + pending_tails: Some(pending_tails), + handoff_floor: None, + }, + } +} + +async fn commit_edit(engine: &MitoEngine, region_id: RegionId, edit: RegionEdit) { + let region = engine.get_region(region_id).unwrap(); + region + .manifest_ctx + .update_manifest( + RegionLeaderState::Writable, + RegionMetaActionList::with_action(RegionMetaAction::Edit(edit.clone())), + false, + ) + .await + .unwrap(); + region + .version_control + .apply_edit(Some(edit), &[], region.file_purger.clone()); +} + +pub(crate) async fn set_pending_tails( + engine: &MitoEngine, + region_id: RegionId, + pending_tails: Vec, +) { + commit_edit(engine, region_id, pending_tails_edit(pending_tails)).await; +} + +async fn reopen(engine: &MitoEngine, region_id: RegionId, request: RegionCreateRequest) { + reopen_region(engine, region_id, request.table_dir, true, request.options).await; +} + +fn wal_span(env: &TestEnv, region_id: RegionId) -> (Option, Option) { + let Some(LogStoreImpl::RaftEngine(log_store)) = env.get_log_store() else { + unreachable!() + }; + log_store.span(&RaftEngineProvider::new(region_id.into())) +} + +#[rstest] +#[case::own_tail(vec![tail(NODE_ID, 0, 2)], None, 1)] +#[case::other_node_tail(vec![tail(NODE_ID + 1, 0, 2)], None, 3)] +#[case::own_floor(vec![], Some(NODE_ID), 1)] +#[case::other_node_floor(vec![], Some(NODE_ID + 1), 3)] +#[tokio::test] +async fn test_open_replays_wal_after_own_tails_and_floor( + #[case] pending_tails: Vec, + #[case] floor_node_id: Option, + #[case] replayed_rows: u64, +) { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let request = create_region_with_entries(&engine, region_id, 3).await; + let has_pending_tails = !pending_tails.is_empty(); + let mut edit = pending_tails_edit(pending_tails); + edit.wal_recovery.handoff_floor = floor_node_id.map(|node_id| HandoffFloor { + node_id, + entry_id: 2, + }); + commit_edit(&engine, region_id, edit).await; + + reopen(&engine, region_id, request).await; + + let region = engine.get_region(region_id).unwrap(); + assert_eq!(replayed_rows, region.version().memtables.num_rows()); + let scan = engine.scanner(region_id, ScanRequest::default()).await; + assert_eq!( + has_pending_tails, + matches!(scan, Err(Error::RegionPendingTails { .. })) + ); +} + +#[tokio::test] +async fn test_open_keeps_last_entry_id_at_flushed_entry_id() { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let request = create_region_with_entries(&engine, region_id, 3).await; + flush_region(&engine, region_id, None).await; + let listed = PendingTail { + node_id: NODE_ID, + after: 3, + upto: 100, + upto_kind: UptoKind::Listed, + }; + set_pending_tails(&engine, region_id, vec![listed]).await; + + reopen(&engine, region_id, request).await; + + let region = engine.get_region(region_id).unwrap(); + assert_eq!(3, region.version_control.current().last_entry_id); +} + +#[tokio::test] +async fn test_flush_keeps_wal_after_own_tails() { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + create_region_with_entries(&engine, region_id, 3).await; + set_pending_tails(&engine, region_id, vec![tail(NODE_ID, 1, 2)]).await; + + flush_region(&engine, region_id, None).await; + + assert_eq!((Some(2), Some(3)), wal_span(&env, region_id)); +} + +#[tokio::test] +async fn test_open_keeps_wal_after_own_tails() { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let request = create_region_with_entries(&engine, region_id, 3).await; + set_pending_tails(&engine, region_id, vec![tail(NODE_ID, 1, 3)]).await; + + reopen(&engine, region_id, request).await; + + assert_eq!((Some(2), Some(3)), wal_span(&env, region_id)); +} + +#[tokio::test] +async fn test_drop_releases_wal_with_pending_tails() { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + create_region_with_entries(&engine, region_id, 3).await; + set_pending_tails(&engine, region_id, vec![tail(NODE_ID, 0, 3)]).await; + + engine + .handle_request( + region_id, + RegionRequest::Drop(RegionDropRequest { + fast_path: false, + force: false, + partial_drop: false, + }), + ) + .await + .unwrap(); + + assert_eq!((None, None), wal_span(&env, region_id)); +} + +/// Creates a region with rows in WAL entries `1..=num_entries` and one pending tail. +async fn region_with_pending_tails(num_entries: usize) -> (TestEnv, MitoEngine, RegionId) { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + create_region_with_entries(&engine, region_id, num_entries).await; + set_pending_tails(&engine, region_id, vec![tail(NODE_ID, 0, 1)]).await; + (env, engine, region_id) +} + +fn assert_pending_tails_error(err: &BoxedError) { + let err = err.as_any().downcast_ref::(); + assert!( + matches!(err, Some(Error::RegionPendingTails { .. })), + "{err:?}" + ); +} + +fn add_field(default_constraint: Option) -> AlterKind { + let column_schema = ColumnSchema::new("field_1", ConcreteDataType::float64_datatype(), true) + .with_default_constraint(default_constraint) + .unwrap(); + AlterKind::AddColumns { + columns: vec![AddColumn { + column_metadata: ColumnMetadata { + column_schema, + semantic_type: SemanticType::Field, + column_id: 10, + }, + location: None, + }], + } +} + +#[tokio::test] +async fn test_scan_rejects_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + + let Err(err) = engine.scanner(region_id, ScanRequest::default()).await else { + panic!("scan succeeded with pending tails"); + }; + + assert!(matches!(err, Error::RegionPendingTails { .. }), "{err:?}"); + assert_eq!(StatusCode::RegionNotReady, err.status_code()); + assert_eq!(RetryHint::Retryable, err.retry_hint()); +} + +#[tokio::test] +async fn test_memtable_and_sst_stats_reject_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + + let result = engine.find_memtable_and_sst_stats(region_id); + + assert!(matches!(result, Err(Error::RegionPendingTails { .. }))); +} + +#[tokio::test] +async fn test_manual_compaction_skips_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + flush_region(&engine, region_id, None).await; + let rows = Rows { + schema: rows_schema(&CreateRequestBuilder::new().build()), + rows: build_rows(1, 2), + }; + put_rows(&engine, region_id, rows).await; + flush_region(&engine, region_id, None).await; + + let request = RegionCompactRequest { + options: Options::StrictWindow(StrictWindow { + window_seconds: 3600, + }), + ..Default::default() + }; + engine + .handle_request(region_id, RegionRequest::Compact(request)) + .await + .unwrap(); + + let region = engine.get_region(region_id).unwrap(); + assert_eq!(2, region.version().ssts.owned_num_files(region_id)); +} + +#[rstest] +#[case::all(RegionTruncateRequest::All)] +#[case::unflushed(RegionTruncateRequest::Unflushed)] +#[case::by_time_ranges(RegionTruncateRequest::ByTimeRanges { + time_ranges: vec![(Timestamp::new_millisecond(0), Timestamp::new_millisecond(1000))], +})] +#[tokio::test] +async fn test_truncate_rejects_region_with_pending_tails(#[case] request: RegionTruncateRequest) { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + + let err = engine + .handle_request(region_id, RegionRequest::Truncate(request)) + .await + .unwrap_err(); + + assert_pending_tails_error(&err); + let region = engine.get_region(region_id).unwrap(); + assert_eq!(1, region.version().memtables.num_rows()); +} + +#[rstest] +#[case::nullable_column(add_field(None), true)] +#[case::region_options(AlterKind::SetRegionOptions { + options: vec![SetRegionOption::AppendMode(true)], +}, true)] +#[case::default_null_column(add_field(Some(ColumnDefaultConstraint::null_value())), false)] +#[case::drop_column(AlterKind::DropColumns { + names: vec!["field_0".to_string()], +}, false)] +#[tokio::test] +async fn test_alter_with_pending_tails(#[case] kind: AlterKind, #[case] allowed: bool) { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + let metadata = engine.get_region(region_id).unwrap().metadata(); + + let result = engine + .handle_request(region_id, RegionRequest::Alter(RegionAlterRequest { kind })) + .await; + + if allowed { + result.unwrap(); + } else { + assert_pending_tails_error(&result.unwrap_err()); + let region = engine.get_region(region_id).unwrap(); + assert_eq!(metadata, region.metadata()); + assert_eq!(1, region.version().memtables.num_rows()); + } +} + +#[tokio::test] +async fn test_alter_rejects_no_op_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + let column_metadatas = engine + .get_region(region_id) + .unwrap() + .metadata() + .column_metadatas + .clone(); + + let kind = AlterKind::SyncColumns { column_metadatas }; + let err = engine + .handle_request(region_id, RegionRequest::Alter(RegionAlterRequest { kind })) + .await + .unwrap_err(); + + assert_pending_tails_error(&err); +} + +#[tokio::test] +async fn test_enter_staging_rejects_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + + let request = EnterStagingRequest { + partition_directive: StagingPartitionDirective::RejectAllWrites, + }; + let err = engine + .handle_request(region_id, RegionRequest::EnterStaging(request)) + .await + .unwrap_err(); + + assert_pending_tails_error(&err); + let region = engine.get_region(region_id).unwrap(); + assert!(!region.is_staging()); + assert_eq!(1, region.version().memtables.num_rows()); +} + +#[tokio::test] +async fn test_repartition_flush_rejects_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(0).await; + + let request = RegionFlushRequest { + reason: Some(RegionFlushReason::Repartition), + ..Default::default() + }; + let err = engine + .handle_request(region_id, RegionRequest::Flush(request)) + .await + .unwrap_err(); + + assert_pending_tails_error(&err); +} + +#[tokio::test] +async fn test_close_with_flush_rejects_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + + let request = RegionCloseRequest { + flush_on_close: true, + }; + let err = engine + .handle_request(region_id, RegionRequest::Close(request)) + .await + .unwrap_err(); + + assert_pending_tails_error(&err); + assert!(engine.get_region(region_id).is_some()); +} diff --git a/src/mito2/src/engine/scan_test.rs b/src/mito2/src/engine/scan_test.rs index e495184b112..8324e3a03e5 100644 --- a/src/mito2/src/engine/scan_test.rs +++ b/src/mito2/src/engine/scan_test.rs @@ -3057,6 +3057,7 @@ async fn test_exact_sequence_range_unsupported_when_legacy_file() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; engine.edit_region(region_id, edit).await.unwrap(); @@ -3144,6 +3145,7 @@ async fn test_exact_sequence_range_accepts_foreign_file_with_target_barrier() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await @@ -4244,6 +4246,7 @@ async fn test_compaction_output_non_preserve_not_laundered_from_legacy_input( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/staging_test.rs b/src/mito2/src/engine/staging_test.rs index 10351f918d9..a0501362ea8 100644 --- a/src/mito2/src/engine/staging_test.rs +++ b/src/mito2/src/engine/staging_test.rs @@ -769,6 +769,7 @@ async fn test_staging_exit_success_with_manifests_with_format(flat_format: bool) flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await @@ -907,6 +908,7 @@ async fn test_staging_exit_success_with_manifests_with_format(flat_format: bool) flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })]), true, ) @@ -1147,6 +1149,7 @@ async fn test_staging_exit_conflict_partition_expr_change_and_change_with_format flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), ]), true, @@ -1627,6 +1630,7 @@ fn edit_adding(file: FileMeta) -> RegionEdit { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), } } @@ -1986,6 +1990,7 @@ async fn test_successful_staging_end_keeps_rows_of_rejected_flush() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await @@ -2410,6 +2415,7 @@ async fn test_close_after_end_of_staging_is_queued_keeps_committed_result() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/sync_test.rs b/src/mito2/src/engine/sync_test.rs index ee0cd7ebe64..1d3fff9216e 100644 --- a/src/mito2/src/engine/sync_test.rs +++ b/src/mito2/src/engine/sync_test.rs @@ -210,6 +210,7 @@ async fn test_sync_after_flush_region_with_format(flat_format: bool) { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs index 5085ca8f78c..18cc8fcf814 100644 --- a/src/mito2/src/error.rs +++ b/src/mito2/src/error.rs @@ -730,6 +730,13 @@ pub enum Error { location: Location, }, + #[snafu(display("Region {} still has WAL tails to recover", region_id))] + RegionPendingTails { + region_id: RegionId, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Region {} does not hold a valid leader lease", region_id))] RegionLeaseExpired { region_id: RegionId, @@ -1693,6 +1700,7 @@ impl ErrorExt for Error { InvalidRegionRequest { source, .. } => source.status_code(), RegionState { .. } | UpdateManifest { .. } => StatusCode::RegionNotReady, RegionLeaseExpired { .. } | UnacknowledgedWrite { .. } => StatusCode::RegionNotReady, + RegionPendingTails { .. } => StatusCode::RegionNotReady, JsonOptions { .. } => StatusCode::InvalidArguments, EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound, ConvertValue { source, .. } => source.status_code(), @@ -1805,6 +1813,7 @@ impl ErrorExt for Error { | UpdateManifest { .. } | RegionStopped { .. } | RegionBusy { .. } + | RegionPendingTails { .. } | ManualCompactionAlreadyRunning { .. } | FlushableRegionState { .. } | ManifestDeltaNotFound { .. } diff --git a/src/mito2/src/flush.rs b/src/mito2/src/flush.rs index 97bdf6dbb47..d0c19c13ec8 100644 --- a/src/mito2/src/flush.rs +++ b/src/mito2/src/flush.rs @@ -533,6 +533,7 @@ impl RegionFlushTask { flushed_entry_id: Some(version_data.last_entry_id), flushed_sequence: Some(version_data.committed_sequence), committed_sequence: None, + wal_recovery: Default::default(), }; info!("Applying {edit:?} to region {}", self.region_id); @@ -1940,6 +1941,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, memtables_to_remove: smallvec![], }), @@ -2194,6 +2196,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[0], builder.file_purger(), @@ -2502,6 +2505,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[0], builder.file_purger(), @@ -2576,6 +2580,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[0], builder.file_purger(), diff --git a/src/mito2/src/gc.rs b/src/mito2/src/gc.rs index 227875ef3ff..3541485c25f 100644 --- a/src/mito2/src/gc.rs +++ b/src/mito2/src/gc.rs @@ -464,7 +464,8 @@ impl LocalGcWorker { ); let manifest = if let Some(region) = ®ion { - let manifest = region.manifest_ctx.manifest().await; + let manager = region.manifest_ctx.manifest_manager.read().await; + let manifest = manager.manifest(); // If the manifest version does not match, skip GC for this region to avoid deleting files that are still in use. let file_ref_manifest_version = self .file_ref_manifest @@ -487,6 +488,24 @@ impl LocalGcWorker { needs_retry: false, }); } + // SSTs written for pending tails but not committed yet, and SSTs of a newer + // manifest, are not in this snapshot, so a full listing would take them for orphans. + if self.full_file_listing + && (!manifest.pending_tails.is_empty() + || manager.has_update().await.unwrap_or_else(|e| { + warn!(e; "Failed to check manifest updates of region {}", region_id); + true + })) + { + info!( + "Skip full listing GC for region {}: it has pending WAL tails or a newer manifest may exist", + region_id + ); + return Ok(RegionGcOutcome { + removed_files: vec![], + needs_retry: false, + }); + } Some(manifest) } else { None diff --git a/src/mito2/src/gc/worker_test.rs b/src/mito2/src/gc/worker_test.rs index 55a04dbad5e..d7e41561d29 100644 --- a/src/mito2/src/gc/worker_test.rs +++ b/src/mito2/src/gc/worker_test.rs @@ -20,9 +20,13 @@ use api::v1::Rows; use async_trait::async_trait; use bytes::Bytes; use common_base::Plugins; +use common_datasource::compression::CompressionType; use common_telemetry::init_default_ut_logging; use common_time::Timestamp; use futures::TryStreamExt; +use object_store::layers::mock::{ + Error as MockError, ErrorKind, MockLayerBuilder, Result as MockResult, oio, +}; use object_store::{Entry, ObjectStore, services}; use store_api::metadata::RegionMetadataRef; use store_api::region_engine::{RegionEngine as _, RegionRole}; @@ -33,10 +37,12 @@ use crate::access_layer::AccessLayerRef; use crate::config::MitoConfig; use crate::engine::MitoEngine; use crate::engine::compaction_test::{delete_and_flush, put_and_flush}; +use crate::engine::pending_tail_test::{set_pending_tails, tail}; use crate::engine::region_hook::{RegionGcInfo, RegionHook, RegionHookRef}; use crate::error::UnexpectedSnafu; use crate::gc::{GcConfig, LocalGcWorker, should_delete_file}; use crate::manifest::action::{RegionEdit, RegionMetaAction, RegionMetaActionList, RemovedFile}; +use crate::manifest::storage::{delta_file, gen_path}; use crate::metrics::GC_SKIPPED_UNPARSABLE_FILES; use crate::region::{MitoRegionRef, RegionLeaderState}; use crate::sst::file::{ColumnIndexMetadata, FileHandle, FileMeta, IndexType, RegionIndexId}; @@ -319,6 +325,7 @@ async fn test_gc_reclaims_obsolete_index_after_readers_and_linger_release( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })), false, ) @@ -1788,3 +1795,83 @@ async fn test_on_region_gc_failure_routes_region_to_retry() { "a failed extension cleanup must not report deleted_files for the region" ); } + +struct FailingLister; + +impl oio::List for FailingLister { + async fn next(&mut self) -> MockResult> { + Err(MockError::new(ErrorKind::Unexpected, "mock list failure")) + } +} + +#[rstest::rstest] +#[case::pending_tails("pending_tails")] +#[case::newer_manifest("newer_manifest")] +#[case::manifest_list_failure("manifest_list_failure")] +#[tokio::test] +async fn test_full_listing_gc_skips_region(#[case] cause: &str) { + let fail_manifest_lists = Arc::new(AtomicBool::new(false)); + let fail = fail_manifest_lists.clone(); + let mock_layer = MockLayerBuilder::default() + .lister_factory(Arc::new(move |path, _, inner| { + if fail.load(Ordering::Relaxed) && path.contains("/manifest/") { + Box::new(FailingLister) + } else { + inner + } + })) + .build() + .unwrap(); + let mut env = TestEnv::new().await.with_mock_layer(mock_layer); + let hook = Arc::new(GcCountingHook::default()); + let plugins = Plugins::new(); + plugins.insert(hook.clone() as RegionHookRef); + let config = MitoConfig { + gc: GcConfig { + enable: true, + ..Default::default() + }, + ..Default::default() + }; + let engine = env.create_engine_with_plugins(config, plugins).await; + let region_id = RegionId::new(1, 1); + engine + .handle_request( + region_id, + RegionRequest::Create(CreateRequestBuilder::new().build()), + ) + .await + .unwrap(); + let region = engine.get_region(region_id).unwrap(); + let run_gc = async || { + let file_ref_manifest = FileRefsManifest { + file_refs: Default::default(), + manifest_version: [(region_id, region.manifest_ctx.manifest_version().await)].into(), + cross_region_refs: HashMap::new(), + }; + let regions = BTreeMap::from([(region_id, Some(region.clone()))]); + let gc_worker = create_gc_worker(&engine, regions, &file_ref_manifest, true).await; + gc_worker.run().await.unwrap(); + }; + run_gc().await; + assert_eq!(1, hook.gc_calls.load(Ordering::Relaxed)); + + match cause { + "pending_tails" => set_pending_tails(&engine, region_id, vec![tail(1, 0, 1)]).await, + "newer_manifest" => { + let store = region.manifest_ctx.manifest_manager.read().await.store(); + let version = region.manifest_ctx.manifest_version().await; + let path = gen_path( + store.manifest_dir(), + &delta_file(version + 1), + CompressionType::Uncompressed, + ); + let object_store = region.access_layer.object_store(); + object_store.write(&path, b"".as_slice()).await.unwrap(); + } + _ => fail_manifest_lists.store(true, Ordering::Relaxed), + } + run_gc().await; + + assert_eq!(1, hook.gc_calls.load(Ordering::Relaxed)); +} diff --git a/src/mito2/src/manifest/action.rs b/src/mito2/src/manifest/action.rs index 5d5620bdab8..cd00237087a 100644 --- a/src/mito2/src/manifest/action.rs +++ b/src/mito2/src/manifest/action.rs @@ -14,7 +14,7 @@ //! Defines [RegionMetaAction] related structs and [RegionCheckpoint]. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::time::Duration; use chrono::Utc; @@ -95,6 +95,62 @@ pub struct RegionEdit { pub flushed_entry_id: Option, pub flushed_sequence: Option, pub committed_sequence: Option, + #[serde(flatten)] + pub wal_recovery: WalRecoveryEdit, +} + +/// Changes an edit makes to the WAL ranges the region still has to recover. +#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)] +pub struct WalRecoveryEdit { + /// The new [RegionManifest::pending_tails], whose segments of each node keep the + /// order and bounds described there. `None` keeps them unchanged. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pending_tails: Option>, + /// Replaces the handoff floor of one node. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub handoff_floor: Option, +} + +/// WAL entries of one node in `(after, upto]` that the region still has to recover. +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct PendingTail { + /// Id of the datanode whose WAL holds the entries. + pub node_id: u64, + pub after: EntryId, + pub upto: EntryId, + pub upto_kind: UptoKind, +} + +/// Where the upper bound of a [PendingTail] comes from. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum UptoKind { + /// The last entry id the node reported, which names an entry that can be replayed. + Reported, + /// An upper bound found by listing the WAL objects of the node, which may name no entry. + Listed, +} + +/// Entries of a node at or below `entry_id` are never recovered from its WAL, except +/// those in a pending tail of the node. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +pub struct HandoffFloor { + pub node_id: u64, + pub entry_id: EntryId, +} + +/// Caps a release of the WAL of node `node_id` at `entry_id`, so that it keeps every +/// pending tail of the node. +pub(crate) fn cap_wal_release( + pending_tails: &[PendingTail], + node_id: Option, + entry_id: EntryId, +) -> EntryId { + pending_tails + .iter() + .filter(|tail| Some(tail.node_id) == node_id) + .map(|tail| tail.after) + .fold(entry_id, EntryId::min) } impl RegionEdit { @@ -191,6 +247,38 @@ pub struct RegionManifest { /// Whether the region is in append mode. #[serde(default)] pub append_mode: Option, + /// WAL ranges the region still has to recover, in the order they were added. + /// + /// The segments of one node are increasing and do not overlap, and each has + /// `after < upto`: a new segment of a node takes as `after` the largest of the + /// flushed entry id, the `upto`s of the node's existing segments and its handoff + /// floor, and is added only if that is below its `upto`. So the entries of a node + /// between two of its segments are flushed or at or below a handoff floor of the + /// node, and need no replay. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub pending_tails: Vec, + /// The [HandoffFloor] of each node, removed once the flushed entry id reaches it. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub handoff_floors: BTreeMap, +} + +impl RegionManifest { + /// Returns the entry id after which node `node_id` replays its WAL when it opens + /// the region. Entries up to it are flushed, left to the pending tails, or below + /// the handoff floor of the node. + /// + /// Entries of the node in `(after, upto]` of one of its tails belong to that tail. + /// Whoever adds a tail ensures that every entry id the node allocates afterwards is + /// above the tail's `upto`, so no later write of the node falls in a skipped range. + pub(crate) fn wal_replay_start(&self, node_id: Option) -> EntryId { + let floor = node_id.and_then(|node_id| self.handoff_floors.get(&node_id).copied()); + self.pending_tails + .iter() + .filter(|tail| Some(tail.node_id) == node_id) + .map(|tail| tail.upto) + .chain(floor) + .fold(self.flushed_entry_id, EntryId::max) + } } #[cfg(test)] @@ -204,6 +292,8 @@ impl PartialEq for RegionManifest { && self.truncated_entry_id == other.truncated_entry_id && self.compaction_time_window == other.compaction_time_window && self.committed_sequence == other.committed_sequence + && self.pending_tails == other.pending_tails + && self.handoff_floors == other.handoff_floors } } @@ -229,6 +319,8 @@ pub struct RegionManifestBuilder { committed_sequence: Option, sst_format: FormatType, append_mode: Option, + pending_tails: Vec, + handoff_floors: BTreeMap, } impl RegionManifestBuilder { @@ -256,6 +348,8 @@ impl RegionManifestBuilder { committed_sequence: s.committed_sequence, sst_format: s.sst_format, append_mode: s.append_mode, + pending_tails: s.pending_tails, + handoff_floors: s.handoff_floors, } } else { Default::default() @@ -335,6 +429,12 @@ impl RegionManifestBuilder { if let Some(window) = edit.compaction_time_window { self.compaction_time_window = Some(window); } + if let Some(pending_tails) = edit.wal_recovery.pending_tails { + self.pending_tails = pending_tails; + } + if let Some(floor) = edit.wal_recovery.handoff_floor { + self.handoff_floors.insert(floor.node_id, floor.entry_id); + } } pub fn apply_truncate(&mut self, manifest_version: ManifestVersion, truncate: RegionTruncate) { @@ -389,8 +489,11 @@ impl RegionManifestBuilder { self.metadata.is_some() } - pub fn try_build(self) -> Result { + pub fn try_build(mut self) -> Result { let metadata = self.metadata.context(RegionMetadataNotFoundSnafu)?; + // A floor at or below the flushed entry id no longer raises the replay start. + self.handoff_floors + .retain(|_, floor| *floor > self.flushed_entry_id); Ok(RegionManifest { metadata, files: self.files, @@ -403,6 +506,8 @@ impl RegionManifestBuilder { compaction_time_window: self.compaction_time_window, sst_format: self.sst_format, append_mode: self.append_mode, + pending_tails: self.pending_tails, + handoff_floors: self.handoff_floors, }) } } @@ -597,6 +702,7 @@ impl RegionMetaActionList { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; let mut partition_expr_change = None; let mut region_change = None; @@ -655,6 +761,7 @@ impl RegionMetaActionList { #[cfg(test)] mod tests { + use std::sync::Arc; use common_time::Timestamp; @@ -991,6 +1098,8 @@ mod tests { }, sst_format: FormatType::PrimaryKey, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), }; let json = serde_json::to_string(&manifest).unwrap(); @@ -1026,6 +1135,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ); builder.apply_edit( @@ -1038,6 +1148,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ); @@ -1057,6 +1168,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ); builder.apply_truncate( @@ -1171,6 +1283,8 @@ mod tests { compaction_time_window: None, sst_format: FormatType::PrimaryKey, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), } ); @@ -1186,6 +1300,8 @@ mod tests { compaction_time_window: None, sst_format: FormatType::PrimaryKey, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), }; let json = serde_json::to_string(&new_manifest).unwrap(); let old_from_new: RegionManifestV1 = serde_json::from_str(&json).unwrap(); @@ -1230,6 +1346,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, new_from_old ); @@ -1243,6 +1360,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; let new_json = serde_json::to_string(&new).unwrap(); @@ -1403,4 +1521,273 @@ mod tests { .contains(&RemovedFile::File(file_id, None)) ); } + + fn tail(node_id: u64, after: EntryId, upto: EntryId) -> PendingTail { + PendingTail { + node_id, + after, + upto, + upto_kind: UptoKind::Reported, + } + } + + fn wal_recovery_edit( + flushed_entry_id: Option, + wal_recovery: WalRecoveryEdit, + ) -> RegionEdit { + RegionEdit { + files_to_add: vec![], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id, + flushed_sequence: None, + committed_sequence: None, + wal_recovery, + } + } + + fn manifest_builder() -> RegionManifestBuilder { + let mut builder = RegionManifestBuilder::default(); + builder.apply_change( + 0, + RegionChange { + metadata: Arc::new(crate::manifest::tests::utils::basic_region_metadata()), + sst_format: FormatType::PrimaryKey, + append_mode: None, + }, + ); + builder + } + + #[test] + fn test_pending_tails_change_only_when_set() { + let mut builder = manifest_builder(); + builder.apply_edit( + 1, + wal_recovery_edit( + None, + WalRecoveryEdit { + pending_tails: Some(vec![tail(1, 10, 20)]), + handoff_floor: None, + }, + ), + ); + builder.apply_edit(2, wal_recovery_edit(Some(5), WalRecoveryEdit::default())); + assert_eq!(builder.pending_tails, vec![tail(1, 10, 20)]); + + builder.apply_edit( + 3, + wal_recovery_edit( + None, + WalRecoveryEdit { + pending_tails: Some(vec![]), + handoff_floor: None, + }, + ), + ); + assert!(builder.pending_tails.is_empty()); + } + + #[test] + fn test_flushed_entry_id_drops_reached_floors_and_keeps_tails() { + let mut builder = manifest_builder(); + for (node_id, entry_id) in [(1, 20), (2, 30)] { + builder.apply_edit( + 1, + wal_recovery_edit( + None, + WalRecoveryEdit { + pending_tails: Some(vec![tail(1, 10, 20)]), + handoff_floor: Some(HandoffFloor { node_id, entry_id }), + }, + ), + ); + } + builder.apply_edit(2, wal_recovery_edit(Some(20), WalRecoveryEdit::default())); + + let manifest = builder.try_build().unwrap(); + assert_eq!(manifest.pending_tails, vec![tail(1, 10, 20)]); + assert_eq!(manifest.handoff_floors, BTreeMap::from([(2, 30)])); + } + + #[test] + fn test_truncate_drops_reached_floors() { + let mut builder = manifest_builder(); + builder.apply_edit( + 1, + wal_recovery_edit( + None, + WalRecoveryEdit { + pending_tails: None, + handoff_floor: Some(HandoffFloor { + node_id: 1, + entry_id: 20, + }), + }, + ), + ); + builder.apply_truncate( + 2, + RegionTruncate { + region_id: RegionId::new(1, 1), + kind: TruncateKind::All { + truncated_entry_id: 20, + truncated_sequence: 20, + }, + timestamp_ms: None, + }, + ); + + assert!(builder.try_build().unwrap().handoff_floors.is_empty()); + } + + #[test] + fn test_wal_replay_start() { + let mut builder = manifest_builder(); + // Node 1 was handed off at 40 after its first tail, so its second tail starts + // there and entries in (20, 40] belong to no tail. + builder.apply_edit( + 1, + wal_recovery_edit( + Some(10), + WalRecoveryEdit { + pending_tails: Some(vec![tail(1, 10, 20), tail(2, 30, 60), tail(1, 40, 50)]), + handoff_floor: Some(HandoffFloor { + node_id: 1, + entry_id: 40, + }), + }, + ), + ); + builder.apply_edit( + 2, + wal_recovery_edit( + None, + WalRecoveryEdit { + pending_tails: None, + handoff_floor: Some(HandoffFloor { + node_id: 3, + entry_id: 40, + }), + }, + ), + ); + let manifest = builder.try_build().unwrap(); + + assert_eq!(manifest.wal_replay_start(Some(1)), 50); + assert_eq!(manifest.wal_replay_start(Some(2)), 60); + assert_eq!(manifest.wal_replay_start(Some(3)), 40); + assert_eq!(manifest.wal_replay_start(Some(4)), 10); + assert_eq!(manifest.wal_replay_start(None), 10); + } + + #[test] + fn test_cap_wal_release() { + let tails = [tail(1, 20, 25), tail(2, 5, 60), tail(1, 30, 40)]; + + assert_eq!(cap_wal_release(&tails, Some(1), 50), 20); + assert_eq!(cap_wal_release(&tails, Some(1), 10), 10); + assert_eq!(cap_wal_release(&tails, Some(3), 50), 50); + assert_eq!(cap_wal_release(&tails, None, 50), 50); + } + + const PENDING_TAILS_DELTA: &str = + include_str!("../../../../tests/compatibility/manifest/pending_tails/delta.json"); + const PENDING_TAILS_CHECKPOINT: &str = + include_str!("../../../../tests/compatibility/manifest/pending_tails/checkpoint.json"); + + fn fixture_pending_tails() -> Vec { + vec![ + tail(1, 1 << 32, (1 << 32) + 5), + PendingTail { + upto_kind: UptoKind::Listed, + ..tail(2, 2 << 32, 3 << 32) + }, + ] + } + + #[test] + fn test_pending_tails_fixtures() { + let delta = PENDING_TAILS_DELTA.trim_end(); + let actions = RegionMetaActionList::decode(delta.as_bytes()).unwrap(); + let [RegionMetaAction::Edit(edit)] = actions.actions.as_slice() else { + panic!("unexpected actions: {actions:?}"); + }; + let expected = WalRecoveryEdit { + pending_tails: Some(fixture_pending_tails()), + handoff_floor: Some(HandoffFloor { + node_id: 3, + entry_id: 4 << 32, + }), + }; + assert_eq!(edit.wal_recovery, expected); + assert_eq!(actions.encode().unwrap(), delta.as_bytes()); + + let checkpoint = PENDING_TAILS_CHECKPOINT.trim_end(); + let decoded = RegionCheckpoint::decode(checkpoint.as_bytes()).unwrap(); + let manifest = decoded.checkpoint.as_ref().unwrap(); + assert_eq!(manifest.pending_tails, fixture_pending_tails()); + assert_eq!(manifest.handoff_floors, BTreeMap::from([(3, 4 << 32)])); + let encoded: serde_json::Value = + serde_json::from_slice(&decoded.encode().unwrap()).unwrap(); + let frozen: serde_json::Value = serde_json::from_str(checkpoint).unwrap(); + for key in ["pending_tails", "handoff_floors"] { + assert_eq!(encoded["checkpoint"][key], frozen["checkpoint"][key]); + } + } + + /// Checks the fixtures against the shapes before pending tails were added. + #[test] + fn test_pending_tails_old_shape_compat() { + #[derive(Serialize, Deserialize)] + struct OldRegionEdit { + files_to_add: Vec, + files_to_remove: Vec, + #[serde(default)] + timestamp_ms: Option, + #[serde(with = "humantime_serde")] + compaction_time_window: Option, + flushed_entry_id: Option, + flushed_sequence: Option, + committed_sequence: Option, + } + + #[derive(Serialize, Deserialize)] + struct OldRegionManifest { + metadata: RegionMetadataRef, + files: HashMap, + #[serde(default)] + removed_files: RemovedFilesRecord, + flushed_entry_id: EntryId, + flushed_sequence: SequenceNumber, + committed_sequence: Option, + manifest_version: ManifestVersion, + truncated_entry_id: Option, + #[serde(with = "humantime_serde")] + compaction_time_window: Option, + #[serde(default)] + sst_format: FormatType, + #[serde(default)] + append_mode: Option, + } + + let delta: serde_json::Value = serde_json::from_str(PENDING_TAILS_DELTA).unwrap(); + let old_edit: OldRegionEdit = + serde_json::from_value(delta["actions"][0]["Edit"].clone()).unwrap(); + assert_eq!(old_edit.committed_sequence, Some((1 << 40) + 5)); + let old_json = serde_json::to_string(&old_edit).unwrap(); + let edit: RegionEdit = serde_json::from_str(&old_json).unwrap(); + assert_eq!(edit.wal_recovery, WalRecoveryEdit::default()); + assert_eq!(serde_json::to_string(&edit).unwrap(), old_json); + + let checkpoint: serde_json::Value = serde_json::from_str(PENDING_TAILS_CHECKPOINT).unwrap(); + let old_manifest: OldRegionManifest = + serde_json::from_value(checkpoint["checkpoint"].clone()).unwrap(); + assert_eq!(old_manifest.flushed_entry_id, 1 << 32); + let old_json = serde_json::to_string(&old_manifest).unwrap(); + let manifest: RegionManifest = serde_json::from_str(&old_json).unwrap(); + assert!(manifest.pending_tails.is_empty() && manifest.handoff_floors.is_empty()); + assert_eq!(serde_json::to_string(&manifest).unwrap(), old_json); + } } diff --git a/src/mito2/src/manifest/manager.rs b/src/mito2/src/manifest/manager.rs index 7531c433191..5cebfbc9b11 100644 --- a/src/mito2/src/manifest/manager.rs +++ b/src/mito2/src/manifest/manager.rs @@ -234,6 +234,7 @@ impl RegionManifestManager { flushed_entry_id: Some(flushed_entry_id), flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })); } @@ -1067,6 +1068,7 @@ mod test { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })]), false, ) diff --git a/src/mito2/src/manifest/tests/checkpoint.rs b/src/mito2/src/manifest/tests/checkpoint.rs index 72f4377221f..708ca51ceb4 100644 --- a/src/mito2/src/manifest/tests/checkpoint.rs +++ b/src/mito2/src/manifest/tests/checkpoint.rs @@ -28,7 +28,8 @@ use strum::IntoEnumIterator; use crate::error::Error::{ChecksumMismatch, ManifestDeltaNotFound}; use crate::manifest::action::{ - RegionCheckpoint, RegionEdit, RegionMetaAction, RegionMetaActionList, + HandoffFloor, PendingTail, RegionCheckpoint, RegionEdit, RegionMetaAction, + RegionMetaActionList, UptoKind, WalRecoveryEdit, }; use crate::manifest::manager::RegionManifestManager; use crate::manifest::storage::checkpoint::CheckpointMetadata; @@ -185,6 +186,55 @@ async fn manager_with_checkpoint_distance_1() { assert_eq!(10, manager.manifest().manifest_version); } +#[rstest::rstest] +#[case::delta(0)] +#[case::checkpoint(1)] +#[tokio::test] +async fn wal_recovery_records_survive_reopen(#[case] checkpoint_distance: u64) { + let (env, mut manager) = + build_manager(checkpoint_distance, CompressionType::Uncompressed).await; + let pending_tails = vec![PendingTail { + node_id: 1, + after: 10, + upto: 20, + upto_kind: UptoKind::Listed, + }]; + let edit = RegionEdit { + files_to_add: vec![], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: Some(1 << 40), + wal_recovery: WalRecoveryEdit { + pending_tails: Some(pending_tails.clone()), + handoff_floor: Some(HandoffFloor { + node_id: 1, + entry_id: 30, + }), + }, + }; + manager + .update( + RegionMetaActionList::with_action(RegionMetaAction::Edit(edit)), + false, + ) + .await + .unwrap(); + wait_for_manifest_checkpoint(&manager).await; + let checkpoint = manager.store().load_last_checkpoint().await.unwrap(); + assert_eq!(checkpoint.is_some(), checkpoint_distance > 0); + manager.stop().await; + + let manifest = reopen_manager(&env, checkpoint_distance, CompressionType::Uncompressed) + .await + .manifest(); + assert_eq!(manifest.pending_tails, pending_tails); + assert_eq!(manifest.handoff_floors, [(1, 30)].into()); + assert_eq!(manifest.committed_sequence, Some(1 << 40)); +} + #[tokio::test] async fn test_corrupted_data_causing_checksum_error() { // Initialize manager @@ -266,6 +316,7 @@ async fn checkpoint_with_different_compression_types() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })]); actions.push(action); } @@ -332,6 +383,7 @@ fn generate_action_lists(num: usize) -> (Vec, Vec) flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })]); actions.push(action); } diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs index c092c7ef4e7..2282eba0036 100644 --- a/src/mito2/src/region.rs +++ b/src/mito2/src/region.rs @@ -1509,6 +1509,7 @@ impl ManifestContext { flushed_entry_id: None, committed_sequence: None, compaction_time_window: None, + wal_recovery: Default::default(), }; let action_list = RegionMetaActionList::with_action(RegionMetaAction::Edit(edit)); let manifest_version = self.update_and_fire(manager, action_list, false).await?; @@ -2526,6 +2527,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), } } @@ -2567,6 +2569,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })); region diff --git a/src/mito2/src/region/catchup.rs b/src/mito2/src/region/catchup.rs index 4b8f1288c4b..144dbc0e23e 100644 --- a/src/mito2/src/region/catchup.rs +++ b/src/mito2/src/region/catchup.rs @@ -107,6 +107,7 @@ impl RegionCatchupTask { wal_entry_reader, region_id, replay_from_entry_id, + replay_from_entry_id, &self.region.version_control, self.allow_stale_entries, on_region_opened, diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs index 85881028ab9..c306c9198c9 100644 --- a/src/mito2/src/region/opener.rs +++ b/src/mito2/src/region/opener.rs @@ -56,7 +56,7 @@ use crate::error::{ ObjectStoreNotFoundSnafu, RegionCorruptedSnafu, Result, StaleLogEntrySnafu, }; use crate::lease::NodeLease; -use crate::manifest::action::RegionManifest; +use crate::manifest::action::{RegionManifest, cap_wal_release}; use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions}; use crate::memtable::bulk::part::BulkPart; use crate::memtable::time_partition::TimePartitions; @@ -627,10 +627,13 @@ impl RegionOpener { .max(flushed_entry_id); let topic_latest_entry_id = if !self.skip_wal_replay { check_wal_history(wal, &provider, &version_control.current().version).await?; + let node_id = self.file_ref_manager.node_id(); + let replay_start = replay_from_entry_id.max(manifest.wal_replay_start(node_id)); info!( - "Start replaying memtable at replay_from_entry_id: {} for region {}, manifest version: {}, flushed entry id: {}, elapsed: {:?}", - replay_from_entry_id, + "Start replaying memtable at replay_start: {} for region {}, replay_from_entry_id: {}, manifest version: {}, flushed entry id: {}, elapsed: {:?}", + replay_start, region_id, + replay_from_entry_id, manifest.manifest_version, flushed_entry_id, now.elapsed() @@ -639,7 +642,8 @@ impl RegionOpener { &provider, wal_entry_reader, region_id, - replay_from_entry_id, + replay_start, + cap_wal_release(&manifest.pending_tails, node_id, replay_start), &version_control, config.allow_stale_entries, on_region_opened, @@ -813,6 +817,7 @@ pub(crate) fn version_builder_from_manifest( .truncated_entry_id(manifest.truncated_entry_id) .compaction_time_window(manifest.compaction_time_window) .options(region_options) + .pending_tails(manifest.pending_tails.clone()) } /// Updates region options by persistent options. @@ -972,12 +977,15 @@ pub(crate) async fn check_wal_history( .await } -/// Replays the mutations from WAL and inserts mutations to memtable of given region. +/// Replays the mutations from WAL and inserts mutations to memtable of given region, +/// then releases the WAL up to `release_entry_id`. +#[allow(clippy::too_many_arguments)] pub(crate) async fn replay_memtable( provider: &Provider, mut wal_entry_reader: Box, region_id: RegionId, flushed_entry_id: EntryId, + release_entry_id: EntryId, version_control: &VersionControlRef, allow_stale_entries: bool, on_region_opened: F, @@ -1072,7 +1080,7 @@ where // TODO(weny): We need to update `flushed_entry_id` in the region manifest // to avoid reading potentially incomplete entries in the future. - (on_region_opened)(region_id, flushed_entry_id, provider).await?; + (on_region_opened)(region_id, release_entry_id, provider).await?; let series_count = version_control.current().series_count(); info!( @@ -1539,6 +1547,8 @@ mod tests { compaction_time_window: None, sst_format, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), } } diff --git a/src/mito2/src/region/version.rs b/src/mito2/src/region/version.rs index c1940b31ab0..a9866ecd748 100644 --- a/src/mito2/src/region/version.rs +++ b/src/mito2/src/region/version.rs @@ -27,11 +27,12 @@ use std::sync::{Arc, RwLock}; use std::time::Duration; use common_telemetry::info; +use snafu::ensure; use store_api::metadata::RegionMetadataRef; use store_api::storage::{RegionId, SequenceNumber}; -use crate::error::Result; -use crate::manifest::action::{RegionEdit, TruncateKind}; +use crate::error::{RegionPendingTailsSnafu, Result}; +use crate::manifest::action::{PendingTail, RegionEdit, TruncateKind}; use crate::memtable::time_partition::TimePartitions; use crate::memtable::version::{MemtableVersion, MemtableVersionRef}; use crate::memtable::{MemtableBuilderRef, MemtableId}; @@ -399,6 +400,24 @@ pub(crate) struct Version { pub(crate) compaction_time_window: Option, /// Options of the region. pub(crate) options: RegionOptions, + /// WAL ranges the region still has to recover. + pub(crate) pending_tails: Vec, +} + +impl Version { + pub(crate) fn has_pending_tails(&self) -> bool { + !self.pending_tails.is_empty() + } + + pub(crate) fn ensure_no_pending_tails(&self) -> Result<()> { + ensure!( + !self.has_pending_tails(), + RegionPendingTailsSnafu { + region_id: self.metadata.region_id, + } + ); + Ok(()) + } } pub(crate) type VersionRef = Arc; @@ -413,6 +432,7 @@ pub(crate) struct VersionBuilder { truncated_entry_id: Option, compaction_time_window: Option, options: RegionOptions, + pending_tails: Vec, } impl VersionBuilder { @@ -427,6 +447,7 @@ impl VersionBuilder { truncated_entry_id: None, compaction_time_window: None, options: RegionOptions::default(), + pending_tails: Vec::new(), } } @@ -441,6 +462,7 @@ impl VersionBuilder { truncated_entry_id: version.truncated_entry_id, compaction_time_window: version.compaction_time_window, options: version.options.clone(), + pending_tails: version.pending_tails.clone(), } } @@ -489,6 +511,11 @@ impl VersionBuilder { self } + pub(crate) fn pending_tails(mut self, pending_tails: Vec) -> Self { + self.pending_tails = pending_tails; + self + } + /// Apply edit to the builder. pub(crate) fn apply_edit(mut self, edit: RegionEdit, file_purger: FilePurgerRef) -> Self { if let Some(entry_id) = edit.flushed_entry_id { @@ -500,6 +527,9 @@ impl VersionBuilder { if let Some(window) = edit.compaction_time_window { self.compaction_time_window = Some(window); } + if let Some(pending_tails) = edit.wal_recovery.pending_tails { + self.pending_tails = pending_tails; + } if !edit.files_to_add.is_empty() || !edit.files_to_remove.is_empty() { let mut ssts = (*self.ssts).clone(); ssts.add_files(file_purger, edit.files_to_add.into_iter()); @@ -573,6 +603,7 @@ impl VersionBuilder { truncated_entry_id: self.truncated_entry_id, compaction_time_window, options: self.options, + pending_tails: self.pending_tails, } } } @@ -621,6 +652,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: Some(sequence), + wal_recovery: Default::default(), }), &[], builder.file_purger(), diff --git a/src/mito2/src/remap_manifest.rs b/src/mito2/src/remap_manifest.rs index 596e60c6ddf..fec0cad0547 100644 --- a/src/mito2/src/remap_manifest.rs +++ b/src/mito2/src/remap_manifest.rs @@ -116,6 +116,8 @@ impl RemapManifest { compaction_time_window: None, sst_format, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), }; new_manifests.insert(*region_id, manifest); @@ -467,6 +469,8 @@ mod tests { committed_sequence: None, sst_format: FormatType::PrimaryKey, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), } } diff --git a/src/mito2/src/series_index/task.rs b/src/mito2/src/series_index/task.rs index 6ce340827a1..eeb8ef50de6 100644 --- a/src/mito2/src/series_index/task.rs +++ b/src/mito2/src/series_index/task.rs @@ -813,6 +813,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], crate::test_util::new_noop_file_purger(), diff --git a/src/mito2/src/series_index/tests.rs b/src/mito2/src/series_index/tests.rs index 5f13d0b4b2d..feb8962ecf9 100644 --- a/src/mito2/src/series_index/tests.rs +++ b/src/mito2/src/series_index/tests.rs @@ -294,6 +294,7 @@ async fn test_reconcile_range_index_disabled( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], crate::test_util::new_noop_file_purger(), @@ -403,6 +404,7 @@ async fn test_reconcile_range_only_build_and_removal() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], crate::test_util::new_noop_file_purger(), @@ -455,6 +457,7 @@ async fn test_reconcile_replaces_whole_bucket_after_failure(#[case] failure: &st flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; region.version_control.apply_edit( Some(edit(false)), diff --git a/src/mito2/src/sst/file_ref.rs b/src/mito2/src/sst/file_ref.rs index 68b0facaab4..b82a7b9c217 100644 --- a/src/mito2/src/sst/file_ref.rs +++ b/src/mito2/src/sst/file_ref.rs @@ -34,7 +34,8 @@ struct RegionFileRefs { /// Tracks reader references to SST metadata views in one datanode for object-store GC. #[derive(Debug)] pub struct FileReferenceManager { - /// Datanode id. used to determine tmp ref file name. + /// Id of this datanode. Names the tmp ref files and identifies the WAL ranges of + /// this node in region manifests. node_id: Option, files_per_region: DashMap, /// Whether global GC is enabled. diff --git a/src/mito2/src/sst/index.rs b/src/mito2/src/sst/index.rs index 4fb0f08e217..635d21f1cb4 100644 --- a/src/mito2/src/sst/index.rs +++ b/src/mito2/src/sst/index.rs @@ -1527,6 +1527,7 @@ mod tests { flushed_entry_id: None, committed_sequence: None, compaction_time_window: None, + wal_recovery: Default::default(), })), false, ) diff --git a/src/mito2/src/sst/primary_key.rs b/src/mito2/src/sst/primary_key.rs index 2d3f8c90e3b..eafa8473c14 100644 --- a/src/mito2/src/sst/primary_key.rs +++ b/src/mito2/src/sst/primary_key.rs @@ -741,6 +741,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, new_noop_file_purger(), ) diff --git a/src/mito2/src/test_util.rs b/src/mito2/src/test_util.rs index 29718163528..be8f3b64003 100644 --- a/src/mito2/src/test_util.rs +++ b/src/mito2/src/test_util.rs @@ -105,6 +105,7 @@ pub(crate) fn nop_action() -> RegionMetaActionList { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })]) } diff --git a/src/mito2/src/test_util/version_util.rs b/src/mito2/src/test_util/version_util.rs index 44827b35cc1..136bf11db28 100644 --- a/src/mito2/src/test_util/version_util.rs +++ b/src/mito2/src/test_util/version_util.rs @@ -226,6 +226,7 @@ pub(crate) fn apply_edit( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], purger, diff --git a/src/mito2/src/worker/handle_alter.rs b/src/mito2/src/worker/handle_alter.rs index 935dbd39413..592b508249e 100644 --- a/src/mito2/src/worker/handle_alter.rs +++ b/src/mito2/src/worker/handle_alter.rs @@ -167,6 +167,13 @@ impl RegionWorkerLoop { return; } + if !is_allowed_with_pending_tails(&request.kind) + && let Err(e) = version.ensure_no_pending_tails() + { + sender.send(Err(e)); + return; + } + // Checks whether we need to alter the region. if !request.need_alter(&version.metadata) { warn!( @@ -425,6 +432,19 @@ impl RegionWorkerLoop { } } +/// Returns true if `kind` only changes region options or only adds nullable columns +/// without a default value, which rows replayed from a pending tail can lack. +fn is_allowed_with_pending_tails(kind: &AlterKind) -> bool { + match kind { + AlterKind::AddColumns { columns } => columns.iter().all(|column| { + let schema = &column.column_metadata.column_schema; + schema.is_nullable() && schema.default_constraint().is_none() + }), + AlterKind::SetRegionOptions { .. } | AlterKind::UnsetRegionOptions { .. } => true, + _ => false, + } +} + fn skip_wal_value(kind: &AlterKind) -> Option { let AlterKind::SetRegionOptions { options } = kind else { return None; diff --git a/src/mito2/src/worker/handle_apply_staging.rs b/src/mito2/src/worker/handle_apply_staging.rs index 2598640b9f2..73b9decd128 100644 --- a/src/mito2/src/worker/handle_apply_staging.rs +++ b/src/mito2/src/worker/handle_apply_staging.rs @@ -124,6 +124,7 @@ impl RegionWorkerLoop { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; let (tx, rx) = oneshot::channel(); diff --git a/src/mito2/src/worker/handle_close.rs b/src/mito2/src/worker/handle_close.rs index c10ac761f43..1e447e3ea54 100644 --- a/src/mito2/src/worker/handle_close.rs +++ b/src/mito2/src/worker/handle_close.rs @@ -36,6 +36,13 @@ impl RegionWorkerLoop { info!("Try to close region {}, worker: {}", region_id, self.id); + if request.flush_on_close + && let Err(e) = region.version().ensure_no_pending_tails() + { + sender.send(Err(e)); + return; + } + if region.is_staging_or_staged_edit() { // A staging region cannot flush, so it leaves staging before the close-time flush // below. The manifest lock lets a successful end of staging in progress finish first, diff --git a/src/mito2/src/worker/handle_copy_region.rs b/src/mito2/src/worker/handle_copy_region.rs index 2204cec8e1c..fbd02e8210b 100644 --- a/src/mito2/src/worker/handle_copy_region.rs +++ b/src/mito2/src/worker/handle_copy_region.rs @@ -264,6 +264,7 @@ impl RegionWorkerLoop { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; if edit .files_to_add diff --git a/src/mito2/src/worker/handle_enter_staging.rs b/src/mito2/src/worker/handle_enter_staging.rs index e5b0674aaac..119f07241f6 100644 --- a/src/mito2/src/worker/handle_enter_staging.rs +++ b/src/mito2/src/worker/handle_enter_staging.rs @@ -63,6 +63,10 @@ impl RegionWorkerLoop { sender.send(Ok(0)); return; } + if let Err(e) = region.version().ensure_no_pending_tails() { + sender.send(Err(e)); + return; + } // Retain the compaction fence while the DDL later waits for flush. let (sender, partition_directive) = match self.compaction_scheduler.try_cancel_and_add_ddl( diff --git a/src/mito2/src/worker/handle_flush.rs b/src/mito2/src/worker/handle_flush.rs index 984b551d7bb..32cd19a4142 100644 --- a/src/mito2/src/worker/handle_flush.rs +++ b/src/mito2/src/worker/handle_flush.rs @@ -27,6 +27,7 @@ use store_api::storage::RegionId; use crate::config::{IndexBuildMode, MitoConfig}; use crate::error::{RegionBusySnafu, RegionNotFoundSnafu, Result}; use crate::flush::{FlushReason, RegionFlushTask}; +use crate::manifest::action::cap_wal_release; use crate::region::version::VersionRef; use crate::region::{MitoRegion, MitoRegionRef}; use crate::request::{BuildIndexRequest, FlushFailed, FlushFinished, OnFailure, OptionOutputTx}; @@ -364,6 +365,13 @@ impl RegionWorkerLoop { } }; + if request.reason == Some(RegionFlushReason::Repartition) + && let Err(e) = region.version().ensure_no_pending_tails() + { + sender.send(Err(e)); + return; + } + // `update_topic_latest_entry_id` updates `topic_latest_entry_id` when memtables are empty. // But the flush is skipped if memtables are empty. Thus should update the `topic_latest_entry_id` // when handling flush request instead of in `schedule_flush` or `flush_finished`. @@ -484,13 +492,18 @@ impl RegionWorkerLoop { self.update_topic_latest_entry_id(®ion); // Delete wal. + let obsolete_entry_id = cap_wal_release( + ®ion.version().pending_tails, + self.file_ref_manager.node_id(), + request.flushed_entry_id, + ); info!( "Region {} flush finished, tries to bump wal to {}", - region_id, request.flushed_entry_id + region_id, obsolete_entry_id ); if let Err(e) = self .wal - .obsolete(region_id, request.flushed_entry_id, ®ion.provider) + .obsolete(region_id, obsolete_entry_id, ®ion.provider) .await { error!(e; "Failed to write wal, region: {}", region_id); diff --git a/src/mito2/src/worker/handle_manifest.rs b/src/mito2/src/worker/handle_manifest.rs index d8f4ecf8c16..f1a575d4433 100644 --- a/src/mito2/src/worker/handle_manifest.rs +++ b/src/mito2/src/worker/handle_manifest.rs @@ -35,6 +35,7 @@ use crate::error::{ }; use crate::manifest::action::{ RegionChange, RegionEdit, RegionMetaAction, RegionMetaActionList, RegionTruncate, TruncateKind, + WalRecoveryEdit, }; use crate::memtable::MemtableBuilderProvider; use crate::metrics::WRITE_CACHE_INFLIGHT_DOWNLOAD; @@ -115,6 +116,7 @@ impl RegionEditQueue { && edit.flushed_entry_id.is_none() && edit.flushed_sequence.is_none() && edit.committed_sequence.is_none() + && edit.wal_recovery == WalRecoveryEdit::default() } let mut merged = self.requests.pop_front()?; @@ -693,6 +695,7 @@ impl RegionWorkerLoop { flushed_entry_id: Some(discarded_entry_id), flushed_sequence: Some(discarded_sequence), committed_sequence: None, + wal_recovery: Default::default(), }; let action_list = RegionMetaActionList::with_action(RegionMetaAction::Edit(edit)); let result = match durability_barrier.wait(discarded_entry_id).await { diff --git a/src/mito2/src/worker/handle_rebuild_index.rs b/src/mito2/src/worker/handle_rebuild_index.rs index 10f55c78943..917a58d0c68 100644 --- a/src/mito2/src/worker/handle_rebuild_index.rs +++ b/src/mito2/src/worker/handle_rebuild_index.rs @@ -399,6 +399,7 @@ impl RegionWorkerLoop { flushed_entry_id: None, committed_sequence: None, compaction_time_window: None, + wal_recovery: Default::default(), }), &[], region.file_purger.clone(), diff --git a/src/mito2/src/worker/handle_truncate.rs b/src/mito2/src/worker/handle_truncate.rs index 409998592d1..a21bb0578e3 100644 --- a/src/mito2/src/worker/handle_truncate.rs +++ b/src/mito2/src/worker/handle_truncate.rs @@ -39,6 +39,11 @@ impl RegionWorkerLoop { return; } }; + // Truncating releases the WAL without keeping the pending tails. + if let Err(e) = region.version().ensure_no_pending_tails() { + sender.send(Err(e)); + return; + } // Fence compaction before any later flush wait so handed-off DDLs keep admission paused. let (sender, req) = match self.compaction_scheduler.try_cancel_and_add_ddl( diff --git a/tests/compatibility/README.md b/tests/compatibility/README.md index b84b55c92e5..0dfcba20215 100644 --- a/tests/compatibility/README.md +++ b/tests/compatibility/README.md @@ -48,6 +48,9 @@ writer that emits these descriptors; downgrade only to readers that recognize th The OSS build does not generate sketch payloads, so SQL restart cases cannot create these descriptors through an OSS writer. +Frozen region manifests that carry pending WAL tails are checked by the manifest tests of `mito2`. +No SQL statement records a pending tail. See the [fixture notes](manifest/pending_tails/README.md). + ## Quick Start ```shell diff --git a/tests/compatibility/manifest/pending_tails/README.md b/tests/compatibility/manifest/pending_tails/README.md new file mode 100644 index 00000000000..b788ad35bbf --- /dev/null +++ b/tests/compatibility/manifest/pending_tails/README.md @@ -0,0 +1,27 @@ +# Pending tail manifest fixtures + +Frozen region manifests with pending WAL tails and a handoff floor, checked by the manifest tests +in `src/mito2/src/manifest/action.rs`, which never rewrite them: + +```shell +cargo nextest run -p mito2 -E 'test(test_pending_tails_fixtures) | test(test_pending_tails_old_shape_compat)' +``` + +The first test decodes both files. It requires the encoder to reproduce `delta.json` byte for byte, +and the `pending_tails` and `handoff_floors` of `checkpoint.json` exactly; the rest of the checkpoint +embeds region metadata, whose encoding may gain compatible fields. The second test decodes both +files with the shapes from before pending tails existed, which must ignore the new fields, and +requires a manifest without pending tails to encode exactly as those shapes do. + +## `delta.json` + +A delta with one edit. It carries committed sequence `(1 << 40) + 5`, two pending tails, and the +handoff floor `4 << 32` of node 3. The tails are node 1, entries `(1 << 32, (1 << 32) + 5]` with a +reported upper bound, and node 2, entries `(2 << 32, 3 << 32]` with a listed upper bound. 424 bytes, +SHA-256 `520cbdc2cf51de4fb960d3c2f2194c126e3d23f02b4c511ed687705f48073c80`. + +## `checkpoint.json` + +A checkpoint of version 7 holding the same two tails and floor, flushed entry id `1 << 32`, no +files, and the region metadata of `basic_region_metadata` in `src/mito2/src/manifest/tests/utils.rs`. +1,245 bytes, SHA-256 `088f3f4caf04dbbf2734739aeb72a9a742e5ba8fb20f840dfd24123fb4027dd7`. diff --git a/tests/compatibility/manifest/pending_tails/checkpoint.json b/tests/compatibility/manifest/pending_tails/checkpoint.json new file mode 100644 index 00000000000..a58882debcb --- /dev/null +++ b/tests/compatibility/manifest/pending_tails/checkpoint.json @@ -0,0 +1 @@ +{"last_version":7,"compacted_actions":7,"checkpoint":{"metadata":{"column_metadatas":[{"column_schema":{"name":"ts","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Timestamp","column_id":45},{"column_schema":{"name":"pk","data_type":{"String":{"size_type":"Utf8"}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Tag","column_id":36},{"column_schema":{"name":"val","data_type":{"Float64":{}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Field","column_id":251}],"primary_key":[36],"region_id":98784247841,"schema_version":0,"primary_key_encoding":"dense","partition_expr":null},"files":{},"removed_files":{"removed_files":[]},"flushed_entry_id":4294967296,"flushed_sequence":100,"committed_sequence":1099511627781,"manifest_version":7,"truncated_entry_id":null,"compaction_time_window":null,"sst_format":"primary_key","append_mode":null,"pending_tails":[{"node_id":1,"after":4294967296,"upto":4294967301,"upto_kind":"reported"},{"node_id":2,"after":8589934592,"upto":12884901888,"upto_kind":"listed"}],"handoff_floors":{"3":17179869184}}} diff --git a/tests/compatibility/manifest/pending_tails/delta.json b/tests/compatibility/manifest/pending_tails/delta.json new file mode 100644 index 00000000000..2305629cab9 --- /dev/null +++ b/tests/compatibility/manifest/pending_tails/delta.json @@ -0,0 +1 @@ +{"actions":[{"Edit":{"files_to_add":[],"files_to_remove":[],"timestamp_ms":1762732800000,"compaction_time_window":null,"flushed_entry_id":null,"flushed_sequence":null,"committed_sequence":1099511627781,"pending_tails":[{"node_id":1,"after":4294967296,"upto":4294967301,"upto_kind":"reported"},{"node_id":2,"after":8589934592,"upto":12884901888,"upto_kind":"listed"}],"handoff_floor":{"node_id":3,"entry_id":17179869184}}}]}