diff --git a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs index 0b09d1634ca..fc004c2af13 100644 --- a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs +++ b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs @@ -512,6 +512,8 @@ pub(super) async fn run_direct_sst(args: DirectArgs) { compaction_time_window: None, sst_format: format, append_mode: table.append_mode, + pending_tails: Default::default(), + handoff_floors: Default::default(), }; let checkpoint = RegionCheckpoint { last_version: args.checkpoint_version, diff --git a/src/cmd/src/datanode/sst_replace.rs b/src/cmd/src/datanode/sst_replace.rs index df13fcd6f25..d72a12a6002 100644 --- a/src/cmd/src/datanode/sst_replace.rs +++ b/src/cmd/src/datanode/sst_replace.rs @@ -208,6 +208,7 @@ impl SstReplaceCommand { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; let version = manifest_manager .update( diff --git a/src/datanode/src/region_server/catalog/inspect.rs b/src/datanode/src/region_server/catalog/inspect.rs index da3ebdb130b..875d7271b37 100644 --- a/src/datanode/src/region_server/catalog/inspect.rs +++ b/src/datanode/src/region_server/catalog/inspect.rs @@ -426,6 +426,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/datanode/src/region_server/catalog/tests/manifest.rs b/src/datanode/src/region_server/catalog/tests/manifest.rs index 91b9f757a3d..81288b506a6 100644 --- a/src/datanode/src/region_server/catalog/tests/manifest.rs +++ b/src/datanode/src/region_server/catalog/tests/manifest.rs @@ -74,6 +74,7 @@ async fn manifest_engine(regions: u32, files_per_region: usize) -> (TestEnv, Mit flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await @@ -189,6 +190,7 @@ async fn test_manifest_referring_region_and_origin_path() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/metric-engine/src/engine/sync/region.rs b/src/metric-engine/src/engine/sync/region.rs index c7b3ac8682c..dbf696db9ec 100644 --- a/src/metric-engine/src/engine/sync/region.rs +++ b/src/metric-engine/src/engine/sync/region.rs @@ -122,6 +122,7 @@ impl MetricEngineInner { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; self.mito .edit_region(target_metadata_region_id, edit) diff --git a/src/mito2/AGENTS.md b/src/mito2/AGENTS.md index 9f12cd750cd..ffd674917a2 100644 --- a/src/mito2/AGENTS.md +++ b/src/mito2/AGENTS.md @@ -82,7 +82,8 @@ pending. DDL cancellation retains its existing scheduling fence and priority. follower replay. Keep it backward compatible. Compaction task state does not belong in the manifest; publication uses ordinary file edits after locked input checks. Shared SSTs require Enterprise capability. Compatibility fixtures: - `tests/compatibility/manifest/table_compaction/`. + `tests/compatibility/manifest/table_compaction/` and + `tests/compatibility/manifest/pending_tails/`. - **SST/Parquet layout** (`sst/`): readers must stay compatible with existing files. - **Series-index disk limit**: tasks estimate usage from entry sizes in open-region snapshots and defer new builds when the shared estimate is full. Cleanup still prunes obsolete @@ -111,8 +112,21 @@ Tests live next to the code as `*_test.rs` (e.g. `src/mito2/src/engine/flush_tes depend on this. Do not change assignment lightly. - Manifest version is monotonic — never reset or skip it. - In regions whose writes require a lease and whose WAL is the object store WAL, - each manifest version has one fixed object name and only one writer can create - it (`manifest/storage.rs`). + each manifest version has one fixed object name, created conditionally. A late + create can succeed again once checkpoint cleanup frees the name, so a commit + checks its result (`ensure_version_not_checkpointed` in `manifest/storage.rs`). +- Pending tails (`RegionManifest::pending_tails`) are WAL ranges a region still has + to recover; the flushed entry id never prunes them. Opening a region replays its + WAL after `RegionManifest::wal_replay_start`. The WAL releases at open and after + a flush stay below this node's tails (`cap_wal_release`); truncate, drop, offline + cleanup and catchup release without that cap. Handoff floors are dropped once the + flushed entry id reaches them, in `RegionManifestBuilder::try_build`. While a + region has pending tails, scans and memtable/SST stats fail with + `RegionPendingTails`, and so do truncate, entering staging, a repartition flush, + a close with `flush_on_close`, and every ALTER other than adding nullable columns + without a default or changing options; compaction pauses. These checks run + before any region state changes. Full-listing GC skips a region that has pending + tails or whose manifest `has_update()` reports newer, or fails to check. - Lock ordering: take the manifest lock before updating `version_control`; the reverse deadlocks against concurrent flush/compaction. - All region I/O runs on tokio workers; never `block_on` inside a worker. diff --git a/src/mito2/src/compaction/compactor.rs b/src/mito2/src/compaction/compactor.rs index d0d2950018b..6cd51dae9a1 100644 --- a/src/mito2/src/compaction/compactor.rs +++ b/src/mito2/src/compaction/compactor.rs @@ -452,6 +452,7 @@ impl CompactionContext { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; let action_list = RegionMetaActionList::with_action(RegionMetaAction::Edit(edit.clone())); @@ -1144,6 +1145,7 @@ mod tests { compaction_time_window: None, flushed_entry_id: None, flushed_sequence: None, + wal_recovery: Default::default(), })), false, ) diff --git a/src/mito2/src/compaction/scheduler.rs b/src/mito2/src/compaction/scheduler.rs index 44886f6ccdb..c7a370df623 100644 --- a/src/mito2/src/compaction/scheduler.rs +++ b/src/mito2/src/compaction/scheduler.rs @@ -577,6 +577,14 @@ impl CompactionScheduler { waiter.send(Ok(0)); return Ok(false); } + if status.has_pending_tails() { + info!( + "Skipping compaction for region {} with pending WAL tails, options: {:?}", + region_id, compact_options + ); + waiter.send(Ok(0)); + return Ok(false); + } if status.is_compacting() { match trigger { @@ -644,6 +652,12 @@ impl CompactionScheduler { let Some(pending_request) = std::mem::take(&mut status.pending_request) else { return false; }; + if status.has_pending_tails() { + status.pending_task_state = None; + status.publish_state(); + pending_request.waiter.send(Ok(0)); + return false; + } let PendingCompaction { options, @@ -737,6 +751,9 @@ impl CompactionScheduler { let Some(status) = self.region_status.get_mut(®ion_id) else { return false; }; + if status.has_pending_tails() { + return false; + } // An external automatic trigger requires one unrestricted follow-up pick. let request = status.new_compaction_request( self.request_sender.clone(), diff --git a/src/mito2/src/compaction/scheduler/state.rs b/src/mito2/src/compaction/scheduler/state.rs index 3e232138610..f0afea33b7e 100644 --- a/src/mito2/src/compaction/scheduler/state.rs +++ b/src/mito2/src/compaction/scheduler/state.rs @@ -483,6 +483,11 @@ impl CompactionStatus { !self.pending_ddl_requests.is_empty() || self.executing_ddl.is_some() } + /// Compaction pauses while the region has WAL tails to recover. + pub(crate) fn has_pending_tails(&self) -> bool { + self.version_control.current().version.has_pending_tails() + } + /// Ends the current cycle without losing region or DDL execution state. pub(crate) fn become_idle(&mut self) { self.active = ActiveCompaction::idle(); diff --git a/src/mito2/src/compaction/scheduler_test.rs b/src/mito2/src/compaction/scheduler_test.rs index a14cf1afa88..a4d310a688f 100644 --- a/src/mito2/src/compaction/scheduler_test.rs +++ b/src/mito2/src/compaction/scheduler_test.rs @@ -41,6 +41,7 @@ use crate::compaction::test_util::new_file_handle; use crate::compaction::twcs::PickClass; use crate::compaction::{CompactionOutput, find_dynamic_options}; use crate::config::MitoConfig; +use crate::engine::pending_tail_test::{pending_tails_edit, tail}; use crate::error::InvalidSchedulerStateSnafu; use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions}; use crate::metrics::COMPACTION_MEMORY_REJECTED; @@ -51,9 +52,9 @@ use crate::schedule::remote_job_scheduler::{RemoteJob, RemoteJobSchedulerRef}; use crate::schedule::scheduler::{Job, Scheduler}; use crate::sst::FormatType; use crate::sst::file::FileHandle; -use crate::test_util::mock_schema_metadata_manager; use crate::test_util::scheduler_util::{SchedulerEnv, VecScheduler}; use crate::test_util::version_util::{VersionControlBuilder, apply_edit}; +use crate::test_util::{mock_schema_metadata_manager, new_noop_file_purger}; struct FailingScheduler; @@ -1193,6 +1194,7 @@ async fn test_pick_result_refreshes_replaced_selected_file() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], stale.file_purger(), @@ -3360,6 +3362,7 @@ async fn test_compaction_tasks_partial_memory_admission() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], builder.file_purger(), @@ -3664,6 +3667,110 @@ async fn test_compaction_tasks_linear_local_publication(#[case] reject_after_fir assert!(!scheduler.is_compacting(region_id)); } +#[tokio::test] +async fn test_finishing_compaction_with_pending_tails_starts_nothing() { + let env = SchedulerEnv::new().await; + let (tx, _rx) = mpsc::channel(4); + let mut scheduler = env.mock_compaction_scheduler(tx); + let builder = VersionControlBuilder::new(); + let version_control = Arc::new(builder.build()); + let region_id = builder.region_id(); + let manifest_ctx = env + .mock_manifest_context(version_control.current().version.metadata.clone()) + .await; + let (schema_metadata_manager, _kv_backend) = mock_schema_metadata_manager(); + let (waiter_tx, mut waiter_rx) = oneshot::channel(); + let mut status = + CompactionStatus::for_test(region_id, version_control.clone(), env.access_layer.clone()); + status.start_local_task(); + status.set_pending_request(PendingCompaction { + options: Options::Regular(Default::default()), + waiter: OptionOutputTx::from(waiter_tx), + max_parallelism: 1, + time_range: None, + }); + scheduler.region_status.insert(region_id, status); + let pending_id = scheduler.inspect_tasks(&MetadataFilter::default(), None)[0] + .task_id + .clone(); + version_control.apply_edit( + Some(pending_tails_edit(vec![tail(1, 0, 1)])), + &[], + builder.file_purger(), + ); + + scheduler + .on_compaction_finished(region_id, &manifest_ctx, schema_metadata_manager, true) + .await; + + assert_eq!(0, waiter_rx.try_recv().unwrap().unwrap()); + assert!(!scheduler.is_compacting(region_id)); + assert!( + scheduler + .inspect_tasks(&MetadataFilter::default(), None) + .is_empty() + ); + assert_eq!( + scheduler.cancel_task(region_id, &pending_id), + CancelCompactionResult::NotFound + ); +} + +#[tokio::test] +async fn test_cancelled_compaction_with_pending_tails_starts_nothing() { + let env = SchedulerEnv::new().await; + let (tx, mut rx) = mpsc::channel(8); + let mut scheduler = env.mock_compaction_scheduler(tx); + let version = compactable_version(); + let region_id = version.region_id(); + let (finished, manifest, schemas) = + begin_pick_result(&env, &mut scheduler, &mut rx, &version).await; + let active_id = scheduler.inspect_tasks(&MetadataFilter::default(), None)[0] + .task_id + .clone(); + let (sender, mut receiver) = oneshot::channel(); + scheduler + .schedule_manual_compaction( + Options::Regular(Default::default()), + &version, + &env.access_layer, + sender.into(), + &manifest, + schemas.clone(), + 1, + None, + ) + .unwrap(); + let pending_id = scheduler.inspect_tasks(&MetadataFilter::default(), None)[1] + .task_id + .clone(); + assert_eq!( + scheduler.cancel_task(region_id, &active_id), + CancelCompactionResult::CancelIssued + ); + version.apply_edit( + Some(pending_tails_edit(vec![tail(1, 0, 1)])), + &[], + new_noop_file_purger(), + ); + + scheduler + .handle_compaction_pick_finished(finished, &manifest, schemas) + .await; + + assert_eq!(0, receiver.try_recv().unwrap().unwrap()); + assert!(!scheduler.is_compacting(region_id)); + assert!( + scheduler + .inspect_tasks(&MetadataFilter::default(), None) + .is_empty() + ); + assert_eq!( + scheduler.cancel_task(region_id, &pending_id), + CancelCompactionResult::NotFound + ); +} + /// Pending task identity survives promotion, but replacement invalidates the old identity. #[tokio::test] async fn test_cancel_compaction_task_identity_and_pending() { diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs index 7c426c5e63b..8ff2b0f6ce0 100644 --- a/src/mito2/src/engine.rs +++ b/src/mito2/src/engine.rs @@ -63,6 +63,8 @@ mod open_test; #[cfg(test)] mod parallel_test; #[cfg(test)] +pub(crate) mod pending_tail_test; +#[cfg(test)] mod projection_test; #[cfg(test)] mod prune_test; @@ -159,7 +161,7 @@ use crate::error::{ #[cfg(feature = "enterprise")] use crate::extension::BoxedExtensionRangeProviderFactory; use crate::gc::GcLimiterRef; -use crate::manifest::action::RegionEdit; +use crate::manifest::action::{RegionEdit, WalRecoveryEdit}; use crate::memtable::MemtableStats; use crate::metrics::{ HANDLE_REQUEST_ELAPSED, SCAN_MEMORY_EXHAUSTED_TOTAL, SCAN_MEMORY_USAGE_BYTES, @@ -586,6 +588,7 @@ impl MitoEngine { .context(RegionNotFoundSnafu { region_id })?; let version = region.version(); + version.ensure_no_pending_tails()?; let memtable_stats = version .memtables .list_memtables() @@ -841,6 +844,7 @@ fn is_valid_region_edit(edit: &RegionEdit) -> bool { .. } ) + && edit.wal_recovery == WalRecoveryEdit::default() } /// Inner struct of [MitoEngine]. @@ -1172,6 +1176,7 @@ impl EngineInner { }); let version_data = region.version_control.current(); let version = version_data.version; + version.ensure_no_pending_tails()?; if request.snapshot_on_scan && request.memtable_max_sequence.is_none() { request.memtable_max_sequence = Some(version_data.committed_sequence); @@ -1889,6 +1894,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(is_valid_region_edit(&edit)); @@ -1901,6 +1907,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(!is_valid_region_edit(&edit)); @@ -1913,6 +1920,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(is_valid_region_edit(&edit)); @@ -1925,6 +1933,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(is_valid_region_edit(&edit)); @@ -1937,6 +1946,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(!is_valid_region_edit(&edit)); let edit = RegionEdit { @@ -1947,6 +1957,7 @@ mod tests { flushed_entry_id: Some(1), flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; assert!(!is_valid_region_edit(&edit)); let edit = RegionEdit { @@ -1957,6 +1968,21 @@ mod tests { flushed_entry_id: None, flushed_sequence: Some(1), committed_sequence: None, + wal_recovery: Default::default(), + }; + assert!(!is_valid_region_edit(&edit)); + let edit = RegionEdit { + files_to_add: vec![FileMeta::default()], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + wal_recovery: WalRecoveryEdit { + pending_tails: Some(vec![]), + handoff_floor: None, + }, }; assert!(!is_valid_region_edit(&edit)); } diff --git a/src/mito2/src/engine/apply_staging_manifest_test.rs b/src/mito2/src/engine/apply_staging_manifest_test.rs index e6ac2bc24b4..be6d651b48e 100644 --- a/src/mito2/src/engine/apply_staging_manifest_test.rs +++ b/src/mito2/src/engine/apply_staging_manifest_test.rs @@ -861,6 +861,7 @@ async fn test_apply_staging_manifest_change_edit_different_columns_fails_with_fo flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), ]), true, diff --git a/src/mito2/src/engine/basic_test.rs b/src/mito2/src/engine/basic_test.rs index ebc9647adc8..72f4949b12d 100644 --- a/src/mito2/src/engine/basic_test.rs +++ b/src/mito2/src/engine/basic_test.rs @@ -2216,6 +2216,7 @@ async fn add_manifest_scan_region( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; region .manifest_ctx diff --git a/src/mito2/src/engine/bump_committed_sequence_test.rs b/src/mito2/src/engine/bump_committed_sequence_test.rs index 23a5af88650..b5b7c23b8e4 100644 --- a/src/mito2/src/engine/bump_committed_sequence_test.rs +++ b/src/mito2/src/engine/bump_committed_sequence_test.rs @@ -84,6 +84,7 @@ async fn test_bump_committed_sequence_with_format(flat_format: bool) { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/catchup_test.rs b/src/mito2/src/engine/catchup_test.rs index b470db5e12a..2e2cdad014e 100644 --- a/src/mito2/src/engine/catchup_test.rs +++ b/src/mito2/src/engine/catchup_test.rs @@ -270,6 +270,7 @@ async fn test_catchup_preserves_region_edit_sequence( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/edit_region_test.rs b/src/mito2/src/engine/edit_region_test.rs index 68db3def4cf..f58daa55d34 100644 --- a/src/mito2/src/engine/edit_region_test.rs +++ b/src/mito2/src/engine/edit_region_test.rs @@ -137,6 +137,7 @@ async fn test_edit_region_schedule_compaction_with_format( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; engine .edit_region(region.region_id, new_edit(&[0, 10, 50, 80])) @@ -266,6 +267,7 @@ async fn test_edit_region_fill_cache( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; engine.edit_region(region.region_id, edit).await.unwrap(); @@ -775,6 +777,7 @@ fn test_region_edit(region_id: RegionId, file_id: FileId) -> RegionEdit { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), } } @@ -864,6 +867,7 @@ async fn test_edit_region_concurrently_with_format(flat_format: bool) { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; engine .edit_region(self.region.region_id, edit) diff --git a/src/mito2/src/engine/file_ref_test.rs b/src/mito2/src/engine/file_ref_test.rs index 3979c14553c..3bc1829c86b 100644 --- a/src/mito2/src/engine/file_ref_test.rs +++ b/src/mito2/src/engine/file_ref_test.rs @@ -114,6 +114,7 @@ async fn get_snapshot_preserves_related_region_old_index_readers() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })), false, ) @@ -219,6 +220,7 @@ async fn get_snapshot_preserves_old_index_readers_until_release( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })), false, ) diff --git a/src/mito2/src/engine/flush_test.rs b/src/mito2/src/engine/flush_test.rs index 399b5857700..8435a38c4a3 100644 --- a/src/mito2/src/engine/flush_test.rs +++ b/src/mito2/src/engine/flush_test.rs @@ -426,6 +426,7 @@ async fn test_ingested_parquet_bytes() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/index_build_test.rs b/src/mito2/src/engine/index_build_test.rs index 2877ac109d9..36ae38330ae 100644 --- a/src/mito2/src/engine/index_build_test.rs +++ b/src/mito2/src/engine/index_build_test.rs @@ -700,6 +700,7 @@ async fn test_index_build_uses_physical_file_region_and_logical_manifest_region( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/merge_mode_test.rs b/src/mito2/src/engine/merge_mode_test.rs index f5dd078103d..b628e2bab4e 100644 --- a/src/mito2/src/engine/merge_mode_test.rs +++ b/src/mito2/src/engine/merge_mode_test.rs @@ -127,6 +127,7 @@ async fn test_admitted_sst_compaction_preserves_intermediate_memtable( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/pending_tail_test.rs b/src/mito2/src/engine/pending_tail_test.rs new file mode 100644 index 00000000000..5f2627f0633 --- /dev/null +++ b/src/mito2/src/engine/pending_tail_test.rs @@ -0,0 +1,434 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use api::v1::region::StrictWindow; +use api::v1::region::compact_request::Options; +use api::v1::{Rows, SemanticType}; +use common_error::ext::{BoxedError, ErrorExt, RetryHint}; +use common_error::status_code::StatusCode; +use common_time::Timestamp; +use datatypes::prelude::ConcreteDataType; +use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema}; +use rstest::rstest; +use store_api::logstore::provider::RaftEngineProvider; +use store_api::metadata::ColumnMetadata; +use store_api::region_engine::RegionEngine; +use store_api::region_request::{ + AddColumn, AlterKind, EnterStagingRequest, RegionAlterRequest, RegionCloseRequest, + RegionCompactRequest, RegionCreateRequest, RegionDropRequest, RegionFlushReason, + RegionFlushRequest, RegionRequest, RegionTruncateRequest, SetRegionOption, + StagingPartitionDirective, +}; +use store_api::storage::{RegionId, ScanRequest}; + +use crate::config::MitoConfig; +use crate::engine::MitoEngine; +use crate::error::Error; +use crate::manifest::action::{ + HandoffFloor, PendingTail, RegionEdit, RegionMetaAction, RegionMetaActionList, UptoKind, + WalRecoveryEdit, +}; +use crate::region::RegionLeaderState; +use crate::test_util::{ + CreateRequestBuilder, LogStoreImpl, TestEnv, build_rows, flush_region, put_rows, reopen_region, + rows_schema, +}; +use crate::wal::EntryId; + +const NODE_ID: u64 = 1; + +pub(crate) fn tail(node_id: u64, after: EntryId, upto: EntryId) -> PendingTail { + PendingTail { + node_id, + after, + upto, + upto_kind: UptoKind::Reported, + } +} + +/// Creates a region and writes one row in each of WAL entries `1..=num_entries`. +async fn create_region_with_entries( + engine: &MitoEngine, + region_id: RegionId, + num_entries: usize, +) -> RegionCreateRequest { + let request = CreateRequestBuilder::new().build(); + let schema = rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request.clone())) + .await + .unwrap(); + for i in 0..num_entries { + let rows = Rows { + schema: schema.clone(), + rows: build_rows(i, i + 1), + }; + put_rows(engine, region_id, rows).await; + } + request +} + +pub(crate) fn pending_tails_edit(pending_tails: Vec) -> RegionEdit { + RegionEdit { + files_to_add: vec![], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + wal_recovery: WalRecoveryEdit { + pending_tails: Some(pending_tails), + handoff_floor: None, + }, + } +} + +async fn commit_edit(engine: &MitoEngine, region_id: RegionId, edit: RegionEdit) { + let region = engine.get_region(region_id).unwrap(); + region + .manifest_ctx + .update_manifest( + RegionLeaderState::Writable, + RegionMetaActionList::with_action(RegionMetaAction::Edit(edit.clone())), + false, + ) + .await + .unwrap(); + region + .version_control + .apply_edit(Some(edit), &[], region.file_purger.clone()); +} + +pub(crate) async fn set_pending_tails( + engine: &MitoEngine, + region_id: RegionId, + pending_tails: Vec, +) { + commit_edit(engine, region_id, pending_tails_edit(pending_tails)).await; +} + +async fn reopen(engine: &MitoEngine, region_id: RegionId, request: RegionCreateRequest) { + reopen_region(engine, region_id, request.table_dir, true, request.options).await; +} + +fn wal_span(env: &TestEnv, region_id: RegionId) -> (Option, Option) { + let Some(LogStoreImpl::RaftEngine(log_store)) = env.get_log_store() else { + unreachable!() + }; + log_store.span(&RaftEngineProvider::new(region_id.into())) +} + +#[rstest] +#[case::own_tail(vec![tail(NODE_ID, 0, 2)], None, 1)] +#[case::other_node_tail(vec![tail(NODE_ID + 1, 0, 2)], None, 3)] +#[case::own_floor(vec![], Some(NODE_ID), 1)] +#[case::other_node_floor(vec![], Some(NODE_ID + 1), 3)] +#[tokio::test] +async fn test_open_replays_wal_after_own_tails_and_floor( + #[case] pending_tails: Vec, + #[case] floor_node_id: Option, + #[case] replayed_rows: u64, +) { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let request = create_region_with_entries(&engine, region_id, 3).await; + let has_pending_tails = !pending_tails.is_empty(); + let mut edit = pending_tails_edit(pending_tails); + edit.wal_recovery.handoff_floor = floor_node_id.map(|node_id| HandoffFloor { + node_id, + entry_id: 2, + }); + commit_edit(&engine, region_id, edit).await; + + reopen(&engine, region_id, request).await; + + let region = engine.get_region(region_id).unwrap(); + assert_eq!(replayed_rows, region.version().memtables.num_rows()); + let scan = engine.scanner(region_id, ScanRequest::default()).await; + assert_eq!( + has_pending_tails, + matches!(scan, Err(Error::RegionPendingTails { .. })) + ); +} + +#[tokio::test] +async fn test_open_keeps_last_entry_id_at_flushed_entry_id() { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let request = create_region_with_entries(&engine, region_id, 3).await; + flush_region(&engine, region_id, None).await; + let listed = PendingTail { + node_id: NODE_ID, + after: 3, + upto: 100, + upto_kind: UptoKind::Listed, + }; + set_pending_tails(&engine, region_id, vec![listed]).await; + + reopen(&engine, region_id, request).await; + + let region = engine.get_region(region_id).unwrap(); + assert_eq!(3, region.version_control.current().last_entry_id); +} + +#[tokio::test] +async fn test_flush_keeps_wal_after_own_tails() { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + create_region_with_entries(&engine, region_id, 3).await; + set_pending_tails(&engine, region_id, vec![tail(NODE_ID, 1, 2)]).await; + + flush_region(&engine, region_id, None).await; + + assert_eq!((Some(2), Some(3)), wal_span(&env, region_id)); +} + +#[tokio::test] +async fn test_open_keeps_wal_after_own_tails() { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let request = create_region_with_entries(&engine, region_id, 3).await; + set_pending_tails(&engine, region_id, vec![tail(NODE_ID, 1, 3)]).await; + + reopen(&engine, region_id, request).await; + + assert_eq!((Some(2), Some(3)), wal_span(&env, region_id)); +} + +#[tokio::test] +async fn test_drop_releases_wal_with_pending_tails() { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + create_region_with_entries(&engine, region_id, 3).await; + set_pending_tails(&engine, region_id, vec![tail(NODE_ID, 0, 3)]).await; + + engine + .handle_request( + region_id, + RegionRequest::Drop(RegionDropRequest { + fast_path: false, + force: false, + partial_drop: false, + }), + ) + .await + .unwrap(); + + assert_eq!((None, None), wal_span(&env, region_id)); +} + +/// Creates a region with rows in WAL entries `1..=num_entries` and one pending tail. +async fn region_with_pending_tails(num_entries: usize) -> (TestEnv, MitoEngine, RegionId) { + let mut env = TestEnv::new().await.with_node_id(NODE_ID); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + create_region_with_entries(&engine, region_id, num_entries).await; + set_pending_tails(&engine, region_id, vec![tail(NODE_ID, 0, 1)]).await; + (env, engine, region_id) +} + +fn assert_pending_tails_error(err: &BoxedError) { + let err = err.as_any().downcast_ref::(); + assert!( + matches!(err, Some(Error::RegionPendingTails { .. })), + "{err:?}" + ); +} + +fn add_field(default_constraint: Option) -> AlterKind { + let column_schema = ColumnSchema::new("field_1", ConcreteDataType::float64_datatype(), true) + .with_default_constraint(default_constraint) + .unwrap(); + AlterKind::AddColumns { + columns: vec![AddColumn { + column_metadata: ColumnMetadata { + column_schema, + semantic_type: SemanticType::Field, + column_id: 10, + }, + location: None, + }], + } +} + +#[tokio::test] +async fn test_scan_rejects_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + + let Err(err) = engine.scanner(region_id, ScanRequest::default()).await else { + panic!("scan succeeded with pending tails"); + }; + + assert!(matches!(err, Error::RegionPendingTails { .. }), "{err:?}"); + assert_eq!(StatusCode::RegionNotReady, err.status_code()); + assert_eq!(RetryHint::Retryable, err.retry_hint()); +} + +#[tokio::test] +async fn test_memtable_and_sst_stats_reject_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + + let result = engine.find_memtable_and_sst_stats(region_id); + + assert!(matches!(result, Err(Error::RegionPendingTails { .. }))); +} + +#[tokio::test] +async fn test_manual_compaction_skips_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + flush_region(&engine, region_id, None).await; + let rows = Rows { + schema: rows_schema(&CreateRequestBuilder::new().build()), + rows: build_rows(1, 2), + }; + put_rows(&engine, region_id, rows).await; + flush_region(&engine, region_id, None).await; + + let request = RegionCompactRequest { + options: Options::StrictWindow(StrictWindow { + window_seconds: 3600, + }), + ..Default::default() + }; + engine + .handle_request(region_id, RegionRequest::Compact(request)) + .await + .unwrap(); + + let region = engine.get_region(region_id).unwrap(); + assert_eq!(2, region.version().ssts.owned_num_files(region_id)); +} + +#[rstest] +#[case::all(RegionTruncateRequest::All)] +#[case::unflushed(RegionTruncateRequest::Unflushed)] +#[case::by_time_ranges(RegionTruncateRequest::ByTimeRanges { + time_ranges: vec![(Timestamp::new_millisecond(0), Timestamp::new_millisecond(1000))], +})] +#[tokio::test] +async fn test_truncate_rejects_region_with_pending_tails(#[case] request: RegionTruncateRequest) { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + + let err = engine + .handle_request(region_id, RegionRequest::Truncate(request)) + .await + .unwrap_err(); + + assert_pending_tails_error(&err); + let region = engine.get_region(region_id).unwrap(); + assert_eq!(1, region.version().memtables.num_rows()); +} + +#[rstest] +#[case::nullable_column(add_field(None), true)] +#[case::region_options(AlterKind::SetRegionOptions { + options: vec![SetRegionOption::AppendMode(true)], +}, true)] +#[case::default_null_column(add_field(Some(ColumnDefaultConstraint::null_value())), false)] +#[case::drop_column(AlterKind::DropColumns { + names: vec!["field_0".to_string()], +}, false)] +#[tokio::test] +async fn test_alter_with_pending_tails(#[case] kind: AlterKind, #[case] allowed: bool) { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + let metadata = engine.get_region(region_id).unwrap().metadata(); + + let result = engine + .handle_request(region_id, RegionRequest::Alter(RegionAlterRequest { kind })) + .await; + + if allowed { + result.unwrap(); + } else { + assert_pending_tails_error(&result.unwrap_err()); + let region = engine.get_region(region_id).unwrap(); + assert_eq!(metadata, region.metadata()); + assert_eq!(1, region.version().memtables.num_rows()); + } +} + +#[tokio::test] +async fn test_alter_rejects_no_op_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + let column_metadatas = engine + .get_region(region_id) + .unwrap() + .metadata() + .column_metadatas + .clone(); + + let kind = AlterKind::SyncColumns { column_metadatas }; + let err = engine + .handle_request(region_id, RegionRequest::Alter(RegionAlterRequest { kind })) + .await + .unwrap_err(); + + assert_pending_tails_error(&err); +} + +#[tokio::test] +async fn test_enter_staging_rejects_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + + let request = EnterStagingRequest { + partition_directive: StagingPartitionDirective::RejectAllWrites, + }; + let err = engine + .handle_request(region_id, RegionRequest::EnterStaging(request)) + .await + .unwrap_err(); + + assert_pending_tails_error(&err); + let region = engine.get_region(region_id).unwrap(); + assert!(!region.is_staging()); + assert_eq!(1, region.version().memtables.num_rows()); +} + +#[tokio::test] +async fn test_repartition_flush_rejects_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(0).await; + + let request = RegionFlushRequest { + reason: Some(RegionFlushReason::Repartition), + ..Default::default() + }; + let err = engine + .handle_request(region_id, RegionRequest::Flush(request)) + .await + .unwrap_err(); + + assert_pending_tails_error(&err); +} + +#[tokio::test] +async fn test_close_with_flush_rejects_region_with_pending_tails() { + let (_env, engine, region_id) = region_with_pending_tails(1).await; + + let request = RegionCloseRequest { + flush_on_close: true, + }; + let err = engine + .handle_request(region_id, RegionRequest::Close(request)) + .await + .unwrap_err(); + + assert_pending_tails_error(&err); + assert!(engine.get_region(region_id).is_some()); +} diff --git a/src/mito2/src/engine/scan_test.rs b/src/mito2/src/engine/scan_test.rs index e495184b112..8324e3a03e5 100644 --- a/src/mito2/src/engine/scan_test.rs +++ b/src/mito2/src/engine/scan_test.rs @@ -3057,6 +3057,7 @@ async fn test_exact_sequence_range_unsupported_when_legacy_file() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; engine.edit_region(region_id, edit).await.unwrap(); @@ -3144,6 +3145,7 @@ async fn test_exact_sequence_range_accepts_foreign_file_with_target_barrier() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await @@ -4244,6 +4246,7 @@ async fn test_compaction_output_non_preserve_not_laundered_from_legacy_input( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/staging_test.rs b/src/mito2/src/engine/staging_test.rs index 10351f918d9..a0501362ea8 100644 --- a/src/mito2/src/engine/staging_test.rs +++ b/src/mito2/src/engine/staging_test.rs @@ -769,6 +769,7 @@ async fn test_staging_exit_success_with_manifests_with_format(flat_format: bool) flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await @@ -907,6 +908,7 @@ async fn test_staging_exit_success_with_manifests_with_format(flat_format: bool) flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })]), true, ) @@ -1147,6 +1149,7 @@ async fn test_staging_exit_conflict_partition_expr_change_and_change_with_format flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), ]), true, @@ -1627,6 +1630,7 @@ fn edit_adding(file: FileMeta) -> RegionEdit { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), } } @@ -1986,6 +1990,7 @@ async fn test_successful_staging_end_keeps_rows_of_rejected_flush() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await @@ -2410,6 +2415,7 @@ async fn test_close_after_end_of_staging_is_queued_keeps_committed_result() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/engine/sync_test.rs b/src/mito2/src/engine/sync_test.rs index ee0cd7ebe64..1d3fff9216e 100644 --- a/src/mito2/src/engine/sync_test.rs +++ b/src/mito2/src/engine/sync_test.rs @@ -210,6 +210,7 @@ async fn test_sync_after_flush_region_with_format(flat_format: bool) { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ) .await diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs index 5085ca8f78c..18cc8fcf814 100644 --- a/src/mito2/src/error.rs +++ b/src/mito2/src/error.rs @@ -730,6 +730,13 @@ pub enum Error { location: Location, }, + #[snafu(display("Region {} still has WAL tails to recover", region_id))] + RegionPendingTails { + region_id: RegionId, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Region {} does not hold a valid leader lease", region_id))] RegionLeaseExpired { region_id: RegionId, @@ -1693,6 +1700,7 @@ impl ErrorExt for Error { InvalidRegionRequest { source, .. } => source.status_code(), RegionState { .. } | UpdateManifest { .. } => StatusCode::RegionNotReady, RegionLeaseExpired { .. } | UnacknowledgedWrite { .. } => StatusCode::RegionNotReady, + RegionPendingTails { .. } => StatusCode::RegionNotReady, JsonOptions { .. } => StatusCode::InvalidArguments, EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound, ConvertValue { source, .. } => source.status_code(), @@ -1805,6 +1813,7 @@ impl ErrorExt for Error { | UpdateManifest { .. } | RegionStopped { .. } | RegionBusy { .. } + | RegionPendingTails { .. } | ManualCompactionAlreadyRunning { .. } | FlushableRegionState { .. } | ManifestDeltaNotFound { .. } diff --git a/src/mito2/src/flush.rs b/src/mito2/src/flush.rs index 97bdf6dbb47..d0c19c13ec8 100644 --- a/src/mito2/src/flush.rs +++ b/src/mito2/src/flush.rs @@ -533,6 +533,7 @@ impl RegionFlushTask { flushed_entry_id: Some(version_data.last_entry_id), flushed_sequence: Some(version_data.committed_sequence), committed_sequence: None, + wal_recovery: Default::default(), }; info!("Applying {edit:?} to region {}", self.region_id); @@ -1940,6 +1941,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, memtables_to_remove: smallvec![], }), @@ -2194,6 +2196,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[0], builder.file_purger(), @@ -2502,6 +2505,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[0], builder.file_purger(), @@ -2576,6 +2580,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[0], builder.file_purger(), diff --git a/src/mito2/src/gc.rs b/src/mito2/src/gc.rs index 227875ef3ff..3541485c25f 100644 --- a/src/mito2/src/gc.rs +++ b/src/mito2/src/gc.rs @@ -464,7 +464,8 @@ impl LocalGcWorker { ); let manifest = if let Some(region) = ®ion { - let manifest = region.manifest_ctx.manifest().await; + let manager = region.manifest_ctx.manifest_manager.read().await; + let manifest = manager.manifest(); // If the manifest version does not match, skip GC for this region to avoid deleting files that are still in use. let file_ref_manifest_version = self .file_ref_manifest @@ -487,6 +488,24 @@ impl LocalGcWorker { needs_retry: false, }); } + // SSTs written for pending tails but not committed yet, and SSTs of a newer + // manifest, are not in this snapshot, so a full listing would take them for orphans. + if self.full_file_listing + && (!manifest.pending_tails.is_empty() + || manager.has_update().await.unwrap_or_else(|e| { + warn!(e; "Failed to check manifest updates of region {}", region_id); + true + })) + { + info!( + "Skip full listing GC for region {}: it has pending WAL tails or a newer manifest may exist", + region_id + ); + return Ok(RegionGcOutcome { + removed_files: vec![], + needs_retry: false, + }); + } Some(manifest) } else { None diff --git a/src/mito2/src/gc/worker_test.rs b/src/mito2/src/gc/worker_test.rs index 55a04dbad5e..d7e41561d29 100644 --- a/src/mito2/src/gc/worker_test.rs +++ b/src/mito2/src/gc/worker_test.rs @@ -20,9 +20,13 @@ use api::v1::Rows; use async_trait::async_trait; use bytes::Bytes; use common_base::Plugins; +use common_datasource::compression::CompressionType; use common_telemetry::init_default_ut_logging; use common_time::Timestamp; use futures::TryStreamExt; +use object_store::layers::mock::{ + Error as MockError, ErrorKind, MockLayerBuilder, Result as MockResult, oio, +}; use object_store::{Entry, ObjectStore, services}; use store_api::metadata::RegionMetadataRef; use store_api::region_engine::{RegionEngine as _, RegionRole}; @@ -33,10 +37,12 @@ use crate::access_layer::AccessLayerRef; use crate::config::MitoConfig; use crate::engine::MitoEngine; use crate::engine::compaction_test::{delete_and_flush, put_and_flush}; +use crate::engine::pending_tail_test::{set_pending_tails, tail}; use crate::engine::region_hook::{RegionGcInfo, RegionHook, RegionHookRef}; use crate::error::UnexpectedSnafu; use crate::gc::{GcConfig, LocalGcWorker, should_delete_file}; use crate::manifest::action::{RegionEdit, RegionMetaAction, RegionMetaActionList, RemovedFile}; +use crate::manifest::storage::{delta_file, gen_path}; use crate::metrics::GC_SKIPPED_UNPARSABLE_FILES; use crate::region::{MitoRegionRef, RegionLeaderState}; use crate::sst::file::{ColumnIndexMetadata, FileHandle, FileMeta, IndexType, RegionIndexId}; @@ -319,6 +325,7 @@ async fn test_gc_reclaims_obsolete_index_after_readers_and_linger_release( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })), false, ) @@ -1788,3 +1795,83 @@ async fn test_on_region_gc_failure_routes_region_to_retry() { "a failed extension cleanup must not report deleted_files for the region" ); } + +struct FailingLister; + +impl oio::List for FailingLister { + async fn next(&mut self) -> MockResult> { + Err(MockError::new(ErrorKind::Unexpected, "mock list failure")) + } +} + +#[rstest::rstest] +#[case::pending_tails("pending_tails")] +#[case::newer_manifest("newer_manifest")] +#[case::manifest_list_failure("manifest_list_failure")] +#[tokio::test] +async fn test_full_listing_gc_skips_region(#[case] cause: &str) { + let fail_manifest_lists = Arc::new(AtomicBool::new(false)); + let fail = fail_manifest_lists.clone(); + let mock_layer = MockLayerBuilder::default() + .lister_factory(Arc::new(move |path, _, inner| { + if fail.load(Ordering::Relaxed) && path.contains("/manifest/") { + Box::new(FailingLister) + } else { + inner + } + })) + .build() + .unwrap(); + let mut env = TestEnv::new().await.with_mock_layer(mock_layer); + let hook = Arc::new(GcCountingHook::default()); + let plugins = Plugins::new(); + plugins.insert(hook.clone() as RegionHookRef); + let config = MitoConfig { + gc: GcConfig { + enable: true, + ..Default::default() + }, + ..Default::default() + }; + let engine = env.create_engine_with_plugins(config, plugins).await; + let region_id = RegionId::new(1, 1); + engine + .handle_request( + region_id, + RegionRequest::Create(CreateRequestBuilder::new().build()), + ) + .await + .unwrap(); + let region = engine.get_region(region_id).unwrap(); + let run_gc = async || { + let file_ref_manifest = FileRefsManifest { + file_refs: Default::default(), + manifest_version: [(region_id, region.manifest_ctx.manifest_version().await)].into(), + cross_region_refs: HashMap::new(), + }; + let regions = BTreeMap::from([(region_id, Some(region.clone()))]); + let gc_worker = create_gc_worker(&engine, regions, &file_ref_manifest, true).await; + gc_worker.run().await.unwrap(); + }; + run_gc().await; + assert_eq!(1, hook.gc_calls.load(Ordering::Relaxed)); + + match cause { + "pending_tails" => set_pending_tails(&engine, region_id, vec![tail(1, 0, 1)]).await, + "newer_manifest" => { + let store = region.manifest_ctx.manifest_manager.read().await.store(); + let version = region.manifest_ctx.manifest_version().await; + let path = gen_path( + store.manifest_dir(), + &delta_file(version + 1), + CompressionType::Uncompressed, + ); + let object_store = region.access_layer.object_store(); + object_store.write(&path, b"".as_slice()).await.unwrap(); + } + _ => fail_manifest_lists.store(true, Ordering::Relaxed), + } + run_gc().await; + + assert_eq!(1, hook.gc_calls.load(Ordering::Relaxed)); +} diff --git a/src/mito2/src/manifest/action.rs b/src/mito2/src/manifest/action.rs index 5d5620bdab8..cd00237087a 100644 --- a/src/mito2/src/manifest/action.rs +++ b/src/mito2/src/manifest/action.rs @@ -14,7 +14,7 @@ //! Defines [RegionMetaAction] related structs and [RegionCheckpoint]. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::time::Duration; use chrono::Utc; @@ -95,6 +95,62 @@ pub struct RegionEdit { pub flushed_entry_id: Option, pub flushed_sequence: Option, pub committed_sequence: Option, + #[serde(flatten)] + pub wal_recovery: WalRecoveryEdit, +} + +/// Changes an edit makes to the WAL ranges the region still has to recover. +#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)] +pub struct WalRecoveryEdit { + /// The new [RegionManifest::pending_tails], whose segments of each node keep the + /// order and bounds described there. `None` keeps them unchanged. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pending_tails: Option>, + /// Replaces the handoff floor of one node. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub handoff_floor: Option, +} + +/// WAL entries of one node in `(after, upto]` that the region still has to recover. +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct PendingTail { + /// Id of the datanode whose WAL holds the entries. + pub node_id: u64, + pub after: EntryId, + pub upto: EntryId, + pub upto_kind: UptoKind, +} + +/// Where the upper bound of a [PendingTail] comes from. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum UptoKind { + /// The last entry id the node reported, which names an entry that can be replayed. + Reported, + /// An upper bound found by listing the WAL objects of the node, which may name no entry. + Listed, +} + +/// Entries of a node at or below `entry_id` are never recovered from its WAL, except +/// those in a pending tail of the node. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +pub struct HandoffFloor { + pub node_id: u64, + pub entry_id: EntryId, +} + +/// Caps a release of the WAL of node `node_id` at `entry_id`, so that it keeps every +/// pending tail of the node. +pub(crate) fn cap_wal_release( + pending_tails: &[PendingTail], + node_id: Option, + entry_id: EntryId, +) -> EntryId { + pending_tails + .iter() + .filter(|tail| Some(tail.node_id) == node_id) + .map(|tail| tail.after) + .fold(entry_id, EntryId::min) } impl RegionEdit { @@ -191,6 +247,38 @@ pub struct RegionManifest { /// Whether the region is in append mode. #[serde(default)] pub append_mode: Option, + /// WAL ranges the region still has to recover, in the order they were added. + /// + /// The segments of one node are increasing and do not overlap, and each has + /// `after < upto`: a new segment of a node takes as `after` the largest of the + /// flushed entry id, the `upto`s of the node's existing segments and its handoff + /// floor, and is added only if that is below its `upto`. So the entries of a node + /// between two of its segments are flushed or at or below a handoff floor of the + /// node, and need no replay. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub pending_tails: Vec, + /// The [HandoffFloor] of each node, removed once the flushed entry id reaches it. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub handoff_floors: BTreeMap, +} + +impl RegionManifest { + /// Returns the entry id after which node `node_id` replays its WAL when it opens + /// the region. Entries up to it are flushed, left to the pending tails, or below + /// the handoff floor of the node. + /// + /// Entries of the node in `(after, upto]` of one of its tails belong to that tail. + /// Whoever adds a tail ensures that every entry id the node allocates afterwards is + /// above the tail's `upto`, so no later write of the node falls in a skipped range. + pub(crate) fn wal_replay_start(&self, node_id: Option) -> EntryId { + let floor = node_id.and_then(|node_id| self.handoff_floors.get(&node_id).copied()); + self.pending_tails + .iter() + .filter(|tail| Some(tail.node_id) == node_id) + .map(|tail| tail.upto) + .chain(floor) + .fold(self.flushed_entry_id, EntryId::max) + } } #[cfg(test)] @@ -204,6 +292,8 @@ impl PartialEq for RegionManifest { && self.truncated_entry_id == other.truncated_entry_id && self.compaction_time_window == other.compaction_time_window && self.committed_sequence == other.committed_sequence + && self.pending_tails == other.pending_tails + && self.handoff_floors == other.handoff_floors } } @@ -229,6 +319,8 @@ pub struct RegionManifestBuilder { committed_sequence: Option, sst_format: FormatType, append_mode: Option, + pending_tails: Vec, + handoff_floors: BTreeMap, } impl RegionManifestBuilder { @@ -256,6 +348,8 @@ impl RegionManifestBuilder { committed_sequence: s.committed_sequence, sst_format: s.sst_format, append_mode: s.append_mode, + pending_tails: s.pending_tails, + handoff_floors: s.handoff_floors, } } else { Default::default() @@ -335,6 +429,12 @@ impl RegionManifestBuilder { if let Some(window) = edit.compaction_time_window { self.compaction_time_window = Some(window); } + if let Some(pending_tails) = edit.wal_recovery.pending_tails { + self.pending_tails = pending_tails; + } + if let Some(floor) = edit.wal_recovery.handoff_floor { + self.handoff_floors.insert(floor.node_id, floor.entry_id); + } } pub fn apply_truncate(&mut self, manifest_version: ManifestVersion, truncate: RegionTruncate) { @@ -389,8 +489,11 @@ impl RegionManifestBuilder { self.metadata.is_some() } - pub fn try_build(self) -> Result { + pub fn try_build(mut self) -> Result { let metadata = self.metadata.context(RegionMetadataNotFoundSnafu)?; + // A floor at or below the flushed entry id no longer raises the replay start. + self.handoff_floors + .retain(|_, floor| *floor > self.flushed_entry_id); Ok(RegionManifest { metadata, files: self.files, @@ -403,6 +506,8 @@ impl RegionManifestBuilder { compaction_time_window: self.compaction_time_window, sst_format: self.sst_format, append_mode: self.append_mode, + pending_tails: self.pending_tails, + handoff_floors: self.handoff_floors, }) } } @@ -597,6 +702,7 @@ impl RegionMetaActionList { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; let mut partition_expr_change = None; let mut region_change = None; @@ -655,6 +761,7 @@ impl RegionMetaActionList { #[cfg(test)] mod tests { + use std::sync::Arc; use common_time::Timestamp; @@ -991,6 +1098,8 @@ mod tests { }, sst_format: FormatType::PrimaryKey, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), }; let json = serde_json::to_string(&manifest).unwrap(); @@ -1026,6 +1135,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ); builder.apply_edit( @@ -1038,6 +1148,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ); @@ -1057,6 +1168,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, ); builder.apply_truncate( @@ -1171,6 +1283,8 @@ mod tests { compaction_time_window: None, sst_format: FormatType::PrimaryKey, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), } ); @@ -1186,6 +1300,8 @@ mod tests { compaction_time_window: None, sst_format: FormatType::PrimaryKey, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), }; let json = serde_json::to_string(&new_manifest).unwrap(); let old_from_new: RegionManifestV1 = serde_json::from_str(&json).unwrap(); @@ -1230,6 +1346,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, new_from_old ); @@ -1243,6 +1360,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; let new_json = serde_json::to_string(&new).unwrap(); @@ -1403,4 +1521,273 @@ mod tests { .contains(&RemovedFile::File(file_id, None)) ); } + + fn tail(node_id: u64, after: EntryId, upto: EntryId) -> PendingTail { + PendingTail { + node_id, + after, + upto, + upto_kind: UptoKind::Reported, + } + } + + fn wal_recovery_edit( + flushed_entry_id: Option, + wal_recovery: WalRecoveryEdit, + ) -> RegionEdit { + RegionEdit { + files_to_add: vec![], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id, + flushed_sequence: None, + committed_sequence: None, + wal_recovery, + } + } + + fn manifest_builder() -> RegionManifestBuilder { + let mut builder = RegionManifestBuilder::default(); + builder.apply_change( + 0, + RegionChange { + metadata: Arc::new(crate::manifest::tests::utils::basic_region_metadata()), + sst_format: FormatType::PrimaryKey, + append_mode: None, + }, + ); + builder + } + + #[test] + fn test_pending_tails_change_only_when_set() { + let mut builder = manifest_builder(); + builder.apply_edit( + 1, + wal_recovery_edit( + None, + WalRecoveryEdit { + pending_tails: Some(vec![tail(1, 10, 20)]), + handoff_floor: None, + }, + ), + ); + builder.apply_edit(2, wal_recovery_edit(Some(5), WalRecoveryEdit::default())); + assert_eq!(builder.pending_tails, vec![tail(1, 10, 20)]); + + builder.apply_edit( + 3, + wal_recovery_edit( + None, + WalRecoveryEdit { + pending_tails: Some(vec![]), + handoff_floor: None, + }, + ), + ); + assert!(builder.pending_tails.is_empty()); + } + + #[test] + fn test_flushed_entry_id_drops_reached_floors_and_keeps_tails() { + let mut builder = manifest_builder(); + for (node_id, entry_id) in [(1, 20), (2, 30)] { + builder.apply_edit( + 1, + wal_recovery_edit( + None, + WalRecoveryEdit { + pending_tails: Some(vec![tail(1, 10, 20)]), + handoff_floor: Some(HandoffFloor { node_id, entry_id }), + }, + ), + ); + } + builder.apply_edit(2, wal_recovery_edit(Some(20), WalRecoveryEdit::default())); + + let manifest = builder.try_build().unwrap(); + assert_eq!(manifest.pending_tails, vec![tail(1, 10, 20)]); + assert_eq!(manifest.handoff_floors, BTreeMap::from([(2, 30)])); + } + + #[test] + fn test_truncate_drops_reached_floors() { + let mut builder = manifest_builder(); + builder.apply_edit( + 1, + wal_recovery_edit( + None, + WalRecoveryEdit { + pending_tails: None, + handoff_floor: Some(HandoffFloor { + node_id: 1, + entry_id: 20, + }), + }, + ), + ); + builder.apply_truncate( + 2, + RegionTruncate { + region_id: RegionId::new(1, 1), + kind: TruncateKind::All { + truncated_entry_id: 20, + truncated_sequence: 20, + }, + timestamp_ms: None, + }, + ); + + assert!(builder.try_build().unwrap().handoff_floors.is_empty()); + } + + #[test] + fn test_wal_replay_start() { + let mut builder = manifest_builder(); + // Node 1 was handed off at 40 after its first tail, so its second tail starts + // there and entries in (20, 40] belong to no tail. + builder.apply_edit( + 1, + wal_recovery_edit( + Some(10), + WalRecoveryEdit { + pending_tails: Some(vec![tail(1, 10, 20), tail(2, 30, 60), tail(1, 40, 50)]), + handoff_floor: Some(HandoffFloor { + node_id: 1, + entry_id: 40, + }), + }, + ), + ); + builder.apply_edit( + 2, + wal_recovery_edit( + None, + WalRecoveryEdit { + pending_tails: None, + handoff_floor: Some(HandoffFloor { + node_id: 3, + entry_id: 40, + }), + }, + ), + ); + let manifest = builder.try_build().unwrap(); + + assert_eq!(manifest.wal_replay_start(Some(1)), 50); + assert_eq!(manifest.wal_replay_start(Some(2)), 60); + assert_eq!(manifest.wal_replay_start(Some(3)), 40); + assert_eq!(manifest.wal_replay_start(Some(4)), 10); + assert_eq!(manifest.wal_replay_start(None), 10); + } + + #[test] + fn test_cap_wal_release() { + let tails = [tail(1, 20, 25), tail(2, 5, 60), tail(1, 30, 40)]; + + assert_eq!(cap_wal_release(&tails, Some(1), 50), 20); + assert_eq!(cap_wal_release(&tails, Some(1), 10), 10); + assert_eq!(cap_wal_release(&tails, Some(3), 50), 50); + assert_eq!(cap_wal_release(&tails, None, 50), 50); + } + + const PENDING_TAILS_DELTA: &str = + include_str!("../../../../tests/compatibility/manifest/pending_tails/delta.json"); + const PENDING_TAILS_CHECKPOINT: &str = + include_str!("../../../../tests/compatibility/manifest/pending_tails/checkpoint.json"); + + fn fixture_pending_tails() -> Vec { + vec![ + tail(1, 1 << 32, (1 << 32) + 5), + PendingTail { + upto_kind: UptoKind::Listed, + ..tail(2, 2 << 32, 3 << 32) + }, + ] + } + + #[test] + fn test_pending_tails_fixtures() { + let delta = PENDING_TAILS_DELTA.trim_end(); + let actions = RegionMetaActionList::decode(delta.as_bytes()).unwrap(); + let [RegionMetaAction::Edit(edit)] = actions.actions.as_slice() else { + panic!("unexpected actions: {actions:?}"); + }; + let expected = WalRecoveryEdit { + pending_tails: Some(fixture_pending_tails()), + handoff_floor: Some(HandoffFloor { + node_id: 3, + entry_id: 4 << 32, + }), + }; + assert_eq!(edit.wal_recovery, expected); + assert_eq!(actions.encode().unwrap(), delta.as_bytes()); + + let checkpoint = PENDING_TAILS_CHECKPOINT.trim_end(); + let decoded = RegionCheckpoint::decode(checkpoint.as_bytes()).unwrap(); + let manifest = decoded.checkpoint.as_ref().unwrap(); + assert_eq!(manifest.pending_tails, fixture_pending_tails()); + assert_eq!(manifest.handoff_floors, BTreeMap::from([(3, 4 << 32)])); + let encoded: serde_json::Value = + serde_json::from_slice(&decoded.encode().unwrap()).unwrap(); + let frozen: serde_json::Value = serde_json::from_str(checkpoint).unwrap(); + for key in ["pending_tails", "handoff_floors"] { + assert_eq!(encoded["checkpoint"][key], frozen["checkpoint"][key]); + } + } + + /// Checks the fixtures against the shapes before pending tails were added. + #[test] + fn test_pending_tails_old_shape_compat() { + #[derive(Serialize, Deserialize)] + struct OldRegionEdit { + files_to_add: Vec, + files_to_remove: Vec, + #[serde(default)] + timestamp_ms: Option, + #[serde(with = "humantime_serde")] + compaction_time_window: Option, + flushed_entry_id: Option, + flushed_sequence: Option, + committed_sequence: Option, + } + + #[derive(Serialize, Deserialize)] + struct OldRegionManifest { + metadata: RegionMetadataRef, + files: HashMap, + #[serde(default)] + removed_files: RemovedFilesRecord, + flushed_entry_id: EntryId, + flushed_sequence: SequenceNumber, + committed_sequence: Option, + manifest_version: ManifestVersion, + truncated_entry_id: Option, + #[serde(with = "humantime_serde")] + compaction_time_window: Option, + #[serde(default)] + sst_format: FormatType, + #[serde(default)] + append_mode: Option, + } + + let delta: serde_json::Value = serde_json::from_str(PENDING_TAILS_DELTA).unwrap(); + let old_edit: OldRegionEdit = + serde_json::from_value(delta["actions"][0]["Edit"].clone()).unwrap(); + assert_eq!(old_edit.committed_sequence, Some((1 << 40) + 5)); + let old_json = serde_json::to_string(&old_edit).unwrap(); + let edit: RegionEdit = serde_json::from_str(&old_json).unwrap(); + assert_eq!(edit.wal_recovery, WalRecoveryEdit::default()); + assert_eq!(serde_json::to_string(&edit).unwrap(), old_json); + + let checkpoint: serde_json::Value = serde_json::from_str(PENDING_TAILS_CHECKPOINT).unwrap(); + let old_manifest: OldRegionManifest = + serde_json::from_value(checkpoint["checkpoint"].clone()).unwrap(); + assert_eq!(old_manifest.flushed_entry_id, 1 << 32); + let old_json = serde_json::to_string(&old_manifest).unwrap(); + let manifest: RegionManifest = serde_json::from_str(&old_json).unwrap(); + assert!(manifest.pending_tails.is_empty() && manifest.handoff_floors.is_empty()); + assert_eq!(serde_json::to_string(&manifest).unwrap(), old_json); + } } diff --git a/src/mito2/src/manifest/manager.rs b/src/mito2/src/manifest/manager.rs index 7531c433191..5cebfbc9b11 100644 --- a/src/mito2/src/manifest/manager.rs +++ b/src/mito2/src/manifest/manager.rs @@ -234,6 +234,7 @@ impl RegionManifestManager { flushed_entry_id: Some(flushed_entry_id), flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })); } @@ -1067,6 +1068,7 @@ mod test { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })]), false, ) diff --git a/src/mito2/src/manifest/tests/checkpoint.rs b/src/mito2/src/manifest/tests/checkpoint.rs index 72f4377221f..708ca51ceb4 100644 --- a/src/mito2/src/manifest/tests/checkpoint.rs +++ b/src/mito2/src/manifest/tests/checkpoint.rs @@ -28,7 +28,8 @@ use strum::IntoEnumIterator; use crate::error::Error::{ChecksumMismatch, ManifestDeltaNotFound}; use crate::manifest::action::{ - RegionCheckpoint, RegionEdit, RegionMetaAction, RegionMetaActionList, + HandoffFloor, PendingTail, RegionCheckpoint, RegionEdit, RegionMetaAction, + RegionMetaActionList, UptoKind, WalRecoveryEdit, }; use crate::manifest::manager::RegionManifestManager; use crate::manifest::storage::checkpoint::CheckpointMetadata; @@ -185,6 +186,55 @@ async fn manager_with_checkpoint_distance_1() { assert_eq!(10, manager.manifest().manifest_version); } +#[rstest::rstest] +#[case::delta(0)] +#[case::checkpoint(1)] +#[tokio::test] +async fn wal_recovery_records_survive_reopen(#[case] checkpoint_distance: u64) { + let (env, mut manager) = + build_manager(checkpoint_distance, CompressionType::Uncompressed).await; + let pending_tails = vec![PendingTail { + node_id: 1, + after: 10, + upto: 20, + upto_kind: UptoKind::Listed, + }]; + let edit = RegionEdit { + files_to_add: vec![], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: Some(1 << 40), + wal_recovery: WalRecoveryEdit { + pending_tails: Some(pending_tails.clone()), + handoff_floor: Some(HandoffFloor { + node_id: 1, + entry_id: 30, + }), + }, + }; + manager + .update( + RegionMetaActionList::with_action(RegionMetaAction::Edit(edit)), + false, + ) + .await + .unwrap(); + wait_for_manifest_checkpoint(&manager).await; + let checkpoint = manager.store().load_last_checkpoint().await.unwrap(); + assert_eq!(checkpoint.is_some(), checkpoint_distance > 0); + manager.stop().await; + + let manifest = reopen_manager(&env, checkpoint_distance, CompressionType::Uncompressed) + .await + .manifest(); + assert_eq!(manifest.pending_tails, pending_tails); + assert_eq!(manifest.handoff_floors, [(1, 30)].into()); + assert_eq!(manifest.committed_sequence, Some(1 << 40)); +} + #[tokio::test] async fn test_corrupted_data_causing_checksum_error() { // Initialize manager @@ -266,6 +316,7 @@ async fn checkpoint_with_different_compression_types() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })]); actions.push(action); } @@ -332,6 +383,7 @@ fn generate_action_lists(num: usize) -> (Vec, Vec) flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })]); actions.push(action); } diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs index c092c7ef4e7..2282eba0036 100644 --- a/src/mito2/src/region.rs +++ b/src/mito2/src/region.rs @@ -1509,6 +1509,7 @@ impl ManifestContext { flushed_entry_id: None, committed_sequence: None, compaction_time_window: None, + wal_recovery: Default::default(), }; let action_list = RegionMetaActionList::with_action(RegionMetaAction::Edit(edit)); let manifest_version = self.update_and_fire(manager, action_list, false).await?; @@ -2526,6 +2527,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), } } @@ -2567,6 +2569,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })); region diff --git a/src/mito2/src/region/catchup.rs b/src/mito2/src/region/catchup.rs index 4b8f1288c4b..144dbc0e23e 100644 --- a/src/mito2/src/region/catchup.rs +++ b/src/mito2/src/region/catchup.rs @@ -107,6 +107,7 @@ impl RegionCatchupTask { wal_entry_reader, region_id, replay_from_entry_id, + replay_from_entry_id, &self.region.version_control, self.allow_stale_entries, on_region_opened, diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs index 85881028ab9..c306c9198c9 100644 --- a/src/mito2/src/region/opener.rs +++ b/src/mito2/src/region/opener.rs @@ -56,7 +56,7 @@ use crate::error::{ ObjectStoreNotFoundSnafu, RegionCorruptedSnafu, Result, StaleLogEntrySnafu, }; use crate::lease::NodeLease; -use crate::manifest::action::RegionManifest; +use crate::manifest::action::{RegionManifest, cap_wal_release}; use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions}; use crate::memtable::bulk::part::BulkPart; use crate::memtable::time_partition::TimePartitions; @@ -627,10 +627,13 @@ impl RegionOpener { .max(flushed_entry_id); let topic_latest_entry_id = if !self.skip_wal_replay { check_wal_history(wal, &provider, &version_control.current().version).await?; + let node_id = self.file_ref_manager.node_id(); + let replay_start = replay_from_entry_id.max(manifest.wal_replay_start(node_id)); info!( - "Start replaying memtable at replay_from_entry_id: {} for region {}, manifest version: {}, flushed entry id: {}, elapsed: {:?}", - replay_from_entry_id, + "Start replaying memtable at replay_start: {} for region {}, replay_from_entry_id: {}, manifest version: {}, flushed entry id: {}, elapsed: {:?}", + replay_start, region_id, + replay_from_entry_id, manifest.manifest_version, flushed_entry_id, now.elapsed() @@ -639,7 +642,8 @@ impl RegionOpener { &provider, wal_entry_reader, region_id, - replay_from_entry_id, + replay_start, + cap_wal_release(&manifest.pending_tails, node_id, replay_start), &version_control, config.allow_stale_entries, on_region_opened, @@ -813,6 +817,7 @@ pub(crate) fn version_builder_from_manifest( .truncated_entry_id(manifest.truncated_entry_id) .compaction_time_window(manifest.compaction_time_window) .options(region_options) + .pending_tails(manifest.pending_tails.clone()) } /// Updates region options by persistent options. @@ -972,12 +977,15 @@ pub(crate) async fn check_wal_history( .await } -/// Replays the mutations from WAL and inserts mutations to memtable of given region. +/// Replays the mutations from WAL and inserts mutations to memtable of given region, +/// then releases the WAL up to `release_entry_id`. +#[allow(clippy::too_many_arguments)] pub(crate) async fn replay_memtable( provider: &Provider, mut wal_entry_reader: Box, region_id: RegionId, flushed_entry_id: EntryId, + release_entry_id: EntryId, version_control: &VersionControlRef, allow_stale_entries: bool, on_region_opened: F, @@ -1072,7 +1080,7 @@ where // TODO(weny): We need to update `flushed_entry_id` in the region manifest // to avoid reading potentially incomplete entries in the future. - (on_region_opened)(region_id, flushed_entry_id, provider).await?; + (on_region_opened)(region_id, release_entry_id, provider).await?; let series_count = version_control.current().series_count(); info!( @@ -1539,6 +1547,8 @@ mod tests { compaction_time_window: None, sst_format, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), } } diff --git a/src/mito2/src/region/version.rs b/src/mito2/src/region/version.rs index c1940b31ab0..a9866ecd748 100644 --- a/src/mito2/src/region/version.rs +++ b/src/mito2/src/region/version.rs @@ -27,11 +27,12 @@ use std::sync::{Arc, RwLock}; use std::time::Duration; use common_telemetry::info; +use snafu::ensure; use store_api::metadata::RegionMetadataRef; use store_api::storage::{RegionId, SequenceNumber}; -use crate::error::Result; -use crate::manifest::action::{RegionEdit, TruncateKind}; +use crate::error::{RegionPendingTailsSnafu, Result}; +use crate::manifest::action::{PendingTail, RegionEdit, TruncateKind}; use crate::memtable::time_partition::TimePartitions; use crate::memtable::version::{MemtableVersion, MemtableVersionRef}; use crate::memtable::{MemtableBuilderRef, MemtableId}; @@ -399,6 +400,24 @@ pub(crate) struct Version { pub(crate) compaction_time_window: Option, /// Options of the region. pub(crate) options: RegionOptions, + /// WAL ranges the region still has to recover. + pub(crate) pending_tails: Vec, +} + +impl Version { + pub(crate) fn has_pending_tails(&self) -> bool { + !self.pending_tails.is_empty() + } + + pub(crate) fn ensure_no_pending_tails(&self) -> Result<()> { + ensure!( + !self.has_pending_tails(), + RegionPendingTailsSnafu { + region_id: self.metadata.region_id, + } + ); + Ok(()) + } } pub(crate) type VersionRef = Arc; @@ -413,6 +432,7 @@ pub(crate) struct VersionBuilder { truncated_entry_id: Option, compaction_time_window: Option, options: RegionOptions, + pending_tails: Vec, } impl VersionBuilder { @@ -427,6 +447,7 @@ impl VersionBuilder { truncated_entry_id: None, compaction_time_window: None, options: RegionOptions::default(), + pending_tails: Vec::new(), } } @@ -441,6 +462,7 @@ impl VersionBuilder { truncated_entry_id: version.truncated_entry_id, compaction_time_window: version.compaction_time_window, options: version.options.clone(), + pending_tails: version.pending_tails.clone(), } } @@ -489,6 +511,11 @@ impl VersionBuilder { self } + pub(crate) fn pending_tails(mut self, pending_tails: Vec) -> Self { + self.pending_tails = pending_tails; + self + } + /// Apply edit to the builder. pub(crate) fn apply_edit(mut self, edit: RegionEdit, file_purger: FilePurgerRef) -> Self { if let Some(entry_id) = edit.flushed_entry_id { @@ -500,6 +527,9 @@ impl VersionBuilder { if let Some(window) = edit.compaction_time_window { self.compaction_time_window = Some(window); } + if let Some(pending_tails) = edit.wal_recovery.pending_tails { + self.pending_tails = pending_tails; + } if !edit.files_to_add.is_empty() || !edit.files_to_remove.is_empty() { let mut ssts = (*self.ssts).clone(); ssts.add_files(file_purger, edit.files_to_add.into_iter()); @@ -573,6 +603,7 @@ impl VersionBuilder { truncated_entry_id: self.truncated_entry_id, compaction_time_window, options: self.options, + pending_tails: self.pending_tails, } } } @@ -621,6 +652,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: Some(sequence), + wal_recovery: Default::default(), }), &[], builder.file_purger(), diff --git a/src/mito2/src/remap_manifest.rs b/src/mito2/src/remap_manifest.rs index 596e60c6ddf..fec0cad0547 100644 --- a/src/mito2/src/remap_manifest.rs +++ b/src/mito2/src/remap_manifest.rs @@ -116,6 +116,8 @@ impl RemapManifest { compaction_time_window: None, sst_format, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), }; new_manifests.insert(*region_id, manifest); @@ -467,6 +469,8 @@ mod tests { committed_sequence: None, sst_format: FormatType::PrimaryKey, append_mode: None, + pending_tails: Default::default(), + handoff_floors: Default::default(), } } diff --git a/src/mito2/src/series_index/task.rs b/src/mito2/src/series_index/task.rs index 6ce340827a1..eeb8ef50de6 100644 --- a/src/mito2/src/series_index/task.rs +++ b/src/mito2/src/series_index/task.rs @@ -813,6 +813,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], crate::test_util::new_noop_file_purger(), diff --git a/src/mito2/src/series_index/tests.rs b/src/mito2/src/series_index/tests.rs index 5f13d0b4b2d..feb8962ecf9 100644 --- a/src/mito2/src/series_index/tests.rs +++ b/src/mito2/src/series_index/tests.rs @@ -294,6 +294,7 @@ async fn test_reconcile_range_index_disabled( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], crate::test_util::new_noop_file_purger(), @@ -403,6 +404,7 @@ async fn test_reconcile_range_only_build_and_removal() { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], crate::test_util::new_noop_file_purger(), @@ -455,6 +457,7 @@ async fn test_reconcile_replaces_whole_bucket_after_failure(#[case] failure: &st flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; region.version_control.apply_edit( Some(edit(false)), diff --git a/src/mito2/src/sst/file_ref.rs b/src/mito2/src/sst/file_ref.rs index 68b0facaab4..b82a7b9c217 100644 --- a/src/mito2/src/sst/file_ref.rs +++ b/src/mito2/src/sst/file_ref.rs @@ -34,7 +34,8 @@ struct RegionFileRefs { /// Tracks reader references to SST metadata views in one datanode for object-store GC. #[derive(Debug)] pub struct FileReferenceManager { - /// Datanode id. used to determine tmp ref file name. + /// Id of this datanode. Names the tmp ref files and identifies the WAL ranges of + /// this node in region manifests. node_id: Option, files_per_region: DashMap, /// Whether global GC is enabled. diff --git a/src/mito2/src/sst/index.rs b/src/mito2/src/sst/index.rs index 4fb0f08e217..635d21f1cb4 100644 --- a/src/mito2/src/sst/index.rs +++ b/src/mito2/src/sst/index.rs @@ -1527,6 +1527,7 @@ mod tests { flushed_entry_id: None, committed_sequence: None, compaction_time_window: None, + wal_recovery: Default::default(), })), false, ) diff --git a/src/mito2/src/sst/primary_key.rs b/src/mito2/src/sst/primary_key.rs index 2d3f8c90e3b..eafa8473c14 100644 --- a/src/mito2/src/sst/primary_key.rs +++ b/src/mito2/src/sst/primary_key.rs @@ -741,6 +741,7 @@ mod tests { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }, new_noop_file_purger(), ) diff --git a/src/mito2/src/test_util.rs b/src/mito2/src/test_util.rs index 29718163528..be8f3b64003 100644 --- a/src/mito2/src/test_util.rs +++ b/src/mito2/src/test_util.rs @@ -105,6 +105,7 @@ pub(crate) fn nop_action() -> RegionMetaActionList { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), })]) } diff --git a/src/mito2/src/test_util/version_util.rs b/src/mito2/src/test_util/version_util.rs index 44827b35cc1..136bf11db28 100644 --- a/src/mito2/src/test_util/version_util.rs +++ b/src/mito2/src/test_util/version_util.rs @@ -226,6 +226,7 @@ pub(crate) fn apply_edit( flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }), &[], purger, diff --git a/src/mito2/src/worker/handle_alter.rs b/src/mito2/src/worker/handle_alter.rs index 935dbd39413..592b508249e 100644 --- a/src/mito2/src/worker/handle_alter.rs +++ b/src/mito2/src/worker/handle_alter.rs @@ -167,6 +167,13 @@ impl RegionWorkerLoop { return; } + if !is_allowed_with_pending_tails(&request.kind) + && let Err(e) = version.ensure_no_pending_tails() + { + sender.send(Err(e)); + return; + } + // Checks whether we need to alter the region. if !request.need_alter(&version.metadata) { warn!( @@ -425,6 +432,19 @@ impl RegionWorkerLoop { } } +/// Returns true if `kind` only changes region options or only adds nullable columns +/// without a default value, which rows replayed from a pending tail can lack. +fn is_allowed_with_pending_tails(kind: &AlterKind) -> bool { + match kind { + AlterKind::AddColumns { columns } => columns.iter().all(|column| { + let schema = &column.column_metadata.column_schema; + schema.is_nullable() && schema.default_constraint().is_none() + }), + AlterKind::SetRegionOptions { .. } | AlterKind::UnsetRegionOptions { .. } => true, + _ => false, + } +} + fn skip_wal_value(kind: &AlterKind) -> Option { let AlterKind::SetRegionOptions { options } = kind else { return None; diff --git a/src/mito2/src/worker/handle_apply_staging.rs b/src/mito2/src/worker/handle_apply_staging.rs index 2598640b9f2..73b9decd128 100644 --- a/src/mito2/src/worker/handle_apply_staging.rs +++ b/src/mito2/src/worker/handle_apply_staging.rs @@ -124,6 +124,7 @@ impl RegionWorkerLoop { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; let (tx, rx) = oneshot::channel(); diff --git a/src/mito2/src/worker/handle_close.rs b/src/mito2/src/worker/handle_close.rs index c10ac761f43..1e447e3ea54 100644 --- a/src/mito2/src/worker/handle_close.rs +++ b/src/mito2/src/worker/handle_close.rs @@ -36,6 +36,13 @@ impl RegionWorkerLoop { info!("Try to close region {}, worker: {}", region_id, self.id); + if request.flush_on_close + && let Err(e) = region.version().ensure_no_pending_tails() + { + sender.send(Err(e)); + return; + } + if region.is_staging_or_staged_edit() { // A staging region cannot flush, so it leaves staging before the close-time flush // below. The manifest lock lets a successful end of staging in progress finish first, diff --git a/src/mito2/src/worker/handle_copy_region.rs b/src/mito2/src/worker/handle_copy_region.rs index 2204cec8e1c..fbd02e8210b 100644 --- a/src/mito2/src/worker/handle_copy_region.rs +++ b/src/mito2/src/worker/handle_copy_region.rs @@ -264,6 +264,7 @@ impl RegionWorkerLoop { flushed_entry_id: None, flushed_sequence: None, committed_sequence: None, + wal_recovery: Default::default(), }; if edit .files_to_add diff --git a/src/mito2/src/worker/handle_enter_staging.rs b/src/mito2/src/worker/handle_enter_staging.rs index e5b0674aaac..119f07241f6 100644 --- a/src/mito2/src/worker/handle_enter_staging.rs +++ b/src/mito2/src/worker/handle_enter_staging.rs @@ -63,6 +63,10 @@ impl RegionWorkerLoop { sender.send(Ok(0)); return; } + if let Err(e) = region.version().ensure_no_pending_tails() { + sender.send(Err(e)); + return; + } // Retain the compaction fence while the DDL later waits for flush. let (sender, partition_directive) = match self.compaction_scheduler.try_cancel_and_add_ddl( diff --git a/src/mito2/src/worker/handle_flush.rs b/src/mito2/src/worker/handle_flush.rs index 984b551d7bb..32cd19a4142 100644 --- a/src/mito2/src/worker/handle_flush.rs +++ b/src/mito2/src/worker/handle_flush.rs @@ -27,6 +27,7 @@ use store_api::storage::RegionId; use crate::config::{IndexBuildMode, MitoConfig}; use crate::error::{RegionBusySnafu, RegionNotFoundSnafu, Result}; use crate::flush::{FlushReason, RegionFlushTask}; +use crate::manifest::action::cap_wal_release; use crate::region::version::VersionRef; use crate::region::{MitoRegion, MitoRegionRef}; use crate::request::{BuildIndexRequest, FlushFailed, FlushFinished, OnFailure, OptionOutputTx}; @@ -364,6 +365,13 @@ impl RegionWorkerLoop { } }; + if request.reason == Some(RegionFlushReason::Repartition) + && let Err(e) = region.version().ensure_no_pending_tails() + { + sender.send(Err(e)); + return; + } + // `update_topic_latest_entry_id` updates `topic_latest_entry_id` when memtables are empty. // But the flush is skipped if memtables are empty. Thus should update the `topic_latest_entry_id` // when handling flush request instead of in `schedule_flush` or `flush_finished`. @@ -484,13 +492,18 @@ impl RegionWorkerLoop { self.update_topic_latest_entry_id(®ion); // Delete wal. + let obsolete_entry_id = cap_wal_release( + ®ion.version().pending_tails, + self.file_ref_manager.node_id(), + request.flushed_entry_id, + ); info!( "Region {} flush finished, tries to bump wal to {}", - region_id, request.flushed_entry_id + region_id, obsolete_entry_id ); if let Err(e) = self .wal - .obsolete(region_id, request.flushed_entry_id, ®ion.provider) + .obsolete(region_id, obsolete_entry_id, ®ion.provider) .await { error!(e; "Failed to write wal, region: {}", region_id); diff --git a/src/mito2/src/worker/handle_manifest.rs b/src/mito2/src/worker/handle_manifest.rs index d8f4ecf8c16..f1a575d4433 100644 --- a/src/mito2/src/worker/handle_manifest.rs +++ b/src/mito2/src/worker/handle_manifest.rs @@ -35,6 +35,7 @@ use crate::error::{ }; use crate::manifest::action::{ RegionChange, RegionEdit, RegionMetaAction, RegionMetaActionList, RegionTruncate, TruncateKind, + WalRecoveryEdit, }; use crate::memtable::MemtableBuilderProvider; use crate::metrics::WRITE_CACHE_INFLIGHT_DOWNLOAD; @@ -115,6 +116,7 @@ impl RegionEditQueue { && edit.flushed_entry_id.is_none() && edit.flushed_sequence.is_none() && edit.committed_sequence.is_none() + && edit.wal_recovery == WalRecoveryEdit::default() } let mut merged = self.requests.pop_front()?; @@ -693,6 +695,7 @@ impl RegionWorkerLoop { flushed_entry_id: Some(discarded_entry_id), flushed_sequence: Some(discarded_sequence), committed_sequence: None, + wal_recovery: Default::default(), }; let action_list = RegionMetaActionList::with_action(RegionMetaAction::Edit(edit)); let result = match durability_barrier.wait(discarded_entry_id).await { diff --git a/src/mito2/src/worker/handle_rebuild_index.rs b/src/mito2/src/worker/handle_rebuild_index.rs index 10f55c78943..917a58d0c68 100644 --- a/src/mito2/src/worker/handle_rebuild_index.rs +++ b/src/mito2/src/worker/handle_rebuild_index.rs @@ -399,6 +399,7 @@ impl RegionWorkerLoop { flushed_entry_id: None, committed_sequence: None, compaction_time_window: None, + wal_recovery: Default::default(), }), &[], region.file_purger.clone(), diff --git a/src/mito2/src/worker/handle_truncate.rs b/src/mito2/src/worker/handle_truncate.rs index 409998592d1..a21bb0578e3 100644 --- a/src/mito2/src/worker/handle_truncate.rs +++ b/src/mito2/src/worker/handle_truncate.rs @@ -39,6 +39,11 @@ impl RegionWorkerLoop { return; } }; + // Truncating releases the WAL without keeping the pending tails. + if let Err(e) = region.version().ensure_no_pending_tails() { + sender.send(Err(e)); + return; + } // Fence compaction before any later flush wait so handed-off DDLs keep admission paused. let (sender, req) = match self.compaction_scheduler.try_cancel_and_add_ddl( diff --git a/tests/compatibility/README.md b/tests/compatibility/README.md index b84b55c92e5..0dfcba20215 100644 --- a/tests/compatibility/README.md +++ b/tests/compatibility/README.md @@ -48,6 +48,9 @@ writer that emits these descriptors; downgrade only to readers that recognize th The OSS build does not generate sketch payloads, so SQL restart cases cannot create these descriptors through an OSS writer. +Frozen region manifests that carry pending WAL tails are checked by the manifest tests of `mito2`. +No SQL statement records a pending tail. See the [fixture notes](manifest/pending_tails/README.md). + ## Quick Start ```shell diff --git a/tests/compatibility/manifest/pending_tails/README.md b/tests/compatibility/manifest/pending_tails/README.md new file mode 100644 index 00000000000..b788ad35bbf --- /dev/null +++ b/tests/compatibility/manifest/pending_tails/README.md @@ -0,0 +1,27 @@ +# Pending tail manifest fixtures + +Frozen region manifests with pending WAL tails and a handoff floor, checked by the manifest tests +in `src/mito2/src/manifest/action.rs`, which never rewrite them: + +```shell +cargo nextest run -p mito2 -E 'test(test_pending_tails_fixtures) | test(test_pending_tails_old_shape_compat)' +``` + +The first test decodes both files. It requires the encoder to reproduce `delta.json` byte for byte, +and the `pending_tails` and `handoff_floors` of `checkpoint.json` exactly; the rest of the checkpoint +embeds region metadata, whose encoding may gain compatible fields. The second test decodes both +files with the shapes from before pending tails existed, which must ignore the new fields, and +requires a manifest without pending tails to encode exactly as those shapes do. + +## `delta.json` + +A delta with one edit. It carries committed sequence `(1 << 40) + 5`, two pending tails, and the +handoff floor `4 << 32` of node 3. The tails are node 1, entries `(1 << 32, (1 << 32) + 5]` with a +reported upper bound, and node 2, entries `(2 << 32, 3 << 32]` with a listed upper bound. 424 bytes, +SHA-256 `520cbdc2cf51de4fb960d3c2f2194c126e3d23f02b4c511ed687705f48073c80`. + +## `checkpoint.json` + +A checkpoint of version 7 holding the same two tails and floor, flushed entry id `1 << 32`, no +files, and the region metadata of `basic_region_metadata` in `src/mito2/src/manifest/tests/utils.rs`. +1,245 bytes, SHA-256 `088f3f4caf04dbbf2734739aeb72a9a742e5ba8fb20f840dfd24123fb4027dd7`. diff --git a/tests/compatibility/manifest/pending_tails/checkpoint.json b/tests/compatibility/manifest/pending_tails/checkpoint.json new file mode 100644 index 00000000000..a58882debcb --- /dev/null +++ b/tests/compatibility/manifest/pending_tails/checkpoint.json @@ -0,0 +1 @@ +{"last_version":7,"compacted_actions":7,"checkpoint":{"metadata":{"column_metadatas":[{"column_schema":{"name":"ts","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Timestamp","column_id":45},{"column_schema":{"name":"pk","data_type":{"String":{"size_type":"Utf8"}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Tag","column_id":36},{"column_schema":{"name":"val","data_type":{"Float64":{}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Field","column_id":251}],"primary_key":[36],"region_id":98784247841,"schema_version":0,"primary_key_encoding":"dense","partition_expr":null},"files":{},"removed_files":{"removed_files":[]},"flushed_entry_id":4294967296,"flushed_sequence":100,"committed_sequence":1099511627781,"manifest_version":7,"truncated_entry_id":null,"compaction_time_window":null,"sst_format":"primary_key","append_mode":null,"pending_tails":[{"node_id":1,"after":4294967296,"upto":4294967301,"upto_kind":"reported"},{"node_id":2,"after":8589934592,"upto":12884901888,"upto_kind":"listed"}],"handoff_floors":{"3":17179869184}}} diff --git a/tests/compatibility/manifest/pending_tails/delta.json b/tests/compatibility/manifest/pending_tails/delta.json new file mode 100644 index 00000000000..2305629cab9 --- /dev/null +++ b/tests/compatibility/manifest/pending_tails/delta.json @@ -0,0 +1 @@ +{"actions":[{"Edit":{"files_to_add":[],"files_to_remove":[],"timestamp_ms":1762732800000,"compaction_time_window":null,"flushed_entry_id":null,"flushed_sequence":null,"committed_sequence":1099511627781,"pending_tails":[{"node_id":1,"after":4294967296,"upto":4294967301,"upto_kind":"reported"},{"node_id":2,"after":8589934592,"upto":12884901888,"upto_kind":"listed"}],"handoff_floor":{"node_id":3,"entry_id":17179869184}}}]}