diff --git a/src/mito2/src/engine/close_test.rs b/src/mito2/src/engine/close_test.rs index dd9ea8cf710..f113bb1e611 100644 --- a/src/mito2/src/engine/close_test.rs +++ b/src/mito2/src/engine/close_test.rs @@ -14,12 +14,13 @@ use std::sync::Arc; use std::sync::atomic::Ordering; +use std::time::Duration; use api::v1::Rows; use common_base::Plugins; use common_recordbatch::RecordBatches; use store_api::region_engine::RegionEngine; -use store_api::region_request::{RegionCloseRequest, RegionRequest}; +use store_api::region_request::{PathType, RegionCloseRequest, RegionOpenRequest, RegionRequest}; use store_api::storage::{RegionId, ScanRequest}; use crate::config::MitoConfig; @@ -27,7 +28,8 @@ use crate::engine::flush_test::MockRegionHook; use crate::engine::listener::AlterFlushListener; use crate::engine::region_hook::RegionHookRef; use crate::test_util::{ - CreateRequestBuilder, TestEnv, build_rows, flush_region, put_rows, rows_schema, + CheckpointTaskBlocker, CreateRequestBuilder, TestEnv, build_rows, flush_region, put_rows, + rows_schema, }; #[tokio::test] @@ -411,3 +413,76 @@ async fn test_repeated_close_during_edit_preserves_flush_requirement( ); engine.stop().await.unwrap(); } + +#[tokio::test(flavor = "multi_thread")] +async fn test_engine_stop_waits_for_checkpoint() { + let (blocker, mock_layer) = CheckpointTaskBlocker::block_last_checkpoint_write(); + let mut env = TestEnv::with_prefix("stop-waits-for-checkpoint") + .await + .with_mock_layer(mock_layer); + let config = MitoConfig { + manifest_checkpoint_distance: 1, + ..Default::default() + }; + let engine = env.create_engine(config.clone()).await; + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().build(); + let table_dir = request.table_dir.clone(); + let column_schemas = rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + put_rows( + &engine, + region_id, + Rows { + schema: column_schemas, + rows: build_rows(0, 3), + }, + ) + .await; + // The flush starts a checkpoint, which blocks after it has created `_last_checkpoint` + // and before it has written the pointer. + flush_region(&engine, region_id, None).await; + blocker.wait_until_blocked().await; + let wait_started = engine + .get_region(region_id) + .unwrap() + .manifest_ctx + .manifest_manager + .read() + .await + .checkpointer() + .pending_checkpoint_wait_started(); + + let mut reopen = tokio::spawn(async move { + let engine = env.reopen_engine(engine, config).await; + let open = RegionRequest::Open(RegionOpenRequest { + engine: String::new(), + table_dir, + path_type: PathType::Bare, + options: Default::default(), + skip_wal_replay: false, + checkpoint: None, + requirements: Default::default(), + }); + let result = engine.handle_request(region_id, open).await; + (env, result) + }); + tokio::time::timeout(Duration::from_secs(10), wait_started.notified()) + .await + .expect("the engine stop does not wait for the checkpoint"); + // The new engine would read the empty pointer while the old checkpoint is blocked. + assert!( + tokio::time::timeout(Duration::from_millis(500), &mut reopen) + .await + .is_err() + ); + blocker.release(); + let (_env, result) = tokio::time::timeout(Duration::from_secs(10), reopen) + .await + .unwrap() + .unwrap(); + result.unwrap(); +} diff --git a/src/mito2/src/engine/object_store_wal_test.rs b/src/mito2/src/engine/object_store_wal_test.rs index b03e3aefd14..b1db5b36a7a 100644 --- a/src/mito2/src/engine/object_store_wal_test.rs +++ b/src/mito2/src/engine/object_store_wal_test.rs @@ -1092,6 +1092,49 @@ async fn test_reload_waits_for_held_upload() { assert_eq!(3, count_rows(&engine, region_id).await); } +#[tokio::test] +async fn test_reload_waits_for_running_checkpoint() { + let (blocker, layer) = CheckpointTaskBlocker::block_last_checkpoint_write(); + let mut env = TestEnv::with_prefix("reload-waits-for-running-checkpoint") + .await + .with_mock_layer(layer); + let region_id = RegionId::new(1, 1); + let (engine, _log_store, _, schema, _) = create_leased_region(&mut env, region_id).await; + put_range(&engine, region_id, &schema, 0, 3).await; + // The flush starts a checkpoint, which blocks after it has created `_last_checkpoint` + // and before it has written the pointer. + flush_region(&engine, region_id, None).await; + blocker.wait_until_blocked().await; + let old = engine.get_region(region_id).unwrap(); + let wait_started = old + .manifest_ctx + .manifest_manager + .read() + .await + .checkpointer() + .pending_checkpoint_wait_started(); + fence(&old).await; + + notify_fenced(&engine, region_id).await; + tokio::time::timeout(Duration::from_secs(10), wait_started.notified()) + .await + .expect("the reload does not wait for the checkpoint"); + // The worker takes the manifest lock of the fenced instance, for example to close it, + // while the reload waits. + let manager = tokio::time::timeout( + Duration::from_secs(10), + old.manifest_ctx.manifest_manager.write(), + ) + .await + .expect("the reload holds the manifest lock while it waits"); + drop(manager); + assert!(!is_replaced(&engine, &old)); + blocker.release(); + + wait_until(|| is_replaced(&engine, &old)).await; + assert_eq!(3, count_rows(&engine, region_id).await); +} + #[tokio::test] async fn test_reload_waits_for_running_flush() { let mut env = TestEnv::with_prefix("reload-waits-for-flush").await; diff --git a/src/mito2/src/manifest/checkpointer.rs b/src/mito2/src/manifest/checkpointer.rs index 3d33942a2f9..8f5a33c8c7c 100644 --- a/src/mito2/src/manifest/checkpointer.rs +++ b/src/mito2/src/manifest/checkpointer.rs @@ -22,6 +22,7 @@ use store_api::storage::RegionId; use store_api::{MIN_VERSION, ManifestVersion}; #[cfg(test)] use tokio::sync::Notify; +use tokio::sync::watch; use crate::error::Result; use crate::manifest::action::{RegionCheckpoint, RegionManifest}; @@ -34,11 +35,19 @@ use crate::metrics::MANIFEST_OP_ELAPSED; pub(crate) struct Checkpointer { manifest_options: RegionManifestOptions, inner: Arc, - checkpoint_task: Option>, + checkpoint_task: Option, #[cfg(test)] pending_checkpoint_wait_started: Arc, } +/// A spawned checkpoint task and the signal that it has ended. +#[derive(Debug)] +struct CheckpointTask { + handle: JoinHandle<()>, + /// Closes when the task ends, so that it can be awaited without borrowing the checkpointer. + done: watch::Receiver<()>, +} + #[derive(Debug)] struct Inner { region_id: RegionId, @@ -142,7 +151,7 @@ impl Checkpointer { if self .checkpoint_task .as_ref() - .is_some_and(|handle| !handle.is_finished()) + .is_some_and(|task| !task.handle.is_finished()) { return; } @@ -187,9 +196,36 @@ impl Checkpointer { fn do_checkpoint(&mut self, checkpoint: RegionCheckpoint) { let inner = self.inner.clone(); - self.checkpoint_task = Some(common_runtime::spawn_global(async move { + let (done_sender, done) = watch::channel(()); + let handle = common_runtime::spawn_global(async move { + // The sender is dropped however the task ends, which closes the channel. + let _done_sender = done_sender; inner.do_checkpoint(checkpoint).await; - })); + }); + self.checkpoint_task = Some(CheckpointTask { handle, done }); + } + + /// Returns a future that resolves once the checkpoint task running now has ended. + /// + /// Unlike [Self::wait_for_pending_checkpoint], which borrows the checkpointer mutably and reaps + /// the task handle, this does neither, so the caller can await the future without holding the + /// manifest lock. + pub(crate) fn pending_checkpoint_completion(&self) -> impl Future + Send + use<> { + let done = self.checkpoint_task.as_ref().map(|task| task.done.clone()); + let region_id = self.inner.region_id(); + #[cfg(test)] + let wait_started = self.pending_checkpoint_wait_started.clone(); + async move { + // The channel is closed once the task has ended. + let Some(mut done) = done.filter(|receiver| receiver.has_changed().is_ok()) else { + return; + }; + #[cfg(test)] + wait_started.notify_one(); + info!("Waiting for the pending checkpoint of region {}", region_id); + // Nothing is ever sent, so this returns once the task drops the sender. + let _ = done.changed().await; + } } /// Waits for the current checkpoint task without removing its handle first. @@ -197,14 +233,14 @@ impl Checkpointer { /// Keeping the handle in `self` while awaiting is important. If the caller is /// cancelled, another lifecycle transition can still wait for the same task. pub(crate) async fn wait_for_pending_checkpoint(&mut self) { - let Some(handle) = self.checkpoint_task.as_mut() else { + let Some(task) = self.checkpoint_task.as_mut() else { return; }; #[cfg(test)] self.pending_checkpoint_wait_started.notify_one(); - let result = (&mut *handle).await; + let result = (&mut task.handle).await; // There is no cancellation point between observing completion and // clearing the handle. self.checkpoint_task = None; @@ -218,7 +254,16 @@ impl Checkpointer { pub(crate) fn is_doing_checkpoint(&self) -> bool { self.checkpoint_task .as_ref() - .is_some_and(|handle| !handle.is_finished()) + .is_some_and(|task| !task.handle.is_finished()) + } + + #[cfg(test)] + pub(crate) fn abort_checkpoint_task(&self) { + self.checkpoint_task + .as_ref() + .expect("checkpoint task must exist") + .handle + .abort(); } #[cfg(test)] diff --git a/src/mito2/src/manifest/manager.rs b/src/mito2/src/manifest/manager.rs index 6f5e36683f6..b15fbe4916b 100644 --- a/src/mito2/src/manifest/manager.rs +++ b/src/mito2/src/manifest/manager.rs @@ -442,7 +442,12 @@ impl RegionManifestManager { Ok(Some(manager)) } - /// Stops the manager. + /// Stops the manager. A stopped manager takes no update and starts no checkpoint, but a + /// checkpoint already running keeps writing the manifest directory until it ends. + /// + /// A caller that hands the storage to another instance waits for that checkpoint with + /// `pending_checkpoint_completion` after the stop, as engine stop and reload do. Close and + /// drop, which run in the region worker loop, deliberately do not. pub async fn stop(&mut self) { self.stopped = true; } @@ -849,6 +854,13 @@ impl RegionManifestManager { self.checkpointer.wait_for_pending_checkpoint().await; } + /// Returns a future that resolves once the in-flight checkpoint has finished, including its + /// cleanup. Unlike [Self::wait_for_pending_checkpoint], which borrows the manager mutably and + /// reaps the task handle, this does neither. + pub(crate) fn pending_checkpoint_completion(&self) -> impl Future + Send + use<> { + self.checkpointer.pending_checkpoint_completion() + } + /// Clear deleted files from manifest's `removed_files` field without update version. Notice if datanode exit before checkpoint then new manifest by open region may still contain these deleted files, which is acceptable for gc process. pub fn clear_deleted_files(&mut self, deleted_files: Vec) { let mut manifest = (*self.manifest()).clone(); diff --git a/src/mito2/src/manifest/tests/checkpoint.rs b/src/mito2/src/manifest/tests/checkpoint.rs index 86f9329ffbe..0a8a7bdab59 100644 --- a/src/mito2/src/manifest/tests/checkpoint.rs +++ b/src/mito2/src/manifest/tests/checkpoint.rs @@ -14,7 +14,7 @@ use std::assert_matches; use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use common_datasource::compression::CompressionType; @@ -25,6 +25,7 @@ use object_store::layers::mock::{ }; use store_api::storage::{FileId, RegionId}; use strum::IntoEnumIterator; +use tokio::sync::Notify; use crate::error::Error::{ChecksumMismatch, ManifestDeltaNotFound}; use crate::manifest::action::{ @@ -828,6 +829,132 @@ async fn completed_checkpoint_is_reaped_before_scheduling_the_next_one() { assert_eq!(2, manager.checkpointer().last_checkpoint_version()); } +#[rstest::rstest] +#[case::cleanup_finished(false)] +#[case::task_aborted(true)] +#[tokio::test] +async fn pending_checkpoint_completion_waits_until_the_task_ends(#[case] abort: bool) { + let (blocker, mock_layer) = CheckpointTaskBlocker::block_cleanup(); + let env = TestEnv::new().await.with_mock_layer(mock_layer); + let metadata = Arc::new(basic_region_metadata()); + let mut manager = env + .create_manifest_manager(CompressionType::Uncompressed, 1, Some(metadata)) + .await + .unwrap() + .unwrap(); + + manager.update(nop_action(), false).await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), blocker.wait_until_blocked()) + .await + .expect("checkpoint cleanup did not start"); + + // Two waiters stay on their own tasks until the checkpoint ends; a third is cancelled. + let waiters = [ + manager.pending_checkpoint_completion(), + manager.pending_checkpoint_completion(), + ] + .map(tokio::spawn); + let mut cancelled = Box::pin(manager.pending_checkpoint_completion()); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut cancelled) + .await + .is_err(), + "the cancelled wait ended before the checkpoint cleanup" + ); + drop(cancelled); + assert!( + waiters.iter().all(|waiter| !waiter.is_finished()), + "a spawned wait ended before the checkpoint cleanup" + ); + // The cleanup stays blocked in the abort case, so only the abort can end the task. + if abort { + manager.checkpointer().abort_checkpoint_task(); + } else { + blocker.release(); + } + for waiter in waiters { + tokio::time::timeout(Duration::from_secs(5), waiter) + .await + .expect("the wait did not end with the checkpoint task") + .unwrap(); + } + let wait_started = manager.checkpointer().pending_checkpoint_wait_started(); + // Consumes the permit the waiters above left. + let _ = tokio::time::timeout(Duration::ZERO, wait_started.notified()).await; + tokio::time::timeout(Duration::ZERO, manager.pending_checkpoint_completion()) + .await + .expect("a wait after the checkpoint task ended did not return at once"); + assert!( + tokio::time::timeout(Duration::ZERO, wait_started.notified()) + .await + .is_err(), + "a wait after the checkpoint task ended started waiting" + ); +} + +/// Panics on the first delete, which a checkpoint issues in its cleanup, once released. +struct PanickingDeleter { + entered: Arc, + release: Arc, + panicked: Arc, +} + +impl oio::Delete for PanickingDeleter { + async fn delete(&mut self, _path: &str, _args: OpDelete) -> MockResult<()> { + self.entered.notify_one(); + self.release.notified().await; + self.panicked.store(true, Ordering::Relaxed); + panic!("mock manifest delete panic"); + } + + async fn close(&mut self) -> MockResult<()> { + Ok(()) + } +} + +#[tokio::test] +async fn pending_checkpoint_completion_ends_when_the_task_panics() { + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let panicked = Arc::new(AtomicBool::new(false)); + let (factory_entered, factory_release, factory_panicked) = + (entered.clone(), release.clone(), panicked.clone()); + let mock_layer = MockLayerBuilder::default() + .deleter_factory(Arc::new(move |_inner| { + Box::new(PanickingDeleter { + entered: factory_entered.clone(), + release: factory_release.clone(), + panicked: factory_panicked.clone(), + }) + })) + .build() + .unwrap(); + let env = TestEnv::new().await.with_mock_layer(mock_layer); + let metadata = Arc::new(basic_region_metadata()); + let mut manager = env + .create_manifest_manager(CompressionType::Uncompressed, 1, Some(metadata)) + .await + .unwrap() + .unwrap(); + + manager.update(nop_action(), false).await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), entered.notified()) + .await + .expect("checkpoint cleanup did not start"); + let mut done = Box::pin(manager.pending_checkpoint_completion()); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut done) + .await + .is_err(), + "the wait ended before the checkpoint task panicked" + ); + release.notify_one(); + tokio::time::timeout(Duration::from_secs(5), done) + .await + .expect("the wait did not end after the checkpoint task panicked"); + assert!(panicked.load(Ordering::Relaxed)); +} + /// Retention runs inside the checkpoint task, so waiting for task completion also /// makes logical size accounting observable to the assertions below. async fn wait_for_manifest_checkpoint(manager: &RegionManifestManager) { diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs index 0fc8acb8e24..7b3c308807c 100644 --- a/src/mito2/src/region.rs +++ b/src/mito2/src/region.rs @@ -1528,6 +1528,21 @@ impl ManifestContext { self.manifest_manager.read().await.has_update().await } + /// Waits for the in-flight checkpoint of a stopped manager. A stopped manager starts no + /// checkpoint, so this instance no longer writes the manifest directory afterwards. + /// + /// Callers must not hold the manifest lock: this takes its read lock, and the lock is not + /// reentrant. Under the lock, use [RegionManifestManager::wait_for_pending_checkpoint] + /// instead. The wait itself holds no lock. + pub(crate) async fn wait_for_checkpoint_after_stop(&self) { + let done = { + let manager = self.manifest_manager.read().await; + debug_assert!(manager.is_stopped()); + manager.pending_checkpoint_completion() + }; + done.await; + } + /// Returns the current region role state. pub(crate) fn current_state(&self) -> RegionRoleState { self.state.load() diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs index fc2f04aa87e..a29e0c99e06 100644 --- a/src/mito2/src/worker.rs +++ b/src/mito2/src/worker.rs @@ -1847,10 +1847,19 @@ impl RegionWorkerLoop { } // Closes remaining regions. let regions = self.regions.list_regions(); - for region in regions { + for region in ®ions { region.deactivate_region_manifest_size_metric(); region.stop().await; } + // Another engine may open these regions from the same storage after the stop. Regions + // closed earlier are not waited for; a datanode or standalone shutdown closes every region + // first, so there this only covers regions whose close failed. + join_all( + regions + .iter() + .map(|region| region.manifest_ctx.wait_for_checkpoint_after_stop()), + ) + .await; self.regions.clear(); } diff --git a/src/mito2/src/worker/handle_reload.rs b/src/mito2/src/worker/handle_reload.rs index 948e79fa0a8..fc5dc996645 100644 --- a/src/mito2/src/worker/handle_reload.rs +++ b/src/mito2/src/worker/handle_reload.rs @@ -66,6 +66,9 @@ impl RegionWorkerLoop { let sender = self.sender.clone(); common_runtime::spawn_global(async move { let result = async { + // The fenced instance is stopped, but its last checkpoint may still be writing + // the manifest that the reload reads. + region.manifest_ctx.wait_for_checkpoint_after_stop().await; // The replay reads only what the log store has made durable. durability_barrier.wait(last_entry_id).await?; opener.open(&config, &wal).await