fix(mito2): wait for a stopped region's checkpoint before reopening it (#10376)

* fix(mito2): wait for a stopped region's checkpoint before reopening it

A region's manifest checkpoint runs as a background task that writes the
checkpoint, then `_last_checkpoint`, then deletes older manifest files:
the deltas up to the previous checkpoint, or every older delta at the
region's first checkpoint. Stopping the manifest manager does not wait
for it, so a new instance of the region could read the manifest while the
old task still wrote it. On a store that writes files in place the open
could read an empty `_last_checkpoint`. On any store, an open that found
no checkpoint could have deltas it had just listed deleted by the first
checkpoint's cleanup.

Wait for the stopped instance's checkpoint at engine stop, after the
worker loop ends, and before a fenced region is reloaded. The wait does
not hold the manifest lock, which the worker may take meanwhile: the
checkpoint task owns the sender of a watch channel, which closes however
the task ends.

Fixes the flaky test_change_region_compaction_window (#7135).

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* test(mito2): cover the checkpoint wait through cleanup, panic and abort

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* refactor(mito2): align checkpoint wait names and log the wait

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* test(mito2): cover concurrent and late waits on the checkpoint

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* test(mito2): keep two checkpoint waiters through the end of the task

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* test(mito2): check that a wait after the checkpoint ended does not start waiting

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* fix(mito2): address review on the stopped checkpoint wait

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* docs(mito2): state the stop contract and assert it before the checkpoint wait

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* refactor(mito2): name the checkpoint completion future and keep its task state together

Rename `pending_checkpoint_done` to `pending_checkpoint_completion` on
`Checkpointer` and `RegionManifestManager`, and state how it differs from
`wait_for_pending_checkpoint`. Keep the task handle and its completion
receiver in one `CheckpointTask`, so reaping the handle also drops the
receiver. Use plain backticks in the `stop` doc, which is public and
cannot link to a crate-private method.

Narrow the comment in `RegionWorkerLoop::clean`: a datanode or standalone
shutdown closes every region before it stops the engines, so there the
wait only covers regions whose close failed.

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

---------

Signed-off-by: jeremyhi <fengjiachun@gmail.com>
This commit is contained in:
jeremyhi
2026-11-16 10:52:27 +00:00
committed by GitHub
parent 89e14ab936
commit dd941a8e33
8 changed files with 341 additions and 12 deletions
+77 -2
View File
@@ -14,12 +14,13 @@
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Duration;
use api::v1::Rows;
use common_base::Plugins;
use common_recordbatch::RecordBatches;
use store_api::region_engine::RegionEngine;
use store_api::region_request::{RegionCloseRequest, RegionRequest};
use store_api::region_request::{PathType, RegionCloseRequest, RegionOpenRequest, RegionRequest};
use store_api::storage::{RegionId, ScanRequest};
use crate::config::MitoConfig;
@@ -27,7 +28,8 @@ use crate::engine::flush_test::MockRegionHook;
use crate::engine::listener::AlterFlushListener;
use crate::engine::region_hook::RegionHookRef;
use crate::test_util::{
CreateRequestBuilder, TestEnv, build_rows, flush_region, put_rows, rows_schema,
CheckpointTaskBlocker, CreateRequestBuilder, TestEnv, build_rows, flush_region, put_rows,
rows_schema,
};
#[tokio::test]
@@ -411,3 +413,76 @@ async fn test_repeated_close_during_edit_preserves_flush_requirement(
);
engine.stop().await.unwrap();
}
#[tokio::test(flavor = "multi_thread")]
async fn test_engine_stop_waits_for_checkpoint() {
let (blocker, mock_layer) = CheckpointTaskBlocker::block_last_checkpoint_write();
let mut env = TestEnv::with_prefix("stop-waits-for-checkpoint")
.await
.with_mock_layer(mock_layer);
let config = MitoConfig {
manifest_checkpoint_distance: 1,
..Default::default()
};
let engine = env.create_engine(config.clone()).await;
let region_id = RegionId::new(1, 1);
let request = CreateRequestBuilder::new().build();
let table_dir = request.table_dir.clone();
let column_schemas = rows_schema(&request);
engine
.handle_request(region_id, RegionRequest::Create(request))
.await
.unwrap();
put_rows(
&engine,
region_id,
Rows {
schema: column_schemas,
rows: build_rows(0, 3),
},
)
.await;
// The flush starts a checkpoint, which blocks after it has created `_last_checkpoint`
// and before it has written the pointer.
flush_region(&engine, region_id, None).await;
blocker.wait_until_blocked().await;
let wait_started = engine
.get_region(region_id)
.unwrap()
.manifest_ctx
.manifest_manager
.read()
.await
.checkpointer()
.pending_checkpoint_wait_started();
let mut reopen = tokio::spawn(async move {
let engine = env.reopen_engine(engine, config).await;
let open = RegionRequest::Open(RegionOpenRequest {
engine: String::new(),
table_dir,
path_type: PathType::Bare,
options: Default::default(),
skip_wal_replay: false,
checkpoint: None,
requirements: Default::default(),
});
let result = engine.handle_request(region_id, open).await;
(env, result)
});
tokio::time::timeout(Duration::from_secs(10), wait_started.notified())
.await
.expect("the engine stop does not wait for the checkpoint");
// The new engine would read the empty pointer while the old checkpoint is blocked.
assert!(
tokio::time::timeout(Duration::from_millis(500), &mut reopen)
.await
.is_err()
);
blocker.release();
let (_env, result) = tokio::time::timeout(Duration::from_secs(10), reopen)
.await
.unwrap()
.unwrap();
result.unwrap();
}
@@ -1092,6 +1092,49 @@ async fn test_reload_waits_for_held_upload() {
assert_eq!(3, count_rows(&engine, region_id).await);
}
#[tokio::test]
async fn test_reload_waits_for_running_checkpoint() {
let (blocker, layer) = CheckpointTaskBlocker::block_last_checkpoint_write();
let mut env = TestEnv::with_prefix("reload-waits-for-running-checkpoint")
.await
.with_mock_layer(layer);
let region_id = RegionId::new(1, 1);
let (engine, _log_store, _, schema, _) = create_leased_region(&mut env, region_id).await;
put_range(&engine, region_id, &schema, 0, 3).await;
// The flush starts a checkpoint, which blocks after it has created `_last_checkpoint`
// and before it has written the pointer.
flush_region(&engine, region_id, None).await;
blocker.wait_until_blocked().await;
let old = engine.get_region(region_id).unwrap();
let wait_started = old
.manifest_ctx
.manifest_manager
.read()
.await
.checkpointer()
.pending_checkpoint_wait_started();
fence(&old).await;
notify_fenced(&engine, region_id).await;
tokio::time::timeout(Duration::from_secs(10), wait_started.notified())
.await
.expect("the reload does not wait for the checkpoint");
// The worker takes the manifest lock of the fenced instance, for example to close it,
// while the reload waits.
let manager = tokio::time::timeout(
Duration::from_secs(10),
old.manifest_ctx.manifest_manager.write(),
)
.await
.expect("the reload holds the manifest lock while it waits");
drop(manager);
assert!(!is_replaced(&engine, &old));
blocker.release();
wait_until(|| is_replaced(&engine, &old)).await;
assert_eq!(3, count_rows(&engine, region_id).await);
}
#[tokio::test]
async fn test_reload_waits_for_running_flush() {
let mut env = TestEnv::with_prefix("reload-waits-for-flush").await;
+52 -7
View File
@@ -22,6 +22,7 @@ use store_api::storage::RegionId;
use store_api::{MIN_VERSION, ManifestVersion};
#[cfg(test)]
use tokio::sync::Notify;
use tokio::sync::watch;
use crate::error::Result;
use crate::manifest::action::{RegionCheckpoint, RegionManifest};
@@ -34,11 +35,19 @@ use crate::metrics::MANIFEST_OP_ELAPSED;
pub(crate) struct Checkpointer {
manifest_options: RegionManifestOptions,
inner: Arc<Inner>,
checkpoint_task: Option<JoinHandle<()>>,
checkpoint_task: Option<CheckpointTask>,
#[cfg(test)]
pending_checkpoint_wait_started: Arc<Notify>,
}
/// A spawned checkpoint task and the signal that it has ended.
#[derive(Debug)]
struct CheckpointTask {
handle: JoinHandle<()>,
/// Closes when the task ends, so that it can be awaited without borrowing the checkpointer.
done: watch::Receiver<()>,
}
#[derive(Debug)]
struct Inner {
region_id: RegionId,
@@ -142,7 +151,7 @@ impl Checkpointer {
if self
.checkpoint_task
.as_ref()
.is_some_and(|handle| !handle.is_finished())
.is_some_and(|task| !task.handle.is_finished())
{
return;
}
@@ -187,9 +196,36 @@ impl Checkpointer {
fn do_checkpoint(&mut self, checkpoint: RegionCheckpoint) {
let inner = self.inner.clone();
self.checkpoint_task = Some(common_runtime::spawn_global(async move {
let (done_sender, done) = watch::channel(());
let handle = common_runtime::spawn_global(async move {
// The sender is dropped however the task ends, which closes the channel.
let _done_sender = done_sender;
inner.do_checkpoint(checkpoint).await;
}));
});
self.checkpoint_task = Some(CheckpointTask { handle, done });
}
/// Returns a future that resolves once the checkpoint task running now has ended.
///
/// Unlike [Self::wait_for_pending_checkpoint], which borrows the checkpointer mutably and reaps
/// the task handle, this does neither, so the caller can await the future without holding the
/// manifest lock.
pub(crate) fn pending_checkpoint_completion(&self) -> impl Future<Output = ()> + Send + use<> {
let done = self.checkpoint_task.as_ref().map(|task| task.done.clone());
let region_id = self.inner.region_id();
#[cfg(test)]
let wait_started = self.pending_checkpoint_wait_started.clone();
async move {
// The channel is closed once the task has ended.
let Some(mut done) = done.filter(|receiver| receiver.has_changed().is_ok()) else {
return;
};
#[cfg(test)]
wait_started.notify_one();
info!("Waiting for the pending checkpoint of region {}", region_id);
// Nothing is ever sent, so this returns once the task drops the sender.
let _ = done.changed().await;
}
}
/// Waits for the current checkpoint task without removing its handle first.
@@ -197,14 +233,14 @@ impl Checkpointer {
/// Keeping the handle in `self` while awaiting is important. If the caller is
/// cancelled, another lifecycle transition can still wait for the same task.
pub(crate) async fn wait_for_pending_checkpoint(&mut self) {
let Some(handle) = self.checkpoint_task.as_mut() else {
let Some(task) = self.checkpoint_task.as_mut() else {
return;
};
#[cfg(test)]
self.pending_checkpoint_wait_started.notify_one();
let result = (&mut *handle).await;
let result = (&mut task.handle).await;
// There is no cancellation point between observing completion and
// clearing the handle.
self.checkpoint_task = None;
@@ -218,7 +254,16 @@ impl Checkpointer {
pub(crate) fn is_doing_checkpoint(&self) -> bool {
self.checkpoint_task
.as_ref()
.is_some_and(|handle| !handle.is_finished())
.is_some_and(|task| !task.handle.is_finished())
}
#[cfg(test)]
pub(crate) fn abort_checkpoint_task(&self) {
self.checkpoint_task
.as_ref()
.expect("checkpoint task must exist")
.handle
.abort();
}
#[cfg(test)]
+13 -1
View File
@@ -442,7 +442,12 @@ impl RegionManifestManager {
Ok(Some(manager))
}
/// Stops the manager.
/// Stops the manager. A stopped manager takes no update and starts no checkpoint, but a
/// checkpoint already running keeps writing the manifest directory until it ends.
///
/// A caller that hands the storage to another instance waits for that checkpoint with
/// `pending_checkpoint_completion` after the stop, as engine stop and reload do. Close and
/// drop, which run in the region worker loop, deliberately do not.
pub async fn stop(&mut self) {
self.stopped = true;
}
@@ -849,6 +854,13 @@ impl RegionManifestManager {
self.checkpointer.wait_for_pending_checkpoint().await;
}
/// Returns a future that resolves once the in-flight checkpoint has finished, including its
/// cleanup. Unlike [Self::wait_for_pending_checkpoint], which borrows the manager mutably and
/// reaps the task handle, this does neither.
pub(crate) fn pending_checkpoint_completion(&self) -> impl Future<Output = ()> + Send + use<> {
self.checkpointer.pending_checkpoint_completion()
}
/// Clear deleted files from manifest's `removed_files` field without update version. Notice if datanode exit before checkpoint then new manifest by open region may still contain these deleted files, which is acceptable for gc process.
pub fn clear_deleted_files(&mut self, deleted_files: Vec<RemovedFile>) {
let mut manifest = (*self.manifest()).clone();
+128 -1
View File
@@ -14,7 +14,7 @@
use std::assert_matches;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Duration;
use common_datasource::compression::CompressionType;
@@ -25,6 +25,7 @@ use object_store::layers::mock::{
};
use store_api::storage::{FileId, RegionId};
use strum::IntoEnumIterator;
use tokio::sync::Notify;
use crate::error::Error::{ChecksumMismatch, ManifestDeltaNotFound};
use crate::manifest::action::{
@@ -828,6 +829,132 @@ async fn completed_checkpoint_is_reaped_before_scheduling_the_next_one() {
assert_eq!(2, manager.checkpointer().last_checkpoint_version());
}
#[rstest::rstest]
#[case::cleanup_finished(false)]
#[case::task_aborted(true)]
#[tokio::test]
async fn pending_checkpoint_completion_waits_until_the_task_ends(#[case] abort: bool) {
let (blocker, mock_layer) = CheckpointTaskBlocker::block_cleanup();
let env = TestEnv::new().await.with_mock_layer(mock_layer);
let metadata = Arc::new(basic_region_metadata());
let mut manager = env
.create_manifest_manager(CompressionType::Uncompressed, 1, Some(metadata))
.await
.unwrap()
.unwrap();
manager.update(nop_action(), false).await.unwrap();
tokio::time::timeout(Duration::from_secs(5), blocker.wait_until_blocked())
.await
.expect("checkpoint cleanup did not start");
// Two waiters stay on their own tasks until the checkpoint ends; a third is cancelled.
let waiters = [
manager.pending_checkpoint_completion(),
manager.pending_checkpoint_completion(),
]
.map(tokio::spawn);
let mut cancelled = Box::pin(manager.pending_checkpoint_completion());
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut cancelled)
.await
.is_err(),
"the cancelled wait ended before the checkpoint cleanup"
);
drop(cancelled);
assert!(
waiters.iter().all(|waiter| !waiter.is_finished()),
"a spawned wait ended before the checkpoint cleanup"
);
// The cleanup stays blocked in the abort case, so only the abort can end the task.
if abort {
manager.checkpointer().abort_checkpoint_task();
} else {
blocker.release();
}
for waiter in waiters {
tokio::time::timeout(Duration::from_secs(5), waiter)
.await
.expect("the wait did not end with the checkpoint task")
.unwrap();
}
let wait_started = manager.checkpointer().pending_checkpoint_wait_started();
// Consumes the permit the waiters above left.
let _ = tokio::time::timeout(Duration::ZERO, wait_started.notified()).await;
tokio::time::timeout(Duration::ZERO, manager.pending_checkpoint_completion())
.await
.expect("a wait after the checkpoint task ended did not return at once");
assert!(
tokio::time::timeout(Duration::ZERO, wait_started.notified())
.await
.is_err(),
"a wait after the checkpoint task ended started waiting"
);
}
/// Panics on the first delete, which a checkpoint issues in its cleanup, once released.
struct PanickingDeleter {
entered: Arc<Notify>,
release: Arc<Notify>,
panicked: Arc<AtomicBool>,
}
impl oio::Delete for PanickingDeleter {
async fn delete(&mut self, _path: &str, _args: OpDelete) -> MockResult<()> {
self.entered.notify_one();
self.release.notified().await;
self.panicked.store(true, Ordering::Relaxed);
panic!("mock manifest delete panic");
}
async fn close(&mut self) -> MockResult<()> {
Ok(())
}
}
#[tokio::test]
async fn pending_checkpoint_completion_ends_when_the_task_panics() {
let entered = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let panicked = Arc::new(AtomicBool::new(false));
let (factory_entered, factory_release, factory_panicked) =
(entered.clone(), release.clone(), panicked.clone());
let mock_layer = MockLayerBuilder::default()
.deleter_factory(Arc::new(move |_inner| {
Box::new(PanickingDeleter {
entered: factory_entered.clone(),
release: factory_release.clone(),
panicked: factory_panicked.clone(),
})
}))
.build()
.unwrap();
let env = TestEnv::new().await.with_mock_layer(mock_layer);
let metadata = Arc::new(basic_region_metadata());
let mut manager = env
.create_manifest_manager(CompressionType::Uncompressed, 1, Some(metadata))
.await
.unwrap()
.unwrap();
manager.update(nop_action(), false).await.unwrap();
tokio::time::timeout(Duration::from_secs(5), entered.notified())
.await
.expect("checkpoint cleanup did not start");
let mut done = Box::pin(manager.pending_checkpoint_completion());
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut done)
.await
.is_err(),
"the wait ended before the checkpoint task panicked"
);
release.notify_one();
tokio::time::timeout(Duration::from_secs(5), done)
.await
.expect("the wait did not end after the checkpoint task panicked");
assert!(panicked.load(Ordering::Relaxed));
}
/// Retention runs inside the checkpoint task, so waiting for task completion also
/// makes logical size accounting observable to the assertions below.
async fn wait_for_manifest_checkpoint(manager: &RegionManifestManager) {
+15
View File
@@ -1528,6 +1528,21 @@ impl ManifestContext {
self.manifest_manager.read().await.has_update().await
}
/// Waits for the in-flight checkpoint of a stopped manager. A stopped manager starts no
/// checkpoint, so this instance no longer writes the manifest directory afterwards.
///
/// Callers must not hold the manifest lock: this takes its read lock, and the lock is not
/// reentrant. Under the lock, use [RegionManifestManager::wait_for_pending_checkpoint]
/// instead. The wait itself holds no lock.
pub(crate) async fn wait_for_checkpoint_after_stop(&self) {
let done = {
let manager = self.manifest_manager.read().await;
debug_assert!(manager.is_stopped());
manager.pending_checkpoint_completion()
};
done.await;
}
/// Returns the current region role state.
pub(crate) fn current_state(&self) -> RegionRoleState {
self.state.load()
+10 -1
View File
@@ -1847,10 +1847,19 @@ impl<S> RegionWorkerLoop<S> {
}
// Closes remaining regions.
let regions = self.regions.list_regions();
for region in regions {
for region in &regions {
region.deactivate_region_manifest_size_metric();
region.stop().await;
}
// Another engine may open these regions from the same storage after the stop. Regions
// closed earlier are not waited for; a datanode or standalone shutdown closes every region
// first, so there this only covers regions whose close failed.
join_all(
regions
.iter()
.map(|region| region.manifest_ctx.wait_for_checkpoint_after_stop()),
)
.await;
self.regions.clear();
}
+3
View File
@@ -66,6 +66,9 @@ impl<S: LogStore> RegionWorkerLoop<S> {
let sender = self.sender.clone();
common_runtime::spawn_global(async move {
let result = async {
// The fenced instance is stopped, but its last checkpoint may still be writing
// the manifest that the reload reads.
region.manifest_ctx.wait_for_checkpoint_after_stop().await;
// The replay reads only what the log store has made durable.
durability_barrier.wait(last_entry_id).await?;
opener.open(&config, &wal).await