diff --git a/src/mito2/src/engine/skip_wal_test.rs b/src/mito2/src/engine/skip_wal_test.rs index e64581b2145..21710dc7e1a 100644 --- a/src/mito2/src/engine/skip_wal_test.rs +++ b/src/mito2/src/engine/skip_wal_test.rs @@ -287,6 +287,66 @@ async fn test_close_region_skip_wal_while_flush_in_flight_closes_region() { assert!(!engine.is_region_exists(region_id)); } +#[tokio::test] +async fn test_concurrent_close_region_skip_wal_while_flush_in_flight_succeeds() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("concurrent-close-skip-wal-while-flush-in-flight").await; + let listener = Arc::new(AlterFlushListener::default()); + let engine = env + .create_engine_with(MitoConfig::default(), None, Some(listener.clone()), None) + .await; + + let region_id = RegionId::new(1, 1); + let mut request = CreateRequestBuilder::new().build(); + request.options.insert( + WAL_OPTIONS_KEY.to_string(), + serde_json::to_string(&WalOptions::Noop).unwrap(), + ); + + engine + .handle_request(region_id, RegionRequest::Create(request.clone())) + .await + .unwrap(); + + let rows = Rows { + schema: rows_schema(&request), + rows: build_rows(0, 3), + }; + put_rows(&engine, region_id, rows).await; + + let request_count = listener.request_count(); + let engine_cloned = engine.clone(); + let first_close = tokio::spawn(async move { + engine_cloned + .handle_request(region_id, RegionRequest::Close(RegionCloseRequest {})) + .await + .unwrap(); + }); + listener.wait_flush_begin().await; + listener.wait_request_count(request_count + 1).await; + + let engine_cloned = engine.clone(); + let second_close = tokio::spawn(async move { + engine_cloned + .handle_request(region_id, RegionRequest::Close(RegionCloseRequest {})) + .await + .unwrap(); + }); + listener.wait_request_count(request_count + 2).await; + + listener.wake_flush(); + tokio::time::timeout(Duration::from_secs(5), first_close) + .await + .unwrap() + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), second_close) + .await + .unwrap() + .unwrap(); + + assert!(!engine.is_region_exists(region_id)); +} + #[tokio::test] async fn test_close_region_after_truncate_skip_wal() { common_telemetry::init_default_ut_logging(); diff --git a/src/mito2/src/flush.rs b/src/mito2/src/flush.rs index 0158a157228..163e9e196ff 100644 --- a/src/mito2/src/flush.rs +++ b/src/mito2/src/flush.rs @@ -57,7 +57,7 @@ use crate::region::options::{IndexOptions, MergeMode, RegionOptions}; use crate::region::version::{VersionControlData, VersionControlRef, VersionRef}; use crate::region::{ManifestContextRef, RegionLeaderState, RegionRoleState, parse_partition_expr}; use crate::request::{ - BackgroundNotify, FlushFailed, FlushFinished, OnFailure, OptionOutputTx, OutputTx, + BackgroundNotify, DdlRequest, FlushFailed, FlushFinished, OnFailure, OptionOutputTx, OutputTx, SenderBulkRequest, SenderDdlRequest, SenderWriteRequest, WorkerRequest, WorkerRequestWithTime, }; use crate::schedule::scheduler::{Job, SchedulerRef}; @@ -1283,7 +1283,11 @@ impl FlushScheduler { /// Notifies the scheduler that the region is closed. pub(crate) fn on_region_closed(&mut self, region_id: RegionId) { - self.remove_region_on_failure(region_id, Arc::new(RegionClosedSnafu { region_id }.build())); + let Some(flush_status) = self.region_status.remove(®ion_id) else { + return; + }; + + flush_status.on_region_closed(Arc::new(RegionClosedSnafu { region_id }.build())); } /// Notifies the scheduler that the region is truncated. @@ -1408,6 +1412,34 @@ impl FlushStatus { })); } } + + fn on_region_closed(self, err: Arc) { + if let Some(mut task) = self.pending_task { + if task.reason == FlushReason::Closing { + task.on_success(); + } else { + task.on_failure(err.clone()); + } + } + + for ddl in self.pending_ddls { + if matches!(ddl.request, DdlRequest::Close(_)) { + ddl.sender.send(Ok(0)); + } else { + ddl.sender.send(Err(err.clone()).context(FlushRegionSnafu { + region_id: self.region_id, + })); + } + } + + for write_req in self.pending_writes { + write_req + .sender + .send(Err(err.clone()).context(FlushRegionSnafu { + region_id: self.region_id, + })); + } + } } #[cfg(test)]