diff --git a/src/mito2/src/engine/flush_test.rs b/src/mito2/src/engine/flush_test.rs index b6688ec5044..b71ce9f9fc0 100644 --- a/src/mito2/src/engine/flush_test.rs +++ b/src/mito2/src/engine/flush_test.rs @@ -17,7 +17,7 @@ use std::assert_matches; use std::collections::HashMap; use std::sync::Arc; -use std::sync::atomic::{AtomicI64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering}; use std::time::Duration; use api::v1::Rows; @@ -41,10 +41,11 @@ use store_api::region_request::{ use store_api::storage::{RegionId, ScanRequest}; use tokio::sync::Notify; +use crate::access_layer::AccessLayerRef; use crate::config::MitoConfig; use crate::engine::MitoEngine; use crate::engine::listener::{EventListener, FlushListener, StallListener}; -use crate::engine::region_hook::{RegionHook, RegionHookRef, SstFileInfo}; +use crate::engine::region_hook::{RegionGcInfo, RegionHook, RegionHookRef, SstFileInfo}; use crate::error::Error; use crate::manifest::action::RegionMetaActionList; use crate::test_util::{ @@ -1606,6 +1607,12 @@ pub(super) struct MockRegionHook { pub(super) closed_count: AtomicUsize, pub(super) dropped_count: AtomicUsize, pub(super) files_removed_count: AtomicUsize, + pub(super) gc_count: AtomicUsize, + pub(super) last_gc_is_region_dropped: AtomicBool, + pub(super) last_gc_full_file_listing: AtomicBool, + /// When set, `on_region_gc` returns `Err` (after recording the call), to + /// exercise the callers' retry/propagation behavior. + pub(super) gc_should_fail: AtomicBool, notify: Notify, dropped_notify: Notify, files_removed_notify: Notify, @@ -1620,6 +1627,10 @@ impl MockRegionHook { closed_count: AtomicUsize::new(0), dropped_count: AtomicUsize::new(0), files_removed_count: AtomicUsize::new(0), + gc_count: AtomicUsize::new(0), + last_gc_is_region_dropped: AtomicBool::new(false), + last_gc_full_file_listing: AtomicBool::new(false), + gc_should_fail: AtomicBool::new(false), notify: Notify::new(), dropped_notify: Notify::new(), files_removed_notify: Notify::new(), @@ -1721,6 +1732,33 @@ impl RegionHook for MockRegionHook { ); self.files_removed_notify.notify_one(); } + + async fn on_region_gc( + &self, + region_id: RegionId, + _region_metadata: Option<&RegionMetadataRef>, + _access_layer: &AccessLayerRef, + info: &RegionGcInfo<'_>, + ) -> Result<(), Error> { + self.gc_count.fetch_add(1, Ordering::Relaxed); + self.last_gc_is_region_dropped + .store(info.is_region_dropped, Ordering::Relaxed); + self.last_gc_full_file_listing + .store(info.full_file_listing, Ordering::Relaxed); + common_telemetry::info!( + "MockRegionHook::on_region_gc: region={}, is_region_dropped={}, full_file_listing={}", + region_id, + info.is_region_dropped, + info.full_file_listing + ); + if self.gc_should_fail.load(Ordering::Relaxed) { + return crate::error::UnexpectedSnafu { + reason: "simulated offline-cleanup hook failure".to_string(), + } + .fail(); + } + Ok(()) + } } #[tokio::test] diff --git a/src/mito2/src/engine/open_test.rs b/src/mito2/src/engine/open_test.rs index f5ee4dd19f7..55cd92e6eda 100644 --- a/src/mito2/src/engine/open_test.rs +++ b/src/mito2/src/engine/open_test.rs @@ -183,6 +183,181 @@ async fn test_engine_offline_cleanup_rejects_opened_region() { assert!(object_store.exists(®ion_dir).await.unwrap()); } +/// Offline cleanup (soft-drop PURGE) removes the region directory directly, +/// bypassing the drop GC worker, so it must notify extensions via +/// `on_region_gc` — otherwise sidecar state (e.g. an Iceberg export) leaks. +/// +/// Covers the wiring: `handle_offline_cleanup_request` fires `on_region_gc` +/// with `is_region_dropped = true` and `full_file_listing = true` once the +/// region dir is physically removed, and does NOT fire `on_region_files_removed` +/// (the region is offline and carries no `RegionMetadataRef`). +#[tokio::test] +async fn test_engine_offline_cleanup_fires_on_region_gc() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("offline-cleanup-gc-hook").await; + + let hook = Arc::new(MockRegionHook::new()); + let plugins = Plugins::new(); + plugins.insert(hook.clone() as RegionHookRef); + let engine = env + .create_engine_with_plugins(MitoConfig::default(), plugins) + .await; + + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().build(); + let table_dir = request.table_dir.clone(); + let path_type = request.path_type; + let options = request.options.clone(); + let region_dir = region_dir_from_table_dir(&table_dir, region_id, path_type); + let object_store = env.get_object_store().unwrap(); + + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + assert!(object_store.exists(®ion_dir).await.unwrap()); + + // Close the region first — offline cleanup refuses a still-registered region. + engine + .handle_request( + region_id, + RegionRequest::Close(RegionCloseRequest::default()), + ) + .await + .unwrap(); + assert!(!engine.is_region_exists(region_id)); + + let gc_before = hook.gc_count.load(Ordering::Relaxed); + let files_removed_before = hook.files_removed_count.load(Ordering::Relaxed); + + engine + .handle_request( + region_id, + RegionRequest::CleanUp(RegionCleanUpRequest { + engine: String::new(), + table_dir, + path_type, + options, + }), + ) + .await + .unwrap(); + + // The region directory is gone … + assert!(!engine.is_region_exists(region_id)); + assert!(!object_store.exists(®ion_dir).await.unwrap()); + + // … and the extension was notified exactly once via `on_region_gc`, with the + // flags that authorize full sidecar reclamation. `on_region_files_removed` + // must NOT fire for the offline path (no region metadata is available). + assert_eq!( + hook.gc_count.load(Ordering::Relaxed), + gc_before + 1, + "offline cleanup must fire on_region_gc" + ); + assert!( + hook.last_gc_is_region_dropped.load(Ordering::Relaxed), + "on_region_gc during offline cleanup must report is_region_dropped = true" + ); + assert!( + hook.last_gc_full_file_listing.load(Ordering::Relaxed), + "on_region_gc during offline cleanup must report full_file_listing = true" + ); + assert_eq!( + hook.files_removed_count.load(Ordering::Relaxed), + files_removed_before, + "offline cleanup must not fire on_region_files_removed" + ); +} + +/// A failing `on_region_gc` during offline cleanup must surface as an error so +/// the caller (`PurgeDroppedTableProcedure`) retries the `CleanUp`, and the +/// retry must succeed once the hook stops failing — the handler is idempotent +/// (it re-runs the directory removal, WAL obsolete and the hook safely). +#[tokio::test] +async fn test_engine_offline_cleanup_propagates_on_region_gc_error() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("offline-cleanup-gc-hook-retry").await; + + let hook = Arc::new(MockRegionHook::new()); + let plugins = Plugins::new(); + plugins.insert(hook.clone() as RegionHookRef); + let engine = env + .create_engine_with_plugins(MitoConfig::default(), plugins) + .await; + + let region_id = RegionId::new(2, 1); + let request = CreateRequestBuilder::new().build(); + let table_dir = request.table_dir.clone(); + let path_type = request.path_type; + let options = request.options.clone(); + let region_dir = region_dir_from_table_dir(&table_dir, region_id, path_type); + let object_store = env.get_object_store().unwrap(); + + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + engine + .handle_request( + region_id, + RegionRequest::Close(RegionCloseRequest::default()), + ) + .await + .unwrap(); + + // Force the extension hook to fail: the offline cleanup must return the + // error so the caller retries, instead of swallowing it. + hook.gc_should_fail.store(true, Ordering::Relaxed); + let gc_before = hook.gc_count.load(Ordering::Relaxed); + let err = engine + .handle_request( + region_id, + RegionRequest::CleanUp(RegionCleanUpRequest { + engine: String::new(), + table_dir: table_dir.clone(), + path_type, + options: options.clone(), + }), + ) + .await + .unwrap_err(); + assert_eq!( + err.status_code(), + StatusCode::Unexpected, + "offline cleanup must propagate the on_region_gc error" + ); + assert_eq!( + hook.gc_count.load(Ordering::Relaxed), + gc_before + 1, + "on_region_gc must fire even when it fails" + ); + + // Retry: clear the failure and the same CleanUp must now succeed. The + // directory removal and WAL obsolete are idempotent, so the second attempt + // re-runs them and the hook without complaint. + hook.gc_should_fail.store(false, Ordering::Relaxed); + engine + .handle_request( + region_id, + RegionRequest::CleanUp(RegionCleanUpRequest { + engine: String::new(), + table_dir, + path_type, + options, + }), + ) + .await + .unwrap(); + assert!(!engine.is_region_exists(region_id)); + assert!(!object_store.exists(®ion_dir).await.unwrap()); + assert_eq!( + hook.gc_count.load(Ordering::Relaxed), + gc_before + 2, + "on_region_gc must fire again on the successful retry" + ); +} + #[tokio::test] async fn test_engine_open_existing() { test_engine_open_existing_with_format(false).await; diff --git a/src/mito2/src/engine/region_hook.rs b/src/mito2/src/engine/region_hook.rs index 4d816463b77..90a15c6aa60 100644 --- a/src/mito2/src/engine/region_hook.rs +++ b/src/mito2/src/engine/region_hook.rs @@ -89,7 +89,7 @@ //! | Close | [`on_region_closed`] | A close request (or a close-after-flush) removes the region from the active set. Data files, manifest and WAL state are **preserved**; the region may be reopened. | //! | Logical drop | [`on_region_dropped`] | A drop request has been handled: the region leaves the active set and its WAL entries are marked obsolete. Data files are **not yet deleted**. | //! | Physical file removal | [`on_region_files_removed`] | The drop GC worker has deleted the region directory. Terminal file-lifecycle event. | -//! | Global GC pass | [`on_region_gc`] | The datanode's global GC worker finished a GC pass for a region — both periodic GC for live regions and the global reclamation of dropped/repartitioned regions (`is_region_dropped`). | +//! | Global GC pass | [`on_region_gc`] | The datanode's global GC worker finished a GC pass for a region — both periodic GC for live regions and the global reclamation of dropped/repartitioned regions (`is_region_dropped`). Also fired by the offline cleanup path (`handle_offline_cleanup_request`, i.e. soft-drop PURGE) once the region directory is removed, with `is_region_dropped = true` and `full_file_listing = true`. | //! //! Notes: //! - `on_region_closed` / `on_region_dropped` run **inline in the region worker loop**, @@ -104,6 +104,12 @@ //! - When global GC is enabled and a normal table region is dropped with `partial_drop`, its //! directory is left for global reclamation and `on_region_files_removed` is **not** fired //! by the drop worker (observe it via the global GC path instead). +//! - The offline cleanup path (`handle_offline_cleanup_request`, reached by a soft-drop +//! `PURGE`) force-removes the region directory but has no `RegionMetadataRef` for the +//! offline region, so it cannot fire `on_region_files_removed`. It fires `on_region_gc` +//! instead — with `is_region_dropped = true` and `full_file_listing = true` — so +//! extensions can reclaim sidecar state for the now-removed region. A hook `Err` is +//! propagated so the caller retries the (idempotent) cleanup. //! - Logical file removal (compaction, region edit, truncate) is already observable via //! [`on_manifest_updated`] (`Edit.files_to_remove` / `Truncate` action); only the drop //! worker's physical directory deletion needs a dedicated file hook. @@ -128,8 +134,9 @@ //! ## Future work //! //! `on_region_files_removed` currently covers only the **drop** GC worker's physical -//! directory removal. A broader per-file `on_files_removed` hook covering compaction -//! removal and truncate is not yet implemented +//! directory removal. The global-GC reclamation path and the offline-cleanup path +//! (soft-drop PURGE) are covered by `on_region_gc` instead. A broader per-file +//! `on_files_removed` hook covering compaction removal and truncate is not yet implemented //! (though logical file removal is already observable via `on_manifest_updated`, //! and the global GC reclamation path is covered by `on_region_gc`). //! Role/leadership transitions (`on_region_role_changed`) are also not hooked. @@ -390,7 +397,9 @@ pub trait RegionHook: Send + Sync + Debug { /// Called after the datanode's global GC worker (`LocalGcWorker`) finishes a /// GC pass for a region — live regions (periodic GC) or dropped/repartitioned - /// regions (the global reclamation path). Lets extensions with sidecar files + /// regions (the global reclamation path). Also fired by the offline cleanup + /// path (`handle_offline_cleanup_request`, i.e. soft-drop PURGE) once the + /// region directory has been removed. Lets extensions with sidecar files /// outside mito2's region dir clean up residual files. /// /// Always scoped to [`RegionGcInfo::removed_files`]: clean only sidecar @@ -412,7 +421,30 @@ pub trait RegionHook: Send + Sync + Debug { /// same `removed_files`; treat live cleanup as opportunistic. /// /// `region_metadata` is `None` for dropped regions; use `region_id` + - /// `access_layer`. Idempotent; runs on the background GC task. + /// `access_layer`. Idempotent. + /// + /// # Execution context + /// + /// On the global-GC path this runs on the background GC task, outside the + /// region worker loop. The offline cleanup path + /// (`handle_offline_cleanup_request`, reached by soft-drop PURGE) instead + /// runs it **inline in the region worker loop** and propagates `Err` so the + /// caller retries the (idempotent) cleanup. So — like `on_region_closed` / + /// `on_region_dropped` — implementations **must be fast and must not block + /// indefinitely** while awaited there: while the callback is pending, every + /// subsequent DDL request for every region on that worker is blocked. + /// Extension authors who wrote their hook against the "background GC task" + /// contract must account for this second, latency-sensitive trigger. + /// + /// The offline cleanup path fires this callback once the region directory is + /// confirmed **absent** — including no-op purges where nothing was actually + /// removed this call (a retry after the directory was already gone, or a + /// region that never had files on this datanode, since + /// `remove_region_dir_for_full_drop` returns `Ok` for an absent/empty + /// prefix). `RegionGcInfo::removed_files` is empty in that case; do **not** + /// interpret the call as "files were deleted in this call". This is + /// asymmetric with the GC path, which fires only when files were deleted or + /// a full listing was performed. async fn on_region_gc( &self, region_id: RegionId, diff --git a/src/mito2/src/worker/handle_open.rs b/src/mito2/src/worker/handle_open.rs index 802ab9c1a83..8193acdfc13 100644 --- a/src/mito2/src/worker/handle_open.rs +++ b/src/mito2/src/worker/handle_open.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use std::time::Instant; -use common_telemetry::info; +use common_telemetry::{info, warn}; use object_store::util::{join_path, normalize_dir}; use snafu::{OptionExt, ResultExt}; use store_api::logstore::LogStore; @@ -25,6 +25,8 @@ use store_api::region_request::{AffectedRows, RegionCleanUpRequest, RegionOpenRe use store_api::storage::RegionId; use table::requests::STORAGE_KEY; +use crate::access_layer::AccessLayer; +use crate::engine::region_hook::{RegionGcInfo, RegionHookRef}; use crate::error::{ ObjectStoreNotFoundSnafu, OpenDalSnafu, OpenRegionSnafu, RegionBusySnafu, RegionNotFoundSnafu, Result, @@ -67,6 +69,49 @@ impl RegionWorkerLoop { let region_dir = region_dir_from_table_dir(&table_dir, region_id, request.path_type); remove_region_dir_for_full_drop(®ion_dir, &object_store).await?; + // The region directory is gone. Offline cleanup (soft-drop PURGE) is a + // terminal physical removal, identical in effect to the drop GC + // worker's directory deletion — but unlike the drop worker it never + // fires `on_region_files_removed`, because the region is offline and + // carries no `RegionMetadataRef`. Fire `on_region_gc` instead: it + // accepts an absent region and lets extensions with sidecar files + // outside the region dir (e.g. an Iceberg export) reclaim their + // per-region state. + // + // Propagate the hook's error so the caller retries the cleanup, mirroring + // how the global GC worker keeps a region for the next pass when + // `on_region_gc` fails. The whole handler is idempotent (the existing + // `test_engine_offline_cleanup_closed_region` already drives it twice in + // a row), so a retry re-runs the directory removal, WAL obsolete and the + // hook safely. Runs inline, consistent with the blocking directory + // removal above; offline cleanup is already a slow path. + if let Some(hook) = self.plugins.get::() { + let access_layer = Arc::new(AccessLayer::new( + table_dir.clone(), + request.path_type, + object_store.clone(), + self.puffin_manager_factory.clone(), + self.intermediate_manager.clone(), + )); + let gc_info = RegionGcInfo { + removed_files: &[], + is_region_dropped: true, + full_file_listing: true, + }; + if let Err(err) = hook + .on_region_gc(region_id, None, &access_layer, &gc_info) + .await + { + warn!( + err; + "Region hook on_region_gc failed during offline cleanup for region {}; \ + returning the error so the caller retries the cleanup", + region_id, + ); + return Err(err); + } + } + self.cleanup_dropped_region_runtime_state(region_id).await; self.dropping_regions.remove_region(region_id); cleanup_region_file_artifacts(