feat: invoke gc hook for offline region cleanup (#8613)

I hereby agree to the terms of the [GreptimeDB
CLA](https://github.com/GreptimeTeam/.github/blob/main/CLA.md).

## Refer to a related PR or issue link (optional)

## What's changed and what's your intention?

This patch adds region hook to soft-drop cleanup phase so we can remove
extension files managed by the hook.

some open questions:

- this cleanup logic sits in `handle_open.rs`, which is a little
confusing
- can we merge this cleanup with our current GC concept to reduce the
complexity?

## PR Checklist
Please convert it to a draft if some of the following conditions are not
met.

- [ ] I have written the necessary rustdoc comments.
- [ ] I have added the necessary unit tests and integration tests.
- [ ] This PR requires documentation updates.
- [ ] API changes are backward compatible.
- [ ] Schema or data changes are backward compatible.
This commit is contained in:
Lei, HUANG
2026-07-23 11:55:37 +00:00
committed by GitHub
4 changed files with 298 additions and 8 deletions
+40 -2
View File
@@ -17,7 +17,7 @@
use std::assert_matches;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicI64, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering};
use std::time::Duration;
use api::v1::Rows;
@@ -41,10 +41,11 @@ use store_api::region_request::{
use store_api::storage::{RegionId, ScanRequest};
use tokio::sync::Notify;
use crate::access_layer::AccessLayerRef;
use crate::config::MitoConfig;
use crate::engine::MitoEngine;
use crate::engine::listener::{EventListener, FlushListener, StallListener};
use crate::engine::region_hook::{RegionHook, RegionHookRef, SstFileInfo};
use crate::engine::region_hook::{RegionGcInfo, RegionHook, RegionHookRef, SstFileInfo};
use crate::error::Error;
use crate::manifest::action::RegionMetaActionList;
use crate::test_util::{
@@ -1606,6 +1607,12 @@ pub(super) struct MockRegionHook {
pub(super) closed_count: AtomicUsize,
pub(super) dropped_count: AtomicUsize,
pub(super) files_removed_count: AtomicUsize,
pub(super) gc_count: AtomicUsize,
pub(super) last_gc_is_region_dropped: AtomicBool,
pub(super) last_gc_full_file_listing: AtomicBool,
/// When set, `on_region_gc` returns `Err` (after recording the call), to
/// exercise the callers' retry/propagation behavior.
pub(super) gc_should_fail: AtomicBool,
notify: Notify,
dropped_notify: Notify,
files_removed_notify: Notify,
@@ -1620,6 +1627,10 @@ impl MockRegionHook {
closed_count: AtomicUsize::new(0),
dropped_count: AtomicUsize::new(0),
files_removed_count: AtomicUsize::new(0),
gc_count: AtomicUsize::new(0),
last_gc_is_region_dropped: AtomicBool::new(false),
last_gc_full_file_listing: AtomicBool::new(false),
gc_should_fail: AtomicBool::new(false),
notify: Notify::new(),
dropped_notify: Notify::new(),
files_removed_notify: Notify::new(),
@@ -1721,6 +1732,33 @@ impl RegionHook for MockRegionHook {
);
self.files_removed_notify.notify_one();
}
async fn on_region_gc(
&self,
region_id: RegionId,
_region_metadata: Option<&RegionMetadataRef>,
_access_layer: &AccessLayerRef,
info: &RegionGcInfo<'_>,
) -> Result<(), Error> {
self.gc_count.fetch_add(1, Ordering::Relaxed);
self.last_gc_is_region_dropped
.store(info.is_region_dropped, Ordering::Relaxed);
self.last_gc_full_file_listing
.store(info.full_file_listing, Ordering::Relaxed);
common_telemetry::info!(
"MockRegionHook::on_region_gc: region={}, is_region_dropped={}, full_file_listing={}",
region_id,
info.is_region_dropped,
info.full_file_listing
);
if self.gc_should_fail.load(Ordering::Relaxed) {
return crate::error::UnexpectedSnafu {
reason: "simulated offline-cleanup hook failure".to_string(),
}
.fail();
}
Ok(())
}
}
#[tokio::test]
+175
View File
@@ -183,6 +183,181 @@ async fn test_engine_offline_cleanup_rejects_opened_region() {
assert!(object_store.exists(&region_dir).await.unwrap());
}
/// Offline cleanup (soft-drop PURGE) removes the region directory directly,
/// bypassing the drop GC worker, so it must notify extensions via
/// `on_region_gc` — otherwise sidecar state (e.g. an Iceberg export) leaks.
///
/// Covers the wiring: `handle_offline_cleanup_request` fires `on_region_gc`
/// with `is_region_dropped = true` and `full_file_listing = true` once the
/// region dir is physically removed, and does NOT fire `on_region_files_removed`
/// (the region is offline and carries no `RegionMetadataRef`).
#[tokio::test]
async fn test_engine_offline_cleanup_fires_on_region_gc() {
common_telemetry::init_default_ut_logging();
let mut env = TestEnv::with_prefix("offline-cleanup-gc-hook").await;
let hook = Arc::new(MockRegionHook::new());
let plugins = Plugins::new();
plugins.insert(hook.clone() as RegionHookRef);
let engine = env
.create_engine_with_plugins(MitoConfig::default(), plugins)
.await;
let region_id = RegionId::new(1, 1);
let request = CreateRequestBuilder::new().build();
let table_dir = request.table_dir.clone();
let path_type = request.path_type;
let options = request.options.clone();
let region_dir = region_dir_from_table_dir(&table_dir, region_id, path_type);
let object_store = env.get_object_store().unwrap();
engine
.handle_request(region_id, RegionRequest::Create(request))
.await
.unwrap();
assert!(object_store.exists(&region_dir).await.unwrap());
// Close the region first — offline cleanup refuses a still-registered region.
engine
.handle_request(
region_id,
RegionRequest::Close(RegionCloseRequest::default()),
)
.await
.unwrap();
assert!(!engine.is_region_exists(region_id));
let gc_before = hook.gc_count.load(Ordering::Relaxed);
let files_removed_before = hook.files_removed_count.load(Ordering::Relaxed);
engine
.handle_request(
region_id,
RegionRequest::CleanUp(RegionCleanUpRequest {
engine: String::new(),
table_dir,
path_type,
options,
}),
)
.await
.unwrap();
// The region directory is gone …
assert!(!engine.is_region_exists(region_id));
assert!(!object_store.exists(&region_dir).await.unwrap());
// … and the extension was notified exactly once via `on_region_gc`, with the
// flags that authorize full sidecar reclamation. `on_region_files_removed`
// must NOT fire for the offline path (no region metadata is available).
assert_eq!(
hook.gc_count.load(Ordering::Relaxed),
gc_before + 1,
"offline cleanup must fire on_region_gc"
);
assert!(
hook.last_gc_is_region_dropped.load(Ordering::Relaxed),
"on_region_gc during offline cleanup must report is_region_dropped = true"
);
assert!(
hook.last_gc_full_file_listing.load(Ordering::Relaxed),
"on_region_gc during offline cleanup must report full_file_listing = true"
);
assert_eq!(
hook.files_removed_count.load(Ordering::Relaxed),
files_removed_before,
"offline cleanup must not fire on_region_files_removed"
);
}
/// A failing `on_region_gc` during offline cleanup must surface as an error so
/// the caller (`PurgeDroppedTableProcedure`) retries the `CleanUp`, and the
/// retry must succeed once the hook stops failing — the handler is idempotent
/// (it re-runs the directory removal, WAL obsolete and the hook safely).
#[tokio::test]
async fn test_engine_offline_cleanup_propagates_on_region_gc_error() {
common_telemetry::init_default_ut_logging();
let mut env = TestEnv::with_prefix("offline-cleanup-gc-hook-retry").await;
let hook = Arc::new(MockRegionHook::new());
let plugins = Plugins::new();
plugins.insert(hook.clone() as RegionHookRef);
let engine = env
.create_engine_with_plugins(MitoConfig::default(), plugins)
.await;
let region_id = RegionId::new(2, 1);
let request = CreateRequestBuilder::new().build();
let table_dir = request.table_dir.clone();
let path_type = request.path_type;
let options = request.options.clone();
let region_dir = region_dir_from_table_dir(&table_dir, region_id, path_type);
let object_store = env.get_object_store().unwrap();
engine
.handle_request(region_id, RegionRequest::Create(request))
.await
.unwrap();
engine
.handle_request(
region_id,
RegionRequest::Close(RegionCloseRequest::default()),
)
.await
.unwrap();
// Force the extension hook to fail: the offline cleanup must return the
// error so the caller retries, instead of swallowing it.
hook.gc_should_fail.store(true, Ordering::Relaxed);
let gc_before = hook.gc_count.load(Ordering::Relaxed);
let err = engine
.handle_request(
region_id,
RegionRequest::CleanUp(RegionCleanUpRequest {
engine: String::new(),
table_dir: table_dir.clone(),
path_type,
options: options.clone(),
}),
)
.await
.unwrap_err();
assert_eq!(
err.status_code(),
StatusCode::Unexpected,
"offline cleanup must propagate the on_region_gc error"
);
assert_eq!(
hook.gc_count.load(Ordering::Relaxed),
gc_before + 1,
"on_region_gc must fire even when it fails"
);
// Retry: clear the failure and the same CleanUp must now succeed. The
// directory removal and WAL obsolete are idempotent, so the second attempt
// re-runs them and the hook without complaint.
hook.gc_should_fail.store(false, Ordering::Relaxed);
engine
.handle_request(
region_id,
RegionRequest::CleanUp(RegionCleanUpRequest {
engine: String::new(),
table_dir,
path_type,
options,
}),
)
.await
.unwrap();
assert!(!engine.is_region_exists(region_id));
assert!(!object_store.exists(&region_dir).await.unwrap());
assert_eq!(
hook.gc_count.load(Ordering::Relaxed),
gc_before + 2,
"on_region_gc must fire again on the successful retry"
);
}
#[tokio::test]
async fn test_engine_open_existing() {
test_engine_open_existing_with_format(false).await;
+37 -5
View File
@@ -89,7 +89,7 @@
//! | Close | [`on_region_closed`] | A close request (or a close-after-flush) removes the region from the active set. Data files, manifest and WAL state are **preserved**; the region may be reopened. |
//! | Logical drop | [`on_region_dropped`] | A drop request has been handled: the region leaves the active set and its WAL entries are marked obsolete. Data files are **not yet deleted**. |
//! | Physical file removal | [`on_region_files_removed`] | The drop GC worker has deleted the region directory. Terminal file-lifecycle event. |
//! | Global GC pass | [`on_region_gc`] | The datanode's global GC worker finished a GC pass for a region — both periodic GC for live regions and the global reclamation of dropped/repartitioned regions (`is_region_dropped`). |
//! | Global GC pass | [`on_region_gc`] | The datanode's global GC worker finished a GC pass for a region — both periodic GC for live regions and the global reclamation of dropped/repartitioned regions (`is_region_dropped`). Also fired by the offline cleanup path (`handle_offline_cleanup_request`, i.e. soft-drop PURGE) once the region directory is removed, with `is_region_dropped = true` and `full_file_listing = true`. |
//!
//! Notes:
//! - `on_region_closed` / `on_region_dropped` run **inline in the region worker loop**,
@@ -104,6 +104,12 @@
//! - When global GC is enabled and a normal table region is dropped with `partial_drop`, its
//! directory is left for global reclamation and `on_region_files_removed` is **not** fired
//! by the drop worker (observe it via the global GC path instead).
//! - The offline cleanup path (`handle_offline_cleanup_request`, reached by a soft-drop
//! `PURGE`) force-removes the region directory but has no `RegionMetadataRef` for the
//! offline region, so it cannot fire `on_region_files_removed`. It fires `on_region_gc`
//! instead — with `is_region_dropped = true` and `full_file_listing = true` — so
//! extensions can reclaim sidecar state for the now-removed region. A hook `Err` is
//! propagated so the caller retries the (idempotent) cleanup.
//! - Logical file removal (compaction, region edit, truncate) is already observable via
//! [`on_manifest_updated`] (`Edit.files_to_remove` / `Truncate` action); only the drop
//! worker's physical directory deletion needs a dedicated file hook.
@@ -128,8 +134,9 @@
//! ## Future work
//!
//! `on_region_files_removed` currently covers only the **drop** GC worker's physical
//! directory removal. A broader per-file `on_files_removed` hook covering compaction
//! removal and truncate is not yet implemented
//! directory removal. The global-GC reclamation path and the offline-cleanup path
//! (soft-drop PURGE) are covered by `on_region_gc` instead. A broader per-file
//! `on_files_removed` hook covering compaction removal and truncate is not yet implemented
//! (though logical file removal is already observable via `on_manifest_updated`,
//! and the global GC reclamation path is covered by `on_region_gc`).
//! Role/leadership transitions (`on_region_role_changed`) are also not hooked.
@@ -390,7 +397,9 @@ pub trait RegionHook: Send + Sync + Debug {
/// Called after the datanode's global GC worker (`LocalGcWorker`) finishes a
/// GC pass for a region — live regions (periodic GC) or dropped/repartitioned
/// regions (the global reclamation path). Lets extensions with sidecar files
/// regions (the global reclamation path). Also fired by the offline cleanup
/// path (`handle_offline_cleanup_request`, i.e. soft-drop PURGE) once the
/// region directory has been removed. Lets extensions with sidecar files
/// outside mito2's region dir clean up residual files.
///
/// Always scoped to [`RegionGcInfo::removed_files`]: clean only sidecar
@@ -412,7 +421,30 @@ pub trait RegionHook: Send + Sync + Debug {
/// same `removed_files`; treat live cleanup as opportunistic.
///
/// `region_metadata` is `None` for dropped regions; use `region_id` +
/// `access_layer`. Idempotent; runs on the background GC task.
/// `access_layer`. Idempotent.
///
/// # Execution context
///
/// On the global-GC path this runs on the background GC task, outside the
/// region worker loop. The offline cleanup path
/// (`handle_offline_cleanup_request`, reached by soft-drop PURGE) instead
/// runs it **inline in the region worker loop** and propagates `Err` so the
/// caller retries the (idempotent) cleanup. So — like `on_region_closed` /
/// `on_region_dropped` — implementations **must be fast and must not block
/// indefinitely** while awaited there: while the callback is pending, every
/// subsequent DDL request for every region on that worker is blocked.
/// Extension authors who wrote their hook against the "background GC task"
/// contract must account for this second, latency-sensitive trigger.
///
/// The offline cleanup path fires this callback once the region directory is
/// confirmed **absent** — including no-op purges where nothing was actually
/// removed this call (a retry after the directory was already gone, or a
/// region that never had files on this datanode, since
/// `remove_region_dir_for_full_drop` returns `Ok` for an absent/empty
/// prefix). `RegionGcInfo::removed_files` is empty in that case; do **not**
/// interpret the call as "files were deleted in this call". This is
/// asymmetric with the GC path, which fires only when files were deleted or
/// a full listing was performed.
async fn on_region_gc(
&self,
region_id: RegionId,
+46 -1
View File
@@ -17,7 +17,7 @@
use std::sync::Arc;
use std::time::Instant;
use common_telemetry::info;
use common_telemetry::{info, warn};
use object_store::util::{join_path, normalize_dir};
use snafu::{OptionExt, ResultExt};
use store_api::logstore::LogStore;
@@ -25,6 +25,8 @@ use store_api::region_request::{AffectedRows, RegionCleanUpRequest, RegionOpenRe
use store_api::storage::RegionId;
use table::requests::STORAGE_KEY;
use crate::access_layer::AccessLayer;
use crate::engine::region_hook::{RegionGcInfo, RegionHookRef};
use crate::error::{
ObjectStoreNotFoundSnafu, OpenDalSnafu, OpenRegionSnafu, RegionBusySnafu, RegionNotFoundSnafu,
Result,
@@ -67,6 +69,49 @@ impl<S: LogStore> RegionWorkerLoop<S> {
let region_dir = region_dir_from_table_dir(&table_dir, region_id, request.path_type);
remove_region_dir_for_full_drop(&region_dir, &object_store).await?;
// The region directory is gone. Offline cleanup (soft-drop PURGE) is a
// terminal physical removal, identical in effect to the drop GC
// worker's directory deletion — but unlike the drop worker it never
// fires `on_region_files_removed`, because the region is offline and
// carries no `RegionMetadataRef`. Fire `on_region_gc` instead: it
// accepts an absent region and lets extensions with sidecar files
// outside the region dir (e.g. an Iceberg export) reclaim their
// per-region state.
//
// Propagate the hook's error so the caller retries the cleanup, mirroring
// how the global GC worker keeps a region for the next pass when
// `on_region_gc` fails. The whole handler is idempotent (the existing
// `test_engine_offline_cleanup_closed_region` already drives it twice in
// a row), so a retry re-runs the directory removal, WAL obsolete and the
// hook safely. Runs inline, consistent with the blocking directory
// removal above; offline cleanup is already a slow path.
if let Some(hook) = self.plugins.get::<RegionHookRef>() {
let access_layer = Arc::new(AccessLayer::new(
table_dir.clone(),
request.path_type,
object_store.clone(),
self.puffin_manager_factory.clone(),
self.intermediate_manager.clone(),
));
let gc_info = RegionGcInfo {
removed_files: &[],
is_region_dropped: true,
full_file_listing: true,
};
if let Err(err) = hook
.on_region_gc(region_id, None, &access_layer, &gc_info)
.await
{
warn!(
err;
"Region hook on_region_gc failed during offline cleanup for region {}; \
returning the error so the caller retries the cleanup",
region_id,
);
return Err(err);
}
}
self.cleanup_dropped_region_runtime_state(region_id).await;
self.dropping_regions.remove_region(region_id);
cleanup_region_file_artifacts(