mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-21 10:14:06 +00:00
feat: invoke gc hook for offline region cleanup (#8613)
I hereby agree to the terms of the [GreptimeDB CLA](https://github.com/GreptimeTeam/.github/blob/main/CLA.md). ## Refer to a related PR or issue link (optional) ## What's changed and what's your intention? This patch adds region hook to soft-drop cleanup phase so we can remove extension files managed by the hook. some open questions: - this cleanup logic sits in `handle_open.rs`, which is a little confusing - can we merge this cleanup with our current GC concept to reduce the complexity? ## PR Checklist Please convert it to a draft if some of the following conditions are not met. - [ ] I have written the necessary rustdoc comments. - [ ] I have added the necessary unit tests and integration tests. - [ ] This PR requires documentation updates. - [ ] API changes are backward compatible. - [ ] Schema or data changes are backward compatible.
This commit is contained in:
@@ -17,7 +17,7 @@
|
||||
use std::assert_matches;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicI64, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use api::v1::Rows;
|
||||
@@ -41,10 +41,11 @@ use store_api::region_request::{
|
||||
use store_api::storage::{RegionId, ScanRequest};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use crate::access_layer::AccessLayerRef;
|
||||
use crate::config::MitoConfig;
|
||||
use crate::engine::MitoEngine;
|
||||
use crate::engine::listener::{EventListener, FlushListener, StallListener};
|
||||
use crate::engine::region_hook::{RegionHook, RegionHookRef, SstFileInfo};
|
||||
use crate::engine::region_hook::{RegionGcInfo, RegionHook, RegionHookRef, SstFileInfo};
|
||||
use crate::error::Error;
|
||||
use crate::manifest::action::RegionMetaActionList;
|
||||
use crate::test_util::{
|
||||
@@ -1606,6 +1607,12 @@ pub(super) struct MockRegionHook {
|
||||
pub(super) closed_count: AtomicUsize,
|
||||
pub(super) dropped_count: AtomicUsize,
|
||||
pub(super) files_removed_count: AtomicUsize,
|
||||
pub(super) gc_count: AtomicUsize,
|
||||
pub(super) last_gc_is_region_dropped: AtomicBool,
|
||||
pub(super) last_gc_full_file_listing: AtomicBool,
|
||||
/// When set, `on_region_gc` returns `Err` (after recording the call), to
|
||||
/// exercise the callers' retry/propagation behavior.
|
||||
pub(super) gc_should_fail: AtomicBool,
|
||||
notify: Notify,
|
||||
dropped_notify: Notify,
|
||||
files_removed_notify: Notify,
|
||||
@@ -1620,6 +1627,10 @@ impl MockRegionHook {
|
||||
closed_count: AtomicUsize::new(0),
|
||||
dropped_count: AtomicUsize::new(0),
|
||||
files_removed_count: AtomicUsize::new(0),
|
||||
gc_count: AtomicUsize::new(0),
|
||||
last_gc_is_region_dropped: AtomicBool::new(false),
|
||||
last_gc_full_file_listing: AtomicBool::new(false),
|
||||
gc_should_fail: AtomicBool::new(false),
|
||||
notify: Notify::new(),
|
||||
dropped_notify: Notify::new(),
|
||||
files_removed_notify: Notify::new(),
|
||||
@@ -1721,6 +1732,33 @@ impl RegionHook for MockRegionHook {
|
||||
);
|
||||
self.files_removed_notify.notify_one();
|
||||
}
|
||||
|
||||
async fn on_region_gc(
|
||||
&self,
|
||||
region_id: RegionId,
|
||||
_region_metadata: Option<&RegionMetadataRef>,
|
||||
_access_layer: &AccessLayerRef,
|
||||
info: &RegionGcInfo<'_>,
|
||||
) -> Result<(), Error> {
|
||||
self.gc_count.fetch_add(1, Ordering::Relaxed);
|
||||
self.last_gc_is_region_dropped
|
||||
.store(info.is_region_dropped, Ordering::Relaxed);
|
||||
self.last_gc_full_file_listing
|
||||
.store(info.full_file_listing, Ordering::Relaxed);
|
||||
common_telemetry::info!(
|
||||
"MockRegionHook::on_region_gc: region={}, is_region_dropped={}, full_file_listing={}",
|
||||
region_id,
|
||||
info.is_region_dropped,
|
||||
info.full_file_listing
|
||||
);
|
||||
if self.gc_should_fail.load(Ordering::Relaxed) {
|
||||
return crate::error::UnexpectedSnafu {
|
||||
reason: "simulated offline-cleanup hook failure".to_string(),
|
||||
}
|
||||
.fail();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -183,6 +183,181 @@ async fn test_engine_offline_cleanup_rejects_opened_region() {
|
||||
assert!(object_store.exists(®ion_dir).await.unwrap());
|
||||
}
|
||||
|
||||
/// Offline cleanup (soft-drop PURGE) removes the region directory directly,
|
||||
/// bypassing the drop GC worker, so it must notify extensions via
|
||||
/// `on_region_gc` — otherwise sidecar state (e.g. an Iceberg export) leaks.
|
||||
///
|
||||
/// Covers the wiring: `handle_offline_cleanup_request` fires `on_region_gc`
|
||||
/// with `is_region_dropped = true` and `full_file_listing = true` once the
|
||||
/// region dir is physically removed, and does NOT fire `on_region_files_removed`
|
||||
/// (the region is offline and carries no `RegionMetadataRef`).
|
||||
#[tokio::test]
|
||||
async fn test_engine_offline_cleanup_fires_on_region_gc() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let mut env = TestEnv::with_prefix("offline-cleanup-gc-hook").await;
|
||||
|
||||
let hook = Arc::new(MockRegionHook::new());
|
||||
let plugins = Plugins::new();
|
||||
plugins.insert(hook.clone() as RegionHookRef);
|
||||
let engine = env
|
||||
.create_engine_with_plugins(MitoConfig::default(), plugins)
|
||||
.await;
|
||||
|
||||
let region_id = RegionId::new(1, 1);
|
||||
let request = CreateRequestBuilder::new().build();
|
||||
let table_dir = request.table_dir.clone();
|
||||
let path_type = request.path_type;
|
||||
let options = request.options.clone();
|
||||
let region_dir = region_dir_from_table_dir(&table_dir, region_id, path_type);
|
||||
let object_store = env.get_object_store().unwrap();
|
||||
|
||||
engine
|
||||
.handle_request(region_id, RegionRequest::Create(request))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(object_store.exists(®ion_dir).await.unwrap());
|
||||
|
||||
// Close the region first — offline cleanup refuses a still-registered region.
|
||||
engine
|
||||
.handle_request(
|
||||
region_id,
|
||||
RegionRequest::Close(RegionCloseRequest::default()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!engine.is_region_exists(region_id));
|
||||
|
||||
let gc_before = hook.gc_count.load(Ordering::Relaxed);
|
||||
let files_removed_before = hook.files_removed_count.load(Ordering::Relaxed);
|
||||
|
||||
engine
|
||||
.handle_request(
|
||||
region_id,
|
||||
RegionRequest::CleanUp(RegionCleanUpRequest {
|
||||
engine: String::new(),
|
||||
table_dir,
|
||||
path_type,
|
||||
options,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The region directory is gone …
|
||||
assert!(!engine.is_region_exists(region_id));
|
||||
assert!(!object_store.exists(®ion_dir).await.unwrap());
|
||||
|
||||
// … and the extension was notified exactly once via `on_region_gc`, with the
|
||||
// flags that authorize full sidecar reclamation. `on_region_files_removed`
|
||||
// must NOT fire for the offline path (no region metadata is available).
|
||||
assert_eq!(
|
||||
hook.gc_count.load(Ordering::Relaxed),
|
||||
gc_before + 1,
|
||||
"offline cleanup must fire on_region_gc"
|
||||
);
|
||||
assert!(
|
||||
hook.last_gc_is_region_dropped.load(Ordering::Relaxed),
|
||||
"on_region_gc during offline cleanup must report is_region_dropped = true"
|
||||
);
|
||||
assert!(
|
||||
hook.last_gc_full_file_listing.load(Ordering::Relaxed),
|
||||
"on_region_gc during offline cleanup must report full_file_listing = true"
|
||||
);
|
||||
assert_eq!(
|
||||
hook.files_removed_count.load(Ordering::Relaxed),
|
||||
files_removed_before,
|
||||
"offline cleanup must not fire on_region_files_removed"
|
||||
);
|
||||
}
|
||||
|
||||
/// A failing `on_region_gc` during offline cleanup must surface as an error so
|
||||
/// the caller (`PurgeDroppedTableProcedure`) retries the `CleanUp`, and the
|
||||
/// retry must succeed once the hook stops failing — the handler is idempotent
|
||||
/// (it re-runs the directory removal, WAL obsolete and the hook safely).
|
||||
#[tokio::test]
|
||||
async fn test_engine_offline_cleanup_propagates_on_region_gc_error() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let mut env = TestEnv::with_prefix("offline-cleanup-gc-hook-retry").await;
|
||||
|
||||
let hook = Arc::new(MockRegionHook::new());
|
||||
let plugins = Plugins::new();
|
||||
plugins.insert(hook.clone() as RegionHookRef);
|
||||
let engine = env
|
||||
.create_engine_with_plugins(MitoConfig::default(), plugins)
|
||||
.await;
|
||||
|
||||
let region_id = RegionId::new(2, 1);
|
||||
let request = CreateRequestBuilder::new().build();
|
||||
let table_dir = request.table_dir.clone();
|
||||
let path_type = request.path_type;
|
||||
let options = request.options.clone();
|
||||
let region_dir = region_dir_from_table_dir(&table_dir, region_id, path_type);
|
||||
let object_store = env.get_object_store().unwrap();
|
||||
|
||||
engine
|
||||
.handle_request(region_id, RegionRequest::Create(request))
|
||||
.await
|
||||
.unwrap();
|
||||
engine
|
||||
.handle_request(
|
||||
region_id,
|
||||
RegionRequest::Close(RegionCloseRequest::default()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Force the extension hook to fail: the offline cleanup must return the
|
||||
// error so the caller retries, instead of swallowing it.
|
||||
hook.gc_should_fail.store(true, Ordering::Relaxed);
|
||||
let gc_before = hook.gc_count.load(Ordering::Relaxed);
|
||||
let err = engine
|
||||
.handle_request(
|
||||
region_id,
|
||||
RegionRequest::CleanUp(RegionCleanUpRequest {
|
||||
engine: String::new(),
|
||||
table_dir: table_dir.clone(),
|
||||
path_type,
|
||||
options: options.clone(),
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
err.status_code(),
|
||||
StatusCode::Unexpected,
|
||||
"offline cleanup must propagate the on_region_gc error"
|
||||
);
|
||||
assert_eq!(
|
||||
hook.gc_count.load(Ordering::Relaxed),
|
||||
gc_before + 1,
|
||||
"on_region_gc must fire even when it fails"
|
||||
);
|
||||
|
||||
// Retry: clear the failure and the same CleanUp must now succeed. The
|
||||
// directory removal and WAL obsolete are idempotent, so the second attempt
|
||||
// re-runs them and the hook without complaint.
|
||||
hook.gc_should_fail.store(false, Ordering::Relaxed);
|
||||
engine
|
||||
.handle_request(
|
||||
region_id,
|
||||
RegionRequest::CleanUp(RegionCleanUpRequest {
|
||||
engine: String::new(),
|
||||
table_dir,
|
||||
path_type,
|
||||
options,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!engine.is_region_exists(region_id));
|
||||
assert!(!object_store.exists(®ion_dir).await.unwrap());
|
||||
assert_eq!(
|
||||
hook.gc_count.load(Ordering::Relaxed),
|
||||
gc_before + 2,
|
||||
"on_region_gc must fire again on the successful retry"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_engine_open_existing() {
|
||||
test_engine_open_existing_with_format(false).await;
|
||||
|
||||
@@ -89,7 +89,7 @@
|
||||
//! | Close | [`on_region_closed`] | A close request (or a close-after-flush) removes the region from the active set. Data files, manifest and WAL state are **preserved**; the region may be reopened. |
|
||||
//! | Logical drop | [`on_region_dropped`] | A drop request has been handled: the region leaves the active set and its WAL entries are marked obsolete. Data files are **not yet deleted**. |
|
||||
//! | Physical file removal | [`on_region_files_removed`] | The drop GC worker has deleted the region directory. Terminal file-lifecycle event. |
|
||||
//! | Global GC pass | [`on_region_gc`] | The datanode's global GC worker finished a GC pass for a region — both periodic GC for live regions and the global reclamation of dropped/repartitioned regions (`is_region_dropped`). |
|
||||
//! | Global GC pass | [`on_region_gc`] | The datanode's global GC worker finished a GC pass for a region — both periodic GC for live regions and the global reclamation of dropped/repartitioned regions (`is_region_dropped`). Also fired by the offline cleanup path (`handle_offline_cleanup_request`, i.e. soft-drop PURGE) once the region directory is removed, with `is_region_dropped = true` and `full_file_listing = true`. |
|
||||
//!
|
||||
//! Notes:
|
||||
//! - `on_region_closed` / `on_region_dropped` run **inline in the region worker loop**,
|
||||
@@ -104,6 +104,12 @@
|
||||
//! - When global GC is enabled and a normal table region is dropped with `partial_drop`, its
|
||||
//! directory is left for global reclamation and `on_region_files_removed` is **not** fired
|
||||
//! by the drop worker (observe it via the global GC path instead).
|
||||
//! - The offline cleanup path (`handle_offline_cleanup_request`, reached by a soft-drop
|
||||
//! `PURGE`) force-removes the region directory but has no `RegionMetadataRef` for the
|
||||
//! offline region, so it cannot fire `on_region_files_removed`. It fires `on_region_gc`
|
||||
//! instead — with `is_region_dropped = true` and `full_file_listing = true` — so
|
||||
//! extensions can reclaim sidecar state for the now-removed region. A hook `Err` is
|
||||
//! propagated so the caller retries the (idempotent) cleanup.
|
||||
//! - Logical file removal (compaction, region edit, truncate) is already observable via
|
||||
//! [`on_manifest_updated`] (`Edit.files_to_remove` / `Truncate` action); only the drop
|
||||
//! worker's physical directory deletion needs a dedicated file hook.
|
||||
@@ -128,8 +134,9 @@
|
||||
//! ## Future work
|
||||
//!
|
||||
//! `on_region_files_removed` currently covers only the **drop** GC worker's physical
|
||||
//! directory removal. A broader per-file `on_files_removed` hook covering compaction
|
||||
//! removal and truncate is not yet implemented
|
||||
//! directory removal. The global-GC reclamation path and the offline-cleanup path
|
||||
//! (soft-drop PURGE) are covered by `on_region_gc` instead. A broader per-file
|
||||
//! `on_files_removed` hook covering compaction removal and truncate is not yet implemented
|
||||
//! (though logical file removal is already observable via `on_manifest_updated`,
|
||||
//! and the global GC reclamation path is covered by `on_region_gc`).
|
||||
//! Role/leadership transitions (`on_region_role_changed`) are also not hooked.
|
||||
@@ -390,7 +397,9 @@ pub trait RegionHook: Send + Sync + Debug {
|
||||
|
||||
/// Called after the datanode's global GC worker (`LocalGcWorker`) finishes a
|
||||
/// GC pass for a region — live regions (periodic GC) or dropped/repartitioned
|
||||
/// regions (the global reclamation path). Lets extensions with sidecar files
|
||||
/// regions (the global reclamation path). Also fired by the offline cleanup
|
||||
/// path (`handle_offline_cleanup_request`, i.e. soft-drop PURGE) once the
|
||||
/// region directory has been removed. Lets extensions with sidecar files
|
||||
/// outside mito2's region dir clean up residual files.
|
||||
///
|
||||
/// Always scoped to [`RegionGcInfo::removed_files`]: clean only sidecar
|
||||
@@ -412,7 +421,30 @@ pub trait RegionHook: Send + Sync + Debug {
|
||||
/// same `removed_files`; treat live cleanup as opportunistic.
|
||||
///
|
||||
/// `region_metadata` is `None` for dropped regions; use `region_id` +
|
||||
/// `access_layer`. Idempotent; runs on the background GC task.
|
||||
/// `access_layer`. Idempotent.
|
||||
///
|
||||
/// # Execution context
|
||||
///
|
||||
/// On the global-GC path this runs on the background GC task, outside the
|
||||
/// region worker loop. The offline cleanup path
|
||||
/// (`handle_offline_cleanup_request`, reached by soft-drop PURGE) instead
|
||||
/// runs it **inline in the region worker loop** and propagates `Err` so the
|
||||
/// caller retries the (idempotent) cleanup. So — like `on_region_closed` /
|
||||
/// `on_region_dropped` — implementations **must be fast and must not block
|
||||
/// indefinitely** while awaited there: while the callback is pending, every
|
||||
/// subsequent DDL request for every region on that worker is blocked.
|
||||
/// Extension authors who wrote their hook against the "background GC task"
|
||||
/// contract must account for this second, latency-sensitive trigger.
|
||||
///
|
||||
/// The offline cleanup path fires this callback once the region directory is
|
||||
/// confirmed **absent** — including no-op purges where nothing was actually
|
||||
/// removed this call (a retry after the directory was already gone, or a
|
||||
/// region that never had files on this datanode, since
|
||||
/// `remove_region_dir_for_full_drop` returns `Ok` for an absent/empty
|
||||
/// prefix). `RegionGcInfo::removed_files` is empty in that case; do **not**
|
||||
/// interpret the call as "files were deleted in this call". This is
|
||||
/// asymmetric with the GC path, which fires only when files were deleted or
|
||||
/// a full listing was performed.
|
||||
async fn on_region_gc(
|
||||
&self,
|
||||
region_id: RegionId,
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use common_telemetry::info;
|
||||
use common_telemetry::{info, warn};
|
||||
use object_store::util::{join_path, normalize_dir};
|
||||
use snafu::{OptionExt, ResultExt};
|
||||
use store_api::logstore::LogStore;
|
||||
@@ -25,6 +25,8 @@ use store_api::region_request::{AffectedRows, RegionCleanUpRequest, RegionOpenRe
|
||||
use store_api::storage::RegionId;
|
||||
use table::requests::STORAGE_KEY;
|
||||
|
||||
use crate::access_layer::AccessLayer;
|
||||
use crate::engine::region_hook::{RegionGcInfo, RegionHookRef};
|
||||
use crate::error::{
|
||||
ObjectStoreNotFoundSnafu, OpenDalSnafu, OpenRegionSnafu, RegionBusySnafu, RegionNotFoundSnafu,
|
||||
Result,
|
||||
@@ -67,6 +69,49 @@ impl<S: LogStore> RegionWorkerLoop<S> {
|
||||
let region_dir = region_dir_from_table_dir(&table_dir, region_id, request.path_type);
|
||||
remove_region_dir_for_full_drop(®ion_dir, &object_store).await?;
|
||||
|
||||
// The region directory is gone. Offline cleanup (soft-drop PURGE) is a
|
||||
// terminal physical removal, identical in effect to the drop GC
|
||||
// worker's directory deletion — but unlike the drop worker it never
|
||||
// fires `on_region_files_removed`, because the region is offline and
|
||||
// carries no `RegionMetadataRef`. Fire `on_region_gc` instead: it
|
||||
// accepts an absent region and lets extensions with sidecar files
|
||||
// outside the region dir (e.g. an Iceberg export) reclaim their
|
||||
// per-region state.
|
||||
//
|
||||
// Propagate the hook's error so the caller retries the cleanup, mirroring
|
||||
// how the global GC worker keeps a region for the next pass when
|
||||
// `on_region_gc` fails. The whole handler is idempotent (the existing
|
||||
// `test_engine_offline_cleanup_closed_region` already drives it twice in
|
||||
// a row), so a retry re-runs the directory removal, WAL obsolete and the
|
||||
// hook safely. Runs inline, consistent with the blocking directory
|
||||
// removal above; offline cleanup is already a slow path.
|
||||
if let Some(hook) = self.plugins.get::<RegionHookRef>() {
|
||||
let access_layer = Arc::new(AccessLayer::new(
|
||||
table_dir.clone(),
|
||||
request.path_type,
|
||||
object_store.clone(),
|
||||
self.puffin_manager_factory.clone(),
|
||||
self.intermediate_manager.clone(),
|
||||
));
|
||||
let gc_info = RegionGcInfo {
|
||||
removed_files: &[],
|
||||
is_region_dropped: true,
|
||||
full_file_listing: true,
|
||||
};
|
||||
if let Err(err) = hook
|
||||
.on_region_gc(region_id, None, &access_layer, &gc_info)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
err;
|
||||
"Region hook on_region_gc failed during offline cleanup for region {}; \
|
||||
returning the error so the caller retries the cleanup",
|
||||
region_id,
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
|
||||
self.cleanup_dropped_region_runtime_state(region_id).await;
|
||||
self.dropping_regions.remove_region(region_id);
|
||||
cleanup_region_file_artifacts(
|
||||
|
||||
Reference in New Issue
Block a user