rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-10-28 22:46:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

storcon: also support tenant deletion for safekeepers (#11289)

Browse Source 
If a tenant gets deleted, delete also all of its timelines. We assume
that by the time a tenant is being deleted, no new timelines are being
created, so we don't need to worry about races with creation in this
situation.

Unlike #11233, which was very simple because it listed the timelines and
invoked timeline deletion, this PR obtains a list of safekeepers to
invoke the tenant deletion on, and then invokes tenant deletion on each
safekeeper that has one or multiple timelines.

Alternative to #11233
Builds on #11288
Part of #9011
This commit is contained in:
Arpad Müller
2025-03-20 10:52:21 +00:00
committed by GitHub
parent 9bf59989db
commit 91dad2514f
6 changed files with 243 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
storage_controller/src/service/safekeeper_reconciler.rs
+71 -17
View File
@@ -51,6 +51,22 @@ impl SafekeeperReconcilers {
handle.cancel.cancel();
}
}
/// Cancel ongoing reconciles for the given timeline
///
/// Specifying `None` here only removes reconciles for the tenant-global reconciliation,
/// instead of doing this for all timelines of the tenant.
///
/// Callers must remove the reconciles from the db manually
pub(crate) fn cancel_reconciles_for_timeline(
&mut self,
node_id: NodeId,
tenant_id: TenantId,
timeline_id: Option<TimelineId>,
) {
if let Some(handle) = self.reconcilers.get(&node_id) {
handle.cancel_reconciliation(tenant_id, timeline_id);
}
}
}
/// Initial load of the pending operations from the db
@@ -73,12 +89,21 @@ pub(crate) async fn load_schedule_requests(
};
let sk = Box::new(sk.clone());
let tenant_id = TenantId::from_str(&op_persist.tenant_id)?;
let timeline_id = TimelineId::from_str(&op_persist.timeline_id)?;
let timeline_id = if !op_persist.timeline_id.is_empty() {
Some(TimelineId::from_str(&op_persist.timeline_id)?)
} else {
None
};
let host_list = match op_persist.op_kind {
SafekeeperTimelineOpKind::Delete => Vec::new(),
SafekeeperTimelineOpKind::Exclude => Vec::new(),
SafekeeperTimelineOpKind::Pull => {
// TODO this code is super hacky, it doesn't take migrations into account
let Some(timeline_id) = timeline_id else {
anyhow::bail!(
"timeline_id is empty for `pull` schedule request for {tenant_id}"
);
};
let timeline_persist = service
.persistence
.get_timeline(tenant_id, timeline_id)
@@ -129,14 +154,15 @@ pub(crate) struct ScheduleRequest {
pub(crate) safekeeper: Box<Safekeeper>,
pub(crate) host_list: Vec<(NodeId, String)>,
pub(crate) tenant_id: TenantId,
pub(crate) timeline_id: TimelineId,
pub(crate) timeline_id: Option<TimelineId>,
pub(crate) generation: u32,
pub(crate) kind: SafekeeperTimelineOpKind,
}
struct ReconcilerHandle {
tx: UnboundedSender<(ScheduleRequest, Arc<CancellationToken>)>,
ongoing_tokens: Arc<ClashMap<(TenantId, TimelineId), Arc<CancellationToken>>>,
#[allow(clippy::type_complexity)]
ongoing_tokens: Arc<ClashMap<(TenantId, Option<TimelineId>), Arc<CancellationToken>>>,
cancel: CancellationToken,
}
@@ -145,7 +171,7 @@ impl ReconcilerHandle {
fn new_token_slot(
&self,
tenant_id: TenantId,
timeline_id: TimelineId,
timeline_id: Option<TimelineId>,
) -> Arc<CancellationToken> {
let entry = self.ongoing_tokens.entry((tenant_id, timeline_id));
if let Entry::Occupied(entry) = &entry {
@@ -154,6 +180,12 @@ impl ReconcilerHandle {
}
entry.insert(Arc::new(self.cancel.child_token())).clone()
}
/// Cancel an ongoing reconciliation
fn cancel_reconciliation(&self, tenant_id: TenantId, timeline_id: Option<TimelineId>) {
if let Some((_, cancel)) = self.ongoing_tokens.remove(&(tenant_id, timeline_id)) {
cancel.cancel();
}
}
fn schedule_reconcile(&self, req: ScheduleRequest) {
let cancel = self.new_token_slot(req.tenant_id, req.timeline_id);
let hostname = req.safekeeper.skp.host.clone();
@@ -206,7 +238,7 @@ impl SafekeeperReconciler {
"reconcile_one",
?kind,
%tenant_id,
%timeline_id
?timeline_id
))
.await;
}
@@ -215,6 +247,12 @@ impl SafekeeperReconciler {
let req_host = req.safekeeper.skp.host.clone();
match req.kind {
SafekeeperTimelineOpKind::Pull => {
let Some(timeline_id) = req.timeline_id else {
tracing::warn!(
"ignoring invalid schedule request: timeline_id is empty for `pull`"
);
return;
};
let our_id = req.safekeeper.get_id();
let http_hosts = req
.host_list
@@ -225,7 +263,7 @@ impl SafekeeperReconciler {
let pull_req = PullTimelineRequest {
http_hosts,
tenant_id: req.tenant_id,
timeline_id: req.timeline_id,
timeline_id,
};
self.reconcile_inner(
req,
@@ -243,7 +281,12 @@ impl SafekeeperReconciler {
SafekeeperTimelineOpKind::Exclude => {
// TODO actually exclude instead of delete here
let tenant_id = req.tenant_id;
let timeline_id = req.timeline_id;
let Some(timeline_id) = req.timeline_id else {
tracing::warn!(
"ignoring invalid schedule request: timeline_id is empty for `exclude`"
);
return;
};
self.reconcile_inner(
req,
async |client| client.delete_timeline(tenant_id, timeline_id).await,
@@ -256,16 +299,27 @@ impl SafekeeperReconciler {
}
SafekeeperTimelineOpKind::Delete => {
let tenant_id = req.tenant_id;
let timeline_id = req.timeline_id;
self.reconcile_inner(
req,
async |client| client.delete_timeline(tenant_id, timeline_id).await,
|_resp| {
tracing::info!("deleted timeline from {req_host}");
},
req_cancel,
)
.await;
if let Some(timeline_id) = req.timeline_id {
self.reconcile_inner(
req,
async |client| client.delete_timeline(tenant_id, timeline_id).await,
|_resp| {
tracing::info!("deleted timeline from {req_host}");
},
req_cancel,
)
.await;
} else {
self.reconcile_inner(
req,
async |client| client.delete_tenant(tenant_id).await,
|_resp| {
tracing::info!("deleted tenant from {req_host}");
},
req_cancel,
)
.await;
}
}
}
}
storage_controller/src/service/safekeeper_service.rs
+86 -3
View File
@@ -1,4 +1,5 @@
use std::collections::{HashMap, HashSet};
use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;
@@ -8,6 +9,7 @@ use crate::persistence::{
DatabaseError, SafekeeperTimelineOpKind, TimelinePendingOpPersistence, TimelinePersistence,
};
use crate::safekeeper::Safekeeper;
use anyhow::Context;
use http_utils::error::ApiError;
use pageserver_api::controller_api::{SafekeeperDescribeResponse, SkSchedulingPolicy};
use pageserver_api::models::{self, SafekeeperInfo, SafekeepersInfo, TimelineInfo};
@@ -265,7 +267,8 @@ impl Service {
.get(&sk.id)
.ok_or_else(|| {
ApiError::InternalServerError(anyhow::anyhow!(
"Couldn't find safekeeper with id {remaining_id} to pull from"
"Couldn't find safekeeper with id {} to pull from",
sk.id
))
})?
.base_url(),
@@ -279,7 +282,7 @@ impl Service {
safekeeper: Box::new(sk.clone()),
host_list,
tenant_id,
timeline_id,
timeline_id: Some(timeline_id),
generation: timeline_persist.generation as u32,
kind: crate::persistence::SafekeeperTimelineOpKind::Pull,
};
@@ -340,7 +343,7 @@ impl Service {
// we don't use this for this kind, put a dummy value
host_list: Vec::new(),
tenant_id,
timeline_id,
timeline_id: Some(timeline_id),
generation: tl.generation as u32,
kind,
};
@@ -350,6 +353,86 @@ impl Service {
Ok(())
}
/// Perform tenant deletion on safekeepers.
pub(super) async fn tenant_delete_safekeepers(
self: &Arc<Self>,
tenant_id: TenantId,
) -> Result<(), ApiError> {
let timeline_list = self
.persistence
.list_timelines_for_tenant(tenant_id)
.await?;
if timeline_list.is_empty() {
// Early exit: the tenant is either empty or not migrated to the storcon yet
tracing::info!("Skipping tenant delete as the timeline doesn't exist in db");
return Ok(());
}
let timeline_list = timeline_list
.into_iter()
.map(|timeline| {
let timeline_id = TimelineId::from_str(&timeline.timeline_id)
.context("timeline id loaded from db")
.map_err(ApiError::InternalServerError)?;
Ok((timeline_id, timeline))
})
.collect::<Result<Vec<_>, ApiError>>()?;
// Remove pending ops from db.
// We cancel them in a later iteration once we hold the state lock.
for (timeline_id, _timeline) in timeline_list.iter() {
self.persistence
.remove_pending_ops_for_timeline(tenant_id, Some(*timeline_id))
.await?;
}
let mut locked = self.inner.write().unwrap();
// The list of safekeepers that have any of the timelines
let mut sk_list = HashSet::new();
// List all pending ops for all timelines, cancel them
for (timeline_id, timeline) in timeline_list.iter() {
let sk_iter = timeline
.sk_set
.iter()
.chain(timeline.new_sk_set.iter().flatten())
.map(|id| NodeId(*id as u64));
for sk_id in sk_iter.clone() {
locked
.safekeeper_reconcilers
.cancel_reconciles_for_timeline(sk_id, tenant_id, Some(*timeline_id));
}
sk_list.extend(sk_iter);
}
// unwrap is safe: we return above for an empty timeline list
let max_generation = timeline_list
.iter()
.map(|(_tl_id, tl)| tl.generation as u32)
.max()
.unwrap();
for sk_id in sk_list {
let Some(safekeeper) = locked.safekeepers.get(&sk_id) else {
tracing::warn!("Couldn't find safekeeper with id {sk_id}");
continue;
};
// Add pending op for tenant deletion
let req = ScheduleRequest {
generation: max_generation,
host_list: Vec::new(),
kind: SafekeeperTimelineOpKind::Delete,
safekeeper: Box::new(safekeeper.clone()),
tenant_id,
timeline_id: None,
};
locked.safekeeper_reconcilers.schedule_request(self, req);
}
Ok(())
}
/// Choose safekeepers for the new timeline: 3 in different azs.
pub(crate) async fn safekeepers_for_new_timeline(
&self,