diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 0cf5c36b87..74faee1115 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -503,6 +503,8 @@ pub enum GetTenantError { /// Gets the tenant from the in-memory data, erroring if it's absent or is not fitting to the query. /// `active_only = true` allows to query only tenants that are ready for operations, erroring on other kinds of tenants. +/// +/// This method is cancel-safe. pub async fn get_tenant( tenant_id: TenantId, active_only: bool, diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index 3e407dda57..39f0d03a01 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -328,9 +328,24 @@ impl Timeline { // Make one of the tenant's timelines draw the short straw and run the calculation. // The others wait until the calculation is done so that they take into account the // imitated accesses that the winner made. - let Ok(tenant) = crate::tenant::mgr::get_tenant(self.tenant_id, true).await else { - // likely, we're shutting down - return ControlFlow::Break(()); + // + // It is critical we are responsive to cancellation here. Otherwise, we deadlock with + // tenant deletion (holds TENANTS in read mode) any other task that attempts to + // acquire TENANTS in write mode before we here call get_tenant. + // See https://github.com/neondatabase/neon/issues/5284. + let res = tokio::select! { + _ = cancel.cancelled() => { + return ControlFlow::Break(()); + } + res = crate::tenant::mgr::get_tenant(self.tenant_id, true) => { + res + } + }; + let tenant = match res { + Ok(t) => t, + Err(_) => { + return ControlFlow::Break(()); + } }; let mut state = tenant.eviction_task_tenant_state.lock().await; match state.last_layer_access_imitation {