Adding transition reason to the log when a tenant is moved to Broken state (#3289)

#3160
This commit is contained in:
Shany Pozin
2023-01-09 10:24:50 +02:00
committed by GitHub
parent 23d5e2bdaa
commit 7920b39a27
2 changed files with 11 additions and 7 deletions

View File

@@ -596,7 +596,7 @@ impl Tenant {
match tenant_clone.attach().await {
Ok(_) => {}
Err(e) => {
tenant_clone.set_broken();
tenant_clone.set_broken(&e.to_string());
error!("error attaching tenant: {:?}", e);
}
}
@@ -860,7 +860,7 @@ impl Tenant {
match tenant_clone.load().await {
Ok(()) => {}
Err(err) => {
tenant_clone.set_broken();
tenant_clone.set_broken(&err.to_string());
error!("could not load tenant {tenant_id}: {err:?}");
}
}
@@ -1496,7 +1496,7 @@ impl Tenant {
});
}
pub fn set_broken(&self) {
pub fn set_broken(&self, reason: &str) {
self.state.send_modify(|current_state| {
match *current_state {
TenantState::Active => {
@@ -1505,18 +1505,22 @@ impl Tenant {
// activated should never be marked as broken. We cope with it the best
// we can, but it shouldn't happen.
*current_state = TenantState::Broken;
warn!("Changing Active tenant to Broken state");
warn!("Changing Active tenant to Broken state, reason: {}", reason);
}
TenantState::Broken => {
// This shouldn't happen either
warn!("Tenant is already broken");
warn!("Tenant is already in Broken state");
}
TenantState::Stopping => {
// This shouldn't happen either
*current_state = TenantState::Broken;
warn!("Marking Stopping tenant as Broken");
warn!(
"Marking Stopping tenant as Broken state, reason: {}",
reason
);
}
TenantState::Loading | TenantState::Attaching => {
info!("Setting tenant as Broken state, reason: {}", reason);
*current_state = TenantState::Broken;
}
}

View File

@@ -430,7 +430,7 @@ where
Err(e) => {
let tenants_accessor = TENANTS.read().await;
match tenants_accessor.get(&tenant_id) {
Some(tenant) => tenant.set_broken(),
Some(tenant) => tenant.set_broken(&e.to_string()),
None => warn!("Tenant {tenant_id} got removed from memory"),
}
Err(e)