rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-06-05 14:30:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(storcon): loop in chaos injection (#11004)

Browse Source 
## Problem

Somehow the previous patch loses the loop in the chaos injector function
so everything will only run once.
https://github.com/neondatabase/neon/pull/10934

## Summary of changes

Add back the loop.

Signed-off-by: Alex Chi Z <chi@neon.tech>
This commit is contained in:
Alex Chi Z.
2025-02-28 10:49:15 -05:00
committed by GitHub
parent 23fb8053c5
commit f79ee0bb88
1 changed files with 37 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
storage_controller/src/service/chaos_injector.rs
View File

@@ -46,48 +46,51 @@ impl ChaosInjector {
}
}
fn get_cron_interval_sleep_future(&self) -> Option<tokio::time::Sleep> {
if let Some(ref chaos_exit_crontab) = self.chaos_exit_crontab {
match cron_to_next_duration(chaos_exit_crontab) {
Ok(interval_exit) => Some(interval_exit),
Err(e) => {
tracing::error!("Error processing the cron schedule: {e}");
None
}
}
} else {
None
}
}
pub async fn run(&mut self, cancel: CancellationToken) {
let mut interval = tokio::time::interval(self.interval);
let cron_interval = {
if let Some(ref chaos_exit_crontab) = self.chaos_exit_crontab {
match cron_to_next_duration(chaos_exit_crontab) {
Ok(interval_exit) => Some(interval_exit),
Err(e) => {
tracing::error!("Error processing the cron schedule: {e}");
None
}
}
} else {
None
}
};
#[derive(Debug)]
enum ChaosEvent {
ShuffleTenant,
ForceKill,
}
let chaos_type = tokio::select! {
_ = interval.tick() => {
ChaosEvent::ShuffleTenant
}
Some(_) = maybe_sleep(cron_interval) => {
ChaosEvent::ForceKill
}
_ = cancel.cancelled() => {
tracing::info!("Shutting down");
return;
}
};
match chaos_type {
ChaosEvent::ShuffleTenant => {
self.inject_chaos().await;
}
ChaosEvent::ForceKill => {
self.force_kill().await;
loop {
let cron_interval = self.get_cron_interval_sleep_future();
let chaos_type = tokio::select! {
_ = interval.tick() => {
ChaosEvent::ShuffleTenant
}
Some(_) = maybe_sleep(cron_interval) => {
ChaosEvent::ForceKill
}
_ = cancel.cancelled() => {
tracing::info!("Shutting down");
return;
}
};
tracing::info!("Chaos iteration: {chaos_type:?}...");
match chaos_type {
ChaosEvent::ShuffleTenant => {
self.inject_chaos().await;
}
ChaosEvent::ForceKill => {
self.force_kill().await;
}
}
}
tracing::info!("Chaos iteration...");
}
/// If a shard has a secondary and attached location, then re-assign the secondary to be