From 60e0b6e2fc12dcaf6ba97dbb16c167d16d076353 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 26 May 2026 03:41:38 +0000 Subject: [PATCH] test(queue): use production refresh cadence in long-running sims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refresh_loop helper had a rollback hack that forced re-aggregation every 250 ms, masking the production reality: in the real cluster `maybe_refresh_overloaded` is rate-limited to `ACTIVE_REFRESH_SECS` (2 s when something is capped, 5 s when not). The hack was useful for the short headline test but unfair to the long-running burst / oscillation simulations, which need to expose what victims actually experience between refreshes. Plumbed a `force_refresh` flag through `refresh_loop` and `run_scenario`. The headline `fairness_50_workers_diverse_workload` keeps the fast refresh (its 5 s push window needs the cap to take effect in time). The long simulations now run with the natural 2 s claim guard. Empirical impact of switching to prod cadence on the burst scenario: - cycle becomes ~12 s (vs ~3 s with forced refresh): noisy uncaps for ~3 s per cycle, then is re-capped on the next refresh - during each ~3 s uncap window workers fully drain noisy at capacity (~1000–1100 noisy jobs per window) - worst-second victim p95 jumps from 285 ms to ~2 s - avg per-second victim p95 still well under 1.5 s - victim completion rate remains 100 % The bumped assertions reflect this: avg p95 < 1500 ms (up from 1000 ms) and worst-second p95 < 4 s (up from 3 s). These bounds still catch a meaningful regression while accommodating the real oscillation amplitude. --- backend/tests/workspace_fairness.rs | 65 +++++++++++++++++++---------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/backend/tests/workspace_fairness.rs b/backend/tests/workspace_fairness.rs index 2b276aec81..3cdb05ec4e 100644 --- a/backend/tests/workspace_fairness.rs +++ b/backend/tests/workspace_fairness.rs @@ -548,16 +548,23 @@ async fn noisy_pusher( /// Background task that re-runs the fairness algorithm on a cadence so the /// overloaded set tracks the live workload (mirrors what `maybe_refresh_overloaded` /// does in production). -async fn refresh_loop(db: Pool, shutdown: Arc) { +/// +/// `force_refresh = true` rolls back the DB-side claim guard every iteration, +/// so each call re-runs the heavy aggregation. Use for short-running tests +/// that need fast adaptation. `force_refresh = false` leaves the natural 2 s +/// (`ACTIVE_REFRESH_SECS`) claim guard in place — this is what production +/// behaves like and what the oscillation/burst simulations want. +async fn refresh_loop(db: Pool, shutdown: Arc, force_refresh: bool) { while !shutdown.load(Ordering::Relaxed) { - // Force a refresh every cycle by rolling back the DB-side guard. - let _ = sqlx::query( - "UPDATE background_task_state - SET updated_at = NOW() - INTERVAL '1 hour' - WHERE name = 'workspace_fairness'", - ) - .execute(&db) - .await; + if force_refresh { + let _ = sqlx::query( + "UPDATE background_task_state + SET updated_at = NOW() - INTERVAL '1 hour' + WHERE name = 'workspace_fairness'", + ) + .execute(&db) + .await; + } let _ = refresh_overloaded(&db).await; tokio::time::sleep(Duration::from_millis(250)).await; } @@ -581,6 +588,7 @@ async fn run_scenario( drain_timeout: Duration, n_workers: u32, fairness_window_secs: u32, + force_refresh: bool, ) -> ScenarioResult { reset_fairness_state(); WORKSPACE_FAIRNESS_DURATION_SECS.store(fairness_window_secs, Ordering::Relaxed); @@ -644,9 +652,9 @@ async fn run_scenario( let refresh_handle = if fairness_on { let db = db.clone(); let shutdown = shutdown.clone(); - Some(tokio::spawn( - async move { refresh_loop(db, shutdown).await }, - )) + Some(tokio::spawn(async move { + refresh_loop(db, shutdown, force_refresh).await + })) } else { None }; @@ -806,9 +814,12 @@ async fn fairness_50_workers_diverse_workload(db: Pool) { drain_dur, 50, 3, + true, ) .await; - // TREATMENT: fairness ON. + // TREATMENT: fairness ON. Short test → force refresh every 250ms so the + // cap takes effect inside the 5s window. (Production rate is 2s, which + // would only give ~2 refresh cycles inside a 5s test.) let treatment = run_scenario( &db, "treatment (fairness ON)", @@ -817,6 +828,7 @@ async fn fairness_50_workers_diverse_workload(db: Pool) { drain_dur, 50, 3, + true, ) .await; @@ -1007,14 +1019,19 @@ async fn fairness_oscillation_long_run(db: Pool) { // already includes everything in the active window. let drain_dur = Duration::from_secs(2); + // Use production refresh cadence (force_refresh=false) — the SQL claim's + // 2 s rate limit takes effect, so refresh runs every 2 s like on the + // real cluster instead of every 250 ms. This is what victims actually + // experience. let treatment = run_scenario( &db, - "treatment (fairness ON) — long run, 10s window", + "treatment (fairness ON) — long run, 10s window, prod refresh", true, push_dur, drain_dur, 50, 10, + false, ) .await; @@ -1231,11 +1248,13 @@ async fn fairness_burst_then_stop(sqlx_db: Pool) { })); } - // Refresh task at 250ms cadence. + // Refresh task. force_refresh=false → the SQL claim's 2 s rate limit + // takes effect, matching `ACTIVE_REFRESH_SECS` in production. This is + // the regime the cloud cluster actually sees. let refresh_handle = { let db = db.clone(); let shutdown = shutdown.clone(); - tokio::spawn(async move { refresh_loop(db, shutdown).await }) + tokio::spawn(async move { refresh_loop(db, shutdown, false).await }) }; // Victim pushers: 3 workspaces, 20 jobs/s each, 80 ms durations, @@ -1333,17 +1352,21 @@ async fn fairness_burst_then_stop(sqlx_db: Pool) { } // The actual QoL claim we're testing against the user's hypothesis: - // even though workers fully switch to noisy during each uncapped interval, - // the uncap is bounded by the refresh interval, so the per-second victim - // p95 stays well under 1 s on average and under 3 s in the worst second. + // even though workers fully switch to noisy during each uncapped + // interval, the uncap is bounded by `ACTIVE_REFRESH_SECS` (2 s in + // production). Empirically with the prod-realistic refresh cadence + // the avg per-second p95 stays under 1.5 s and the worst-second p95 + // stays under 4 s. If either of these blows out, the oscillation is + // worse than acceptable and the algorithm needs a softer rate limit + // (e.g. stochastic admission of capped workspaces). assert!( - avg_p95 < 1_000, + avg_p95 < 1_500, "average per-second victim p95 under burst was {} ms — \ oscillation is degrading victim QoL more than expected", avg_p95, ); assert!( - worst_p95 < 3_000, + worst_p95 < 4_000, "worst per-second victim p95 under burst was {} ms — \ uncapped bursts are too long; check ACTIVE_REFRESH_SECS", worst_p95,