diff --git a/backend/tests/workspace_fairness.rs b/backend/tests/workspace_fairness.rs index 2b276aec81..3cdb05ec4e 100644 --- a/backend/tests/workspace_fairness.rs +++ b/backend/tests/workspace_fairness.rs @@ -548,16 +548,23 @@ async fn noisy_pusher( /// Background task that re-runs the fairness algorithm on a cadence so the /// overloaded set tracks the live workload (mirrors what `maybe_refresh_overloaded` /// does in production). -async fn refresh_loop(db: Pool, shutdown: Arc) { +/// +/// `force_refresh = true` rolls back the DB-side claim guard every iteration, +/// so each call re-runs the heavy aggregation. Use for short-running tests +/// that need fast adaptation. `force_refresh = false` leaves the natural 2 s +/// (`ACTIVE_REFRESH_SECS`) claim guard in place — this is what production +/// behaves like and what the oscillation/burst simulations want. +async fn refresh_loop(db: Pool, shutdown: Arc, force_refresh: bool) { while !shutdown.load(Ordering::Relaxed) { - // Force a refresh every cycle by rolling back the DB-side guard. - let _ = sqlx::query( - "UPDATE background_task_state - SET updated_at = NOW() - INTERVAL '1 hour' - WHERE name = 'workspace_fairness'", - ) - .execute(&db) - .await; + if force_refresh { + let _ = sqlx::query( + "UPDATE background_task_state + SET updated_at = NOW() - INTERVAL '1 hour' + WHERE name = 'workspace_fairness'", + ) + .execute(&db) + .await; + } let _ = refresh_overloaded(&db).await; tokio::time::sleep(Duration::from_millis(250)).await; } @@ -581,6 +588,7 @@ async fn run_scenario( drain_timeout: Duration, n_workers: u32, fairness_window_secs: u32, + force_refresh: bool, ) -> ScenarioResult { reset_fairness_state(); WORKSPACE_FAIRNESS_DURATION_SECS.store(fairness_window_secs, Ordering::Relaxed); @@ -644,9 +652,9 @@ async fn run_scenario( let refresh_handle = if fairness_on { let db = db.clone(); let shutdown = shutdown.clone(); - Some(tokio::spawn( - async move { refresh_loop(db, shutdown).await }, - )) + Some(tokio::spawn(async move { + refresh_loop(db, shutdown, force_refresh).await + })) } else { None }; @@ -806,9 +814,12 @@ async fn fairness_50_workers_diverse_workload(db: Pool) { drain_dur, 50, 3, + true, ) .await; - // TREATMENT: fairness ON. + // TREATMENT: fairness ON. Short test → force refresh every 250ms so the + // cap takes effect inside the 5s window. (Production rate is 2s, which + // would only give ~2 refresh cycles inside a 5s test.) let treatment = run_scenario( &db, "treatment (fairness ON)", @@ -817,6 +828,7 @@ async fn fairness_50_workers_diverse_workload(db: Pool) { drain_dur, 50, 3, + true, ) .await; @@ -1007,14 +1019,19 @@ async fn fairness_oscillation_long_run(db: Pool) { // already includes everything in the active window. let drain_dur = Duration::from_secs(2); + // Use production refresh cadence (force_refresh=false) — the SQL claim's + // 2 s rate limit takes effect, so refresh runs every 2 s like on the + // real cluster instead of every 250 ms. This is what victims actually + // experience. let treatment = run_scenario( &db, - "treatment (fairness ON) — long run, 10s window", + "treatment (fairness ON) — long run, 10s window, prod refresh", true, push_dur, drain_dur, 50, 10, + false, ) .await; @@ -1231,11 +1248,13 @@ async fn fairness_burst_then_stop(sqlx_db: Pool) { })); } - // Refresh task at 250ms cadence. + // Refresh task. force_refresh=false → the SQL claim's 2 s rate limit + // takes effect, matching `ACTIVE_REFRESH_SECS` in production. This is + // the regime the cloud cluster actually sees. let refresh_handle = { let db = db.clone(); let shutdown = shutdown.clone(); - tokio::spawn(async move { refresh_loop(db, shutdown).await }) + tokio::spawn(async move { refresh_loop(db, shutdown, false).await }) }; // Victim pushers: 3 workspaces, 20 jobs/s each, 80 ms durations, @@ -1333,17 +1352,21 @@ async fn fairness_burst_then_stop(sqlx_db: Pool) { } // The actual QoL claim we're testing against the user's hypothesis: - // even though workers fully switch to noisy during each uncapped interval, - // the uncap is bounded by the refresh interval, so the per-second victim - // p95 stays well under 1 s on average and under 3 s in the worst second. + // even though workers fully switch to noisy during each uncapped + // interval, the uncap is bounded by `ACTIVE_REFRESH_SECS` (2 s in + // production). Empirically with the prod-realistic refresh cadence + // the avg per-second p95 stays under 1.5 s and the worst-second p95 + // stays under 4 s. If either of these blows out, the oscillation is + // worse than acceptable and the algorithm needs a softer rate limit + // (e.g. stochastic admission of capped workspaces). assert!( - avg_p95 < 1_000, + avg_p95 < 1_500, "average per-second victim p95 under burst was {} ms — \ oscillation is degrading victim QoL more than expected", avg_p95, ); assert!( - worst_p95 < 3_000, + worst_p95 < 4_000, "worst per-second victim p95 under burst was {} ms — \ uncapped bursts are too long; check ACTIVE_REFRESH_SECS", worst_p95,