perf(agent-workers): two-stage prefetch pipeline for agent throughput

Experimental and env-gated (defaults off — stock behavior unchanged).

Read path is made fully non-blocking on both sides so neither the worker
nor the server waits on the stage behind it:
  Postgres -> [server-side buffer] -> HTTP -> [agent-side buffer] -> worker

- batch pull/complete: N jobs per pull, N completions per flush, so the
  per-job HTTP round-trips and per-job dequeue queries are amortized
- server-side prefetch: a single background puller per job-tag-set runs
  the SKIP LOCKED dequeue ahead of demand and coalesces concurrent agents,
  moving the PG query off the request critical path
- agent-side prefetch: a background refiller keeps the local job buffer
  deep (high low-watermark + several refills in flight) so the worker
  loop pops without a network call
- empty-pull backoff fix: don't apply the idle long-sleep to a transient
  empty pull from a prefetching consumer
- AGENT_PROF: per-phase profiler + auto-rendered per-job time-budget SVG

Single agent goes from ~2.5k to ~125k+ j/s. Multi-agent collapse is
reproduced but its root cause is still open (not established to be PG).
See benchmarks/agent_worker_scaling.md.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.gitignore

@@ -33,4 +33,14 @@ backend/chrome_profiler.json
 .fast-check/
 __pycache__/
 .playwright-mcp/
-.codex
+.codex
+
+# Local helm values overlay for wm_sim — holds secrets (EE license key,
+# instance auth, etc). Tracked example sits next to it as `local.example.yaml`.
+benchmarks/sim/values/local.yaml
+
+# Per-run bench output: dashboards, JSONL streams, kubectl-derived data,
+# huge pg.log / pgbadger.html. Regenerated on every bench fire; never checked
+# in.
+benchmarks/reports/
+benchmarks/sim/results/

backend/Cargo.lock

@@ -14050,6 +14050,7 @@ dependencies = [
 name = "windmill-api-agent-workers"
 version = "1.719.0"
 dependencies = [
+ "arc-swap",
  "axum 0.8.9",
  "chrono",
  "http 1.4.1",
@@ -15645,6 +15646,7 @@ dependencies = [
  "itertools 0.14.0",
  "jsonwebtoken 8.3.0",
  "lazy_static",
+ "libc",
  "libffi-sys",
  "libloading",
  "mappable-rc",

backend/windmill-api-agent-workers/Cargo.toml

@@ -22,6 +22,7 @@ windmill-queue.workspace = true
 windmill-worker.workspace = true
 axum.workspace = true
 lazy_static.workspace = true
+arc-swap.workspace = true
 chrono.workspace = true
 http.workspace = true
 hyper.workspace = true

backend/windmill-worker/Cargo.toml

@@ -42,6 +42,7 @@ quickjs = ["windmill-jseval/quickjs"]
 bedrock = ["windmill-ai/bedrock"]
 
 [dependencies]
+libc = "0.2"  # AGENT_PROF: thread CPU time (CLOCK_THREAD_CPUTIME_ID) to split gap into cpu vs parked
 windmill-ai = { workspace = true, default-features = false }
 windmill-queue.workspace = true
 windmill-dep-map.workspace = true

backend/windmill-worker/src/agent_workers.rs

@@ -5,6 +5,180 @@ use windmill_common::{
 };
 use windmill_queue::{JobAndPerms, JobCompleted};
 
+// ---- agent end-to-end profiling (gated behind AGENT_PROF=1, zero-cost when off) ----
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Mutex, Once, OnceLock};
+use std::time::{Duration, Instant};
+
+static PROF: OnceLock<bool> = OnceLock::new();
+fn prof_on() -> bool {
+    *PROF.get_or_init(|| std::env::var("AGENT_PROF").as_deref() == Ok("1"))
+}
+
+static PULL_NS: AtomicU64 = AtomicU64::new(0); // total ns spent in pull HTTP round-trips
+static PULL_CNT: AtomicU64 = AtomicU64::new(0); // pull calls made
+static PULL_NONE: AtomicU64 = AtomicU64::new(0); // pulls that returned no job (would nap)
+static SEND_NS: AtomicU64 = AtomicU64::new(0); // total ns spent in send_result round-trips
+static SEND_CNT: AtomicU64 = AtomicU64::new(0); // jobs completed (= 1 send_result each)
+static LAST_PULL: Mutex<Option<Instant>> = Mutex::new(None); // start time of the previous pull
+static CYCLE_NS: AtomicU64 = AtomicU64::new(0); // sum of inter-pull intervals = independent cycle
+static CYCLE_CNT: AtomicU64 = AtomicU64::new(0);
+// Agent-side buffer health: the worker BLOCKS whenever the pull buffer is empty (must sync-fetch)
+// or the send buffer is full (must flush inline). For the ~1M j/s single-worker target both must
+// be ~0 — pull always has a job ready, push always drains async.
+static PULL_EMPTY: AtomicU64 = AtomicU64::new(0); // worker found PULL_BUF empty -> blocking sync refill
+static SEND_INLINE: AtomicU64 = AtomicU64::new(0); // worker hit full SEND_BUF -> blocking inline flush
+// Split the pull cost: POP = the fast path (buffer had a job) = mutex acquire + pop_front;
+// REFILL = the slow path (buffer empty) = the blocking sync HTTP fetch. pop+refill == PULL_NS.
+static PULL_POP_NS: AtomicU64 = AtomicU64::new(0);
+static PULL_POP_CNT: AtomicU64 = AtomicU64::new(0);
+static PULL_REFILL_NS: AtomicU64 = AtomicU64::new(0); // count == PULL_EMPTY
+// Directly measure the two phases previously left as a residual, so cycle = pull+exec+send+gap
+// EXACTLY (no inference). EXEC = pull_end->send_start (real job processing); GAP = send_end->
+// next pull_start (loop overhead + the agent being parked/descheduled / waiting between jobs).
+static EXEC_NS: AtomicU64 = AtomicU64::new(0);
+static EXEC_CNT: AtomicU64 = AtomicU64::new(0);
+static GAP_NS: AtomicU64 = AtomicU64::new(0);
+static GAP_CNT: AtomicU64 = AtomicU64::new(0);
+// Split the GAP into CPU vs parked: gap_cpu = thread CPU consumed during the gap (real loop work),
+// gap_parked = gap_wall − gap_cpu (suspended: I/O await OR descheduled waiting for a core). Only
+// valid when the task didn't migrate OS threads mid-gap (CLOCK_THREAD_CPUTIME_ID is per-thread) —
+// guarded by comparing ThreadId; migrated gaps are counted but excluded from the cpu split.
+static GAP_CPU_NS: AtomicU64 = AtomicU64::new(0);       // thread CPU during same-thread gaps
+static GAP_CPU_WALL_NS: AtomicU64 = AtomicU64::new(0);  // wall of those same-thread gaps (denominator)
+static GAP_CPU_CNT: AtomicU64 = AtomicU64::new(0);
+static GAP_MIG_CNT: AtomicU64 = AtomicU64::new(0);      // gaps where the task migrated threads
+static LAST_PULL_END: Mutex<Option<Instant>> = Mutex::new(None);
+static LAST_SEND_END: Mutex<Option<(Instant, u64, std::thread::ThreadId)>> = Mutex::new(None);
+// Empty-pull NAP: when a pull returns no job the worker tokio::sleeps (sleep_queue()*10 ms). For a
+// high-throughput batching agent that momentarily drains its buffer this is catastrophic — it
+// sleeps 500ms+ and the whole nap lands in the gap. Count them so the diagram shows it directly.
+static NAP_NS: AtomicU64 = AtomicU64::new(0);   // total wall time slept on empty pulls
+static NAP_CNT: AtomicU64 = AtomicU64::new(0);  // number of empty-pull naps
+static REPORTER: Once = Once::new();
+
+/// Record an empty-pull nap (called from the worker loop's Ok(None) arm). Counts only under AGENT_PROF.
+pub(crate) fn record_nap(elapsed: Duration) {
+    if prof_on() {
+        NAP_NS.fetch_add(elapsed.as_nanos() as u64, Ordering::Relaxed);
+        NAP_CNT.fetch_add(1, Ordering::Relaxed);
+    }
+}
+
+// Per-thread consumed CPU time (nanoseconds). Wall − this = time the thread was NOT running.
+fn thread_cpu_nanos() -> u64 {
+    let mut ts = libc::timespec { tv_sec: 0, tv_nsec: 0 };
+    unsafe { libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, &mut ts) };
+    (ts.tv_sec as u64) * 1_000_000_000 + (ts.tv_nsec as u64)
+}
+
+/// Spawn the 2s rolling reporter once. For a single synchronous agent (NUM_WORKERS=1)
+/// the loop is serial, so cycle = wall/jobs is exact; pull+send come from the timers;
+/// gap = cycle - pull - send = local per-job work (job_dir, perms, exec) + any nap.
+fn start_reporter() {
+    REPORTER.call_once(|| {
+        tokio::spawn(async {
+            let (mut p_ns, mut p_c, mut p_n, mut s_ns, mut s_c, mut cy_ns, mut cy_c) =
+                (0u64, 0u64, 0u64, 0u64, 0u64, 0u64, 0u64);
+            let (mut prev_pe, mut prev_si) = (0u64, 0u64);
+            // prev trackers so the per-job phase costs can be emitted WINDOWED (per-interval delta)
+            // instead of cumulative lifetime — stable on spiky runs.
+            let (mut prev_pop_ns, mut prev_ref_ns, mut prev_exec_ns, mut prev_exec_cnt) = (0u64, 0u64, 0u64, 0u64);
+            let (mut prev_gap_ns, mut prev_gap_cnt) = (0u64, 0u64);
+            let (mut prev_gcpu_ns, mut prev_gcpu_wall, mut prev_gcpu_cnt, mut prev_gmig) = (0u64, 0u64, 0u64, 0u64);
+            let mut prev_nap_ns = 0u64;
+            let start = Instant::now(); // for lifetime/cumulative aggregation
+            let mut t = Instant::now();
+            loop {
+                tokio::time::sleep(Duration::from_secs(2)).await;
+                let now = Instant::now();
+                let dt = now.duration_since(t).as_secs_f64();
+                let (np, cp, nn, ns, cs, ncy, ccy) = (
+                    PULL_NS.load(Ordering::Relaxed),
+                    PULL_CNT.load(Ordering::Relaxed),
+                    PULL_NONE.load(Ordering::Relaxed),
+                    SEND_NS.load(Ordering::Relaxed),
+                    SEND_CNT.load(Ordering::Relaxed),
+                    CYCLE_NS.load(Ordering::Relaxed),
+                    CYCLE_CNT.load(Ordering::Relaxed),
+                );
+                let d_np = np - p_ns;
+                let (d_cp, d_nn, d_ns, d_cs, d_ncy, d_ccy) =
+                    (cp - p_c, nn - p_n, ns - s_ns, cs - s_c, ncy - cy_ns, ccy - cy_c);
+                if d_cs > 0 {
+                    let thr = d_cs as f64 / dt; // independent #1: completed / wall
+                    let pull_ms = if d_cp > 0 { d_np as f64 / d_cp as f64 / 1e6 } else { 0.0 };
+                    let send_ms = d_ns as f64 / d_cs as f64 / 1e6;
+                    let cycle_thr = 1000.0 / thr; // cycle implied by throughput
+                    let cycle_meas = if d_ccy > 0 { d_ncy as f64 / d_ccy as f64 / 1e6 } else { 0.0 }; // independent #2: inter-pull interval
+                    let local_ms = cycle_meas - pull_ms - send_ms;
+                    eprintln!(
+                        "[AGENTPROF] thr={:.0} j/s | cycle: meas={:.3}ms thr-implied={:.3}ms (ratio {:.2}) | pull={:.3} send={:.3} local={:.3} ms | HTTP={:.0}% of cycle | empty_pulls/s={:.0}",
+                        thr, cycle_meas, cycle_thr, cycle_meas / cycle_thr,
+                        pull_ms, send_ms, local_ms,
+                        (pull_ms + send_ms) / cycle_meas * 100.0, d_nn as f64 / dt
+                    );
+                    // Cumulative / lifetime means accumulated over the whole run (not a
+                    // single 2s window): total ns / total count since pulling began.
+                    eprintln!(
+                        "[AGENTPROF-CUM] t={:.0}s served={} pull={:.3}ms send={:.3}ms cycle={:.3}ms (lifetime means)",
+                        start.elapsed().as_secs_f64(),
+                        cs,
+                        if cp > 0 { np as f64 / cp as f64 / 1e6 } else { 0.0 },
+                        if cs > 0 { ns as f64 / cs as f64 / 1e6 } else { 0.0 },
+                        if ccy > 0 { ncy as f64 / ccy as f64 / 1e6 } else { 0.0 }
+                    );
+                    // Agent-side buffer health. depth sampled now (try_lock; -1 = momentarily
+                    // locked). pull_empty/s = worker BLOCKED on a sync refill (pull buffer ran
+                    // dry); inline_flush/s = worker BLOCKED flushing a full send buffer. Both
+                    // must trend to 0 for pull/push to be free.
+                    let pe = PULL_EMPTY.load(Ordering::Relaxed);
+                    let si = SEND_INLINE.load(Ordering::Relaxed);
+                    let pd = pull_buf().try_lock().map(|b| b.len() as i64).unwrap_or(-1);
+                    let sd = send_buf().try_lock().map(|b| b.len() as i64).unwrap_or(-1);
+                    // WINDOWED per-job phase costs: per-interval delta (Δns / Δcount over THIS 2s
+                    // window), NOT cumulative lifetime — so a spiky run's dips don't smear across the
+                    // whole run, and the plot's steady-window aggregation reconciles with throughput.
+                    let r = Ordering::Relaxed;
+                    let pop_ns = PULL_POP_NS.load(r); let ref_ns = PULL_REFILL_NS.load(r);
+                    let exec_ns = EXEC_NS.load(r); let exec_cnt = EXEC_CNT.load(r);
+                    let gap_ns = GAP_NS.load(r); let gap_cnt = GAP_CNT.load(r);
+                    let gcpu_ns = GAP_CPU_NS.load(r); let gcpu_wall = GAP_CPU_WALL_NS.load(r);
+                    let gcpu_cnt = GAP_CPU_CNT.load(r); let gmig = GAP_MIG_CNT.load(r);
+                    let d_cpm = d_cp.max(1) as f64;                                   // ΔPULL_CNT this window
+                    let d_exec_c = (exec_cnt - prev_exec_cnt).max(1) as f64;
+                    let d_gap_c = (gap_cnt - prev_gap_cnt).max(1) as f64;
+                    let d_gcc = (gcpu_cnt - prev_gcpu_cnt).max(1) as f64;
+                    let d_gmig = (gmig - prev_gmig) as f64;
+                    let pop_us = (pop_ns - prev_pop_ns) as f64 / d_cpm / 1000.0;       // amortized over jobs
+                    let ref_us = (ref_ns - prev_ref_ns) as f64 / d_cpm / 1000.0;
+                    let exec_us = (exec_ns - prev_exec_ns) as f64 / d_exec_c / 1000.0;
+                    let gap_us = (gap_ns - prev_gap_ns) as f64 / d_gap_c / 1000.0;
+                    let gap_cpu_us = (gcpu_ns - prev_gcpu_ns) as f64 / d_gcc / 1000.0;
+                    let gap_parked_us = ((gcpu_wall - prev_gcpu_wall).saturating_sub(gcpu_ns - prev_gcpu_ns) as f64 / d_gcc / 1000.0).max(0.0);
+                    let gap_mig_pct = d_gmig / (d_gcc + d_gmig).max(1.0) * 100.0;
+                    // NAP: the empty-pull sleep (part of the gap). nap_us = windowed per-job; total
+                    // count + total ms are cumulative so the diagram can show "N naps = X ms slept".
+                    let nap_ns = NAP_NS.load(r); let nap_cnt = NAP_CNT.load(r);
+                    let nap_us = (nap_ns - prev_nap_ns) as f64 / d_gap_c / 1000.0;
+                    eprintln!(
+                        "[AGENTBUF] pull_depth={} send_depth={} pull_empty/s={:.0} inline_flush/s={:.0} pull_pop_us={:.2} pull_refill_us={:.2} exec_us={:.2} gap_us={:.2} gap_cpu_us={:.2} gap_parked_us={:.2} gap_mig_pct={:.0} nap_us={:.2} nap_cnt={} nap_total_ms={:.0} pull_empty_total={} send_inline_total={}",
+                        pd, sd, (pe - prev_pe) as f64 / dt, (si - prev_si) as f64 / dt, pop_us, ref_us, exec_us, gap_us, gap_cpu_us, gap_parked_us, gap_mig_pct, nap_us, nap_cnt, nap_ns as f64 / 1e6, pe, si
+                    );
+                    prev_pe = pe; prev_si = si;
+                    prev_pop_ns = pop_ns; prev_ref_ns = ref_ns;
+                    prev_exec_ns = exec_ns; prev_exec_cnt = exec_cnt;
+                    prev_gap_ns = gap_ns; prev_gap_cnt = gap_cnt;
+                    prev_gcpu_ns = gcpu_ns; prev_gcpu_wall = gcpu_wall; prev_gcpu_cnt = gcpu_cnt; prev_gmig = gmig;
+                    prev_nap_ns = nap_ns;
+                }
+                t = now;
+                p_ns = np; p_c = cp; p_n = nn; s_ns = ns; s_c = cs; cy_ns = ncy; cy_c = ccy;
+            }
+        });
+    });
+}
+
 pub async fn queue_init_job(client: &HttpClient, content: &str) -> anyhow::Result<Uuid> {
     client
         .post(
@@ -27,27 +201,314 @@ pub async fn queue_periodic_job(client: &HttpClient, content: &str) -> anyhow::R
         .and_then(|x: String| Uuid::parse_str(&x).map_err(|e| anyhow::anyhow!(e)))
 }
 
+// ---- EXPERIMENT 3: agent-side batch pull + batch complete (gated behind AGENT_CLIENT_BATCH>1) ----
+// The agent pulls N jobs per HTTP and buffers completions, flushing N-at-a-time (or on a
+// timeout), so the ~2 HTTP round-trips per job collapse to ~2 per N jobs — amortizing the wire.
+// For a single agent worker (NUM_WORKERS=1) the process-wide buffers below are single-consumer.
+static CLIENT_BATCH: OnceLock<usize> = OnceLock::new();
+fn client_batch() -> usize {
+    *CLIENT_BATCH.get_or_init(|| {
+        std::env::var("AGENT_CLIENT_BATCH").ok().and_then(|v| v.parse().ok()).unwrap_or(0)
+    })
+}
+static CLIENT_BATCH_MS: OnceLock<u64> = OnceLock::new();
+fn client_batch_ms() -> u64 {
+    *CLIENT_BATCH_MS.get_or_init(|| {
+        std::env::var("AGENT_CLIENT_BATCH_MS").ok().and_then(|v| v.parse().ok()).unwrap_or(20)
+    })
+}
+static PULL_BUF: OnceLock<tokio::sync::Mutex<std::collections::VecDeque<JobAndPerms>>> = OnceLock::new();
+fn pull_buf() -> &'static tokio::sync::Mutex<std::collections::VecDeque<JobAndPerms>> {
+    PULL_BUF.get_or_init(|| tokio::sync::Mutex::new(std::collections::VecDeque::new()))
+}
+static SEND_BUF: OnceLock<tokio::sync::Mutex<Vec<JobCompleted>>> = OnceLock::new();
+fn send_buf() -> &'static tokio::sync::Mutex<Vec<JobCompleted>> {
+    SEND_BUF.get_or_init(|| tokio::sync::Mutex::new(Vec::new()))
+}
+static FLUSHER: Once = Once::new();
+static REFILL_INFLIGHT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
+
+// Eager (background) refill of the local pull buffer (gated by AGENT_CLIENT_EAGER=1): top it up
+// when it drops below AGENT_CLIENT_REFILL_PCT% of the batch, so pop() never waits on a fetch.
+static CLIENT_EAGER: OnceLock<bool> = OnceLock::new();
+fn client_eager() -> bool {
+    *CLIENT_EAGER.get_or_init(|| {
+        std::env::var("AGENT_CLIENT_EAGER").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)
+    })
+}
+static CLIENT_REFILL_PCT: OnceLock<usize> = OnceLock::new();
+#[allow(dead_code)]
+fn client_refill_pct() -> usize {
+    *CLIENT_REFILL_PCT.get_or_init(|| {
+        std::env::var("AGENT_CLIENT_REFILL_PCT").ok().and_then(|v| v.parse().ok()).filter(|n| *n <= 100).unwrap_or(50)
+    })
+}
+// Keep this many BATCHES of runway buffered (refill whenever below). Each refill takes ~5ms to
+// land, so the buffer must be deep enough that the worker can't drain it in that time.
+static CLIENT_PREFETCH: OnceLock<usize> = OnceLock::new();
+fn client_prefetch_batches() -> usize {
+    *CLIENT_PREFETCH.get_or_init(|| {
+        std::env::var("AGENT_CLIENT_PREFETCH_BATCHES").ok().and_then(|v| v.parse().ok()).filter(|n| *n >= 1).unwrap_or(2)
+    })
+}
+// Allow this many concurrent refills in flight (one isn't enough when refill latency > drain time).
+static CLIENT_MAX_INFLIGHT: OnceLock<usize> = OnceLock::new();
+fn client_max_inflight() -> usize {
+    *CLIENT_MAX_INFLIGHT.get_or_init(|| {
+        std::env::var("AGENT_CLIENT_MAX_INFLIGHT").ok().and_then(|v| v.parse().ok()).filter(|n| *n >= 1).unwrap_or(3)
+    })
+}
+
+// Record the per-job cycle (interval between pull_job calls) into the AGENT_PROF counters.
+fn prof_cycle() {
+    use std::sync::atomic::Ordering::Relaxed;
+    let now = Instant::now();
+    let mut lp = LAST_PULL.lock().unwrap();
+    if let Some(prev) = *lp {
+        CYCLE_NS.fetch_add(now.duration_since(prev).as_nanos() as u64, Relaxed);
+        CYCLE_CNT.fetch_add(1, Relaxed);
+    }
+    *lp = Some(now);
+}
+
+// Keep the pull buffer DEEP: refill whenever it's below `prefetch_batches` worth of jobs, and
+// allow up to `max_inflight` concurrent refills — so the buffer never drains to 0 even though
+// each refill takes ~5ms to land (otherwise pop() falls into the blocking sync fetch).
+fn maybe_spawn_refill(client: &HttpClient) {
+    use std::sync::atomic::Ordering::{AcqRel, Relaxed};
+    let n = client_batch();
+    let target = (n * client_prefetch_batches()).max(1);
+    let max_inflight = client_max_inflight();
+    let low = match pull_buf().try_lock() {
+        Ok(b) => b.len() < target,
+        Err(_) => false,
+    };
+    if !low {
+        return;
+    }
+    // claim an in-flight slot (cap at max_inflight)
+    let cur = REFILL_INFLIGHT.load(Relaxed);
+    if cur >= max_inflight
+        || REFILL_INFLIGHT.compare_exchange(cur, cur + 1, AcqRel, Relaxed).is_err()
+    {
+        return;
+    }
+    let client = clone_client(client);
+    tokio::spawn(async move {
+        let jobs: Vec<JobAndPerms> = client
+            .post("/api/agent_workers/pull_jobs_batch", None, &n)
+            .await
+            .unwrap_or_default();
+        if !jobs.is_empty() {
+            pull_buf().lock().await.extend(jobs);
+        }
+        REFILL_INFLIGHT.fetch_sub(1, Relaxed);
+    });
+}
+
+fn clone_client(client: &HttpClient) -> HttpClient {
+    HttpClient { client: client.client.clone(), base_internal_url: client.base_internal_url.clone() }
+}
+
+async fn flush_completes(client: &HttpClient, batch: Vec<JobCompleted>) -> anyhow::Result<()> {
+    if batch.is_empty() {
+        return Ok(());
+    }
+    let w_id = batch[0].job.workspace_id.clone();
+    let _: String = client
+        .post(&format!("/api/w/{}/agent_workers/send_results_batch", w_id), None, &batch)
+        .await?;
+    Ok(())
+}
+
+// Background flush so buffered completions (e.g. the tail) don't sit unsent past the timeout.
+fn ensure_flusher(client: &HttpClient) {
+    FLUSHER.call_once(|| {
+        let client = clone_client(client);
+        let ms = client_batch_ms();
+        tokio::spawn(async move {
+            loop {
+                tokio::time::sleep(Duration::from_millis(ms)).await;
+                let batch = {
+                    let mut b = send_buf().lock().await;
+                    if b.is_empty() {
+                        continue;
+                    }
+                    std::mem::take(&mut *b)
+                };
+                let _ = flush_completes(&client, batch).await;
+            }
+        });
+    });
+}
+
+async fn pull_job_batched(client: &HttpClient) -> anyhow::Result<Option<JobAndPerms>> {
+    use std::sync::atomic::Ordering::Relaxed;
+    let prof = prof_on();
+    if prof {
+        start_reporter();
+        prof_cycle();
+        // GAP: between the previous send returning and this pull starting (loop + suspension).
+        if let Some((se, se_cpu, se_tid)) = *LAST_SEND_END.lock().unwrap() {
+            let now = Instant::now();
+            let wall = now.duration_since(se).as_nanos() as u64;
+            GAP_NS.fetch_add(wall, Relaxed);
+            GAP_CNT.fetch_add(1, Relaxed);
+            // CPU split is only valid if the worker task stayed on the same OS thread for the gap.
+            if std::thread::current().id() == se_tid {
+                GAP_CPU_NS.fetch_add(thread_cpu_nanos().saturating_sub(se_cpu), Relaxed);
+                GAP_CPU_WALL_NS.fetch_add(wall, Relaxed);
+                GAP_CPU_CNT.fetch_add(1, Relaxed);
+            } else {
+                GAP_MIG_CNT.fetch_add(1, Relaxed);
+            }
+        }
+    }
+    let t = Instant::now();
+    {
+        let mut b = pull_buf().lock().await;
+        if let Some(j) = b.pop_front() {
+            drop(b);
+            if prof {
+                let e = t.elapsed().as_nanos() as u64;
+                PULL_NS.fetch_add(e, Relaxed);
+                PULL_CNT.fetch_add(1, Relaxed);
+                PULL_POP_NS.fetch_add(e, Relaxed); // fast path = mutex acquire + pop_front
+                PULL_POP_CNT.fetch_add(1, Relaxed);
+                *LAST_PULL_END.lock().unwrap() = Some(Instant::now()); // exec starts after this
+            }
+            if client_eager() {
+                maybe_spawn_refill(client);
+            }
+            return Ok(Some(j));
+        }
+    }
+    // buffer empty (reactive path, or eager fell behind): fetch a batch synchronously.
+    // This is the BLOCKING refill — the worker stalls here until the HTTP fetch returns.
+    if prof {
+        PULL_EMPTY.fetch_add(1, Relaxed);
+    }
+    let n = client_batch();
+    let jobs: Vec<JobAndPerms> = client
+        .post("/api/agent_workers/pull_jobs_batch", None, &n)
+        .await?;
+    if prof {
+        let e = t.elapsed().as_nanos() as u64;
+        PULL_NS.fetch_add(e, Relaxed);
+        PULL_CNT.fetch_add(1, Relaxed);
+        PULL_REFILL_NS.fetch_add(e, Relaxed); // slow path = blocking sync HTTP fetch
+        if jobs.is_empty() {
+            PULL_NONE.fetch_add(1, Relaxed);
+        }
+    }
+    if jobs.is_empty() {
+        return Ok(None);
+    }
+    let mut b = pull_buf().lock().await;
+    b.extend(jobs);
+    let j = b.pop_front();
+    drop(b);
+    if prof && j.is_some() {
+        *LAST_PULL_END.lock().unwrap() = Some(Instant::now()); // exec starts after this
+    }
+    if client_eager() {
+        maybe_spawn_refill(client);
+    }
+    Ok(j)
+}
+
+async fn send_result_batched(client: &HttpClient, jc: JobCompleted) -> anyhow::Result<String> {
+    use std::sync::atomic::Ordering::Relaxed;
+    let prof = prof_on();
+    ensure_flusher(client);
+    if prof {
+        // EXEC: pull_end -> send_start = the real job processing the worker did this cycle.
+        if let Some(pe) = *LAST_PULL_END.lock().unwrap() {
+            EXEC_NS.fetch_add(Instant::now().duration_since(pe).as_nanos() as u64, Relaxed);
+            EXEC_CNT.fetch_add(1, Relaxed);
+        }
+    }
+    let t = Instant::now();
+    let n = client_batch();
+    let flush = {
+        let mut b = send_buf().lock().await;
+        b.push(jc);
+        if b.len() >= n {
+            Some(std::mem::take(&mut *b))
+        } else {
+            None
+        }
+    };
+    let did_flush = flush.is_some();
+    if let Some(batch) = flush {
+        // BLOCKING inline flush — the worker stalls here sending the batch over HTTP.
+        flush_completes(client, batch).await?;
+    }
+    if prof {
+        if did_flush {
+            SEND_INLINE.fetch_add(1, Relaxed);
+        }
+        SEND_NS.fetch_add(t.elapsed().as_nanos() as u64, Relaxed);
+        SEND_CNT.fetch_add(1, Relaxed);
+        // stamp wall + thread-CPU + thread-id so the next pull can split the gap into cpu vs parked.
+        *LAST_SEND_END.lock().unwrap() =
+            Some((Instant::now(), thread_cpu_nanos(), std::thread::current().id()));
+    }
+    Ok("ok".to_string())
+}
+
 pub async fn pull_job(
     client: &HttpClient,
     headers: Option<HeaderMap>,
     body: Option<bool>,
 ) -> anyhow::Result<Option<JobAndPerms>> {
-    client
+    if client_batch() > 1 {
+        return pull_job_batched(client).await;
+    }
+    if !prof_on() {
+        return client
+            .post("/api/agent_workers/pull_job", headers, &body)
+            .await;
+    }
+    start_reporter();
+    {
+        // independent cycle: time between consecutive pull starts (serial agent => true cycle)
+        let now = Instant::now();
+        let mut lp = LAST_PULL.lock().unwrap();
+        if let Some(prev) = *lp {
+            CYCLE_NS.fetch_add(now.duration_since(prev).as_nanos() as u64, Ordering::Relaxed);
+            CYCLE_CNT.fetch_add(1, Ordering::Relaxed);
+        }
+        *lp = Some(now);
+    }
+    let t = Instant::now();
+    let r: anyhow::Result<Option<JobAndPerms>> = client
         .post("/api/agent_workers/pull_job", headers, &body)
-        .await
+        .await;
+    PULL_NS.fetch_add(t.elapsed().as_nanos() as u64, Ordering::Relaxed);
+    PULL_CNT.fetch_add(1, Ordering::Relaxed);
+    if matches!(&r, Ok(None)) {
+        PULL_NONE.fetch_add(1, Ordering::Relaxed);
+    }
+    r
 }
 
 pub async fn send_result(client: &HttpClient, jc: JobCompleted) -> anyhow::Result<String> {
-    client
-        .post(
-            &format!(
-                "/api/w/{}/agent_workers/send_result/{}",
-                jc.job.workspace_id, jc.job.id
-            ),
-            None,
-            &jc,
-        )
-        .await
+    if client_batch() > 1 {
+        return send_result_batched(client, jc).await;
+    }
+    let url = format!(
+        "/api/w/{}/agent_workers/send_result/{}",
+        jc.job.workspace_id, jc.job.id
+    );
+    if !prof_on() {
+        return client.post(&url, None, &jc).await;
+    }
+    let t = Instant::now();
+    let r: anyhow::Result<String> = client.post(&url, None, &jc).await;
+    SEND_NS.fetch_add(t.elapsed().as_nanos() as u64, Ordering::Relaxed);
+    SEND_CNT.fetch_add(1, Ordering::Relaxed);
+    r
 }
 
 #[allow(dead_code)]

backend/windmill-worker/src/worker.rs

@@ -1841,6 +1841,7 @@ fn start_interactive_worker_shell(
                         }
                         _ => Duration::from_millis(sleep_queue() * 10),
                     };
+                    let _nap_start = Instant::now();
                     tokio::select! {
                         _ = tokio::time::sleep(nap_time) => {
                         }
@@ -1848,6 +1849,7 @@ fn start_interactive_worker_shell(
                             break;
                         }
                     }
+                    crate::agent_workers::record_nap(_nap_start.elapsed());
                 }
 
                 Err(err) => {
@@ -2734,10 +2736,14 @@ pub async fn run_worker(
                         }
                     }
 
-                    Connection::Http(client) => crate::agent_workers::pull_job(&client, None, None)
-                        .await
-                        .map_err(|e| error::Error::InternalErr(e.to_string()))
-                        .map(|x| x.map(|y| NextJob::Http(y))),
+                    Connection::Http(client) => {
+                        let _ag_pt = Instant::now();
+                        let _ag_r = crate::agent_workers::pull_job(&client, None, None).await;
+                        tracing::info!(target: "agent_timing", "[agent-timing] pull={}ms got_job={}", _ag_pt.elapsed().as_millis(), _ag_r.as_ref().map(|x| x.is_some()).unwrap_or(false));
+                        _ag_r
+                            .map_err(|e| error::Error::InternalErr(e.to_string()))
+                            .map(|x| x.map(|y| NextJob::Http(y)))
+                    }
                 }
             }
         };
@@ -2887,6 +2893,7 @@ pub async fn run_worker(
                 }
 
                 if matches!(job.kind, JobKind::Noop) {
+                    let _ag_noop = Instant::now();
                     add_time!(bench, "send job completed START");
                     job_completed_tx
                         .send_job(
@@ -2911,6 +2918,7 @@ pub async fn run_worker(
                         .await
                         .expect("send job completed END");
                     add_time!(bench, "sent job completed");
+                    tracing::info!(target: "agent_timing", "[agent-timing] noop send_result={}ms", _ag_noop.elapsed().as_millis());
                 } else {
                     if !was_suspended_job {
                         add_outstanding_wait_time(&conn, &job, *OUTSTANDING_WAIT_TIME_THRESHOLD_MS);
@@ -2972,6 +2980,7 @@ pub async fn run_worker(
                       // fields macro we can make a job id that only appears when
                       // the job is defined?
 
+                    let _ag_t0 = Instant::now();
                     let job_dir = create_job_dir(&worker_dir, job.id).await;
 
                     let same_worker = job.same_worker;
@@ -3053,6 +3062,7 @@ pub async fn run_worker(
                     let span = create_span_with_name(&arc_job, &worker_name, Some(hostname), "job");
                     let log_ctx = log_context_for_job(&arc_job, &worker_name, Some(hostname));
 
+                    let _ag_t2 = Instant::now();
                     let job_result = windmill_common::log_context::with_log_context(
                         log_ctx,
                         async {
@@ -3085,6 +3095,7 @@ pub async fn run_worker(
                         .instrument(span),
                     )
                     .await;
+                    tracing::info!(target: "agent_timing", "[agent-timing] total={}ms handle={}ms", _ag_t0.elapsed().as_millis(), _ag_t2.elapsed().as_millis());
 
                     match job_result {
                         Ok(ref outcome) if !outcome.is_success() && is_init_script => {

benchmarks/.gitignore

@@ -0,0 +1,7 @@
+# Sim run outputs land in dated subfolders here; never commit them.
+results/
+
+# Legacy single-file outputs (pre-dated-folders) if any leak.
+sim_results_*.json
+sim_results_*.md
+sim_results_*.svg

benchmarks/README.md

@@ -69,3 +69,39 @@ Generates SVG graphs from `*_benchmark.json` data files.
 ## CI
 
 The GitHub Actions workflow (`.github/workflows/benchmark.yml`) runs hourly with 1/4/8 worker configurations plus WAC benchmarks. Results are committed to the `benchmarks` branch.
+
+## Cluster benchmarks — sim mode
+
+Provisioning + measuring throughput on a real multi-node Kubernetes cluster
+(minikube + helm + per-pod cgroup sampling + PG analysis) is a separate
+workflow on top of `main.ts`. See [`sim/README.md`](sim/README.md) for:
+
+- bringing up the cluster (`wm_sim up`)
+- firing a phased or flood workload from `workloads/`
+- the consolidated dashboard with throughput, queue depth, per-node CPU +
+  oversaturation, PG latency / conn counts, OOM events, restart events
+- the JSONL poller pipeline + reliability fixes (sampler host-log, rollout-
+  complete readiness, procs_running oversaturation)
+
+Quick path:
+
+```sh
+# Provision cluster + deploy Windmill (foreground; Ctrl-C tears down).
+# Prints `[helm] API reachable at http://127.0.0.1:<port>` — note the port.
+# Port-forward stays alive as a child of wm_sim, so keep this terminal open.
+wm_sim up \
+  --topology    sim/topologies/k8s-4node.json \
+  --helm        ../windmill-helm-charts/charts/windmill \
+  --helm-values sim/values/smoke.yaml \
+  --helm-values sim/values/local.yaml
+
+# In another shell — fire bench against the wm_sim-managed port:
+deno run -A main.ts \
+  --host http://127.0.0.1:<port> \
+  --token <admin-token> \
+  --workload-config workloads/io_150ms_flood.json \
+  --minikube-profile wm-sim-k8s-4node \
+  --wait-ready 60
+
+# Report appears at reports/<timestamp>/dashboard.svg
+```

benchmarks/agent_worker_scaling.md

@@ -0,0 +1,93 @@
+## The core idea: a two-stage prefetch pipeline so nobody ever waits
+
+Jobs flow through four stages:
+
+```
+Postgres  →  [server-side buffer]  →  HTTP  →  [agent-side buffer]  →  worker loop
+```
+
+At **each** stage a background producer keeps the **next** consumer's buffer full *ahead of
+demand*. In steady state every consumer pops a job that's already sitting in front of it and
+never blocks on the stage behind it: the worker never waits on HTTP, and the server never waits
+on Postgres. Everything below is in service of making that true, and then dealing with what's
+left once it is.
+
+**Baseline (stock).** None of this exists yet: the worker is a serial request/response loop —
+pull one job (HTTP to the app), run it, report it (HTTP to the app), repeat — blocking on each
+call. For a noop that's **2 HTTP round-trips per job and ~0 compute**: **2,539 j/s**, ~75% of the
+cycle in the pull RPC, ~25% in the completion RPC. The whole problem is "stop paying a round-trip
+per job," then "stop being limited by the database behind the app."
+
+> All single-agent numbers below are from one environment: a single box, native, with app +
+> bundled Postgres + agent all on `localhost`, prefill-then-drain, tables truncated per run.
+
+## 1. Make the unit of transfer a batch, not a job (both sides)
+
+- **Agent side:** ask for **N jobs in one request** and drain them from a local buffer; accumulate
+  completions and send them back **N in one request** (flush triggered by either a size threshold
+  or a short max-latency timer, so completion latency stays bounded at low rates).
+- **Server side:** answer a pull with a single `LIMIT N … FOR UPDATE SKIP LOCKED` (one dequeue
+  query yields N jobs), and accept N completions in one call.
+
+This amortizes the fixed round-trip cost across N jobs (2 round-trips per *N* jobs) and collapses N
+separate dequeue queries into one. → first cut ~23k j/s; ~62–64k once the bench method was clean.
+
+## 2. Server-side prefetch — the server never queries Postgres on the request path
+
+Even batched, a pull still triggers a fresh dequeue query *while the agent waits*. Fix: a **single
+background "puller" task per job-tag-set** runs the dequeue query in a loop and fills an in-memory
+buffer of ready jobs. An agent's pull is served **from that buffer**, not from a live query, and
+multiple agents sharing the same tags are **coalesced onto the one puller** (so 25 agents don't
+become 25 concurrent `SKIP LOCKED` queries fighting each other). The Postgres work moves off the
+request's critical path and into the background refill.
+
+## 3. Agent-side prefetch — the worker never waits on HTTP
+
+Symmetrically on the agent: a **background refiller** pulls batches from the server ahead of demand
+into the agent's local buffer, and the worker loop pops from that buffer with **no network call**.
+The HTTP latency is now **overlapped with job execution** instead of serialized in front of it.
+→ ~64k → ~89k.
+
+## 4. Keep both buffers deep so prefetch actually stays ahead
+
+A background producer only helps if its buffer never empties — otherwise the consumer falls back to
+a blocking fetch and you've lost the benefit. Two rules, applied to the prefetchers:
+
+- **Refill early, not on empty** — trigger the next refill when the buffer falls below a *high*
+  low-watermark (several batches deep), not when it hits zero.
+- **Allow several refills in flight at once** — a refill takes a few ms to land; if only one can be
+  outstanding, a fast consumer drains the buffer before it arrives. Concurrent refills keep
+  inventory above `drain-rate × refill-latency` with margin.
+
+→ ~89k → ~127k; the agent essentially never blocks (popping + running a job is sub-µs to low-µs).
+**At this point the single-agent path is solved.**
+
+## 5. Where it stands: single agent in good shape, scaling up is the next focus
+
+The two-stage prefetch pipeline takes a single agent from **2,539 j/s → ~125k+** — at that point it
+essentially never blocks, and neither the agent nor the server sits on the network or Postgres on the
+critical path. It can still be optimized further (the completion/write path isn't prefetched yet, and
+there's per-job cost left to squeeze), but we haven't gone there — the more valuable direction is to
+scale up, running many agents together, and that's where we turned next.
+
+That's where the hard part is. Multiple agents don't yet scale cleanly — aggregate throughput is
+unstable and doesn't multiply the way one well-fed agent predicts — and we've spent real effort trying
+to pin down why **without success**. It's a stubborn problem that hasn't yielded to the instrumentation
+we've thrown at it; the leads we have are unverified, and it's not established to be Postgres, possibly:
+
+- An idle-style backoff sleep that fires on a transient empty pull, which a prefetching consumer hits
+  under contention.
+- Periodic exclusive-lock stalls from vacuum reclaiming the empty tail of the queue tables.
+
+## Current state / next
+
+1. **Single agent: in good shape** — fully prefetched on both sides, ~50× over stock, no blocking on
+   the critical path; can still be improved, but with diminishing returns.
+2. **Scaling to many agents: the open problem** — aggregate throughput doesn't multiply cleanly, and
+   we've investigated without pinning the cause (likely not Postgres). This is where the priority is.
+3. **Completion (write) path** is the one part of the agent not yet given the same async/prefetch
+   treatment as the read path — a known candidate to harden, independent of the collapse.
+4. **Everything is experimental** and gated off by default; nothing changes stock behavior. Any of
+   it becoming a default needs the collapse understood first, plus a fairness/latency review of
+   batching.
+

benchmarks/benchmark_suite.ts

@@ -6,11 +6,41 @@ import { main as runBenchmark } from "./benchmark_oneoff.ts";
 
 import { VERSION, loadJsonConfig } from "./lib.ts";
 
-type Config = {
+// Two accepted on-disk shapes:
+//   - array of benchmark entries (the original shape; runs unchanged here).
+//   - object with optional `topology` + `benchmarks`. If `topology` is set the
+//     suite needs the sim layer, so we redirect; if it's an array-equivalent
+//     object (no topology) we accept it and run the same as the array form.
+// `normaliseConfig` returns the array form so the run loop stays unchanged.
+type BenchmarkEntry = {
   kind: string;
   jobs: number;
   noSave?: boolean;
-}[];
+};
+type Config = BenchmarkEntry[];
+type SuiteFile = Config | {
+  topology?: string;
+  benchmarks: BenchmarkEntry[];
+};
+
+function normaliseConfig(raw: SuiteFile, configPath: string): Config {
+  if (Array.isArray(raw)) return raw;
+  if (raw && typeof raw === "object" && Array.isArray(raw.benchmarks)) {
+    if (raw.topology) {
+      console.error(
+        `[benchmark_suite] Suite ${configPath} declares a topology ` +
+        `(${raw.topology}). benchmark_suite.ts only runs against an already-` +
+        `running Windmill; for provisioned topologies use sim/sim.ts instead.`,
+      );
+      Deno.exit(2);
+    }
+    return raw.benchmarks;
+  }
+  throw new Error(
+    `Suite ${configPath}: expected either an array or an object with ` +
+    `a "benchmarks" array.`,
+  );
+}
 
 async function warmUp(
   host: string,
@@ -55,7 +85,8 @@ async function main({
   }
 
   try {
-    const config = await loadJsonConfig<Config>(configPath);
+    const raw = await loadJsonConfig<SuiteFile>(configPath);
+    const config = normaliseConfig(raw, configPath);
     for (const benchmark of config) {
       try {
         console.log(

benchmarks/graph.ts

@@ -5,7 +5,12 @@ type DataPoint = {
   value: number;
   date: Date;
 };
-export function drawGraph(data: DataPoint[], title: string) {
+export function drawGraph(
+  data: DataPoint[],
+  title: string,
+  yLabel: string = "[jobs/s]",
+  yMax?: number,
+) {
   const context = {
     jsdom: new JSDOM(""),
   };
@@ -14,7 +19,11 @@ export function drawGraph(data: DataPoint[], title: string) {
 
   const body = d3.select(document).select("body");
 
-  const width = 400;
+  // Time-series chart — 900px wide so the many-points lines (throughput,
+  // CPU util, memory, PG memory etc.) have room to read individual peaks.
+  // drawBars/drawDonut below stay at 600 since bar charts don't benefit
+  // from extra width.
+  const width = 900;
   const height = 200;
 
   const marginTop = 20;
@@ -45,26 +54,43 @@ export function drawGraph(data: DataPoint[], title: string) {
         return d.date;
       })
     )
-    .nice()
+    // No .nice() on the time scale — it rounds the domain to "nice" tick
+    // boundaries (e.g. extends domain to next 30s mark), pushing the first
+    // few seconds of data off the left edge so phase 1 (idle baseline) and
+    // m02's CPU curve appear to "start from the middle". Exact data extent
+    // keeps the line starting at the actual first sample.
     .range([0, width]);
 
-  const xAxis = d3.axisBottom(x).ticks(5);
+  // Relative-time x-axis: labels show seconds since the earliest data point
+  // in this chart ("0s, 30s, 60s ..."). Wall-clock HH:MM:SS labels were
+  // confusing — they alternated between "03:45" and ":30" formats and made
+  // same-time comparisons across panels hard. Relative units anchor every
+  // panel at 0s.
+  const xDom = x.domain() as [Date, Date];
+  const xOriginMs = xRelativeOriginMs ?? xDom[0].getTime();
+  const xAxis = d3.axisBottom(x).ticks(5).tickFormat((d) => {
+    const sec = Math.round(((d as Date).getTime() - xOriginMs) / 1000);
+    return `${sec}s`;
+  });
 
   svg
     .append("g")
     .attr("transform", "translate(0," + height + ")")
     .call(xAxis);
 
-  // Add Y axis
+  // Add Y axis. When yMax is set, also `.clamp(true)` so out-of-range data
+  // values (e.g. a CPU spike to 250% on a chart capped at 150%) render at the
+  // axis boundary instead of escaping the chart area entirely.
   const y = d3
     .scaleLinear()
     .domain([
       0,
-      d3.max(data, function (d: DataPoint) {
+      yMax !== undefined ? yMax : d3.max(data, function (d: DataPoint) {
         return +d.value;
       }) * 1.5,
     ])
     .range([height, 0])
+    .clamp(yMax !== undefined)
     .nice();
   svg.append("g").call(d3.axisLeft(y));
 
@@ -75,7 +101,7 @@ export function drawGraph(data: DataPoint[], title: string) {
     .attr("transform", "rotate(-90)")
     .attr("y", -marginLeft + 20)
     .attr("x", -height / 2)
-    .text("[jobs/s]");
+    .text(yLabel);
 
   svg
     .append("text")
@@ -107,11 +133,56 @@ export function drawGraph(data: DataPoint[], title: string) {
   return body.node().innerHTML;
 }
 
-interface DataPointMulti extends DataPoint {
+export interface DataPointMulti extends DataPoint {
   kind: string;
 }
 
-export function drawGraphMulti(data: DataPointMulti[], title: string) {
+export function drawGraphMulti(
+  data: DataPointMulti[],
+  title: string,
+  yLabel: string = "[jobs/s]",
+  // When set, fixes the chart Y axis at [0, yMax] instead of auto-scaling to
+  // 1.5x the max sample. Use for percent-based panels (Node CPU at 150 caps
+  // noise spikes that would otherwise compress the rest of the chart).
+  yMax?: number,
+  // Optional vertical dashed lines at given timestamps. Used by the phased
+  // bench to mark phase boundaries (warmup → peak → cooldown, etc.) on every
+  // time-series panel so you can attribute throughput/CPU shifts to phases.
+  // Accepts a plain Date[] (lines + labels drawn) or `{ dates, hideLabels }`
+  // — the renderer suppresses the P1>P2 labels on charts other than the
+  // "primary" one so the dashboard doesn't end up with 27 boundary labels
+  // (3 per chart × 9 charts) which reads as duplication side-by-side.
+  verticalLines?: Date[] | { dates: Date[]; hideLabels?: boolean },
+  // Optional horizontal reference lines at given y-values (in chart units).
+  // Drawn as labeled dashed grey lines — used to show e.g. the 100% per-VM
+  // ceiling on the Node CPU chart so saturation is obvious at a glance.
+  horizontalLines?: { y: number; label: string }[],
+  // Optional translucent shaded rectangles spanning [from, to] on the x-axis.
+  // Used to mark the "push window" on Throughput / Queue depth / Node CPU so
+  // it's visually obvious which range is "bench actively pushing" vs
+  // "drain only" without staring at vertical phase boundaries.
+  shadedZones?: { from: Date; to: Date; fill?: string; label?: string }[],
+  // Optional substring match: if a series' `kind` contains this token, render
+  // a translucent colored area fill under its curve so it stands out as the
+  // "tinted" series. Used to flag the PG-hosting node in Node CPU.
+  highlightKindToken?: string,
+  // Optional ordered list of area fills drawn BEFORE the lines (back to front
+  // in this order, so the first entry is the backmost). Each entry tints any
+  // series whose `kind` exactly equals `kind`. Used for the util group where
+  // we want oversaturation as the backmost orange band with CPU util on top.
+  areaFills?: { kind: string; color: string; opacity?: number }[],
+  // Optional per-kind line color override. Without it, lines use d3's
+  // schemeCategory10 in the order kinds were inserted into sumstat — which
+  // means "oversaturation" wouldn't naturally come out orange. Used by the
+  // util group to pin each series' line color to its area fill color.
+  lineColorOverrides?: Record<string, string>,
+  // Optional shared origin (epoch ms) for the relative-time x-axis. When
+  // unset, each chart uses its own earliest data point — fine in isolation
+  // but inconsistent across panels because pollers / pushers / CPU samples
+  // start at slightly different moments. Pass meta.json's `bench_start_ms`
+  // here so 0s on every panel is the same wall-clock moment.
+  xRelativeOriginMs?: number,
+) {
   const context = {
     jsdom: new JSDOM(""),
   };
@@ -120,7 +191,11 @@ export function drawGraphMulti(data: DataPointMulti[], title: string) {
 
   const body = d3.select(document).select("body");
 
-  const width = 400;
+  // Multi-series time chart — 900px wide for the same reason as drawGraph above.
+  // Throughput, Queue depth, Node CPU/memory, PG memory, Failed jobs, Workers per
+  // node all flow through this; bumping width here is the user-requested
+  // "make the lots-of-points charts 50% wider".
+  const width = 900;
   const height = 200;
 
   const marginTop = 20;
@@ -151,26 +226,43 @@ export function drawGraphMulti(data: DataPointMulti[], title: string) {
         return d.date;
       })
     )
-    .nice()
+    // No .nice() on the time scale — it rounds the domain to "nice" tick
+    // boundaries (e.g. extends domain to next 30s mark), pushing the first
+    // few seconds of data off the left edge so phase 1 (idle baseline) and
+    // m02's CPU curve appear to "start from the middle". Exact data extent
+    // keeps the line starting at the actual first sample.
     .range([0, width]);
 
-  const xAxis = d3.axisBottom(x).ticks(5);
+  // Relative-time x-axis: labels show seconds since the earliest data point
+  // in this chart ("0s, 30s, 60s ..."). Wall-clock HH:MM:SS labels were
+  // confusing — they alternated between "03:45" and ":30" formats and made
+  // same-time comparisons across panels hard. Relative units anchor every
+  // panel at 0s.
+  const xDom = x.domain() as [Date, Date];
+  const xOriginMs = xRelativeOriginMs ?? xDom[0].getTime();
+  const xAxis = d3.axisBottom(x).ticks(5).tickFormat((d) => {
+    const sec = Math.round(((d as Date).getTime() - xOriginMs) / 1000);
+    return `${sec}s`;
+  });
 
   svg
     .append("g")
     .attr("transform", "translate(0," + height + ")")
     .call(xAxis);
 
-  // Add Y axis
+  // Add Y axis. When yMax is set, also `.clamp(true)` so out-of-range data
+  // values (e.g. a CPU spike to 250% on a chart capped at 150%) render at the
+  // axis boundary instead of escaping the chart area entirely.
   const y = d3
     .scaleLinear()
     .domain([
       0,
-      d3.max(data, function (d: DataPoint) {
+      yMax !== undefined ? yMax : d3.max(data, function (d: DataPoint) {
         return +d.value;
       }) * 1.5,
     ])
     .range([height, 0])
+    .clamp(yMax !== undefined)
     .nice();
   svg.append("g").call(d3.axisLeft(y));
 
@@ -181,7 +273,7 @@ export function drawGraphMulti(data: DataPointMulti[], title: string) {
     .attr("transform", "rotate(-90)")
     .attr("y", -marginLeft + 20)
     .attr("x", -height / 2)
-    .text("[jobs/s]");
+    .text(yLabel);
 
   svg
     .append("text")
@@ -212,6 +304,84 @@ export function drawGraphMulti(data: DataPointMulti[], title: string) {
       "#999999",
     ]);
 
+  // Shaded zones (e.g. push window) — drawn BEFORE the lines so lines
+  // paint on top. Translucent fill so the chart underneath stays
+  // readable. Optional label hugs the top-left of the zone.
+  if (shadedZones && shadedZones.length > 0) {
+    svg
+      .selectAll(".shaded-zone")
+      .data(shadedZones)
+      .enter()
+      .append("rect")
+      .attr("class", "shaded-zone")
+      .attr("x", (d: { from: Date }) => x(d.from))
+      .attr("width", (d: { from: Date; to: Date }) => Math.max(0, x(d.to) - x(d.from)))
+      .attr("y", 0)
+      .attr("height", height)
+      .attr("fill", (d: { fill?: string }) => d.fill ?? "#1f77b4")
+      .attr("fill-opacity", 0.08)
+      .attr("stroke", "none");
+    svg
+      .selectAll(".shaded-zone-label")
+      .data(shadedZones.filter((z: { label?: string }) => z.label))
+      .enter()
+      .append("text")
+      .attr("class", "shaded-zone-label")
+      .attr("x", (d: { from: Date }) => x(d.from) + 4)
+      .attr("y", 22)
+      .attr("text-anchor", "start")
+      .style("font-size", "10px")
+      .style("font-weight", "600")
+      .style("fill", "#3a5d8a")
+      .style("font-family", "monospace")
+      .text((d: { label?: string }) => d.label!);
+  }
+
+  // Explicit area-fills list — drawn in order so caller controls back-to-front
+  // z-order. Each is the area under the matching series' curve.
+  if (areaFills && areaFills.length > 0) {
+    const entries = Array.from(sumstat as any) as any[];
+    for (const fill of areaFills) {
+      const match = entries.find((e: any) => String(e[0]) === fill.kind);
+      if (!match) continue;
+      svg.append("path")
+        .attr("class", "area-fill")
+        .attr("fill", fill.color)
+        .attr("fill-opacity", fill.opacity ?? 0.25)
+        .attr("stroke", "none")
+        .attr("d", d3
+          .area()
+          .x((p: any) => x(p.date))
+          .y0(height)
+          .y1((p: any) => y(p.value))(match[1]));
+    }
+  }
+
+  // Tinted area fill under the highlighted series — drawn BEFORE the line so
+  // line stays crisp on top. Translucent so the underlying x-axis and
+  // overlapping series remain visible. sumstat is a d3.InternMap (Map-like);
+  // convert to entries Array to filter.
+  if (highlightKindToken) {
+    const highlightSeries = Array.from(sumstat as any).filter(
+      (d: any) => String(d[0]).includes(highlightKindToken),
+    );
+    svg
+      .selectAll("path.tint")
+      .data(highlightSeries)
+      .join("path")
+      .attr("class", "tint")
+      .attr("fill", function (d: any) { return color(d[0]); })
+      .attr("fill-opacity", 0.18)
+      .attr("stroke", "none")
+      .attr("d", (d: any) => {
+        return d3
+          .area()
+          .x((p: any) => x(p.date))
+          .y0(height)
+          .y1((p: any) => y(p.value))(d[1]);
+      });
+  }
+
   // Add the line
   svg
     .selectAll("path.line")
@@ -220,7 +390,7 @@ export function drawGraphMulti(data: DataPointMulti[], title: string) {
     .attr("class", "line")
     .attr("fill", "none")
     .attr("stroke", function (d) {
-      return color(d[0]);
+      return (lineColorOverrides && lineColorOverrides[String(d[0])]) ?? color(d[0]);
     })
     .attr("stroke-width", 1.5)
     .attr("d", (d) => {
@@ -265,6 +435,313 @@ export function drawGraphMulti(data: DataPointMulti[], title: string) {
     .attr("text-anchor", "left")
     .style("alignment-baseline", "middle");
 
+  // Phase-boundary dashed verticals. Drawn last so they overlay the data lines.
+  // Each line gets a tiny "P{n}>P{n+1}" label at the top so the migration is
+  // visible at a glance — useful for phased benches where throughput / CPU
+  // shifts between phases.
+  // Normalize verticalLines into { dates, hideLabels }.
+  const vlNormalized = Array.isArray(verticalLines)
+    ? { dates: verticalLines, hideLabels: false }
+    : verticalLines;
+  if (vlNormalized && vlNormalized.dates.length > 0) {
+    svg
+      .selectAll(".phase-boundary")
+      .data(vlNormalized.dates)
+      .enter()
+      .append("line")
+      .attr("class", "phase-boundary")
+      .attr("x1", (d: Date) => x(d))
+      .attr("x2", (d: Date) => x(d))
+      .attr("y1", 0)
+      .attr("y2", height)
+      .attr("stroke", "#888")
+      .attr("stroke-width", 1)
+      .attr("stroke-dasharray", "3 3");
+    if (!vlNormalized.hideLabels) {
+      svg
+        .selectAll(".phase-boundary-label")
+        .data(vlNormalized.dates)
+        .enter()
+        .append("text")
+        .attr("class", "phase-boundary-label")
+        .attr("x", (d: Date) => x(d) + 2)
+        .attr("y", 10)
+        .attr("text-anchor", "start")
+        .style("font-size", "9px")
+        .style("fill", "#666")
+        .style("font-family", "monospace")
+        .text((_: Date, i: number) => `P${i + 1}>P${i + 2}`);
+    }
+  }
+
+  // Horizontal reference lines (e.g. CPU 100% ceiling) — labeled dashed
+  // grey, with the label hugging the right edge so it doesn't collide
+  // with the data lines.
+  if (horizontalLines && horizontalLines.length > 0) {
+    svg
+      .selectAll(".h-ref")
+      .data(horizontalLines)
+      .enter()
+      .append("line")
+      .attr("class", "h-ref")
+      .attr("x1", 0)
+      .attr("x2", width)
+      .attr("y1", (d: { y: number; label: string }) => y(d.y))
+      .attr("y2", (d: { y: number; label: string }) => y(d.y))
+      .attr("stroke", "#999")
+      .attr("stroke-width", 1)
+      .attr("stroke-dasharray", "4 4");
+    svg
+      .selectAll(".h-ref-label")
+      .data(horizontalLines)
+      .enter()
+      .append("text")
+      .attr("class", "h-ref-label")
+      .attr("x", width - 4)
+      .attr("y", (d: { y: number; label: string }) => y(d.y) - 4)
+      .attr("text-anchor", "end")
+      .style("font-size", "9px")
+      .style("fill", "#666")
+      .style("font-family", "monospace")
+      .text((d: { y: number; label: string }) => d.label);
+  }
+
+  return body.node().innerHTML;
+}
+
+// Bar chart for one-dimensional distributions. Each bin is { label, count }.
+// Works for both continuous (histogram bins, label = midpoint as string) and
+// categorical (label = category name). Summary stats are rendered as text in
+// the top-right.
+export function drawBars(
+  bins: { label: string; count: number; color?: string }[],
+  title: string,
+  xLabel: string,
+  stats?: { min: number; max: number; avg: number },
+  opts: { rotateDeg?: number; fontSize?: string } = {},
+): string {
+  const context = { jsdom: new JSDOM("") };
+  const { document } = context.jsdom.window;
+  const body = d3.select(document).select("body");
+
+  const width = 600;
+  const height = 200;
+  const marginTop = 30;
+  const marginRight = 30;
+  // 100px bottom margin so long rotated tick labels (e.g. OOM panel pod
+  // names like "windmill-postgresql-0 (12.5G) (cgroup)") aren't cropped.
+  const marginBottom = 100;
+  const marginLeft = 60;
+
+  let svg = body
+    .append("svg")
+    .attr("xmlns", "http://www.w3.org/2000/svg")
+    .attr("width", width + marginLeft + marginRight)
+    .attr("height", height + marginTop + marginBottom);
+
+  svg
+    .append("rect")
+    .attr("width", "100%")
+    .attr("height", "100%")
+    .attr("fill", "white");
+
+  svg = svg
+    .append("g")
+    .attr("transform", "translate(" + marginLeft + "," + marginTop + ")");
+
+  // Title
+  svg
+    .append("text")
+    .attr("x", width / 2)
+    .attr("y", -10)
+    .attr("text-anchor", "middle")
+    .style("font-size", "14px")
+    .style("font-weight", "600")
+    .text(title);
+
+  const x = d3.scaleBand()
+    .domain(bins.map((b) => b.label))
+    .range([0, width])
+    .padding(0.1);
+
+  const y = d3.scaleLinear()
+    .domain([0, d3.max(bins, (b) => b.count) || 1])
+    .nice()
+    .range([height, 0]);
+
+  // X axis — label EVERY bar so the user can read what each one is.
+  // rotateDeg / fontSize are caller-tunable: -45° + 10px is fine for
+  // ~12 bars; -90° + smaller font lets a chart fit ~100 labels (per-pod
+  // memory bars on a ~80-worker cluster).
+  const rotateDeg = opts.rotateDeg ?? -45;
+  const fontSize = opts.fontSize ?? (bins.length > 12 ? "9px" : "10px");
+  svg.append("g")
+    .attr("transform", "translate(0," + height + ")")
+    .call(
+      d3.axisBottom(x).tickValues(bins.map((b) => b.label)),
+    )
+    .selectAll("text")
+    .attr("transform", `rotate(${rotateDeg})`)
+    .style("text-anchor", "end")
+    .style("font-size", fontSize);
+
+  svg.append("text")
+    .attr("x", width / 2)
+    .attr("y", height + 90)   // ↓ below rotated tick labels (was 45)
+    .attr("text-anchor", "middle")
+    .style("font-size", "11px")
+    .text(xLabel);
+
+  svg.append("g").call(d3.axisLeft(y));
+
+  svg.selectAll(".bar")
+    .data(bins)
+    .join("rect")
+    .attr("class", "bar")
+    .attr("x", (b) => x(b.label) || 0)
+    .attr("y", (b) => y(b.count))
+    .attr("width", x.bandwidth())
+    .attr("height", (b) => height - y(b.count))
+    .attr("fill", (b: { color?: string }) => b.color ?? "#377eb8");
+
+  // Dashed vertical divider lines between contiguous groups of same-color
+  // bars. With the Pod inventory chart (one node = one color), this draws a
+  // separator between m02's run of bars and m03's, etc., making it
+  // visually obvious which bars belong to which node.
+  const groupBoundaries: number[] = [];
+  for (let i = 1; i < bins.length; i++) {
+    if (bins[i].color !== bins[i - 1].color) groupBoundaries.push(i);
+  }
+  if (groupBoundaries.length > 0) {
+    const bandStep = x.step();
+    svg.selectAll(".group-divider")
+      .data(groupBoundaries)
+      .enter()
+      .append("line")
+      .attr("class", "group-divider")
+      .attr("x1", (i: number) => (x(bins[i].label) || 0) - bandStep * (1 - x.bandwidth() / bandStep) / 2)
+      .attr("x2", (i: number) => (x(bins[i].label) || 0) - bandStep * (1 - x.bandwidth() / bandStep) / 2)
+      .attr("y1", 0)
+      .attr("y2", height + 5)
+      .attr("stroke", "#aaa")
+      .attr("stroke-width", 1)
+      .attr("stroke-dasharray", "3 3");
+  }
+
+  // Stats text top-right.
+  if (stats) {
+    const fmt = (n: number) =>
+      n >= 1000 ? n.toFixed(0) : n >= 10 ? n.toFixed(1) : n.toFixed(2);
+    const lines = [
+      `min: ${fmt(stats.min)}`,
+      `avg: ${fmt(stats.avg)}`,
+      `max: ${fmt(stats.max)}`,
+    ];
+    svg.selectAll(".stat")
+      .data(lines)
+      .enter()
+      .append("text")
+      .attr("class", "stat")
+      .attr("x", width)
+      .attr("y", (_, i) => 12 + i * 14)
+      .attr("text-anchor", "end")
+      .style("font-size", "11px")
+      .style("font-family", "monospace")
+      .style("fill", "#555")
+      .text((d) => d);
+  }
+
+  return body.node().innerHTML;
+}
+
+// Donut chart for categorical distributions. Slices labeled with category +
+// percentage; legend on the right.
+export function drawDonut(
+  slices: { label: string; count: number }[],
+  title: string,
+): string {
+  const context = { jsdom: new JSDOM("") };
+  const { document } = context.jsdom.window;
+  const body = d3.select(document).select("body");
+
+  const width = 600;
+  const height = 220;
+  const marginTop = 30;
+  const marginBottom = 10;
+  const marginLeft = 20;
+  const marginRight = 20;
+
+  const total = slices.reduce((a, b) => a + b.count, 0) || 1;
+
+  let svg = body
+    .append("svg")
+    .attr("xmlns", "http://www.w3.org/2000/svg")
+    .attr("width", width + marginLeft + marginRight)
+    .attr("height", height + marginTop + marginBottom);
+
+  svg
+    .append("rect")
+    .attr("width", "100%")
+    .attr("height", "100%")
+    .attr("fill", "white");
+
+  // Title
+  svg
+    .append("text")
+    .attr("x", (width + marginLeft + marginRight) / 2)
+    .attr("y", marginTop - 8)
+    .attr("text-anchor", "middle")
+    .style("font-size", "14px")
+    .style("font-weight", "600")
+    .text(title);
+
+  const r = Math.min(width, height) / 2 - 10;
+  const cx = marginLeft + r + 10;
+  const cy = marginTop + height / 2;
+
+  const palette = [
+    "#377eb8", "#e41a1c", "#4daf4a", "#984ea3",
+    "#ff7f00", "#a65628", "#f781bf", "#999999",
+  ];
+  const color = (i: number) => palette[i % palette.length];
+
+  const pie = d3.pie<{ label: string; count: number }>().value((d) => d.count).sort(null);
+  const arc = d3.arc<d3.PieArcDatum<{ label: string; count: number }>>()
+    .innerRadius(r * 0.55)
+    .outerRadius(r);
+
+  const arcs = pie(slices);
+  const g = svg.append("g").attr("transform", `translate(${cx},${cy})`);
+
+  g.selectAll("path")
+    .data(arcs)
+    .join("path")
+    .attr("d", arc as never)
+    .attr("fill", (_, i) => color(i))
+    .attr("stroke", "white")
+    .attr("stroke-width", 2);
+
+  // Legend on the right
+  const legendX = cx + r + 30;
+  const legendY = marginTop + 20;
+  const sw = 12;
+  slices.forEach((s, i) => {
+    const pct = ((s.count / total) * 100).toFixed(1);
+    svg.append("rect")
+      .attr("x", legendX)
+      .attr("y", legendY + i * 22)
+      .attr("width", sw)
+      .attr("height", sw)
+      .attr("fill", color(i));
+    svg.append("text")
+      .attr("x", legendX + sw + 6)
+      .attr("y", legendY + i * 22 + sw - 1)
+      .style("font-size", "12px")
+      .style("font-family", "monospace")
+      .style("fill", "#333")
+      .text(`${s.label}  ${pct}%`);
+  });
+
   return body.node().innerHTML;
 }
 

benchmarks/lib.ts

@@ -91,6 +91,37 @@ export async function createBenchScript(
     scriptContent =
       'export function main(){ return Deno.env.get("WM_JOB_ID"); }';
     language = "deno";
+  } else if (scriptPattern === "deno_sleep_150") {
+    // Sleep-based deno script — every job awaits 150ms. Gives the bench a
+    // realistic non-zero per-job duration so headroom is visible (queue can
+    // actually drain, throughput reflects scheduling, not script cold-start).
+    scriptContent =
+      'export async function main(){ await new Promise(r => setTimeout(r, 150)); return Deno.env.get("WM_JOB_ID"); }';
+    language = "deno";
+  } else if (scriptPattern === "random") {
+    // Synthetic parametric workload: each job carries its own (ram_mb,
+    // duration_ms, mode) tuple sampled by the bench client from a workload
+    // config (benchmarks/workloads/*.json). RAM is allocated AND touched
+    // every 4KB so the kernel actually maps pages (Uint8Array(N) alone is
+    // lazy). `sleep` mode is IO-bound; `busy` mode is CPU-bound.
+    scriptContent =
+      'export async function main(ram_mb: number, duration_ms: number, mode: "sleep" | "busy") {\n' +
+      '  const buf = new Uint8Array(ram_mb * 1024 * 1024);\n' +
+      '  for (let i = 0; i < buf.length; i += 4096) buf[i] = (Math.random() * 256) | 0;\n' +
+      '  if (mode === "sleep") {\n' +
+      '    await new Promise(r => setTimeout(r, duration_ms));\n' +
+      '  } else {\n' +
+      '    const end = performance.now() + duration_ms;\n' +
+      '    while (performance.now() < end) { /* busy */ }\n' +
+      '  }\n' +
+      '  return { ram_mb, duration_ms, mode };\n' +
+      '}';
+    language = "deno";
+    schemaProperties = {
+      ram_mb: { type: "number", description: "MB of RAM to allocate + touch" },
+      duration_ms: { type: "number", description: "Sleep / busy-loop duration" },
+      mode: { type: "string", enum: ["sleep", "busy"], description: "IO-bound or CPU-bound" },
+    };
   } else if (scriptPattern === "nativets") {
     scriptContent =
       '//native\nexport async function main(){ return (await fetch(BASE_URL + "/api/version")).text() }';

benchmarks/main.ts

@@ -8,6 +8,20 @@ import { Action } from "./action.ts";
 import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/upgrade_command.ts";
 import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
 import { VERSION, createBenchScript } from "./lib.ts";
+import { MinikubeProvisioner } from "./sim/k8s_provisioner.ts";
+import { capturePgLog, enableVerbosePgLogging } from "./sim/pg_logging.ts";
+import { runPgbadger } from "./sim/pgbadger.ts";
+import { collectCpuSamples } from "./sim/cpu_sampler_k8s.ts";
+import { capturePodInventory } from "./sim/pod_inventory.ts";
+import { startPodTimeline, type PodTimelinePoller } from "./sim/pod_timeline.ts";
+import { checkReadiness, waitForReady } from "./sim/readiness.ts";
+import { startOomPoller, type OomPoller } from "./sim/oom_poller.ts";
+import { startPgLatencyPoller, type PgLatencyPoller } from "./sim/pg_latency_poller.ts";
+import { startPgConnPoller, type PgConnPoller } from "./sim/pg_conn_poller.ts";
+import { startNodeLoadPoller, type NodeLoadPoller } from "./sim/node_load_poller.ts";
+import { collectOomEvents } from "./sim/oom_events.ts";
+import { collectFailedJobs } from "./sim/failed_jobs.ts";
+import { renderReport } from "./sim/render_report.ts";
 export {
   DenoLandProvider,
   UpgradeCommand,
@@ -26,6 +40,7 @@ export async function main({
   host,
   workers: num_workers,
   seconds,
+  timeout,
   email,
   password,
   token,
@@ -44,10 +59,14 @@ export async function main({
   continous,
   max,
   custom,
+  minikubeProfile,
+  workloadConfig: workloadConfigPath,
+  waitReady,
 }: {
   host: string;
   workers: number;
   seconds: number;
+  timeout?: number;
   email?: string;
   password?: string;
   token?: string;
@@ -66,6 +85,17 @@ export async function main({
   continous?: boolean;
   max?: number;
   custom?: string;
+  // When set, bench captures cluster-side measurements (per-node CPU/mem,
+  // pod inventory, PG log) via kubectl scoped to the bench window, and
+  // composes the full dashboard. Without it, only Throughput + Queue depth.
+  minikubeProfile?: string;
+  // Path to a JSON workload config — required for `--script-pattern random`.
+  // Defines per-parameter distributions (ram_mb, duration_ms, mode).
+  workloadConfig?: string;
+  // Seconds to poll cluster readiness before firing. 0 = single check, fail
+  // immediately on any unmet condition. >0 = poll every 5s until ready or
+  // timeout. See sim/readiness.ts.
+  waitReady?: number;
 }) {
   windmill.setClient("", host);
   const versionResp = await fetch(`${host}/api/version`);
@@ -159,6 +189,45 @@ export async function main({
 
   const per_worker_throughput = maximumThroughput / num_workers;
   const max_per_worker = max ? max / num_workers : undefined;
+  // For `--script-pattern random`: load the workload config so each worker
+  // can sample (ram_mb, duration_ms, mode) per job push from the configured
+  // distributions. Other patterns ignore this.
+  let workloadConfig: unknown = undefined;
+  if (scriptPattern === "random") {
+    if (!workloadConfigPath) {
+      throw new Error("--script-pattern random requires --workload-config <path>");
+    }
+    workloadConfig = JSON.parse(await Deno.readTextFile(workloadConfigPath));
+    console.log(`Loaded workload config from ${workloadConfigPath}`);
+  }
+
+  // Phased workload: take ownership of --seconds and --timeout so every phase
+  // gets to run. --seconds becomes the sum of phase durations; --timeout gets
+  // bumped to phase-sum + 30s drain budget if the user passed a smaller value.
+  // Without this, a short --timeout would silently cut off later phases.
+  const phasedCfg = workloadConfig as { phases?: Array<{ duration_s: number; pushers: number; name?: string }> } | undefined;
+  if (phasedCfg?.phases && phasedCfg.phases.length > 0) {
+    const phaseSumS = phasedCfg.phases.reduce((a, p) => a + p.duration_s, 0);
+    const maxPushers = phasedCfg.phases.reduce((a, p) => Math.max(a, p.pushers), 0);
+    console.log(`[phased] ${phasedCfg.phases.length} phase(s), total push window = ${phaseSumS}s`);
+    for (const p of phasedCfg.phases) {
+      console.log(`  - ${p.name ?? "(unnamed)"}: ${p.duration_s}s @ ${p.pushers} pushers`);
+    }
+    if (num_workers < maxPushers) {
+      console.warn(`[phased] --workers ${num_workers} is below max phase pushers (${maxPushers}); some phases will be undersaturated. Bumping --workers to ${maxPushers}.`);
+      num_workers = maxPushers;
+    }
+    if (seconds !== phaseSumS) {
+      console.log(`[phased] overriding --seconds ${seconds} -> ${phaseSumS} (phase total)`);
+      seconds = phaseSumS;
+    }
+    const minTimeoutS = phaseSumS + 30; // 30s drain budget for queue to empty
+    if (timeout !== undefined && timeout < minTimeoutS) {
+      console.warn(`[phased] --timeout ${timeout}s would cut off later phases (need >= ${minTimeoutS}s for phases + drain). Bumping to ${minTimeoutS}s.`);
+      timeout = minTimeoutS;
+    }
+  }
+
   const shared_config = {
     server: host,
     token: final_token,
@@ -170,12 +239,13 @@ export async function main({
     scriptPattern,
     continous,
     custom: custom_content,
+    workloadConfig,
   };
 
   if (
     !useFlows &&
     (scriptPattern === undefined ||
-      ["deno", "python", "go", "bash", "bun", "dedicated"].includes(
+      ["deno", "deno_sleep_150", "random", "python", "go", "bash", "bun", "dedicated"].includes(
         scriptPattern
       ))
   ) {
@@ -209,6 +279,12 @@ export async function main({
 
   console.log("Initial queue length:", initial_queue_length);
 
+  // Throughput-over-time samples captured at each updateState tick. Stored as
+  // raw cumulative counts; the report renderer computes the windowed rate.
+  // Window-from-start cumulative numbers get diluted by warmup → useless for
+  // dashboard purposes; the renderer does a rolling-window diff instead.
+  const throughputSamples: { ts: number; processed: number; sum: number; queue: number }[] = [];
+
   const updateState = setInterval(async () => {
     const elapsed = start ? Math.ceil((Date.now() - start) / 1000) : 0;
     const sum = jobsSent.reduce((a, b) => a + b, 0);
@@ -224,6 +300,16 @@ export async function main({
         continue;
       }
     }
+    // Only sample after the start clock is armed so warmup ticks aren't
+    // pulled into the windowed series.
+    if (start !== undefined) {
+      throughputSamples.push({
+        ts: Date.now(),
+        processed: sum - queue_length,
+        sum,
+        queue: queue_length,
+      });
+    }
     await Deno.stdout.write(
       enc(
         `elapsed: ${elapsed}/${seconds} | jobs sent: ${JSON.stringify(
@@ -237,6 +323,84 @@ export async function main({
     );
   }, 100);
 
+  // Re-apply pgBadger PG settings before each bench. They live in PGDATA's
+  // postgresql.auto.conf, which a fresh PVC (helm upgrade flipping persistence,
+  // chart-bundled PG image upgrade) silently wipes. Settings are SIGHUP-able,
+  // so this is a no-cost no-op when they're already set.
+  if (minikubeProfile) {
+    try {
+      const provForPg = new MinikubeProvisioner({ profile: minikubeProfile });
+      await enableVerbosePgLogging(provForPg);
+    } catch (e) {
+      console.warn(`[bench] could not enable verbose PG logging: ${(e as Error).message}`);
+    }
+  }
+
+  // Pre-flight readiness check — MUST run BEFORE worker.postMessage() below
+  // (which triggers pushers). Earlier versions ran this after postMessage and
+  // the workers pushed thousands of jobs into PG during the wait window,
+  // making `queue=0` impossible to ever observe → the check timed out
+  // every time even on a perfectly healthy cluster.
+  if (minikubeProfile) {
+    const provForReady = new MinikubeProvisioner({ profile: minikubeProfile });
+    const waitSec = (waitReady ?? 0) as number;
+    const report = waitSec > 0
+      ? await waitForReady(provForReady, { timeoutMs: waitSec * 1000 })
+      : await checkReadiness(provForReady);
+    console.log(`[readiness] samplers=${report.details.samplers_running}/${report.details.samplers_total}, workers=${report.details.workers_ready}/${report.details.workers_total}, pg=${report.details.pg_phase}(responsive=${report.details.pg_responsive}), queue=${report.details.queue_depth}, tox=${report.details.toxiproxy_ready}, app=${report.details.app_ready}`);
+    if (!report.ready) {
+      console.error("[readiness] cluster NOT ready — refusing to start bench:");
+      for (const issue of report.issues) console.error(`  - ${issue}`);
+      console.error("(re-run with --wait-ready <seconds> to poll until ready, or fix the issues and retry.)");
+      Deno.exit(2);
+    }
+    console.log("[readiness] cluster ready, proceeding");
+  }
+
+  // Truncate the sampler host-log file on every worker node so this bench
+  // gets a clean file. The sampler dual-writes to /var/log/wm-sim-cpu-sampler/
+  // sampler.tsv via hostPath; without rotation it grows unbounded across runs
+  // (~1.7 GB/day per node under load) and stale data from prior benches mixes
+  // with the current run's collection. The collector still filters by
+  // ts_ns ≥ bench_start_ms, so failure to truncate is recoverable — best-
+  // effort only.
+  if (minikubeProfile) {
+    try {
+      const provForRotate = new MinikubeProvisioner({ profile: minikubeProfile });
+      const nodesRes = await provForRotate.kubectl([
+        "get", "nodes",
+        "-o", "jsonpath={range .items[*]}{.metadata.name}{\"|\"}{.status.addresses[?(@.type==\"InternalIP\")].address}{\"\\n\"}{end}",
+      ]);
+      if (nodesRes.code === 0) {
+        const home = Deno.env.get("HOME") ?? "";
+        await Promise.all(
+          nodesRes.stdout.split("\n").filter(Boolean).map(async (line) => {
+            const [name, ip] = line.split("|");
+            if (!name || !ip) return;
+            const key = `${home}/.minikube/machines/${name.trim()}/id_rsa`;
+            const cmd = new Deno.Command("ssh", {
+              args: [
+                "-o", "StrictHostKeyChecking=no",
+                "-o", "UserKnownHostsFile=/dev/null",
+                "-o", "ConnectTimeout=5",
+                "-o", "LogLevel=ERROR",
+                "-i", key,
+                `docker@${ip.trim()}`,
+                "sudo truncate -s 0 /var/log/wm-sim-cpu-sampler/sampler.tsv 2>/dev/null || true",
+              ],
+              stdout: "null",
+              stderr: "null",
+            });
+            await cmd.output();
+          }),
+        );
+        console.log("[bench] sampler host-log truncated on all nodes");
+      }
+    } catch (e) {
+      console.warn(`[bench] sampler host-log truncate failed (continuing): ${(e as Error).message}`);
+    }
+  }
+
   workers.forEach((worker, i) => {
     worker.addEventListener("message", (evt: MessageEvent<any>) => {
       if (evt.data.type === "jobs_sent") {
@@ -245,8 +409,101 @@ export async function main({
     });
     worker.postMessage({ ...shared_config, i });
   });
+
+  // outDir is created BEFORE the bench begins so the in-bench pollers (pod
+  // timeline, OOM live) can stream JSONL into it as the run progresses.
+  // Previously this was created post-bench and the pollers crashed with a
+  // TDZ "Cannot access 'outDir' before initialization" — silently failing,
+  // which is why pod_timeline.jsonl was missing on every report and the
+  // Workers-per-node panel fell back to the (wrong) cgroup-derived count.
+  const benchStartIso = new Date(start ?? Date.now()).toISOString();
+  const isoStamp = new Date().toISOString().replace(/[:.]/g, "-").replace(/Z$/, "");
+  const outDir = `reports/${isoStamp}`;
+  await Deno.mkdir(outDir, { recursive: true });
+  console.log(`[bench] reports dir: ${outDir}`);
+
+  // Pin bench-context metadata into the report so the dashboard header can
+  // surface it. Includes paths so the report dir is self-describing.
+  try {
+    const meta = {
+      topology: minikubeProfile ?? "n/a",
+      host,
+      workspace,
+      bench_cmd: ["main.ts", ...Deno.args].join(" "),
+      workload_path: workloadConfigPath ?? null,
+      script_pattern: scriptPattern ?? null,
+      ts_iso: new Date().toISOString(),
+      // Shared relative-time origin. Every chart in render_report.ts uses this
+      // as "0s" on its x-axis, so identical x positions across panels mean the
+      // same wall-clock moment. Without this, each panel picked its own origin
+      // from its earliest sample and panels drifted by tens of seconds (poller
+      // startup vs first-push vs first-CPU-sample timings).
+      bench_start_ms: Date.parse(benchStartIso),
+    };
+    await Deno.writeTextFile(`${outDir}/meta.json`, JSON.stringify(meta, null, 2));
+  } catch (e) {
+    console.warn(`[bench] meta.json write failed: ${(e as Error).message}`);
+  }
+
+  // Start the kubectl-based pod-timeline poller (1Hz) so the renderer can plot
+  // a truly-Ready workers-per-node line — the cgroup sampler's view conflates
+  // "container slice exists" with "worker is healthy", so a CrashLoopBackOff
+  // cluster looks fully staffed when it isn't.
+  let podTimelinePoller: PodTimelinePoller | undefined;
+  let oomPoller: OomPoller | undefined;
+  let pgLatencyPoller: PgLatencyPoller | undefined;
+  let pgConnPoller: PgConnPoller | undefined;
+  let nodeLoadPoller: NodeLoadPoller | undefined;
+  if (minikubeProfile) {
+    try {
+      const provForPoll = new MinikubeProvisioner({ profile: minikubeProfile });
+      podTimelinePoller = startPodTimeline(provForPoll, `${outDir}/pod_timeline.jsonl`);
+    } catch (e) {
+      console.warn(`[bench] pod timeline poller failed to start: ${(e as Error).message}`);
+    }
+    // OOM poller — catches container OOMKills as they happen so the end-of-
+    // bench scan (which only sees the MOST RECENT lastState.terminated) can
+    // merge in mid-bench kills that have already been overwritten by the
+    // post-restart Running state.
+    try {
+      const provForOom = new MinikubeProvisioner({ profile: minikubeProfile });
+      oomPoller = startOomPoller(provForOom, `${outDir}/oom_events_live.jsonl`);
+    } catch (e) {
+      console.warn(`[bench] OOM poller failed to start: ${(e as Error).message}`);
+    }
+    // PG latency poller — every 1s runs `SELECT 1` against PG, records wall-
+    // clock latency. Surfaces PG-responsiveness over time on a dedicated
+    // panel: flat baseline when healthy, spikes when connection storm/
+    // contention hits.
+    try {
+      const provForPg = new MinikubeProvisioner({ profile: minikubeProfile });
+      pgLatencyPoller = startPgLatencyPoller(provForPg, `${outDir}/pg_latency.jsonl`);
+    } catch (e) {
+      console.warn(`[bench] PG latency poller failed to start: ${(e as Error).message}`);
+    }
+    try {
+      const provForConn = new MinikubeProvisioner({ profile: minikubeProfile });
+      pgConnPoller = startPgConnPoller(provForConn, `${outDir}/pg_connections.jsonl`);
+    } catch (e) {
+      console.warn(`[bench] PG conn poller failed to start: ${(e as Error).message}`);
+    }
+    try {
+      const provForLoad = new MinikubeProvisioner({ profile: minikubeProfile });
+      nodeLoadPoller = startNodeLoadPoller(provForLoad, `${outDir}/node_load.jsonl`);
+    } catch (e) {
+      console.warn(`[bench] node-load poller failed to start: ${(e as Error).message}`);
+    }
+  }
+
   start = Date.now();
 
+  // Hard wall-clock deadline for the whole bench (push + drain). When --timeout
+  // is set, every blocking await below caps at the remaining time so the bench
+  // always terminates predictably, regardless of how long the queue takes to
+  // drain. Whatever was sampled by then is what the report uses.
+  const deadlineMs = timeout !== undefined ? start + timeout * 1000 : Infinity;
+  const remainingS = () => Math.max(0, (deadlineMs - Date.now()) / 1000);
+
   console.log("collecting samples...");
   if (continous) {
     while (true) {
@@ -254,15 +511,16 @@ export async function main({
     }
   }
 
-  await sleep(seconds);
-
-  clearInterval(updateState);
+  await sleep(Math.min(seconds, remainingS()));
 
   let sum = jobsSent.reduce((a, b) => a + b, 0);
   await Deno.stdout.write(
     enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`)
   );
 
+  // Tell pusher workers to stop. Sampler interval keeps running through the
+  // drain phase so the throughput / queue panels show the queue actually
+  // emptying instead of stopping at push-end.
   const shutdown_start = Date.now();
   workers.forEach((worker, i) => {
     const l = (evt: MessageEvent<any>) => {
@@ -278,36 +536,49 @@ export async function main({
   });
 
   console.log("waiting for shutdown\n");
-  while (workers.length > 0) {
+  while (workers.length > 0 && remainingS() > 0) {
     await sleep(0.1);
   }
+  if (workers.length > 0) {
+    console.log(`\n[timeout] ${workers.length} driver-worker(s) still running — terminating`);
+    workers.forEach((w) => w.terminate());
+    workers = [];
+  }
 
+  // Drain phase: keep waiting until the queue is empty so the metric reflects
+  // sustained throughput, not "jobs queued / push window." Backstop is 10x the
+  // push duration — bails if the workload genuinely can't keep up so the run
+  // doesn't hang forever. --timeout still applies as the outer wallclock cap.
+  const drainStart = Date.now();
+  const drainDeadlineMs = drainStart + seconds * 10 * 1000;
   let queue_length = await getQueueCount();
-  const updateQueue = setInterval(async () => {
-    queue_length = (
-      await (
-        await fetch(
-          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-          { headers: { ["Authorization"]: "Bearer " + config.token } }
-        )
-      ).json()
-    ).database_length;
-    await Deno.stdout.write(enc(`queue length: ${queue_length}\r`));
-  }, 100);
-  while (queue_length > 0) {
+  while (queue_length > 0 && remainingS() > 0 && Date.now() < drainDeadlineMs) {
     await sleep(0.1);
+    try {
+      queue_length = await getQueueCount();
+    } catch (_) { /* transient — keep going */ }
+    await Deno.stdout.write(enc(`draining: queue=${queue_length}                    \r`));
+  }
+  clearInterval(updateState);
+  if (queue_length > 0) {
+    const reason = remainingS() <= 0 ? "--timeout reached" : "10x push duration safety cap";
+    console.log(`\n[drain] ${queue_length} job(s) still pending (${reason}) — reporting what's done`);
   }
-
-  clearInterval(updateQueue);
 
   sum = jobsSent.reduce((a, b) => a + b, 0);
+  const drainTime = (Date.now() - drainStart) / 1000;
+  const totalTime = (Date.now() - start) / 1000;
+  const processed = sum - queue_length;
 
-  const tts = (Date.now() - shutdown_start) / 1000;
-  const time = seconds + tts;
-  console.log("\ntime to shutdown:", tts);
-  console.log("jobs:", sum);
-  console.log("time (s + tts):", time);
-  console.log("throughput /s (jobs/time):", sum / time);
+  const pushRate = sum / seconds;
+  const sustainedRate = processed / totalTime;
+
+  console.log("\nshutdown wait (s):", (drainStart - shutdown_start) / 1000);
+  console.log("drain time (s):", drainTime);
+  console.log("total wall time (s):", totalTime);
+  console.log("jobs sent:", sum, "  processed:", processed, "  remaining:", queue_length);
+  console.log("push rate (sent/push_seconds):", pushRate.toFixed(2));
+  console.log("sustained throughput (processed/total_time):", sustainedRate.toFixed(2));
 
   console.log(
     "queue length:",
@@ -373,12 +644,173 @@ export async function main({
       f.close();
     }
   }
+  // ---------- post-bench: capture cluster measurements + render report ----------
+  // outDir + benchStartIso were declared above (pre-bench) so pollers could
+  // stream into them as the run progressed.
+
+  if (throughputSamples.length > 0) {
+    await Deno.writeTextFile(
+      `${outDir}/throughput_samples.json`,
+      JSON.stringify(throughputSamples),
+    );
+  }
+
+  // Persist the workload config alongside the report so the renderer can show
+  // the per-parameter distributions.
+  if (workloadConfig) {
+    await Deno.writeTextFile(
+      `${outDir}/workload.json`,
+      JSON.stringify(workloadConfig, null, 2),
+    );
+  }
+
+  // Cluster-side capture only if the caller pointed us at a minikube profile.
+  let cpusPerNode: number | Record<string, number> = 1;
+  if (minikubeProfile) {
+    const prov = new MinikubeProvisioner({ profile: minikubeProfile });
+    // Cores per node — used by the renderer to scale Node CPU to % of VM.
+    // Heterogeneous topologies (small tainted control plane + larger workers)
+    // need a per-node map so each node is normalized against its own capacity.
+    try {
+      const r = await prov.kubectl([
+        "get", "nodes",
+        "-o", "jsonpath={range .items[*]}{.metadata.name}={.status.capacity.cpu}{\"\\n\"}{end}",
+      ]);
+      const byNode: Record<string, number> = {};
+      for (const line of r.stdout.split("\n")) {
+        const [name, cpuStr] = line.split("=");
+        const n = parseInt((cpuStr ?? "").trim());
+        if (name && Number.isFinite(n) && n > 0) byNode[name.trim()] = n;
+      }
+      if (Object.keys(byNode).length > 0) cpusPerNode = byNode;
+    } catch { /* leave default 1 */ }
+    // Stop the pod-timeline poller before grabbing the final inventory — both
+    // write to outDir and we want the poller's last sample on disk first.
+    if (podTimelinePoller) {
+      podTimelinePoller.cont.value = false;
+      try {
+        await podTimelinePoller.done;
+      } catch (e) {
+        console.warn(`[bench] pod timeline finalize failed: ${(e as Error).message}`);
+      }
+    }
+    if (oomPoller) {
+      oomPoller.cont.value = false;
+      try {
+        await oomPoller.done;
+      } catch (e) {
+        console.warn(`[bench] OOM poller finalize failed: ${(e as Error).message}`);
+      }
+    }
+    if (pgLatencyPoller) {
+      pgLatencyPoller.cont.value = false;
+      try {
+        await pgLatencyPoller.done;
+      } catch (e) {
+        console.warn(`[bench] PG latency poller finalize failed: ${(e as Error).message}`);
+      }
+    }
+    if (pgConnPoller) {
+      pgConnPoller.cont.value = false;
+      try {
+        await pgConnPoller.done;
+      } catch (e) {
+        console.warn(`[bench] PG conn poller finalize failed: ${(e as Error).message}`);
+      }
+    }
+    if (nodeLoadPoller) {
+      nodeLoadPoller.cont.value = false;
+      try {
+        await nodeLoadPoller.done;
+      } catch (e) {
+        console.warn(`[bench] node-load poller finalize failed: ${(e as Error).message}`);
+      }
+    }
+    try {
+      await capturePodInventory(prov, `${outDir}/pods.json`);
+    } catch (e) {
+      console.warn(`[bench] pod inventory failed: ${(e as Error).message}`);
+    }
+    try {
+      await collectCpuSamples(prov, `${outDir}/cpu_samples.tsv`, { sinceTime: benchStartIso });
+    } catch (e) {
+      console.warn(`[bench] CPU sample capture failed: ${(e as Error).message}`);
+    }
+    try {
+      await collectOomEvents(prov, `${outDir}/oom_events.json`, { sinceMs: start });
+    } catch (e) {
+      console.warn(`[bench] OOM event capture failed: ${(e as Error).message}`);
+    }
+    try {
+      await collectFailedJobs({
+        host,
+        token: final_token,
+        workspace,
+        sinceMs: start,
+        outPath: `${outDir}/failed_jobs.jsonl`,
+      });
+    } catch (e) {
+      console.warn(`[bench] failed-jobs capture failed: ${(e as Error).message}`);
+    }
+  } else {
+    console.log(`[bench] --minikube-profile not set — skipping cluster-side panels (CPU/mem/workers/PG)`);
+  }
+
+  // Render the dashboard FIRST — it depends only on cpu_samples + pods +
+  // throughput. pgBadger is slow (~30s parse on a busy run's 50MB log) and
+  // produces a separate HTML; do it last so the dashboard is openable
+  // immediately.
+  try {
+    const benchWalltimeS = (Date.now() - start) / 1000;
+    await renderReport({
+      outDir,
+      topology: "bench",
+      walltimeS: benchWalltimeS,
+      finalThroughput: sustainedRate,
+      cpusPerNode,
+    });
+  } catch (e) {
+    console.warn(`[bench] report render failed: ${(e as Error).message}`);
+  }
+
+  // Background the pg-log capture + pgBadger render — the kubectl logs pull
+  // on a busy run takes ~40s and is dominated by network/log volume, not the
+  // bench. Spawn a detached subshell so wm-bench can exit while pgbadger
+  // continues; the report dir gets `pg.log` then `pgbadger.html` when ready.
+  if (minikubeProfile) {
+    const minikube = Deno.env.get("SIM_MINIKUBE_BIN") ?? "minikube";
+    const pgbadger = Deno.env.get("SIM_PGBADGER_BIN") ?? "pgbadger";
+    const logPath = `${outDir}/pg.log`;
+    const htmlPath = `${outDir}/pgbadger.html`;
+    const since = benchStartIso;
+    // Subshell — kubectl logs piped to file, then pgbadger; backgrounded with
+    // `&` so the parent sh exits immediately and the child becomes orphaned
+    // (and inherited by init). Output muted so a closed terminal doesn't kill
+    // it via SIGPIPE.
+    const script =
+      `( ${minikube} kubectl -p ${minikubeProfile} -- -n default logs ` +
+      `-l app=windmill-postgresql-demo-app --all-containers=true --tail=-1 ` +
+      `--since-time=${since} > "${logPath}" 2>/dev/null && ` +
+      `${pgbadger} "${logPath}" -o "${htmlPath}" -q >/dev/null 2>&1 ) </dev/null >/dev/null 2>&1 &`;
+    const cmd = new Deno.Command("sh", {
+      args: ["-c", script],
+      stdin: "null", stdout: "null", stderr: "null",
+    });
+    await cmd.output();
+    console.log(`[bench] pg.log + pgbadger.html will be written to ${outDir}/ in the background.`);
+  }
+
   console.log("done");
   return {
-    throughput: sum / time,
+    throughput: sustainedRate,
+    throughputSamples,
   };
 }
 
+// runWithTopology has moved entirely to `wm_sim up` — wm-bench no longer
+// provisions clusters or installs helm. Point it at a running cluster via
+// `--host` (and optionally `--minikube-profile` for the cluster-side panels).
+
 if (import.meta.main) {
   await new Command()
     .name("wmillbench")
@@ -396,11 +828,16 @@ if (import.meta.main) {
     )
     .option(
       "-s --seconds <seconds:number>",
-      "How long to run the benchmark for (in seconds).",
+      "How long the workers push jobs for (in seconds).",
       {
         default: 30,
       }
     )
+    .option(
+      "--timeout <timeout:number>",
+      "Hard wall-clock cap on the whole bench (push + drain). On hit, sampling stops and the report is rendered from what was collected. Default 120s.",
+      { default: 120 }
+    )
     .option("--max <max:number>", "Maximum number of operations performed.")
     .option("-e --email <email:string>", "The email to use to login.")
     .option("-p --password <password:string>", "The password to use to login.")
@@ -494,7 +931,20 @@ if (import.meta.main) {
       }
     )
     .option("--hide-progress", "Hide worker progress logs")
-    .action(main)
+    .option(
+      "--minikube-profile <name:string>",
+      "Enable cluster-side panels (per-node CPU/mem, workers per node, PG/pgBadger) by giving the minikube profile name. Required for the full dashboard; without it only Throughput + Queue depth are rendered.",
+    )
+    .option(
+      "--workload-config <path:string>",
+      "Path to a JSON workload config (distributions for ram_mb, duration_ms, mode). Required when --script-pattern is `random`. Examples in benchmarks/workloads/.",
+    )
+    .option(
+      "--wait-ready <seconds:number>",
+      "Before firing the bench, poll the cluster every 5s and only start once samplers (4/4 Running), workers (ready==replicas), PG (responsive), toxiproxy + app are healthy AND the queue is empty. Errors with the unmet conditions on timeout. Skip the check entirely with --wait-ready 0.",
+      { default: 0 },
+    )
+    .action((opts: any) => main(opts))
     .command(
       "upgrade",
       new UpgradeCommand({

benchmarks/sim/README.md

@@ -0,0 +1,278 @@
+# Windmill benchmark sim
+
+Provisions a minikube-backed multi-node Kubernetes cluster, deploys Windmill
+on it via Helm, runs workload benches against it, and produces a single SVG
+dashboard with throughput, queue depth, per-node CPU + memory + oversaturation,
+per-pod CPU/memory peaks, PG response latency + connection counts, OOM and
+restart events, and pod inventory.
+
+The goal: realistic, reproducible benches at scales where running Windmill
+on bare containers (`docker run`) hides the production failure modes
+(scheduler saturation, cgroup_mutex contention, PG backend thrash, kubelet
+log rotation, OOM cascades). Everything here exists to make those
+production-shape problems observable in 3 minutes from a clean checkout.
+
+## What you actually get
+
+Single CLI to bring up the cluster + deploy Windmill:
+
+```sh
+wm_sim up \
+  --topology  sim/topologies/k8s-4node.json \
+  --helm      ../windmill-helm-charts/charts/windmill \
+  --helm-values sim/values/smoke.yaml \
+  --helm-values sim/values/local.yaml      # gitignored; carries the EE license
+```
+
+`wm_sim up` opens its own port-forward to `svc/windmill-app` on a free local
+port and prints the URL — look for a line like:
+
+```
+[helm] API reachable at http://127.0.0.1:38291 (port-forward svc/windmill-app)
+```
+
+The forward dies with the wm_sim process, so leave the wm_sim terminal open.
+Then in a second shell, fire a bench against that URL:
+
+```sh
+cd benchmarks && deno run -A main.ts \
+  --host http://127.0.0.1:<port from wm_sim output> \
+  --token <admin-token> \
+  --workload-config workloads/io_150ms_flood.json \
+  --minikube-profile wm-sim-k8s-4node \
+  --wait-ready 60
+```
+
+Bench runs ~3 min. Report lands at `benchmarks/reports/<timestamp>/`, with
+`dashboard.svg` as the entry point. Open it in a browser.
+
+If the port-forward dies mid-bench (kubelet sometimes drops it under heavy
+load), restart it manually in a second shell with the same port:
+
+```sh
+kubectl --context wm-sim-k8s-4node port-forward -n default \
+  svc/windmill-app <same port>:8000
+```
+
+## Prerequisites
+
+- `nix develop` from the repo flake — provisions `minikube`, `kubectl`, `helm`,
+  `pgbadger`, `deno`, and the `wm_sim` wrapper. **No host config required
+  beyond the flake** with one exception: if you want multi-node KVM clusters
+  on NixOS, libvirtd needs to be enabled at the system level — see
+  `sim/nix/kvm2-driver.nix`.
+- Root + libvirtd for the kvm2 driver. Single-node clusters work with the
+  qemu2 driver without root, but multi-node needs kvm2.
+- A clone of `windmill-helm-charts` next to this repo (`../windmill-helm-charts`).
+  The vendored chart was deleted in favor of using the upstream chart with
+  bench-specific overlays in `values/smoke.yaml`. The chart depends on a few
+  knobs (PriorityClass support, `oomImmune`, `maxConnections`) that may not
+  be in upstream yet — see the windmill-helm-charts PR.
+
+## Topology + values layout
+
+```
+sim/
+  topologies/
+    k8s-4node.json          # 1 control plane + 3 workers (4 vCPU, 20 GiB each)
+    k8s-3node.json          # control plane + 2 workers
+    ...                     # several other shapes
+  values/
+    smoke.yaml              # bench-tuned helm overlay (worker reqs, PG sizing,
+                            # priorityClass=wm-critical, maxConnections, etc.)
+    local.example.yaml      # template — copy to local.yaml, fill in your
+                            # EE license key. local.yaml is gitignored.
+```
+
+`smoke.yaml` is the source of truth for chart config across benches. Tune
+worker replica counts, PG memory, max_connections, etc. there.
+
+## Architecture
+
+```
+wm_sim up
+  │
+  ▼ provision
+  ┌────────────────────────────────────────────────────────────────┐
+  │ k8s_provisioner: minikube start + add-nodes + per-node sizing  │
+  │ image_cache:     pre-load images so bringup is offline-safe    │
+  │ helm_deploy:     helm upgrade --install windmill via smoke.yaml│
+  │ toxiproxy_k8s:   per-node toxiproxy DaemonSet for latency inj  │
+  │ cpu_sampler_k8s: per-node sampler DaemonSet (10Hz cpu.stat)    │
+  └────────────────────────────────────────────────────────────────┘
+  │
+  ▼ benchmarks/main.ts (separate shell)
+  ┌────────────────────────────────────────────────────────────────┐
+  │ readiness:        wait for samplers stable, workers Ready,    │
+  │                   PG responsive, queue empty,                 │
+  │                   *deployment rollout complete* — mid-rollout │
+  │                   pod churn starves the sampler under         │
+  │                   cgroup_mutex contention                     │
+  │ start pollers:    pod_timeline, oom, pg_latency, pg_conn,     │
+  │                   node_load (all write JSONL into report dir) │
+  │ truncate sampler  /var/log/wm-sim-cpu-sampler/sampler.tsv on   │
+  │   host log:       each node so this run gets a clean file     │
+  │ push jobs:        pushers → API → workers pull → PG queue     │
+  │ stop pollers:     finalize JSONL                              │
+  │ collect samples:  scp host log from each node (fallback:      │
+  │                   kubectl logs)                               │
+  │ collect pg.log:   kubectl logs PG pod → pgBadger HTML report  │
+  │ render_report:    JSONL + samples → dashboard.svg + report.md │
+  └────────────────────────────────────────────────────────────────┘
+  │
+  ▼ output: benchmarks/reports/<ISO timestamp>/
+```
+
+## Measurement subsystem
+
+| Signal | Source | Cadence | File in report |
+|---|---|---|---|
+| Per-pod CPU + memory | DS sampler reading cgroup cpu.stat / memory.current | 10 Hz | `cpu_samples.tsv` |
+| Workers Ready per node | `kubectl get pods` | 1 Hz | `pod_timeline.jsonl` |
+| OOM events | `kubectl get events` + node-kernel dmesg | live | `oom_events_live.jsonl` |
+| PG query latency | `psql -c "\timing on" -c "SELECT 1"` | 4 Hz | `pg_latency.jsonl` |
+| PG connections by state | `SELECT state, count(*) FROM pg_stat_activity` | 1 Hz | `pg_connections.jsonl` |
+| Node loadavg + procs_running | ssh `/proc/loadavg`, `/proc/stat` | 0.5 Hz | `node_load.jsonl` |
+| Throughput (jobs/s sent + processed) | bench worker `postMessage` | sub-second | `throughput_samples.json` |
+| Failed jobs | `/api/w/{ws}/jobs/completed/list` | post-bench | `failed_jobs.jsonl` |
+| Pod inventory + node placement | `kubectl get pods -o json` | post-bench | `pods.json` |
+| PG slow-query analysis | PG log → pgBadger | post-bench, background | `pgbadger.html` |
+
+Key reliability fixes (don't undo them):
+
+- **Sampler host file**: the sampler dual-writes to stdout AND a hostPath log
+  file. `kubectl logs --since-time=X` loses early data when kubelet rotates
+  log files under heavy benches (m04 was missing the first 90-120s of every
+  busy run). Collector now scp's the host file as primary source.
+- **Readiness rollout check**: `kubectl get deploy windmill-workers-default`
+  is checked for `status.updatedReplicas == spec.replicas`. Mid-rolling-update
+  pod churn floods the kernel's `cgroup_mutex`, starving every other process
+  on the node — including the sampler. Pod-readiness alone isn't enough.
+- **Oversaturation uses procs_running**, not load1. loadavg includes D-state
+  procs (PG backends waiting on disk I/O, kernel mutex contention) which has
+  nothing to do with CPU starvation but inflates "oversaturation" by 5-10x.
+  `procs_running` from `/proc/stat` is the runnable count.
+
+## Dashboard panels
+
+Rendered into `dashboard.svg`. Layout is roughly:
+
+| Section | Panels |
+|---|---|
+| **Run context** | Topology, workload, helm values, bench cmd, report dir |
+| **Throughput** | Jobs sent + processed over time, with push-window shading and phase boundaries |
+| **Queue depth** | Pending jobs in `v2_job_queue` |
+| **Node memory** | Per-node memory time-series + per-phase average bar chart |
+| **PG latency** | Pure SQL time (psql `\timing`) + kubectl-exec roundtrip (separate axes) |
+| **PG connections** | active / idle / idle_in_xact over time |
+| **Util group** | One panel per node, 2 per row. Solid blue CPU util (capped 100%) on top of translucent orange oversaturation. 100% reference line. PG-host node flagged `[PG]` |
+| **Node CPU** | Multi-line per-node CPU, with PG-host node tinted via stroke + label |
+| **Pod inventory** | Bar chart of pod count by node and pod-type donut |
+| **Failed jobs** | Cumulative + categorized bars |
+| **Restart events** | Pod restarts detected from `pod_timeline.jsonl` |
+| **OOM events** | L0 scheduler preemptions, L1 kubelet evictions, L2 cgroup + node-kernel kills |
+
+All time-series share an x-axis origin (`bench_start_ms` written into
+`meta.json`) so the same x-coordinate on every panel means the same wall-clock
+moment. Tick labels are relative seconds ("0s, 30s, …"), not wall-clock.
+
+## Workloads
+
+`benchmarks/workloads/*.json`. Each is a phased JSON spec the bench reads at
+startup. Phases run sequentially, each with its own pusher count, mode mix,
+duration range, and ram_mb distribution.
+
+Examples:
+
+| Workload | Shape | Use case |
+|---|---|---|
+| `io_4phase.json` | idle → 2.5s → 500ms → 150ms IO jobs | Steady-state vs saturation in one bench |
+| `io_150ms_flood.json` | 60 pushers, 150ms IO, 120s | Worker-host CFS saturation regime |
+| `io_300ms_flood.json` | 60 pushers, 300ms IO, 120s | Lower context-switch rate |
+| `io_1s_flood.json`, `io_2s_flood.json` | Same shape, longer jobs | Near-theoretical worker scaling |
+| `ops_day.json` | Mixed phases simulating a day | Realistic load profile |
+| `cpu_heavy.json`, `mixed.json`, `burst.json` | Non-IO and bursty shapes | CPU-bound and spikiness |
+
+## Agent workers (HTTP+JWT path)
+
+`smoke.yaml` declares a second worker group `agent` with `mode: agent`. Agent
+workers reach the windmill API over HTTP+JWT instead of holding a direct
+sqlx pool to PG. Useful for measuring how much of the bench cap is PG
+contention vs. the HTTP-mediated path.
+
+The JWT lives in a K8s Secret `windmill-agent-token` (not in any committed
+yaml). Create it once:
+
+```sh
+kubectl --context wm-sim-k8s-4node create secret generic windmill-agent-token \
+  --from-literal=token='jwt_agent_…'
+```
+
+## Tests
+
+```sh
+cd benchmarks && deno test --allow-import --no-check \
+  sim/util_metrics_test.ts sim/util_panel_snapshot_test.ts
+```
+
+- `sim/util_metrics_test.ts` — 8 tests for `computeOversatPct`, the formula
+  the util panel uses. Specifically guards the procs_running > load1 fallback,
+  the clamp-at-zero, invalid-ncpu inputs.
+- `sim/util_panel_snapshot_test.ts` — 5 assertions on the rendered util-panel
+  SVG: orange-behind-blue z-order, 100% reference line, relative-time ticks
+  (no wall-clock leak), phase-boundary dashed verticals, shared-origin override.
+- `sim/topology_test.ts`, `sim/toxiproxy_test.ts` — earlier coverage of the
+  validator + proxy planner.
+
+Pre-existing typecheck errors in `graph.ts` (untyped d3 callback args,
+implicit `any`) trip `deno test` without `--no-check`. The functional tests
+pass regardless.
+
+## Operational notes
+
+- Cluster name / minikube profile is `wm-sim-k8s-4node` by default. Pass
+  `--minikube-profile <name>` to `main.ts` if you renamed it.
+- Default kubectl context comes from minikube. Use
+  `kubectl --context wm-sim-k8s-4node …` to be explicit.
+- `wm_sim up` runs the port-forward as a child process. Killing wm_sim
+  (Ctrl-C) kills the forward. The kubelet sometimes drops the forward under
+  heavy bench load; if a bench fails partway with "connection refused",
+  start a replacement forward manually with `kubectl port-forward` on the
+  same local port and re-fire.
+- `minikube stop` is safe — VM disks persist, etcd survives, helm releases
+  + Secrets all come back when you `minikube start`. `wm_sim up` does
+  `minikube delete` first, so it's destructive — only use it for clean reprovisions.
+
+## Code layout
+
+```
+sim/
+  sim.ts                  # wm_sim CLI entry (cliffy)
+  k8s_provisioner.ts      # minikube up + node sizing
+  helm_deploy.ts          # helm upgrade --install
+  image_cache.ts          # pre-load images so bringup is offline-safe
+  cpu_sampler_k8s.ts      # sampler DaemonSet + collector
+  toxiproxy_k8s.ts        # per-node toxiproxy DaemonSet
+  pg_logging.ts           # ALTER SYSTEM + SIGHUP for verbose PG logs
+  pgbadger.ts             # post-bench PG log → HTML
+  readiness.ts            # pre-bench cluster health check
+  pod_timeline.ts         # pod readiness poller
+  pod_inventory.ts        # post-bench pod inventory snapshot
+  oom_poller.ts           # live OOM event capture
+  oom_events.ts           # post-bench OOM event parsing
+  pg_latency_poller.ts    # 4Hz PG \timing on SELECT 1
+  pg_conn_poller.ts       # pg_stat_activity by state
+  node_load_poller.ts     # /proc/loadavg + /proc/stat procs_running
+  failed_jobs.ts          # post-bench failed-jobs fetch
+  util_metrics.ts         # pure helpers (oversat formula)
+  render_report.ts        # JSONL → SVG dashboard + report.md
+  dashboard.ts            # SVG layout primitives (grid, sections)
+  svg_to_pdf.ts           # dashboard.svg → dashboard.pdf (best-effort)
+  topology.ts             # topology JSON loader + validator
+  values/                 # helm overlay yamls
+  topologies/             # topology JSON files
+  nix/kvm2-driver.nix     # kvm2 driver overlay for nixpkgs
+```
+
+The bench runner itself is `benchmarks/main.ts` (one level up). It calls
+into `sim/` for measurement + report rendering.

benchmarks/sim/cpu_sampler_k8s.ts

@@ -0,0 +1,489 @@
+// Per-node CPU sampler for the k8s sim path.
+//
+// A privileged DaemonSet (one pod per node) mounts the host's
+// `/sys/fs/cgroup` and reads `cpu.stat` for every Windmill pod's cgroup at a
+// configurable cadence (default 100 ms — busybox shell sleep granularity is
+// the practical floor; finer than ~50 ms tends to drift. For true 10 ms
+// fidelity we'd need a small Go/Rust binary, queued as a follow-on).
+//
+// Output is CSV on stdout, captured at end-of-bench via `kubectl logs`:
+//   <ts_ns> <node> <pod_uid> <container_id> <usage_usec>
+//
+// The sampler walks `/sys/fs/cgroup/kubepods.slice/**/cpu.stat`. cgroup v2's
+// hierarchy under kubepods.slice is:
+//   kubepods.slice/
+//     kubepods-besteffort.slice/
+//       kubepods-besteffort-pod<pod-uid>.slice/
+//         cri-containerd-<container-id>.scope/cpu.stat
+//     kubepods-burstable.slice/...
+//     kubepods.slice/kubepods-pod<pod-uid>.slice/...  (guaranteed)
+// The sampler emits raw paths and lets the consumer post-process.
+
+import { stringify as yamlStringify } from "https://deno.land/std@0.224.0/yaml/mod.ts";
+import { MinikubeProvisioner } from "./k8s_provisioner.ts";
+
+const NAMESPACE = "kube-system";
+const DS_NAME = "wm-sim-cpu-sampler";
+const IMAGE = "python:3.12-alpine";
+
+// Single-process Python sampler. Replaces the prior bash loop that forked
+// awk/cat/find every tick — under heavy node CPU pressure the shell pipeline
+// alone could consume seconds at a stretch (m03 had 13.7s sample gaps even
+// after Guaranteed QoS). Python's persistent FDs + os.read on lseek(0) costs
+// ~0 between samples: ~1 syscall per file, no path resolution, no subprocess.
+const SAMPLE_SCRIPT = `#!/usr/bin/env python3
+import os, sys, time, glob
+
+INTERVAL_S = float(os.environ.get("INTERVAL_S", "0.1"))
+NODE_NAME  = os.environ.get("NODE_NAME", "unknown")
+CG_HOST    = "/host-sys/fs/cgroup"
+CG_KP      = CG_HOST + "/kubepods.slice"
+PROC_MEM   = "/host-proc/meminfo"
+NODE_TAG   = "__node_root__"
+# Host-mounted append-only log. Bypasses kubelet's log file rotation —
+# heavy benches were rotating early samples out of kubelet's 10MB×5
+# window before the bench finished, leaving multi-minute gaps for the
+# busiest node in the report. The collector scp's this file from each
+# node at end-of-bench; kubectl logs is kept as a fallback path.
+HOST_LOG   = "/host-logs/sampler.tsv"
+
+# Rescan kubepods every N seconds to pick up new pods. Cheap glob walk,
+# but no need to do it every tick — pod churn is slow vs the sample rate.
+RESCAN_S = 5.0
+
+# Two timestamps per row:
+#   ts_ns   = CLOCK_REALTIME (wall) — needed for x-axis alignment with
+#             other bench panels (throughput, OOM events, etc).
+#   mono_ns = CLOCK_MONOTONIC — used by the consumer for delta math.
+# Why both: in qemu/minikube guests the wall clock can step backwards on
+# NTP correction, producing apparent dt < real_wall and impossible
+# >100% per-VM CPU spikes (du/dt where du is real but dt is shrunk by
+# the backwards clock jump). MONOTONIC is immune by spec.
+print("ts_ns mono_ns node pod_slice container_scope usage_usec mem_bytes", flush=True)
+# Open host log append-only; line-buffered. The header is printed only once
+# per container life (re-opening on restart appends a new header — collector
+# tolerates this since it filters by ts_ns >= bench_start).
+host_log_fd = None
+try:
+    host_log_fd = open(HOST_LOG, "a", buffering=1)
+    host_log_fd.write("ts_ns mono_ns node pod_slice container_scope usage_usec mem_bytes\\n")
+except OSError:
+    host_log_fd = None
+
+def parse_usage(fd):
+    os.lseek(fd, 0, 0)
+    # cpu.stat first line is "usage_usec <N>"; reads <1KiB
+    buf = os.read(fd, 1024)
+    # bytes split is faster than decode+split here
+    nl = buf.find(b"\\n")
+    line = buf if nl < 0 else buf[:nl]
+    sp = line.find(b" ")
+    return line[sp + 1:].decode() if sp > 0 else ""
+
+def parse_meminfo(fd):
+    os.lseek(fd, 0, 0)
+    buf = os.read(fd, 4096)
+    t = a = -1
+    for line in buf.split(b"\\n"):
+        if line.startswith(b"MemTotal:"):
+            t = int(line.split()[1])
+        elif line.startswith(b"MemAvailable:"):
+            a = int(line.split()[1])
+        if t >= 0 and a >= 0:
+            break
+    return str((t - a) * 1024) if (t >= 0 and a >= 0) else "-"
+
+def read_mem(fd):
+    os.lseek(fd, 0, 0)
+    return os.read(fd, 64).strip().decode()
+
+# Persistent FDs — open once, reuse across ticks.
+root_cpu_fd = os.open(CG_HOST + "/cpu.stat", os.O_RDONLY)
+mem_fd      = os.open(PROC_MEM, os.O_RDONLY)
+# pod_slice -> (cpu_fd, mem_fd or None)
+pods: dict[str, tuple[int, int | None]] = {}
+last_rescan = 0.0
+
+def rescan_pods():
+    """Walk kubepods.slice once, find leaf pod cgroups, open their FDs."""
+    seen = set()
+    # Two-level glob: kubepods-<qos>-pod*.slice + kubepods-pod*.slice (guaranteed).
+    for cpu_stat in glob.iglob(CG_KP + "/**/cpu.stat", recursive=True):
+        d = os.path.dirname(cpu_stat)
+        slice_name = os.path.basename(d)
+        # Skip container scopes (cri-containerd-*.scope); we want pod slices.
+        if not slice_name.endswith(".slice") or "-pod" not in slice_name:
+            continue
+        seen.add(slice_name)
+        if slice_name in pods:
+            continue
+        try:
+            cfd = os.open(cpu_stat, os.O_RDONLY)
+        except OSError:
+            continue
+        mfd = None
+        try:
+            mfd = os.open(d + "/memory.current", os.O_RDONLY)
+        except OSError:
+            pass
+        pods[slice_name] = (cfd, mfd)
+    # Drop FDs for pods that disappeared (terminated).
+    for name in list(pods.keys()):
+        if name not in seen:
+            cfd, mfd = pods.pop(name)
+            try: os.close(cfd)
+            except OSError: pass
+            if mfd is not None:
+                try: os.close(mfd)
+                except OSError: pass
+
+# Build output in-memory then single write — fewer syscalls than print-per-row.
+out_buf = []
+while True:
+    now = time.monotonic()
+    if now - last_rescan >= RESCAN_S:
+        rescan_pods()
+        last_rescan = now
+    ts = time.time_ns()
+    mono = time.monotonic_ns()
+    try:
+        ru = parse_usage(root_cpu_fd)
+    except OSError:
+        ru = "-"
+    try:
+        rm = parse_meminfo(mem_fd)
+    except OSError:
+        rm = "-"
+    out_buf.append(f"{ts} {mono} {NODE_NAME} {NODE_TAG} - {ru} {rm}\\n")
+    for slice_name, (cfd, mfd) in pods.items():
+        try:
+            u = parse_usage(cfd)
+        except OSError:
+            continue
+        if not u:
+            continue
+        try:
+            m = read_mem(mfd) if mfd is not None else "-"
+        except OSError:
+            m = "-"
+        out_buf.append(f"{ts} {mono} {NODE_NAME} {slice_name} - {u} {m}\\n")
+    blob = "".join(out_buf)
+    sys.stdout.write(blob)
+    sys.stdout.flush()
+    if host_log_fd is not None:
+        try:
+            host_log_fd.write(blob)
+        except OSError:
+            pass
+    out_buf.clear()
+    time.sleep(INTERVAL_S)
+`;
+
+function configMapManifest(): string {
+  return yamlStringify({
+    apiVersion: "v1",
+    kind: "ConfigMap",
+    metadata: { name: DS_NAME, namespace: NAMESPACE },
+    data: { "sample.py": SAMPLE_SCRIPT },
+  });
+}
+
+
+function daemonSetManifest(intervalSeconds: number): string {
+  return yamlStringify({
+    apiVersion: "apps/v1",
+    kind: "DaemonSet",
+    metadata: { name: DS_NAME, namespace: NAMESPACE, labels: { app: DS_NAME } },
+    spec: {
+      selector: { matchLabels: { app: DS_NAME } },
+      template: {
+        metadata: { labels: { app: DS_NAME } },
+        spec: {
+          // Schedule onto every node including the control-plane.
+          tolerations: [{ operator: "Exists" }],
+          hostPID: true,
+          // Same priorityClass as PG/app (wm-critical = 1B). Sampler is tiny
+          // (25m/32Mi request) and PG/app are big, so they fit together on
+          // any worker node trivially. The shared priority means scheduler
+          // can preempt workers (priority 0) to land the sampler if the
+          // request budget gets squeezed — and the sampler is never the
+          // pod that loses out to PG/app.
+          priorityClassName: "wm-critical",
+          containers: [{
+            name: "sampler",
+            image: IMAGE,
+            securityContext: { privileged: true },
+            // Burstable QoS: requests give a CPU-scheduling floor, but we
+            // INTENTIONALLY omit limits.cpu — a hard CPU cap throttles the
+            // sampler via CFS, which interacts badly with NTP wall-clock
+            // corrections to produce >100% per-VM artifacts (the sampler
+            // emits ts immediately after a backwards clock jump while the
+            // cgroup counter has accumulated normally).
+            //
+            // Memory: 256Mi limit was still OOM-looping on busy worker nodes
+            // (m02/m03/m04 all hit exit 137 mid-bench, dropping CPU panels
+            // for those nodes). Bumped 5× to 1.25Gi limit / 320Mi request —
+            // way over what an idle sampler uses (~15Mi RSS) but the
+            // observed restart pattern shows transient spikes the previous
+            // ceiling couldn't absorb. CPU bumped 5× too (200m→1000m req)
+            // so the sampler never CPU-throttles even under hot-node load.
+            // Burstable QoS; combined with the postStart oom_score_adj=-999
+            // below, the sampler is also last-to-pick for kernel OOM.
+            // Request is intentionally small so this DaemonSet pod schedules
+            // on heavily-booked worker nodes (where workers' 256Mi×N requests
+            // eat most of the per-node request budget). Limit stays generous
+            // so the sampler can burst when reading lots of cgroup files
+            // under load. Burstable QoS.
+            // Tiny requests so this DaemonSet pod can ALWAYS fit, even on
+            // worker nodes where PG (cpu=3) + workers (33×50m) + other
+            // pods leave only a sliver of CPU/mem request budget.
+            // Without this, m02 (where PG lives) ran out of CPU request
+            // budget and the m02 sampler stayed Pending the entire bench
+            // → no m02 data → Workers-per-node chart wrong.
+            //
+            // Limit bumped to 4Gi after repeated OOMKilled/exit-137 with
+            // peak usage <30Mi — likely node-kernel OOM picking the sampler
+            // before its postStart -999 hook ran. Wide ceiling makes the
+            // sampler an unappealing OOM victim.
+            // 500m (half a core) reserved. Originally tried 1 full core but
+            // m02 hosts PG (cpu=3) so 3+1=4 = the entire node, leaving zero
+            // room for ANY worker AND the sampler couldn't even schedule
+            // there (Pending). 500m fits comfortably on m02 (PG 3 + sampler
+            // 0.5 + ~0.5 for workers = 4), still 20× the idle ~25m, and is
+            // enough headroom that the 100ms tick won't CFS-throttle.
+            //
+            // No memory limit — cgroup OOM kept getting tripped under busy-
+            // node memory pressure even at 4Gi. OOM-immunity now comes from
+            // priorityClassName + oom_score_adj=-999 set inline at PID-1
+            // startup.
+            resources: {
+              requests: { cpu: "500m", memory: "32Mi" },
+            },
+            // OOM-immunity: lower this container's PID-1 oom_score_adj to
+            // -999 so the kernel picks anything else first under node memory
+            // pressure. `privileged: true` already grants CAP_SYS_RESOURCE.
+            // postStart-exec is safe here — python:3.12-alpine ships
+            // /bin/sh, unlike distroless toxiproxy where we had to use a
+            // sidecar approach.
+            lifecycle: {
+              postStart: {
+                exec: { command: ["/bin/sh", "-c", "echo -999 > /proc/1/oom_score_adj || true"] },
+              },
+            },
+            env: [
+              { name: "INTERVAL_S", value: String(intervalSeconds) },
+              {
+                name: "NODE_NAME",
+                valueFrom: { fieldRef: { fieldPath: "spec.nodeName" } },
+              },
+            ],
+            // Setting oom_score_adj as the FIRST thing PID 1 does eliminates
+            // a startup race the postStart hook couldn't: under node memory
+            // pressure (m04 sits at ~98% committed), the kernel OOM-killer
+            // can pick the sampler container BEFORE postStart fires, since
+            // postStart runs after container start. Doing it inline in the
+            // exec means the very first syscall the container makes is the
+            // adj write, before any allocation that could trigger OOM.
+            command: ["/bin/sh", "-c", "echo -999 > /proc/self/oom_score_adj 2>/dev/null || true; exec python3 -u /scripts/sample.py"],
+            volumeMounts: [
+              { name: "cgroup", mountPath: "/host-sys/fs/cgroup", readOnly: true },
+              { name: "hostproc", mountPath: "/host-proc", readOnly: true },
+              { name: "script", mountPath: "/scripts" },
+              { name: "hostlogs", mountPath: "/host-logs" },
+            ],
+          }],
+          volumes: [
+            { name: "cgroup", hostPath: { path: "/sys/fs/cgroup", type: "Directory" } },
+            { name: "hostproc", hostPath: { path: "/proc", type: "Directory" } },
+            { name: "script", configMap: { name: DS_NAME, defaultMode: 0o755 } },
+            // Append-only log dir on the node. DirectoryOrCreate makes
+            // first-time minikube nodes happy. The sampler writes its TSV
+            // here so the collector can scp it instead of relying on kubelet
+            // log rotation (which loses data under heavy benches).
+            { name: "hostlogs", hostPath: { path: "/var/log/wm-sim-cpu-sampler", type: "DirectoryOrCreate" } },
+          ],
+        },
+      },
+    },
+  });
+}
+
+// Apply the sampler. Best-effort — if the cluster doesn't have a usable cgroup
+// layout the sampler pods will run but produce empty output, which we tolerate.
+export async function applyCpuSampler(
+  prov: MinikubeProvisioner,
+  outDir: string,
+  opts: { intervalSeconds?: number } = {},
+): Promise<void> {
+  const intervalSeconds = opts.intervalSeconds ?? 0.1;
+  const cm = configMapManifest();
+  const ds = daemonSetManifest(intervalSeconds);
+  const path = `${outDir}/cpu-sampler.yaml`;
+  await Deno.writeTextFile(path, `${cm}---\n${ds}`);
+  console.log(`[cpu] applying CPU sampler DaemonSet (interval ${intervalSeconds}s)`);
+  const res = await prov.kubectl(["apply", "-f", path]);
+  if (res.code !== 0) {
+    throw new Error(`[cpu] kubectl apply failed: ${res.stdout}`);
+  }
+  await prov.kubectl([
+    "-n", NAMESPACE,
+    "rollout", "status", `daemonset/${DS_NAME}`,
+    "--timeout=120s",
+  ]);
+  console.log("[cpu] sampler running on all nodes");
+}
+
+// Collect the sampler output as CSV. Captures all pods' logs across the
+// DaemonSet via `kubectl logs --selector` so multi-node clusters are merged.
+// `sinceTime` (ISO 8601, RFC 3339) scopes the output to the bench's wall
+// window — the sampler runs continuously between bench runs, so without this
+// every report would include all-time cumulative data.
+export async function collectCpuSamples(
+  prov: MinikubeProvisioner,
+  outPath: string,
+  opts: { sinceTime?: string } = {},
+): Promise<void> {
+  // Primary path: scp the hostPath log file from each node. This bypasses
+  // kubelet log rotation (which loses early samples on heavily-loaded nodes —
+  // saw m04 routinely missing the first ~120s of a bench because the kubelet
+  // log file rotated past those rows before bench end). If scp succeeds for
+  // all nodes we use that data; otherwise fall back to per-pod kubectl logs.
+  const sinceNs = opts.sinceTime ? Date.parse(opts.sinceTime) * 1e6 : 0;
+  const nodesRes = await prov.kubectl([
+    "get", "nodes",
+    "-o", "jsonpath={range .items[*]}{.metadata.name}{\"|\"}{.status.addresses[?(@.type==\"InternalIP\")].address}{\"\\n\"}{end}",
+  ]);
+  const nodes: { name: string; ip: string }[] = [];
+  if (nodesRes.code === 0) {
+    for (const line of nodesRes.stdout.split("\n")) {
+      if (!line.trim()) continue;
+      const [n, ip] = line.split("|");
+      if (n && ip) nodes.push({ name: n.trim(), ip: ip.trim() });
+    }
+  }
+  const home = Deno.env.get("HOME") ?? "";
+  let scpFullSuccess = nodes.length > 0;
+  const scpPieces: string[] = [];
+  for (const { name: n, ip } of nodes) {
+    const key = `${home}/.minikube/machines/${n}/id_rsa`;
+    const cmd = new Deno.Command("ssh", {
+      args: [
+        "-o", "StrictHostKeyChecking=no",
+        "-o", "UserKnownHostsFile=/dev/null",
+        "-o", "ConnectTimeout=5",
+        "-o", "LogLevel=ERROR",
+        "-i", key,
+        `docker@${ip}`,
+        "sudo cat /var/log/wm-sim-cpu-sampler/sampler.tsv 2>/dev/null || true",
+      ],
+      stdout: "piped",
+      stderr: "null",
+    });
+    try {
+      const out = await cmd.output();
+      const txt = new TextDecoder().decode(out.stdout);
+      if (!txt) {
+        scpFullSuccess = false;
+        continue;
+      }
+      // Filter to bench window (ts_ns ≥ sinceNs), prefix lines with the pod
+      // marker the renderer expects ([pod/wm-sim-cpu-sampler-...]).
+      const filtered: string[] = [];
+      const marker = `[pod/wm-sim-cpu-sampler-on-${n}/sampler]`;
+      for (const line of txt.split("\n")) {
+        if (!line) continue;
+        const sp = line.indexOf(" ");
+        if (sp < 0) continue;
+        const tsStr = line.slice(0, sp);
+        if (!/^\d+$/.test(tsStr)) continue; // header / non-data row
+        if (sinceNs && Number(tsStr) < sinceNs) continue;
+        filtered.push(marker + " " + line);
+      }
+      if (filtered.length > 0) scpPieces.push(filtered.join("\n") + "\n");
+    } catch (_e) {
+      scpFullSuccess = false;
+    }
+  }
+  if (scpFullSuccess && scpPieces.length > 0) {
+    await Deno.writeTextFile(outPath, scpPieces.join(""));
+    console.log(`[cpu] collected from host log files via scp (${scpPieces.length} node(s))`);
+    return;
+  }
+  console.warn(`[cpu] scp host-log fetch incomplete (success=${scpFullSuccess}, pieces=${scpPieces.length}) — falling back to kubectl logs`);
+
+  // Discover the sampler pods so we can pull previous-container logs for any
+  // that restarted mid-bench. The `-l` selector form only returns logs from
+  // the *current* container — if a sampler pod restarted during the bench
+  // window, samples from before the restart are silently dropped.
+  const podsRes = await prov.kubectl([
+    "-n", NAMESPACE,
+    "get", "pods",
+    "-l", `app=${DS_NAME}`,
+    "-o", "jsonpath={range .items[*]}{.metadata.name}={.status.containerStatuses[0].restartCount}{\"\\n\"}{end}",
+  ]);
+  const pods: { name: string; restarts: number }[] = [];
+  for (const line of podsRes.stdout.split("\n")) {
+    const [name, rcStr] = line.split("=");
+    const rc = parseInt((rcStr ?? "0").trim());
+    if (name?.trim()) pods.push({ name: name.trim(), restarts: Number.isFinite(rc) ? rc : 0 });
+  }
+  if (pods.length === 0) {
+    throw new Error(`[cpu] no sampler pods found via -l app=${DS_NAME}`);
+  }
+
+  const baseArgs = ["-n", NAMESPACE, "logs", "--tail=-1", "--prefix=true"];
+  if (opts.sinceTime) baseArgs.push(`--since-time=${opts.sinceTime}`);
+
+  const pieces: string[] = [];
+  let restartedPodsWithPrev = 0;
+  let failedPods = 0;
+  // Per-pod fetch is resilient: a single sampler pod's kubectl-logs failing
+  // (commonly transient TLS handshake timeout to a kubelet) must NOT abort
+  // the whole capture — that was costing whole-bench reports (the renderer
+  // skips when cpu_samples.tsv is missing entirely). Falling back to partial
+  // data + warning is the right tradeoff.
+  for (const p of pods) {
+    if (p.restarts > 0) {
+      try {
+        const prev = await prov.kubectl([...baseArgs, p.name, "--previous"]);
+        if (prev.code === 0 && prev.stdout) {
+          pieces.push(prev.stdout);
+          restartedPodsWithPrev++;
+        }
+      } catch (e) {
+        console.warn(`[cpu] --previous fetch for ${p.name} failed: ${(e as Error).message}`);
+      }
+    }
+    try {
+      const cur = await prov.kubectl([...baseArgs, p.name]);
+      if (cur.code !== 0) {
+        console.warn(`[cpu] kubectl logs ${p.name} failed (code ${cur.code}): ${(cur.stderr || cur.stdout).slice(0, 200)} — skipping this pod`);
+        failedPods++;
+        continue;
+      }
+      pieces.push(cur.stdout);
+    } catch (e) {
+      console.warn(`[cpu] kubectl logs ${p.name} threw: ${(e as Error).message} — skipping this pod`);
+      failedPods++;
+    }
+  }
+  await Deno.writeTextFile(outPath, pieces.join(""));
+  const tag = restartedPodsWithPrev > 0
+    ? ` (recovered --previous logs from ${restartedPodsWithPrev} restarted pod(s))`
+    : "";
+  const failTag = failedPods > 0 ? ` — ${failedPods}/${pods.length} pod(s) FAILED, report will have partial data` : "";
+  console.log(`[cpu] CPU samples captured -> ${outPath}${tag}${failTag}`);
+}
+
+export async function removeCpuSampler(prov: MinikubeProvisioner): Promise<void> {
+  await prov.kubectl([
+    "-n", NAMESPACE,
+    "delete", "daemonset", DS_NAME,
+    "--ignore-not-found",
+  ]);
+  await prov.kubectl([
+    "-n", NAMESPACE,
+    "delete", "configmap", DS_NAME,
+    "--ignore-not-found",
+  ]);
+}

benchmarks/sim/dashboard.ts

@@ -0,0 +1,356 @@
+// Stitches per-metric chart SVGs (rendered by graph.ts:drawGraphMulti) into a
+// single dashboard SVG with a Windmill-branded header. No new chart rendering
+// — we just position the existing SVG outputs inside one outer document.
+//
+// Brand applied to the wrapper only (background, fonts, header colors). The
+// embedded charts keep their existing line-color cycle for now.
+
+import { drawGraphMulti } from "../graph.ts";
+
+// Multi-series data point — matches the shape graph.ts:drawGraphMulti consumes.
+// Each entry is one (series, sample) point — `kind` groups points into lines.
+export type DataPointMulti = {
+  value: number;
+  date: Date;
+  kind: string;
+};
+
+// Brand tokens, light mode. Extracted from frontend/brand-guidelines.md so we
+// don't have to import the frontend Tailwind config.
+const BRAND = {
+  surface_primary: "#fbfbfd",
+  surface_tertiary: "#ffffff",
+  border_light: "#e5e7eb",
+  text_emphasis: "#1d2430",
+  text_primary: "#3d4758",
+  text_secondary: "#718096",
+  text_hint: "#8d93a1",
+  font: "Inter, ui-sans-serif, system-ui, -apple-system, Segoe UI, sans-serif",
+  font_mono: "ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace",
+};
+
+export type DashboardMeta = {
+  topology: string;
+  suite: string;
+  generated: string;       // ISO timestamp
+  walltime_s: number;
+  jobs_completed: number;
+  throughput_per_s: number;
+  // Free-form summary rows shown in the header. Use sentence case.
+  summary?: Array<{ label: string; value: string }>;
+};
+
+export type DashboardPanel = {
+  title: string;
+  yLabel: string;
+  // Time-series data — rendered via drawGraphMulti (one line per `kind`).
+  // Pass `[]` if you're providing a pre-rendered svg instead.
+  data: DataPointMulti[];
+  // Optional pre-rendered SVG (e.g. from drawBars for distributions). When
+  // present, used as-is instead of calling drawGraphMulti.
+  svg?: string;
+  // Optional dashed vertical lines drawn on the chart. Used by phased benches
+  // to mark phase boundaries on every time-series panel. Either Date[] (lines
+  // + labels) or { dates, hideLabels: true } to draw lines without P1>P2
+  // text — only the primary chart (Throughput) labels them; other charts
+  // get bare lines so the dashboard isn't visually noisy.
+  verticalLines?: Date[] | { dates: Date[]; hideLabels?: boolean };
+  // Optional dashed horizontal reference lines (e.g. CPU 100% ceiling).
+  horizontalLines?: { y: number; label: string }[];
+  // Optional translucent shaded rectangles spanning [from, to] on the x-axis
+  // (e.g. "push window"). Drawn UNDER the data lines.
+  shadedZones?: { from: Date; to: Date; fill?: string; label?: string }[];
+  // Optional phase grouping. Panels with the same `phaseGroup.index` are
+  // pulled out of the general 2-col grid and rendered together inside a
+  // boxed section labeled with `phaseGroup.label`. `cols` overrides the
+  // default 3-per-row layout for this section (e.g. Node memory uses 2).
+  phaseGroup?: { index: number; label: string; cols?: number };
+  // How many grid columns this panel spans. Default 1; the Node memory
+  // time-series uses 2 so it gets the full first row of its section.
+  colSpan?: number;
+  // Optional list of area fills drawn behind the lines (back→front order).
+  // Each entry tints the area under a series whose `kind` matches exactly.
+  // Used by the per-node util panels to layer oversaturation behind CPU util.
+  areaFills?: { kind: string; color: string; opacity?: number }[];
+  // Optional y-axis cap. When set, fixes the chart y-axis to [0, yMax]
+  // instead of letting drawGraphMulti auto-scale.
+  yMax?: number;
+  // Optional per-kind line color override (matches drawGraphMulti's param).
+  lineColorOverrides?: Record<string, string>;
+};
+
+// drawGraphMulti returns a full <svg>...</svg> document. To embed it inside
+// our outer SVG we strip the outer tag and keep the inner content. The inner
+// content is already wrapped in a transformed <g>, so we just need the
+// dimensions to lay panels out.
+function extractSvgInner(svg: string): { inner: string; width: number; height: number } {
+  const m = svg.match(/<svg[^>]*\swidth="(\d+)"[^>]*\sheight="(\d+)"[^>]*>([\s\S]*)<\/svg>\s*$/);
+  if (!m) {
+    return { inner: svg, width: 530, height: 250 };
+  }
+  return { inner: m[3], width: parseInt(m[1], 10), height: parseInt(m[2], 10) };
+}
+
+// Shared relative-time origin for the whole dashboard (epoch ms). Read from
+// meta.json's bench_start_ms — used as the "0s" reference on every panel's
+// x-axis so the same x position means the same moment across charts.
+export type DashboardOptions = { xRelativeOriginMs?: number };
+
+export function renderDashboard(
+  meta: DashboardMeta,
+  panels: DashboardPanel[],
+  opts: DashboardOptions = {},
+): string {
+  const panelsWithData = panels.filter((p) => p.data.length > 0 || p.svg);
+
+  const PADDING = 40;       // outer margin
+  const COLS = 2;
+  const COL_GAP = 48;
+  const ROW_GAP = 40;
+  const PHASE_BOX_PAD = 40;   // inner padding inside a phase box — bumped from 20
+                              // so rotated x-axis labels on bar charts (drawBars
+                              // uses text-anchor:end transform:rotate(-45) which
+                              // can extend past the panel SVG's declared height/width)
+                              // don't touch the box border.
+  const PHASE_LABEL_H = 56;   // height reserved for phase header text — two
+                              // lines (heading + subtitle) when the label
+                              // contains " — "; single line otherwise.
+  const PHASE_BOX_GAP = 32;   // vertical gap between phase boxes
+  // Header height grows with the summary list. Base block (title + suite +
+  // generated + headline metrics) takes ~80px; each summary row adds 18px.
+  const summaryCount = (meta.summary ?? []).length;
+  const HEADER_H = Math.max(130, 80 + summaryCount * 18 + 24);
+
+  // Split panels: general 2-col grid above, phase-grouped boxes below. Both
+  // share the same panelW/panelH so all charts align visually.
+  const generalPanels = panelsWithData.filter((p) => !p.phaseGroup);
+  const phasedPanels  = panelsWithData.filter((p) => !!p.phaseGroup);
+
+  // Pre-render all panels (general + phased) up front — we need their widths
+  // to compute the dashboard total width before we can lay them out.
+  const allRendered = panelsWithData.map((p) => {
+    const svg = p.svg ?? drawGraphMulti(p.data, p.title, p.yLabel, p.yMax, p.verticalLines, p.horizontalLines, p.shadedZones, undefined, p.areaFills, p.lineColorOverrides, opts.xRelativeOriginMs);
+    return { ...extractSvgInner(svg), title: p.title, phaseGroup: p.phaseGroup };
+  });
+  const renderedGeneral = allRendered.filter((r) => !r.phaseGroup);
+  const renderedPhased  = allRendered.filter((r) =>  r.phaseGroup);
+
+  // Two separate "slot widths" — the general grid (with throughput/CPU/etc.
+  // time-series at 1060px) inflates panelW for the top section, but per-phase
+  // distribution panels (bars/donut, 640-690px) shouldn't be forced to reserve
+  // 1060 each just because some OTHER panel is that wide. Computing each
+  // separately keeps the dashboard from being absurdly wider than its content.
+  const generalPanelW = Math.max(530, ...renderedGeneral.map((r) => r.width));
+  const panelH = Math.max(250, ...allRendered.map((r) => r.height));
+
+  // Group phased panels by index → ordered phase sections (each section will
+  // render as a single boxed row containing N side-by-side panels). The slot
+  // width for a section is the MAX width of its own panels — bar charts get
+  // bar-chart slot widths, not the global time-series slot width.
+  const phaseByIndex = new Map<number, { label: string; panels: typeof renderedPhased; slotW: number }>();
+  for (const r of renderedPhased) {
+    const pg = r.phaseGroup!;
+    let entry = phaseByIndex.get(pg.index);
+    if (!entry) {
+      entry = { label: pg.label, panels: [], slotW: 0 };
+      phaseByIndex.set(pg.index, entry);
+    }
+    entry.panels.push(r);
+    if (r.width > entry.slotW) entry.slotW = r.width;
+  }
+  const phaseSections = [...phaseByIndex.entries()]
+    .sort(([a], [b]) => a - b)
+    .map(([_, v]) => v);
+
+  const generalRows = Math.ceil(renderedGeneral.length / COLS);
+
+  // Per-section column count (default 3, overridable via phaseGroup.cols)
+  // + per-panel colSpan support so a section can be e.g. 1 wide hero +
+  // 2-per-row beneath (Node memory section uses this).
+  const DEFAULT_COLS = 3;
+  const sectionColsOf = (s: typeof phaseSections[number]): number => {
+    for (const r of s.panels) {
+      const pg = r.phaseGroup;
+      if (pg?.cols !== undefined) return pg.cols;
+    }
+    return Math.min(s.panels.length, DEFAULT_COLS);
+  };
+  const colSpansOf = (s: typeof phaseSections[number]): number[] =>
+    s.panels.map((p) => Math.max(1, p.colSpan ?? 1));
+  // Compute grid row count taking colSpan into account.
+  const sectionRowCountOf = (s: typeof phaseSections[number]): number => {
+    const cols = sectionColsOf(s);
+    let row = 0;
+    let usedInRow = 0;
+    for (const span of colSpansOf(s)) {
+      if (usedInRow + span > cols) {
+        row++;
+        usedInRow = span;
+      } else {
+        usedInRow += span;
+      }
+    }
+    return row + 1;
+  };
+  const sectionRowWs = phaseSections.map((s) => {
+    const cols = sectionColsOf(s);
+    return cols * s.slotW + (cols - 1) * COL_GAP + PHASE_BOX_PAD * 2;
+  });
+  const generalGridW = COLS * generalPanelW + (COLS - 1) * COL_GAP;
+  const contentW = Math.max(generalGridW, ...sectionRowWs, 0);
+  const totalW = PADDING * 2 + contentW;
+  const sectionHeights = phaseSections.map((s) => {
+    const rows = sectionRowCountOf(s);
+    return PHASE_LABEL_H + rows * panelH + (rows - 1) * ROW_GAP + PHASE_BOX_PAD * 2;
+  });
+  const phaseSectionH = phaseSections.length > 0
+    ? sectionHeights.reduce((a, h) => a + h, 0)
+      + (phaseSections.length - 1) * PHASE_BOX_GAP
+      + (renderedGeneral.length > 0 ? ROW_GAP : 0)
+    : 0;
+  const totalH = PADDING * 2 + HEADER_H + 16
+    + (generalRows > 0 ? generalRows * panelH + (generalRows - 1) * ROW_GAP : 0)
+    + phaseSectionH;
+
+  // --- Header ---
+  const summaryRows = meta.summary ?? [];
+  const summaryX0 = PADDING;
+  const summaryY0 = PADDING + 64; // below the title block
+  const summaryLineH = 18;
+
+  // Title + suite path + generated timestamp on the left,
+  // headline metrics on the right.
+  const headerSvg = `
+    <rect x="0" y="0" width="${totalW}" height="${totalH}" fill="${BRAND.surface_primary}"/>
+    <rect x="${PADDING}" y="${PADDING}" width="${totalW - PADDING * 2}" height="${HEADER_H}"
+          fill="${BRAND.surface_tertiary}" stroke="${BRAND.border_light}" stroke-width="1"/>
+    <text x="${PADDING + 20}" y="${PADDING + 32}"
+          font-family="${BRAND.font}" font-size="18" font-weight="600"
+          fill="${BRAND.text_emphasis}">Sim run: ${escapeText(meta.topology)}</text>
+    <text x="${PADDING + 20}" y="${PADDING + 52}"
+          font-family="${BRAND.font_mono}" font-size="11"
+          fill="${BRAND.text_secondary}">${escapeText(meta.suite)}</text>
+    <text x="${PADDING + 20}" y="${PADDING + 68}"
+          font-family="${BRAND.font}" font-size="11"
+          fill="${BRAND.text_hint}">${escapeText(meta.generated)}</text>
+
+    ${headlineMetric(totalW - PADDING - 320, PADDING + 32, "Wall time", `${meta.walltime_s.toFixed(2)}s`)}
+    ${headlineMetric(totalW - PADDING - 220, PADDING + 32, "Jobs done", String(meta.jobs_completed))}
+    ${headlineMetric(totalW - PADDING - 100, PADDING + 32, "Throughput", `${meta.throughput_per_s.toFixed(1)}/s`)}
+
+    ${summaryRows.map((row, i) => `
+      <text x="${summaryX0 + 20}" y="${summaryY0 + 18 + i * summaryLineH}"
+            font-family="${BRAND.font}" font-size="12" font-weight="600"
+            fill="${BRAND.text_emphasis}">${escapeText(row.label)}:</text>
+      <text x="${summaryX0 + 140}" y="${summaryY0 + 18 + i * summaryLineH}"
+            font-family="${BRAND.font_mono}" font-size="11"
+            fill="${BRAND.text_primary}">${escapeText(row.value)}</text>
+    `).join("\n")}
+  `;
+
+  // --- General panels (2-col grid, using time-series slot width) ---
+  const generalY0 = PADDING + HEADER_H + 16;
+  const generalSvg = renderedGeneral.map((p, i) => {
+    const col = i % COLS;
+    const row = Math.floor(i / COLS);
+    const x = PADDING + col * (generalPanelW + COL_GAP);
+    const y = generalY0 + row * (panelH + ROW_GAP);
+    return `<g transform="translate(${x}, ${y})">${p.inner}</g>`;
+  }).join("\n");
+
+  // --- Phase sections (one boxed row per phase, no wrap) ---
+  // Cycle through subtle pastel backgrounds so adjacent phases are
+  // visually distinguishable; user explicitly asked for "different
+  // background / outlines" so the eye can compare phases at a glance.
+  const PHASE_PALETTE = [
+    { fill: "#f0f6ff", stroke: "#bcd5ff" }, // soft blue
+    { fill: "#fff7ed", stroke: "#fdba74" }, // soft orange
+    { fill: "#f0fdf4", stroke: "#86efac" }, // soft green
+    { fill: "#fdf4ff", stroke: "#e9d5ff" }, // soft violet
+    { fill: "#fff1f2", stroke: "#fda4af" }, // soft rose
+  ];
+  const phasesY0 = generalY0
+    + (generalRows > 0 ? generalRows * panelH + (generalRows - 1) * ROW_GAP + ROW_GAP : 0);
+  const phasesSvg = phaseSections.map((section, si) => {
+    const palette = PHASE_PALETTE[si % PHASE_PALETTE.length];
+    const boxY = phasesY0
+      + sectionHeights.slice(0, si).reduce((a, h) => a + h + PHASE_BOX_GAP, 0);
+    const phaseBoxH = sectionHeights[si];
+    const slotW = section.slotW;
+    const cols = sectionColsOf(section);
+    const rowW = cols * slotW + (cols - 1) * COL_GAP;
+    const boxX = PADDING;
+    const boxW = contentW;
+    const innerX0 = boxX + Math.max(PHASE_BOX_PAD, (boxW - rowW) / 2);
+    // Lay out with colSpan awareness: panel with colSpan=2 takes two grid
+    // slots (slotW + COL_GAP + slotW) and a single-span panel takes one.
+    // Wrap to next row when current row's used cols would overflow.
+    const spans = colSpansOf(section);
+    let row = 0;
+    let usedInRow = 0;
+    const panelsRow = section.panels.map((p, i) => {
+      const span = spans[i];
+      if (usedInRow + span > cols) {
+        row++;
+        usedInRow = 0;
+      }
+      const col = usedInRow;
+      const x = innerX0 + col * (slotW + COL_GAP);
+      const y = boxY + PHASE_LABEL_H + PHASE_BOX_PAD + row * (panelH + ROW_GAP);
+      usedInRow += span;
+      return `<g transform="translate(${x}, ${y})">${p.inner}</g>`;
+    }).join("\n");
+    return `
+      <rect x="${boxX}" y="${boxY}" width="${boxW}" height="${phaseBoxH}"
+            rx="10" ry="10"
+            fill="${palette.fill}" stroke="${palette.stroke}" stroke-width="1.5"/>
+      ${renderSectionLabel(section.label, boxX + PHASE_BOX_PAD, boxY + 22)}
+      ${panelsRow}
+    `;
+  }).join("\n");
+
+  const panelsSvg = `${generalSvg}\n${phasesSvg}`;
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="${totalW}" height="${totalH}" viewBox="0 0 ${totalW} ${totalH}">
+${headerSvg}
+${panelsSvg}
+</svg>`;
+}
+
+// Section header that breaks a "Foo — bar baz qux" label into two lines
+// (heading + a slightly smaller subtitle). Single-line labels render
+// unchanged. Wrapping keeps long section labels from overflowing the box
+// width without needing dynamic measurement.
+function renderSectionLabel(label: string, x: number, y: number): string {
+  const sep = " — ";
+  const idx = label.indexOf(sep);
+  if (idx < 0) {
+    return `<text x="${x}" y="${y}" font-family="${BRAND.font}" font-size="14"
+      font-weight="600" fill="${BRAND.text_emphasis}">${escapeText(label)}</text>`;
+  }
+  const heading = label.slice(0, idx);
+  const subtitle = label.slice(idx + sep.length);
+  return `
+    <text x="${x}" y="${y}" font-family="${BRAND.font}" font-size="14"
+          font-weight="600" fill="${BRAND.text_emphasis}">${escapeText(heading)}</text>
+    <text x="${x}" y="${y + 18}" font-family="${BRAND.font}" font-size="12"
+          font-weight="500" fill="${BRAND.text_primary}">${escapeText(subtitle)}</text>
+  `;
+}
+
+function headlineMetric(x: number, y: number, label: string, value: string): string {
+  return `
+    <text x="${x}" y="${y}" font-family="${BRAND.font}" font-size="11"
+          fill="${BRAND.text_secondary}">${escapeText(label)}</text>
+    <text x="${x}" y="${y + 22}" font-family="${BRAND.font_mono}" font-size="18" font-weight="600"
+          fill="${BRAND.text_emphasis}">${escapeText(value)}</text>
+  `;
+}
+
+function escapeText(s: string): string {
+  return String(s)
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;");
+}

benchmarks/sim/failed_jobs.ts

@@ -0,0 +1,128 @@
+// Failed-jobs timeline collector. Called at end-of-bench: pages through
+// Windmill's completed-jobs list filtered to `success=false` and within the
+// bench's wall window, writes a JSONL timeline that the renderer plots as a
+// "Failed jobs over time" panel.
+//
+// Output (one JSON object per line):
+//   {"ts_ms": 1780573...0, "success": false, "job_id": "...", "kind": "..."}
+//
+// Why we query the API at end-of-bench instead of sampling during: list
+// endpoints are slow under heavy bench load and would contend with the worker
+// pull queries. One pass at the end is cheaper and complete.
+
+export type FailedJob = {
+  ts_ms: number;
+  success: boolean;
+  job_id?: string;
+  kind?: string;
+  // Categorised failure mode so the renderer can bin them on a "failures
+  // by type" bar chart. Values:
+  //   "sigkill_137"      — deno subprocess SIGKILLed (worker heartbeat
+  //                        reaper or external SIGKILL — exit_code 137)
+  //   "oom_oomkilled"    — container/process explicitly marked OOMKilled
+  //   "timeout"          — job timeout from app side
+  //   "script_error"     — actual script-level exception
+  //   "unknown"          — couldn't categorise
+  category?: string;
+  exit_code?: number;
+  mem_peak_mb?: number;
+};
+
+type ApiJob = {
+  id: string;
+  success: boolean | null;
+  // Various candidates depending on EE/OSS endpoint version; we pick whichever
+  // is present.
+  ended_at?: string;
+  completed_at?: string;
+  duration_ms?: number;
+  created_at?: string;
+  job_kind?: string;
+  kind?: string;
+  mem_peak?: number;
+  result?: {
+    error?: {
+      name?: string;
+      message?: string;
+      exit_code?: number;
+    };
+  };
+};
+
+function jobTs(j: ApiJob): number {
+  for (const k of ["ended_at", "completed_at", "created_at"] as const) {
+    const v = j[k];
+    if (typeof v === "string") {
+      const ms = Date.parse(v);
+      if (Number.isFinite(ms)) return ms;
+    }
+  }
+  return 0;
+}
+
+export async function collectFailedJobs(
+  opts: {
+    host: string;
+    token: string;
+    workspace: string;
+    sinceMs: number;
+    outPath: string;
+  },
+): Promise<void> {
+  const { host, token, workspace, sinceMs, outPath } = opts;
+  // ISO 8601 since-time for the API filter — minus 1s for clock skew between
+  // bench host and cluster.
+  const sinceIso = new Date(sinceMs - 1000).toISOString();
+  const headers = { Authorization: `Bearer ${token}` };
+  const out: FailedJob[] = [];
+
+  // Paginate. Windmill's list_completed supports `success=false`,
+  // `created_after`, page/per_page. Cap pages at 200 (= 200k jobs) as a
+  // sanity guard — a heavy bench shouldn't get near that.
+  for (let page = 1; page <= 200; page++) {
+    // Correct endpoint is `/api/w/{ws}/jobs/completed/list` — the `jobs_u`
+    // routes only have per-id getters, not a list. 404 from the old path is
+    // what silently produced "0 failures" in prior reports even when the
+    // bench actually had a wave of failed jobs.
+    const u = new URL(`${host}/api/w/${workspace}/jobs/completed/list`);
+    u.searchParams.set("page", String(page));
+    u.searchParams.set("per_page", "1000");
+    u.searchParams.set("success", "false");
+    u.searchParams.set("created_after", sinceIso);
+    const res = await fetch(u, { headers });
+    if (!res.ok) {
+      console.warn(`[failed-jobs] page ${page} HTTP ${res.status} — stopping`);
+      break;
+    }
+    const arr = (await res.json()) as ApiJob[];
+    if (!Array.isArray(arr) || arr.length === 0) break;
+    for (const j of arr) {
+      const err = j.result?.error;
+      const msg = err?.message ?? "";
+      const exit = err?.exit_code;
+      let category = "unknown";
+      if (exit === 137 || /signal:?\s*9|SIGKILL/i.test(msg)) category = "sigkill_137";
+      else if (/OOMKilled|out of memory/i.test(msg)) category = "oom_oomkilled";
+      else if (/timeout|did not receive .* ping/i.test(msg)) category = "timeout";
+      else if (err?.name === "ExecutionErr" && exit !== undefined && exit !== 0) category = "exit_" + exit;
+      else if (err) category = "script_error";
+      out.push({
+        ts_ms: jobTs(j),
+        success: !!j.success,
+        job_id: j.id,
+        kind: j.job_kind ?? j.kind,
+        category,
+        exit_code: exit,
+        mem_peak_mb: j.mem_peak,
+      });
+    }
+    if (arr.length < 1000) break;
+  }
+
+  out.sort((a, b) => a.ts_ms - b.ts_ms);
+  await Deno.writeTextFile(
+    outPath,
+    out.map((r) => JSON.stringify(r)).join("\n") + (out.length ? "\n" : ""),
+  );
+  console.log(`[failed-jobs] ${out.length} failures captured -> ${outPath}`);
+}

benchmarks/sim/helm_deploy.ts

@@ -0,0 +1,243 @@
+// Helm deploy of Windmill onto a provisioned minikube cluster, + API URL
+// resolution via port-forward. Part of the k8s sim path (task #49).
+//
+// Chart source (`chart`):
+//   - omitted            -> the public remote chart "windmill/windmill"
+//                           (repo auto-added)
+//   - a local path       -> used directly (e.g. ../windmill-helm-charts/charts/windmill)
+//   - "windmill/<name>"  -> remote; the windmill repo is auto-added
+//   - any other "repo/x" -> used as-is (assumes the repo is already configured)
+//
+// PG: the chart bundles Postgres (postgresql.enabled, via cloudnative-pg). API:
+// the chart exposes a ClusterIP Service `windmill-app` on port 8000; we
+// port-forward it to localhost so the existing bench can hit it over http.
+
+import { parse as yamlParse } from "https://deno.land/std@0.224.0/yaml/mod.ts";
+
+const HELM = Deno.env.get("SIM_HELM_BIN") ?? "helm";
+const MINIKUBE = Deno.env.get("SIM_MINIKUBE_BIN") ?? "minikube";
+const WINDMILL_REPO = "https://windmill-labs.github.io/windmill-helm-charts/";
+const APP_SERVICE = "windmill-app";
+const APP_PORT = 8000;
+
+export type HelmDeployOptions = {
+  profile: string; // minikube profile == kube context
+  chart?: string; // local path or "repo/name"; default = vendored chart at
+                  // benchmarks/sim/charts/windmill (forked from upstream
+                  // 4.0.169 with `priorityClassName`, PG `maxConnections`,
+                  // and `priorityClasses.classes[]` exposed via values).
+  release?: string; // default "wm"
+  namespace?: string; // default "default"
+  set?: string[]; // extra --set key=value
+  valuesFiles?: string[]; // extra -f values.yaml
+  timeoutSec?: number; // helm --wait timeout; default 600
+};
+
+// minikube needs the kvm2 driver dir on PATH + LD_LIBRARY_PATH for libvirt
+// (see k8s_provisioner.ts). `minikube kubectl` inherits it; helm doesn't need it.
+function minikubeEnv(): Record<string, string> {
+  const base = Deno.env.toObject();
+  const driverDir = Deno.env.get("SIM_KVM2_DRIVER_DIR");
+  const libDir = Deno.env.get("SIM_LIBVIRT_LIB_DIR");
+  if (driverDir) base.PATH = `${driverDir}:${base.PATH ?? ""}`;
+  if (libDir) base.LD_LIBRARY_PATH = base.LD_LIBRARY_PATH ? `${libDir}:${base.LD_LIBRARY_PATH}` : libDir;
+  return base;
+}
+
+async function run(
+  bin: string,
+  args: string[],
+  { check = true, env }: { check?: boolean; env?: Record<string, string> } = {},
+): Promise<{ stdout: string; stderr: string; code: number }> {
+  const p = new Deno.Command(bin, { args, env, stdout: "piped", stderr: "piped" });
+  const { code, stdout, stderr } = await p.output();
+  const out = new TextDecoder().decode(stdout);
+  const err = new TextDecoder().decode(stderr);
+  if (check && code !== 0) {
+    throw new Error(`${bin} ${args.join(" ")} failed (code ${code}):\n${err || out}`);
+  }
+  return { stdout: out, stderr: err, code };
+}
+
+async function pathExists(p: string): Promise<boolean> {
+  try {
+    await Deno.stat(p);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function pickFreePort(): Promise<number> {
+  const l = Deno.listen({ port: 0 });
+  const port = (l.addr as Deno.NetAddr).port;
+  l.close();
+  return port;
+}
+
+// Parse a local chart's Chart.yaml, register each dep's helm repo. Idempotent
+// (`helm repo add --force-update` overwrites). Skips oci:// URLs (helm fetches
+// those directly without `repo add`).
+async function registerLocalChartDepRepos(chartDir: string): Promise<void> {
+  let raw: string;
+  try {
+    raw = await Deno.readTextFile(`${chartDir}/Chart.yaml`);
+  } catch (e) {
+    console.warn(`[helm] could not read ${chartDir}/Chart.yaml: ${(e as Error).message}`);
+    return;
+  }
+  let parsed: unknown;
+  try { parsed = yamlParse(raw); } catch (e) {
+    console.warn(`[helm] Chart.yaml parse failed: ${(e as Error).message}`);
+    return;
+  }
+  const deps = (parsed as { dependencies?: Array<{ name?: string; repository?: string }> })
+    .dependencies ?? [];
+  const seen = new Set<string>();
+  for (const d of deps) {
+    const url = d.repository;
+    if (!url) continue;
+    if (!/^https?:\/\//i.test(url)) continue;
+    if (seen.has(url)) continue;
+    seen.add(url);
+    // Use the dep's chart name as the repo alias. Helm allows any alias.
+    const alias = d.name ?? new URL(url).hostname.replace(/[^a-z0-9]/g, "");
+    console.log(`[helm]   repo add ${alias} ${url}`);
+    await run(HELM, ["repo", "add", alias, url, "--force-update"], { check: false });
+  }
+  await run(HELM, ["repo", "update"], { check: false });
+}
+
+// helm install Windmill (bundled PG on). Waits for the release to be ready.
+export async function helmDeployWindmill(opts: HelmDeployOptions): Promise<void> {
+  // Default to the in-repo vendored chart so sim runs are reproducible without
+  // depending on upstream chart version drift (we hit 4.0.165 → 4.0.166 silent
+  // breakage during this project; the vendored copy is our pinned baseline).
+  const chart = opts.chart ?? new URL("./charts/windmill", import.meta.url).pathname;
+  const release = opts.release ?? "wm";
+  const namespace = opts.namespace ?? "default";
+
+  const isLocalPath = chart.startsWith(".") || chart.startsWith("/") || (await pathExists(chart));
+  if (!isLocalPath && chart.startsWith("windmill/")) {
+    // Idempotent: `repo add` errors if it exists with a different URL; --force-update avoids that.
+    await run(HELM, ["repo", "add", "windmill", WINDMILL_REPO, "--force-update"], { check: false });
+    await run(HELM, ["repo", "update", "windmill"], { check: false });
+  }
+  if (isLocalPath) {
+    // The chart has subchart dependencies declared in Chart.yaml. For a remote
+    // chart helm fetches them at pull/install; for a LOCAL chart path we must
+    // (a) register each dependency's repository with helm (it errors otherwise
+    // with "no repository definition for ..."), and (b) `helm dependency build`
+    // to vendor the subcharts into the local chart's ./charts/ dir.
+    console.log(`[helm] resolving chart dependencies for local chart at ${chart}`);
+    await registerLocalChartDepRepos(chart);
+    await run(HELM, ["dependency", "build", chart]);
+  }
+
+  // `upgrade --install`: install when absent, upgrade (reconcile) when
+  // present. Required for the cluster-reuse path — bare `install` errors with
+  // "cannot re-use a name that is still in use" on a second run.
+  //
+  // No `--wait`. With many workers the chart's default --wait blocks until
+  // EVERY deployment is Ready, which deadlocks if workers crashloop on the
+  // default PG max_connections (we bump that AFTER helm in main.ts). The
+  // caller waits for PG + windmill-app explicitly via kubectl wait, which is
+  // both faster and avoids the worker-readiness lock-step.
+  const args = [
+    "--kube-context", opts.profile,
+    "upgrade", "--install", release, chart,
+    "--namespace", namespace, "--create-namespace",
+    "--set", "postgresql.enabled=true",
+    "--timeout", `${opts.timeoutSec ?? 600}s`,
+  ];
+  for (const s of opts.set ?? []) args.push("--set", s);
+  for (const f of opts.valuesFiles ?? []) args.push("-f", f);
+
+  console.log(`[helm] upgrade --install Windmill (release=${release}, chart=${chart}, ns=${namespace})`);
+  await run(HELM, args);
+  console.log(`[helm] manifests applied — caller waits for PG + app readiness`);
+}
+
+// Wait for the bundled PG (StatefulSet or Deployment) to report Ready. Used
+// after `helmDeployWindmill` so the PG-bound steps that follow (max_conn
+// patch, verbose logging) can run.
+export async function waitPgReady(
+  profile: string,
+  namespace = "default",
+  timeoutSec = 300,
+): Promise<void> {
+  // Probe both — only one of these will exist depending on persistence flag.
+  for (const ref of ["statefulset/windmill-postgresql", "deployment/windmill-postgresql-demo-app"]) {
+    const r = await run(
+      MINIKUBE,
+      [
+        "kubectl", "-p", profile, "--",
+        "-n", namespace, "rollout", "status", ref,
+        `--timeout=${timeoutSec}s`,
+      ],
+      { check: false },
+    );
+    if (r.code === 0) {
+      console.log(`[helm] PG ready (${ref})`);
+      return;
+    }
+  }
+  throw new Error("[helm] PG not found or not ready (neither statefulset nor deployment)");
+}
+
+// Wait for windmill-app Deployment to report at least 1 Ready replica.
+export async function waitWindmillAppReady(
+  profile: string,
+  namespace = "default",
+  timeoutSec = 300,
+): Promise<void> {
+  const r = await run(
+    MINIKUBE,
+    [
+      "kubectl", "-p", profile, "--",
+      "-n", namespace, "rollout", "status", "deployment/windmill-app",
+      `--timeout=${timeoutSec}s`,
+    ],
+    { check: false },
+  );
+  if (r.code !== 0) {
+    throw new Error(`[helm] windmill-app not ready after ${timeoutSec}s`);
+  }
+  console.log(`[helm] windmill-app ready`);
+}
+
+export type ApiEndpoint = { host: string; stop: () => void };
+
+// Port-forward svc/windmill-app:8000 to a free localhost port and wait until
+// the API answers. Returns the URL + a stop() to kill the forward.
+export async function portForwardApi(profile: string, namespace = "default"): Promise<ApiEndpoint> {
+  const localPort = await pickFreePort();
+  const child = new Deno.Command(MINIKUBE, {
+    args: [
+      "kubectl", "-p", profile, "--",
+      "port-forward", "-n", namespace, `svc/${APP_SERVICE}`, `${localPort}:${APP_PORT}`,
+    ],
+    env: minikubeEnv(),
+    stdout: "piped",
+    stderr: "piped",
+  }).spawn();
+
+  const host = `http://127.0.0.1:${localPort}`;
+  const deadline = Date.now() + 60_000;
+  let lastErr = "";
+  while (Date.now() < deadline) {
+    try {
+      const r = await fetch(`${host}/api/version`, { signal: AbortSignal.timeout(2000) });
+      if (r.ok) {
+        await r.body?.cancel();
+        console.log(`[helm] API reachable at ${host} (port-forward svc/${APP_SERVICE})`);
+        return { host, stop: () => { try { child.kill("SIGTERM"); } catch { /* already gone */ } } };
+      }
+    } catch (e) {
+      lastErr = (e as Error).message;
+    }
+    await new Promise((r) => setTimeout(r, 1000));
+  }
+  try { child.kill("SIGTERM"); } catch { /* ignore */ }
+  throw new Error(`[helm] port-forward to svc/${APP_SERVICE} never became reachable (last: ${lastErr})`);
+}

benchmarks/sim/image_cache.ts

@@ -0,0 +1,134 @@
+// Host-side cache for container images that the cluster pulls.
+//
+// Cluster containers run inside the minikube VM, and `minikube delete` destroys
+// the VM's container runtime — next bringup re-pulls everything (windmill
+// alone is ~3.85 GB per node). We work around that with two hooks:
+//   - on bringup (after `minikube start`, before `helm install`): for every
+//     `.tar` in the host cache dir, `minikube image load <tar>` into the new
+//     VM(s).
+//   - on teardown (before `minikube delete`): `minikube image save <img> <tar>`
+//     for each workload image currently in the cluster, skipping ones already
+//     cached.
+//
+// Net effect: first run pulls and caches; subsequent runs reuse the cache,
+// trading a multi-GB network pull for a ~30 s disk-to-VM load per node.
+//
+// Default cache dir: $XDG_CACHE_HOME/wm-sim/images (~/.cache/wm-sim/images).
+
+const MINIKUBE = Deno.env.get("SIM_MINIKUBE_BIN") ?? "minikube";
+
+// System images we don't cache — they're small, kube-system-only, and minikube
+// itself often pre-stages them via its ISO/preload bundle.
+const SYSTEM_PREFIXES = [
+  "registry.k8s.io/",
+  "gcr.io/k8s-minikube/",
+  "docker.io/flannel/",
+  "k8s.gcr.io/",
+  "kindest/",
+];
+
+function defaultCacheDir(): string {
+  const xdg = Deno.env.get("XDG_CACHE_HOME");
+  const home = Deno.env.get("HOME") ?? ".";
+  return `${xdg ?? `${home}/.cache`}/wm-sim/images`;
+}
+
+// Same env shape the provisioner uses — minikube needs the kvm2 driver dir on
+// PATH + LD_LIBRARY_PATH for libvirt.
+function minikubeEnv(): Record<string, string> {
+  const base = Deno.env.toObject();
+  const driverDir = Deno.env.get("SIM_KVM2_DRIVER_DIR");
+  const libDir = Deno.env.get("SIM_LIBVIRT_LIB_DIR");
+  if (driverDir) base.PATH = `${driverDir}:${base.PATH ?? ""}`;
+  if (libDir) {
+    base.LD_LIBRARY_PATH = base.LD_LIBRARY_PATH
+      ? `${libDir}:${base.LD_LIBRARY_PATH}`
+      : libDir;
+  }
+  return base;
+}
+
+async function runMinikube(
+  args: string[],
+): Promise<{ code: number; stdout: string; stderr: string }> {
+  const p = new Deno.Command(MINIKUBE, {
+    args,
+    env: minikubeEnv(),
+    stdout: "piped",
+    stderr: "piped",
+  });
+  const { code, stdout, stderr } = await p.output();
+  return {
+    code,
+    stdout: new TextDecoder().decode(stdout),
+    stderr: new TextDecoder().decode(stderr),
+  };
+}
+
+function tarballName(image: string): string {
+  // ghcr.io/windmill-labs/windmill:1.711.0 -> ghcr.io_windmill-labs_windmill_1.711.0.tar
+  return image.replace(/[/:@]/g, "_") + ".tar";
+}
+
+async function pathExists(p: string): Promise<boolean> {
+  try { await Deno.stat(p); return true; } catch { return false; }
+}
+
+function isWorkloadImage(image: string): boolean {
+  return !SYSTEM_PREFIXES.some((p) => image.startsWith(p));
+}
+
+// Load every .tar in cacheDir into the cluster. Best-effort: errors are
+// logged but don't abort — a corrupted tarball shouldn't break bringup.
+export async function loadCachedImages(
+  profile: string,
+  cacheDir = defaultCacheDir(),
+): Promise<number> {
+  if (!(await pathExists(cacheDir))) return 0;
+  let loaded = 0;
+  for await (const ent of Deno.readDir(cacheDir)) {
+    if (!ent.isFile || !ent.name.endsWith(".tar")) continue;
+    const path = `${cacheDir}/${ent.name}`;
+    console.log(`[cache] loading ${ent.name}`);
+    const r = await runMinikube(["-p", profile, "image", "load", path]);
+    if (r.code === 0) loaded++;
+    else console.warn(`[cache] failed to load ${ent.name}: ${r.stderr.trim().split("\n").pop()}`);
+  }
+  if (loaded > 0) console.log(`[cache] loaded ${loaded} image(s) from ${cacheDir}`);
+  return loaded;
+}
+
+// Save each "workload" image in the running cluster to a tarball in cacheDir.
+// Skips images already cached. Discovers images via the supplied kubectl
+// runner (typically `prov.kubectl`).
+export async function saveImagesToCache(
+  profile: string,
+  kubectl: (args: string[]) => Promise<{ stdout: string; code: number }>,
+  cacheDir = defaultCacheDir(),
+): Promise<number> {
+  await Deno.mkdir(cacheDir, { recursive: true });
+
+  const r = await kubectl([
+    "get", "pods", "-A",
+    "-o", "jsonpath={range .items[*].spec.containers[*]}{.image}{\"\\n\"}{end}",
+  ]);
+  if (r.code !== 0) {
+    console.warn("[cache] could not list images — skipping save");
+    return 0;
+  }
+  const images = new Set(
+    r.stdout.split("\n").map((l) => l.trim()).filter((l) => l && isWorkloadImage(l)),
+  );
+
+  let saved = 0;
+  for (const image of images) {
+    const tarPath = `${cacheDir}/${tarballName(image)}`;
+    if (await pathExists(tarPath)) continue;
+    console.log(`[cache] saving ${image}`);
+    const s = await runMinikube(["-p", profile, "image", "save", image, tarPath]);
+    if (s.code === 0) saved++;
+    else console.warn(`[cache] failed to save ${image}: ${s.stderr.trim().split("\n").pop()}`);
+  }
+  if (saved > 0) console.log(`[cache] saved ${saved} new image(s) to ${cacheDir}`);
+  return saved;
+}

benchmarks/sim/k8s_provisioner.ts

@@ -0,0 +1,396 @@
+// minikube-backed provisioner for the k8s sim path.
+//
+// Brings up a local multi-node Kubernetes cluster from the existing topology
+// format using minikube + the kvm2 driver (VM-per-node, which sidesteps the
+// rootless-cgroup wall that blocks k3d/kind on this host). Windmill itself is
+// deployed onto the cluster via helm; toxiproxy and the measurement layer sit
+// on top. This module owns ONLY the cluster substrate: start, wait-ready,
+// teardown.
+//
+// Topology mapping (the topology is the source of truth — no --nodes flag):
+//   - Each topology node  ->  one minikube VM (substrate) node, sized from its
+//     `cpu` (whole vCPUs) and `memory`. Node count = topology.nodes.length.
+//   - Each node's `db_latency_ms` -> that node's RTT, applied via a per-node
+//     toxiproxy (workers scheduled on the node route their DB traffic through
+//     it). Handled in the toxiproxy step, not here.
+//   - The 250 workers are PODS deployed by helm and spread across these nodes
+//     by the scheduler — worker count/resources come from helm, not topology.
+//
+// kvm2 driver + libvirt resolution (flake integration deferred): the caller
+// supplies these via env (or options):
+//   SIM_KVM2_DRIVER_DIR  - dir containing `docker-machine-driver-kvm2`
+//   SIM_LIBVIRT_LIB_DIR  - dir containing `libvirt.so.0` (-> LD_LIBRARY_PATH;
+//                          the libvirt-go binding dlopens it by soname)
+//   SIM_MINIKUBE_BIN     - minikube binary (default: `minikube` on PATH)
+
+import { type Topology, type NodeSpec, parseMemory } from "./topology.ts";
+
+const MINIKUBE = Deno.env.get("SIM_MINIKUBE_BIN") ?? "minikube";
+
+export type K8sProvisionerOptions = {
+  profile?: string; // minikube profile name; default "wm-sim"
+  cni?: string; // default "flannel" — REQUIRED for cross-node pod networking
+  kubernetesVersion?: string; // pin k8s version; optional
+  driverDir?: string; // override SIM_KVM2_DRIVER_DIR
+  libvirtLibDir?: string; // override SIM_LIBVIRT_LIB_DIR
+};
+
+export type ProvisionedCluster = {
+  profile: string;
+  nodes: string[]; // node names as kubectl sees them
+};
+
+// minikube wants whole vCPUs per VM and a plain MB memory value; the topology
+// uses fractional historically and "16G"/"512M" strings. Normalise.
+function cpusOf(n: NodeSpec): number {
+  return Math.max(1, Math.round(n.cpu));
+}
+function memMbOf(n: NodeSpec): string {
+  return `${Math.round(parseMemory(n.memory) / (1024 * 1024))}mb`;
+}
+
+export class MinikubeProvisioner {
+  readonly profile: string;
+  private readonly cni: string;
+  private readonly kubernetesVersion?: string;
+  private readonly driverDir?: string;
+  private readonly libvirtLibDir?: string;
+
+  constructor(opts: K8sProvisionerOptions = {}) {
+    this.profile = opts.profile ?? "wm-sim";
+    this.cni = opts.cni ?? "flannel";
+    this.kubernetesVersion = opts.kubernetesVersion;
+    this.driverDir = opts.driverDir;
+    this.libvirtLibDir = opts.libvirtLibDir;
+  }
+
+  // Environment for every minikube invocation: prepend the kvm2 driver dir to
+  // PATH and set LD_LIBRARY_PATH so the libvirt-go dlopen resolves.
+  private env(): Record<string, string> {
+    const base = Deno.env.toObject();
+    const driverDir = this.driverDir ?? Deno.env.get("SIM_KVM2_DRIVER_DIR");
+    const libDir = this.libvirtLibDir ?? Deno.env.get("SIM_LIBVIRT_LIB_DIR");
+    if (driverDir) base.PATH = `${driverDir}:${base.PATH ?? ""}`;
+    if (libDir) {
+      base.LD_LIBRARY_PATH = base.LD_LIBRARY_PATH
+        ? `${libDir}:${base.LD_LIBRARY_PATH}`
+        : libDir;
+    }
+    return base;
+  }
+
+  private async run(
+    args: string[],
+    { check = true }: { check?: boolean } = {},
+  ): Promise<{ stdout: string; stderr: string; code: number }> {
+    const p = new Deno.Command(MINIKUBE, {
+      args,
+      env: this.env(),
+      stdout: "piped",
+      stderr: "piped",
+    });
+    const { code, stdout, stderr } = await p.output();
+    const out = new TextDecoder().decode(stdout);
+    const err = new TextDecoder().decode(stderr);
+    if (check && code !== 0) {
+      throw new Error(`minikube ${args.join(" ")} failed (code ${code}):\n${err || out}`);
+    }
+    return { stdout: out, stderr: err, code };
+  }
+
+  // `minikube kubectl -p <profile> -- <args>` — minikube's bundled kubectl, so
+  // we don't depend on a separate kubectl on PATH.
+  async kubectl(args: string[]): Promise<{ stdout: string; stderr: string; code: number }> {
+    const { stdout, stderr, code } = await this.run(
+      ["kubectl", "-p", this.profile, "--", ...args],
+      { check: false },
+    );
+    return { stdout, stderr, code };
+  }
+
+  // Best-effort: force-destroy any libvirt domains left over from a prior run
+  // that didn't tear down cleanly (SIGKILL/OOM/host reboot before `minikube
+  // delete` could run). `minikube start` fails with cryptic disk/registration
+  // errors when stale domains squat on the names. Silent no-op if virsh isn't
+  // reachable — the regular start path will surface a real failure.
+  private async sweepLingeringDomains(): Promise<void> {
+    const virsh = Deno.env.get("SIM_VIRSH_BIN") ?? "virsh";
+    const uri = ["-c", "qemu:///system"];
+    let list: { stdout: string; code: number };
+    try {
+      const p = new Deno.Command(virsh, {
+        args: [...uri, "list", "--all", "--name"],
+        stdout: "piped", stderr: "piped",
+      });
+      const out = await p.output();
+      list = { stdout: new TextDecoder().decode(out.stdout), code: out.code };
+    } catch {
+      return; // virsh not on PATH — skip silently
+    }
+    if (list.code !== 0) return;
+    const lingerers = list.stdout
+      .split("\n").map((s) => s.trim())
+      .filter((n) => n === this.profile || n.startsWith(`${this.profile}-m`));
+    if (lingerers.length === 0) return;
+    console.log(`[k8s] sweeping ${lingerers.length} lingering libvirt domain(s): ${lingerers.join(", ")}`);
+    for (const name of lingerers) {
+      // destroy = force power-off; undefine = remove definition. Don't pass
+      // --remove-all-storage — minikube's kvm2 VMs have an unmanaged hdc
+      // (boot2docker.iso) that fails the whole undefine when libvirt tries to
+      // free it. We sweep the leftover machine dir afterwards instead.
+      for (const op of [
+        ["destroy", name],
+        ["undefine", "--managed-save", "--snapshots-metadata", name],
+      ]) {
+        try {
+          await new Deno.Command(virsh, { args: [...uri, ...op], stdout: "null", stderr: "null" }).output();
+        } catch { /* best-effort */ }
+      }
+      // Drop the machine dir minikube created for this VM (boot2docker.iso +
+      // disk image + config). Best-effort — silent if it doesn't exist.
+      const home = Deno.env.get("HOME") ?? "";
+      if (home) {
+        try {
+          await Deno.remove(`${home}/.minikube/machines/${name}`, { recursive: true });
+        } catch { /* not present */ }
+      }
+    }
+  }
+
+  // Bring up a cluster with one VM node per topology node.
+  //
+  // minikube `start` sizes all nodes uniformly, so for heterogeneous topologies
+  // (e.g. a "far" node sized differently) we start the first node, then
+  // `node add` each remaining node with its own cpu/memory. Homogeneous
+  // topologies take the fast `--nodes=N` path.
+  //
+  // NOTE: `minikube start --nodes=N` / `node add` can exit non-zero on a
+  // healthy cluster (worker-label race — it labels the node before the
+  // apiserver registers it). We DON'T trust the exit code; the readiness gate
+  // (kubectl) is the real success signal.
+  async provision(topology: Topology): Promise<ProvisionedCluster> {
+    const nodes = topology.nodes;
+    if (nodes.length === 0) throw new Error("[k8s] topology has no nodes");
+    const n = nodes.length;
+    const homogeneous = nodes.every(
+      (x) => cpusOf(x) === cpusOf(nodes[0]) && memMbOf(x) === memMbOf(nodes[0]),
+    );
+
+    console.log(
+      `[k8s] provisioning ${n} VM node(s) for topology "${topology.name}" ` +
+        `(profile=${this.profile}, cni=${this.cni}, ${homogeneous ? "homogeneous" : "heterogeneous"})`,
+    );
+
+    await this.sweepLingeringDomains();
+
+    // Always 1-by-1: `minikube start --nodes=N` silently creates fewer VMs
+    // than asked under load (3rd node never spawns, label race masks it,
+    // waitNodesReady times out 10min later). `node add` one-by-one is reliable
+    // but slower; we accept the +30s/node for determinism.
+    const startArgs = [
+      "start", "-p", this.profile,
+      "--driver=kvm2",
+      `--cni=${this.cni}`,
+      `--cpus=${cpusOf(nodes[0])}`,
+      `--memory=${memMbOf(nodes[0])}`,
+    ];
+    if (this.kubernetesVersion) startArgs.push(`--kubernetes-version=${this.kubernetesVersion}`);
+
+    const start = await this.run(startArgs, { check: false });
+    this.handleStartResult(start, "start");
+
+    // Add the remaining nodes one at a time. `minikube node add` doesn't
+    // accept --cpus/--memory in this minikube version — added VMs come up
+    // with the minikube defaults (2 vCPU / 2 GiB) regardless of what we want.
+    // We virsh-resize each one after the add so the topology's sizes actually
+    // take effect.
+    for (let i = 1; i < n; i++) {
+      console.log(`[k8s]   adding node for topology node "${nodes[i].id}"`);
+      const add = await this.run(
+        ["node", "add", "-p", this.profile],
+        { check: false },
+      );
+      this.handleStartResult(add, "node add");
+      // Resize the just-added VM to the topology's spec.
+      await this.resizeNodeVm(i, cpusOf(nodes[i]), parseMemory(nodes[i].memory));
+    }
+
+    const ready = await this.waitNodesReady(n);
+    console.log(`[k8s] cluster ready: ${ready.length} node(s) [${ready.join(", ")}]`);
+
+    // Taint the control-plane node so worker pods can't be scheduled there.
+    // Without this, the heavy-bench failure mode is: workers land on the
+    // control-plane VM, CPU-starve apiserver/etcd, kubectl port-forward dies,
+    // bench loses connection and crashes. Static pods (apiserver, etcd, etc.)
+    // and DaemonSets-with-tolerations (kube-proxy, flannel, our sampler) stay.
+    if (ready.length > 1) {
+      const cpName = ready[0]; // minikube's first node is the control plane
+      console.log(`[k8s] tainting control-plane "${cpName}" — workloads go to worker nodes only`);
+      const taint = await this.kubectl([
+        "taint", "node", cpName,
+        "node-role.kubernetes.io/control-plane=:NoSchedule",
+        "--overwrite",
+      ]);
+      if (taint.code !== 0) {
+        console.warn(`[k8s] control-plane taint failed (non-fatal): ${taint.stderr || taint.stdout}`);
+      }
+    }
+
+    return { profile: this.profile, nodes: ready };
+  }
+
+  // Distinguish the cosmetic node-label race (which is harmless — minikube
+  // tries to label the new node a beat before the apiserver registers it,
+  // and exits non-zero even though the cluster converges) from a real
+  // failure. The race always shows BOTH "applying worker node label" and
+  // "nodes \"...\" not found" in stderr; anything else is a real error and
+  // we throw instead of silently moving on to waitNodesReady (which would
+  // then sit for 10 min before timing out).
+  private handleStartResult(
+    res: { code: number; stderr: string },
+    op: string,
+  ): void {
+    if (res.code === 0) return;
+    if (isCosmeticRace(res.stderr)) {
+      console.warn(
+        `[k8s] minikube ${op} exited ${res.code} (cosmetic node-label race) — ` +
+          `kubectl readiness gate will confirm the cluster.`,
+      );
+      return;
+    }
+    const detail = extractRelevantErrorLines(res.stderr);
+    throw new Error(
+      `[k8s] minikube ${op} failed (exit ${res.code}). Not the label-race.\n` +
+        (detail ? `Diagnostics:\n${detail}\n` : "(no diagnostics in stderr)\n") +
+        `Recommended next step: \`minikube delete -p ${this.profile}\` and retry.`,
+    );
+  }
+
+  // Poll `kubectl get nodes` until `expected` nodes are present and all Ready.
+  async waitNodesReady(expected: number, timeoutMs = 600_000): Promise<string[]> {
+    const deadline = Date.now() + timeoutMs;
+    let last = "";
+    while (Date.now() < deadline) {
+      const { stdout, code } = await this.kubectl([
+        "get", "nodes",
+        "-o", "jsonpath={range .items[*]}{.metadata.name}{\"=\"}{.status.conditions[?(@.type==\"Ready\")].status}{\"\\n\"}{end}",
+      ]);
+      if (code === 0) {
+        const lines = stdout.trim().split("\n").filter(Boolean);
+        const ready = lines.filter((l) => l.endsWith("=True")).map((l) => l.split("=")[0]);
+        last = lines.join(", ");
+        if (lines.length >= expected && ready.length >= expected) return ready;
+      }
+      await new Promise((r) => setTimeout(r, 3000));
+    }
+    throw new Error(
+      `[k8s] timed out waiting for ${expected} Ready node(s) after ${timeoutMs}ms. Last: ${last}`,
+    );
+  }
+
+  // Resize a just-added minikube node VM via virsh: shutdown → set max/min
+  // memory + vcpus → start. Required because `minikube node add` ignores
+  // sizing flags in this minikube version, so additional nodes come up at
+  // the default 2 GiB / 2 vCPU regardless of what we want.
+  //
+  // The new node's name is `<profile>-m{02,03,...}` per minikube convention.
+  private async resizeNodeVm(index: number, cpus: number, memBytes: number): Promise<void> {
+    const vmName = `${this.profile}-m${String(index + 1).padStart(2, "0")}`;
+    const memKb = Math.floor(memBytes / 1024);
+    const virsh = Deno.env.get("SIM_VIRSH_BIN") ?? "virsh";
+    const uri = ["-c", "qemu:///system"];
+
+    console.log(`[k8s]   resizing ${vmName} → ${cpus} vCPU / ${Math.round(memBytes / 1024 / 1024 / 1024)} GiB`);
+
+    async function v(args: string[], opts: { check?: boolean } = {}): Promise<{ code: number; stdout: string; stderr: string }> {
+      const p = new Deno.Command(virsh, { args: [...uri, ...args], stdout: "piped", stderr: "piped" });
+      const out = await p.output();
+      const r = {
+        code: out.code,
+        stdout: new TextDecoder().decode(out.stdout),
+        stderr: new TextDecoder().decode(out.stderr),
+      };
+      if (opts.check && r.code !== 0) {
+        throw new Error(`virsh ${args.join(" ")} failed (code ${r.code}): ${r.stderr || r.stdout}`);
+      }
+      return r;
+    }
+
+    // 1. Stop the minikube node first — this runs kubelet/crio teardown
+    //    hooks AND shuts down the VM at the libvirt level. A bare
+    //    `virsh shutdown` skips minikube's bootstrap context, and a bare
+    //    `virsh start` afterward brings the VM up without kubelet starting,
+    //    so the node never re-registers.
+    const nodeName = vmName; // minikube node name == libvirt domain name
+    await this.run(["node", "stop", "-p", this.profile, nodeName], { check: false });
+
+    // Wait for libvirt to confirm it's actually shut off (minikube node stop
+    // returns before libvirt finishes).
+    let shutoff = false;
+    for (let attempt = 0; attempt < 30; attempt++) {
+      const state = await v(["domstate", vmName]);
+      if (state.stdout.trim() === "shut off") { shutoff = true; break; }
+      await new Promise((r) => setTimeout(r, 1000));
+    }
+    if (!shutoff) {
+      console.warn(`[k8s]   ${vmName} did not shut down cleanly — forcing destroy`);
+      await v(["destroy", vmName]);
+    }
+
+    // 2. Resize at the libvirt level. `--config` persists across reboots.
+    //    For vcpus, --maximum must be set before raising the current count.
+    await v(["setmaxmem", vmName, String(memKb), "--config"], { check: true });
+    await v(["setmem", vmName, String(memKb), "--config"], { check: true });
+    await v(["setvcpus", vmName, String(cpus), "--config", "--maximum"], { check: true });
+    await v(["setvcpus", vmName, String(cpus), "--config"], { check: true });
+
+    // 3. Start back via minikube so kubelet/crio bootstrap runs properly.
+    //    `waitNodesReady` will pick it up once kubelet re-registers.
+    await this.run(["node", "start", "-p", this.profile, nodeName], { check: false });
+  }
+
+  async teardown(): Promise<void> {
+    console.log(`[k8s] minikube delete (profile=${this.profile})`);
+    await this.run(["delete", "-p", this.profile], { check: false });
+    // minikube's kvm2 driver doesn't always destroy libvirt domains on delete
+    // (process killed mid-cleanup, driver bug, etc). Sweep them so a follow-up
+    // `provision()` starts from a truly clean slate.
+    await this.sweepLingeringDomains();
+  }
+}
+
+// --- diagnostics helpers (module-level) -----------------------------------
+
+// Actual minikube text: `error applying worker node "m02" label: apply node labels: ...`
+// — the node name is quoted between "node" and "label", so allow chars in between.
+const RACE_LABEL_RE = /error applying worker node\b[\s\S]*?\blabel\b/i;
+const RACE_NOT_FOUND_RE = /nodes? "[^"]+" not found/i;
+
+function isCosmeticRace(stderr: string): boolean {
+  const clean = stripAnsiAndBoxes(stderr);
+  return RACE_LABEL_RE.test(clean) && RACE_NOT_FOUND_RE.test(clean);
+}
+
+// minikube wraps suggestions in a Unicode-box; the literal box characters and
+// ANSI colour codes pollute "stderr tail" output. Strip them so we can show a
+// useful diagnostic.
+function stripAnsiAndBoxes(s: string): string {
+  return s
+    // deno-lint-ignore no-control-regex
+    .replace(/\x1b\[[0-9;]*[A-Za-z]/g, "")
+    .replace(/[│┌┐└┘├┤┬┴┼─╭╮╯╰━]/g, "");
+}
+
+// Pick out the lines that actually describe what went wrong from minikube's
+// long, decorated stderr. We grab error markers + adjacent context.
+function extractRelevantErrorLines(stderr: string): string {
+  const clean = stripAnsiAndBoxes(stderr);
+  const lines = clean.split("\n").map((l) => l.trim()).filter(Boolean);
+  const keepRe = /(^[X!*]\s)|Exiting due to|Error[: ]|FAIL|panic|missing |not found|status [1-9]|Suggestion:|Documentation:/i;
+  const picked: string[] = [];
+  for (let i = 0; i < lines.length; i++) {
+    if (keepRe.test(lines[i])) picked.push(lines[i]);
+    if (picked.length >= 15) break;
+  }
+  return picked.join("\n");
+}

benchmarks/sim/nix/kvm2-driver.nix

@@ -0,0 +1,64 @@
+# minikube's kvm2 driver (docker-machine-driver-kvm2), built from the minikube
+# source. Not packaged in nixpkgs; minikube's auto-download won't run on NixOS
+# (wrong dynamic linker), so we build it here. Used by the k8s sim path (wm_sim
+# / sim/sim.ts) via SIM_KVM2_DRIVER_DIR.
+#
+# IMPORTANT: build with the *same nixpkgs as the system libvirtd*, otherwise the
+# driver links a different glibc than the system-libvirt libs it loads at
+# runtime, and you'll see `GLIBC_ABI_DT_X86_64_PLT not found`. The flake passes
+# `pkgs = pkgsSim` (fresh nixos-unstable, system-matching) for that reason.
+#
+# Gotchas baked in:
+#   - v1.38.1 ships the driver *library* (pkg/drivers/kvm) but NOT the
+#     cmd/drivers/kvm main wrapper (Makefile references it but it's absent),
+#     so we synthesise a minimal main mirroring cmd/drivers/hyperkit/main.go.
+#     Uses 1.38's internalised plugin path (k8s.io/minikube/pkg/libmachine/...).
+#   - Built WITHOUT the `libvirt_dlopen` tag so libvirt is linked (nix sets the
+#     rpath). The libvirt-go binding *still* dlopens libvirt.so.0 by soname at
+#     connect time, so the caller must set LD_LIBRARY_PATH to a libvirt
+#     compatible with the system libvirtd (NixOS has no ld cache).
+{ pkgs ? import <nixpkgs> { } }:
+pkgs.buildGoModule rec {
+  pname = "docker-machine-driver-kvm2";
+  version = "1.38.1";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "kubernetes";
+    repo = "minikube";
+    rev = "v${version}";
+    hash = "sha256-1unwbu2pJviHXukQKalJLgrkHpjf0sRR2nCm2gKv2VU=";
+  };
+
+  vendorHash = "sha256-Oy8cM/foZKC83PxqkJW+o8vVYJhszKxXs9l2eks7FN4=";
+
+  postPatch = ''
+    mkdir -p cmd/drivers/kvm
+    cat > cmd/drivers/kvm/main.go <<'EOF'
+//go:build linux
+
+package main
+
+import (
+	"k8s.io/minikube/pkg/drivers/kvm"
+	"k8s.io/minikube/pkg/libmachine/drivers/plugin"
+)
+
+func main() {
+	plugin.RegisterDriver(kvm.NewDriver("", ""))
+}
+EOF
+  '';
+
+  subPackages = [ "cmd/drivers/kvm" ];
+
+  nativeBuildInputs = [ pkgs.pkg-config ];
+  buildInputs = [ pkgs.libvirt ];
+
+  env.CGO_ENABLED = "1";
+
+  postInstall = ''
+    if [ -e "$out/bin/kvm" ]; then mv "$out/bin/kvm" "$out/bin/docker-machine-driver-kvm2"; fi
+  '';
+
+  meta.description = "minikube kvm2 driver, built from minikube source with synthesised cmd main";
+}

benchmarks/sim/node_load_poller.ts

@@ -0,0 +1,105 @@
+// Polls /proc/loadavg + nproc on each minikube node every intervalMs and
+// writes one JSONL row per node per tick. Used to compute *saturation* (load /
+// ncpu) and *oversaturation* (max(0, load/ncpu - 1)) in the dashboard. PSI is
+// not available in the minikube kernel and cpu.stat throttling is meaningless
+// without limits.cpu, so loadavg is the only saturation signal we have.
+//
+// Output line: {"ts": ms, "node": "wm-sim-k8s-4node-m02", "load1": 44.2,
+//                "load5": 32.1, "load15": 24.4, "ncpu": 4}
+
+import { MinikubeProvisioner } from "./k8s_provisioner.ts";
+
+export type NodeLoadPoller = {
+  cont: { value: boolean };
+  done: Promise<void>;
+};
+
+export function startNodeLoadPoller(
+  prov: MinikubeProvisioner,
+  outPath: string,
+  opts: { intervalMs?: number } = {},
+): NodeLoadPoller {
+  const intervalMs = opts.intervalMs ?? 2000;
+  const cont = { value: true };
+  const f = Deno.openSync(outPath, { write: true, create: true, truncate: true });
+  const enc = new TextEncoder();
+
+  const done = (async () => {
+    // Discover nodes once at startup. The poll loop reuses this list. New
+    // nodes joining mid-bench are rare (we don't auto-scale the cluster).
+    let nodes: { name: string; ip: string }[] = [];
+    try {
+      const r = await prov.kubectl([
+        "get", "nodes",
+        "-o", "jsonpath={range .items[*]}{.metadata.name}{\"|\"}{.status.addresses[?(@.type==\"InternalIP\")].address}{\"\\n\"}{end}",
+      ]);
+      if (r.code === 0) {
+        for (const line of r.stdout.split("\n")) {
+          if (!line.trim()) continue;
+          const [name, ip] = line.split("|");
+          if (name && ip) nodes.push({ name: name.trim(), ip: ip.trim() });
+        }
+      }
+    } catch (e) {
+      console.warn(`[node-load] node discovery failed: ${(e as Error).message}`);
+    }
+
+    while (cont.value) {
+      const startMs = Date.now();
+      // Poll all nodes in parallel — one ssh per node per tick.
+      await Promise.all(nodes.map(async ({ name, ip }) => {
+        try {
+          // Each minikube node has its own ssh key under ~/.minikube/machines.
+          const keyPath = `${Deno.env.get("HOME")}/.minikube/machines/${name}/id_rsa`;
+          const proc = new Deno.Command("ssh", {
+            args: [
+              "-o", "StrictHostKeyChecking=no",
+              "-o", "UserKnownHostsFile=/dev/null",
+              "-o", "ConnectTimeout=2",
+              "-o", "LogLevel=ERROR",
+              "-i", keyPath,
+              `docker@${ip}`,
+              // procs_running is the runnable count (CPU-bound queue) — does
+              // NOT include D-state procs (disk/network wait). loadavg counts
+              // both, so loadavg/ncpu was conflating CPU-starved processes
+              // with PG backends waiting on disk I/O.
+              "cat /proc/loadavg && cat /proc/stat | grep ^procs_running && nproc",
+            ],
+            stdout: "piped",
+            stderr: "null",
+          });
+          const out = await proc.output();
+          const text = new TextDecoder().decode(out.stdout).trim();
+          const lines = text.split("\n");
+          if (lines.length < 3) return;
+          const loadParts = lines[0].split(" ");
+          const load1 = parseFloat(loadParts[0]);
+          const load5 = parseFloat(loadParts[1]);
+          const load15 = parseFloat(loadParts[2]);
+          // "procs_running N" — instantaneous count of runnable processes
+          // (current + queued for CPU). Excludes D-state.
+          const procsRunning = parseInt(lines[1].split(/\s+/)[1] ?? "");
+          const ncpu = parseInt(lines[2].trim());
+          if (!Number.isFinite(load1) || !Number.isFinite(ncpu)) return;
+          const row = {
+            ts: startMs,
+            node: name,
+            load1,
+            load5,
+            load15,
+            procs_running: Number.isFinite(procsRunning) ? procsRunning : null,
+            ncpu,
+          };
+          f.writeSync(enc.encode(JSON.stringify(row) + "\n"));
+        } catch (_e) { /* skip this node this tick */ }
+      }));
+      const elapsed = Date.now() - startMs;
+      if (cont.value && elapsed < intervalMs) {
+        await new Promise((r) => setTimeout(r, intervalMs - elapsed));
+      }
+    }
+    f.close();
+  })();
+
+  return { cont, done };
+}

benchmarks/sim/oom_events.ts

@@ -0,0 +1,294 @@
+// Collect OOM kills that fired during the bench window, from two distinct
+// sources:
+//
+//   - Node-level VM kernel OOM (`kubectl get events --field-selector
+//     reason=SystemOOM`): fires when the sum of pod RSS on a node exceeds
+//     the VM's RAM ceiling. The kernel picks a victim by oom_score across
+//     all cgroups; the event message names the victim process (e.g. "deno").
+//
+//   - Per-container cgroup OOM (pod containerStatus
+//     `lastState.terminated.reason == "OOMKilled"`): fires when one pod
+//     exceeds its own `limits.memory`. Kills only processes in that pod's
+//     cgroup; kubelet marks the container OOMKilled and restarts per policy.
+//
+// Output is a JSON file the renderer turns into a bar chart so heavy-bench
+// reports show "what got killed and how many times" at a glance.
+
+import { MinikubeProvisioner } from "./k8s_provisioner.ts";
+
+export type OomEvent = {
+  ts_ms: number;
+  source: "node_kernel" | "cgroup" | "kubelet_eviction" | "scheduler_preemption";
+  // process name from SystemOOM, or pod name from cgroup OOM / eviction /
+  // preemption
+  victim: string;
+  node: string;
+  // bytes the container was using at eviction time, parsed from the
+  // Evicted message. Only present for `kubelet_eviction` source.
+  bytes_at_kill?: number;
+};
+
+type RawEvent = {
+  metadata?: { creationTimestamp?: string };
+  lastTimestamp?: string;
+  eventTime?: string;
+  involvedObject?: { name?: string };
+  message?: string;
+  reason?: string;
+};
+
+type RawPod = {
+  metadata?: { name?: string };
+  spec?: { nodeName?: string };
+  status?: {
+    containerStatuses?: Array<{
+      restartCount?: number;
+      lastState?: {
+        terminated?: {
+          reason?: string;
+          finishedAt?: string;
+        };
+      };
+    }>;
+  };
+};
+
+function parseTs(s?: string): number {
+  if (!s) return 0;
+  const ms = Date.parse(s);
+  return Number.isFinite(ms) ? ms : 0;
+}
+
+// Pull "victim process: deno" out of the SystemOOM message text.
+function parseVictim(msg: string): string {
+  const m = msg.match(/victim process:\s*([^\s,]+)/);
+  return m ? m[1] : "unknown";
+}
+
+// Read dmesg + uptime from a sampler pod (privileged + hostPID, so it sees
+// the host kernel log buffer) and convert each "Memory cgroup out of memory:
+// Killed process N (CMD)" line to an OomEvent. Catches subprocess kills
+// inside a cgroup that don't surface as containerStatus OOMKilled (because
+// PID1 survived) and don't surface as SystemOOM events (because the OOM
+// scope was cgroup, not whole-VM). These are the ones that produce 500+
+// failed jobs + low CPU util while every other source says "0 OOMs".
+async function dmesgOomsFromSampler(
+  prov: MinikubeProvisioner,
+  samplerNamespace: string,
+  samplerPod: string,
+  nodeName: string,
+  sinceMs: number,
+): Promise<OomEvent[]> {
+  // Two outputs in one exec, separated by a sentinel — saves a round-trip.
+  const res = await prov.kubectl([
+    "-n", samplerNamespace, "exec", samplerPod, "--", "sh", "-c",
+    "cat /proc/uptime; echo '---DMESG---'; dmesg 2>/dev/null | grep 'Memory cgroup out of memory'",
+  ]);
+  if (res.code !== 0 || !res.stdout) return [];
+  const parts = res.stdout.split("---DMESG---");
+  if (parts.length < 2) return [];
+  const uptime_s = parseFloat(parts[0].trim().split(/\s+/)[0] || "0");
+  if (!Number.isFinite(uptime_s) || uptime_s <= 0) return [];
+  const wallNowMs = Date.now();
+  const bootWallMs = wallNowMs - uptime_s * 1000;
+
+  // dmesg line format: `[ 7126.289518] Memory cgroup out of memory: Killed process 354780 (deno) total-vm:...`
+  const re = /^\[\s*(\d+(?:\.\d+)?)\]\s+Memory cgroup out of memory:\s+Killed process \d+ \(([^)]+)\)/;
+  const out: OomEvent[] = [];
+  for (const line of parts[1].split("\n")) {
+    const m = re.exec(line);
+    if (!m) continue;
+    const event_uptime_s = parseFloat(m[1]);
+    const cmd = m[2];
+    if (!Number.isFinite(event_uptime_s)) continue;
+    const ts_ms = Math.round(bootWallMs + event_uptime_s * 1000);
+    if (ts_ms < sinceMs) continue;
+    out.push({ ts_ms, source: "node_kernel", victim: cmd, node: nodeName });
+  }
+  return out;
+}
+
+// Pull "was using 14821920Ki" (or similar size suffixes) out of an Evicted
+// event message. kubelet formats this as `<digits><Ki|Mi|Gi>`. Returns bytes.
+function parseEvictedBytes(msg: string): number | undefined {
+  const m = msg.match(/was using\s+(\d+)(Ki|Mi|Gi)\b/);
+  if (!m) return undefined;
+  const n = parseInt(m[1]);
+  if (!Number.isFinite(n)) return undefined;
+  const mult = m[2] === "Gi" ? 1024 * 1024 * 1024 : m[2] === "Mi" ? 1024 * 1024 : 1024;
+  return n * mult;
+}
+
+export async function collectOomEvents(
+  prov: MinikubeProvisioner,
+  outPath: string,
+  opts: { sinceMs?: number } = {},
+): Promise<void> {
+  const since = opts.sinceMs ?? 0;
+  const out: OomEvent[] = [];
+
+  // 1) Node-level VM kernel OOMs (SystemOOM events).
+  const evRes = await prov.kubectl([
+    "get", "events",
+    "-A",
+    "--field-selector", "reason=SystemOOM",
+    "-o", "json",
+  ]);
+  if (evRes.code === 0 && evRes.stdout) {
+    let evJson: { items?: RawEvent[] } = {};
+    try { evJson = JSON.parse(evRes.stdout); } catch { /* ignore parse errors */ }
+    for (const e of evJson.items ?? []) {
+      const ts = parseTs(e.eventTime ?? e.lastTimestamp ?? e.metadata?.creationTimestamp);
+      if (ts < since) continue;
+      out.push({
+        ts_ms: ts,
+        source: "node_kernel",
+        victim: parseVictim(e.message ?? ""),
+        node: e.involvedObject?.name ?? "unknown",
+      });
+    }
+  }
+
+  // 1b) Kubelet evictions (L1, fires BEFORE kernel OOM under graceful node
+  // pressure). Has a usable memory-at-kill value in the message.
+  const evictedRes = await prov.kubectl([
+    "get", "events",
+    "-A",
+    "--field-selector", "reason=Evicted",
+    "-o", "json",
+  ]);
+  if (evictedRes.code === 0 && evictedRes.stdout) {
+    let evJson: { items?: RawEvent[] } = {};
+    try { evJson = JSON.parse(evictedRes.stdout); } catch { /* ignore parse errors */ }
+    for (const e of evJson.items ?? []) {
+      const ts = parseTs(e.eventTime ?? e.lastTimestamp ?? e.metadata?.creationTimestamp);
+      if (ts < since) continue;
+      out.push({
+        ts_ms: ts,
+        source: "kubelet_eviction",
+        victim: e.involvedObject?.name ?? "unknown",
+        node: "unknown",  // event references the evicted pod, not the node;
+                          // the message has it but parsing is brittle
+        bytes_at_kill: parseEvictedBytes(e.message ?? ""),
+      });
+    }
+  }
+
+  // 1c) Scheduler preemptions (L0, fires when a higher-priority pod can't
+  // fit and the scheduler picks a victim to evict). DIFFERENT from kubelet
+  // eviction: kubelet evicts due to NODE PRESSURE (memory, disk); scheduler
+  // preempts due to a PriorityClass collision. Common case in this cluster:
+  // PG/sampler at wm-critical can't fit → workers (priority 0) get
+  // preempted. Without this we miss the bulk of "worker dying" events.
+  const preemptedRes = await prov.kubectl([
+    "get", "events",
+    "-A",
+    "--field-selector", "reason=Preempted",
+    "-o", "json",
+  ]);
+  if (preemptedRes.code === 0 && preemptedRes.stdout) {
+    let evJson: { items?: RawEvent[] } = {};
+    try { evJson = JSON.parse(preemptedRes.stdout); } catch { /* ignore parse errors */ }
+    for (const e of evJson.items ?? []) {
+      const ts = parseTs(e.eventTime ?? e.lastTimestamp ?? e.metadata?.creationTimestamp);
+      if (ts < since) continue;
+      out.push({
+        ts_ms: ts,
+        source: "scheduler_preemption",
+        victim: e.involvedObject?.name ?? "unknown",
+        node: "unknown",
+      });
+    }
+  }
+
+  // 2) Per-container cgroup OOMs (pod containerStatuses).
+  // Catches at most ONE OOMKill per container — the most recent. For multi-
+  // restart OOM patterns we merge the live poller's JSONL below.
+  const podRes = await prov.kubectl([
+    "get", "pods", "-A",
+    "-o", "json",
+  ]);
+  if (podRes.code === 0 && podRes.stdout) {
+    let podJson: { items?: RawPod[] } = {};
+    try { podJson = JSON.parse(podRes.stdout); } catch { /* ignore parse errors */ }
+    for (const p of podJson.items ?? []) {
+      for (const cs of p.status?.containerStatuses ?? []) {
+        const term = cs.lastState?.terminated;
+        if (term?.reason !== "OOMKilled") continue;
+        const ts = parseTs(term.finishedAt);
+        if (ts < since) continue;
+        out.push({
+          ts_ms: ts,
+          source: "cgroup",
+          victim: p.metadata?.name ?? "unknown",
+          node: p.spec?.nodeName ?? "unknown",
+        });
+      }
+    }
+  }
+
+  // 3) Merge mid-bench OOMKills captured by the live poller. The poller
+  // dedupes by (pod, container, finishedAt) and writes JSONL; we union with
+  // the end-of-bench scan above so OOMs that fired AND recovered mid-bench
+  // (which the lastState scan misses) still show up on the L2 cgroup panel.
+  const liveJsonlPath = outPath.replace(/\.json$/, "_live.jsonl");
+  try {
+    const text = await Deno.readTextFile(liveJsonlPath);
+    const seen = new Set(out.map((e) => `${e.victim}|${e.ts_ms}`));
+    for (const line of text.split("\n")) {
+      if (!line.trim()) continue;
+      try {
+        const r = JSON.parse(line) as OomEvent;
+        if (r.ts_ms < since) continue;
+        const key = `${r.victim}|${r.ts_ms}`;
+        if (seen.has(key)) continue;
+        seen.add(key);
+        out.push(r);
+      } catch { /* skip malformed line */ }
+    }
+  } catch { /* poller wasn't running or file missing — ignore */ }
+
+  // 4) Per-node dmesg scan. Catches subprocess OOM kills inside a cgroup
+  // (where the container's PID 1 survived → no containerStatus OOMKilled →
+  // sources 2/3 miss it, and the OOM was cgroup-scope → no SystemOOM event
+  // → source 1 misses it). This is what produces "lots of failed jobs +
+  // low CPU util while every panel says 0 OOMs" — workers' deno
+  // subprocesses get SIGKILLed mid-job, the worker container keeps polling.
+  const samplerNs = "kube-system";
+  const samplerSelector = "app=wm-sim-cpu-sampler";
+  const samplerListRes = await prov.kubectl([
+    "-n", samplerNs, "get", "pods", "-l", samplerSelector,
+    "-o", "jsonpath={range .items[*]}{.metadata.name}|{.spec.nodeName}\\n{end}",
+  ]);
+  if (samplerListRes.code === 0 && samplerListRes.stdout) {
+    const dmesgEvents: OomEvent[] = [];
+    for (const line of samplerListRes.stdout.split("\n")) {
+      if (!line.trim()) continue;
+      const [samplerPod, nodeName] = line.split("|");
+      if (!samplerPod || !nodeName) continue;
+      try {
+        const evs = await dmesgOomsFromSampler(prov, samplerNs, samplerPod.trim(), nodeName.trim(), since);
+        dmesgEvents.push(...evs);
+      } catch (e) {
+        console.warn(`[oom] dmesg scan on ${nodeName} failed: ${(e as Error).message}`);
+      }
+    }
+    // Dedupe across nodes — extremely close timestamps for the same victim
+    // would be rare but defend against double-counting if a future change
+    // adds dmesg multi-source overlap.
+    const seenDmesg = new Set(out.map((e) => `${e.source}|${e.victim}|${e.ts_ms}`));
+    for (const e of dmesgEvents) {
+      const key = `${e.source}|${e.victim}|${e.ts_ms}`;
+      if (seenDmesg.has(key)) continue;
+      seenDmesg.add(key);
+      out.push(e);
+    }
+  }
+
+  out.sort((a, b) => a.ts_ms - b.ts_ms);
+  await Deno.writeTextFile(outPath, JSON.stringify(out));
+  const ks = out.filter(e => e.source === "node_kernel").length;
+  const cs = out.filter(e => e.source === "cgroup").length;
+  const es = out.filter(e => e.source === "kubelet_eviction").length;
+  console.log(`[oom] ${out.length} kill event(s) captured (${es} L1 evicted, ${cs} L2 cgroup, ${ks} L2 node-kernel) -> ${outPath}`);
+}

benchmarks/sim/oom_poller.ts

@@ -0,0 +1,99 @@
+// In-bench OOMKilled poller. The end-of-bench scan in `oom_events.ts` reads
+// `lastState.terminated` from every pod, but that field only holds the MOST
+// RECENT termination — a pod that OOMKilled mid-bench and then recovered into
+// Running state by end-of-bench shows no OOMKill at all in the final scan,
+// silently zeroing out the L2 cgroup OOM panel.
+//
+// This poller closes the gap: every N seconds it lists all pods, watches for
+// `restartCount` increases on any container whose `lastState.terminated.reason`
+// is "OOMKilled", and appends a JSONL row per newly-seen OOMKill. End-of-bench
+// `collectOomEvents` reads this JSONL and merges it with the final scan so the
+// L2 cgroup OOM panel reflects every kill that fired during the window.
+//
+// Dedupe key: `<pod>|<container>|<finishedAt>` — restartCount can climb across
+// multiple polls before the next OOMKill, but finishedAt is unique per kill.
+
+import { MinikubeProvisioner } from "./k8s_provisioner.ts";
+
+export type OomPoller = {
+  // Mutate this to false to stop the loop. The polling promise resolves when
+  // the in-flight kubectl call finishes after the flip.
+  cont: { value: boolean };
+  done: Promise<void>;
+};
+
+type ContainerStatus = {
+  name?: string;
+  restartCount?: number;
+  lastState?: {
+    terminated?: {
+      reason?: string;
+      finishedAt?: string;
+    };
+  };
+};
+
+type PodSnapshot = {
+  metadata?: { name?: string };
+  spec?: { nodeName?: string };
+  status?: { containerStatuses?: ContainerStatus[] };
+};
+
+export function startOomPoller(
+  prov: MinikubeProvisioner,
+  outPath: string,
+  opts: { intervalMs?: number } = {},
+): OomPoller {
+  const intervalMs = opts.intervalMs ?? 2000;
+  const cont = { value: true };
+  const f = Deno.openSync(outPath, { write: true, create: true, truncate: true });
+  const enc = new TextEncoder();
+  // Dedupe: a kill is uniquely identified by where + when it finished.
+  const seen = new Set<string>();
+
+  const done = (async () => {
+    while (cont.value) {
+      const startMs = Date.now();
+      try {
+        const res = await prov.kubectl(["get", "pods", "-A", "-o", "json"]);
+        if (res.code === 0 && res.stdout) {
+          let parsed: { items?: PodSnapshot[] } = {};
+          try { parsed = JSON.parse(res.stdout); } catch { /* transient — retry next tick */ }
+          for (const p of parsed.items ?? []) {
+            const podName = p.metadata?.name;
+            const node = p.spec?.nodeName ?? "unknown";
+            if (!podName) continue;
+            for (const cs of p.status?.containerStatuses ?? []) {
+              const term = cs.lastState?.terminated;
+              if (term?.reason !== "OOMKilled") continue;
+              const finishedAt = term.finishedAt;
+              if (!finishedAt) continue;
+              const key = `${podName}|${cs.name ?? "?"}|${finishedAt}`;
+              if (seen.has(key)) continue;
+              seen.add(key);
+              const ts = Date.parse(finishedAt);
+              const row = JSON.stringify({
+                ts_ms: Number.isFinite(ts) ? ts : startMs,
+                source: "cgroup",
+                victim: podName,
+                container: cs.name,
+                node,
+              });
+              f.writeSync(enc.encode(row + "\n"));
+            }
+          }
+        }
+      } catch (_e) {
+        // Transient kubectl errors tolerated — bench should not die because
+        // one poll failed. Next iteration retries.
+      }
+      const elapsed = Date.now() - startMs;
+      if (cont.value && elapsed < intervalMs) {
+        await new Promise((r) => setTimeout(r, intervalMs - elapsed));
+      }
+    }
+    f.close();
+  })();
+
+  return { cont, done };
+}

benchmarks/sim/pg_conn_poller.ts

@@ -0,0 +1,88 @@
+// PG connection-count poller. Every interval (default 1s), runs
+//   SELECT state, count(*) FROM pg_stat_activity GROUP BY state
+// against the bundled PG. Writes one JSONL line per poll with per-state
+// breakdown so the dashboard can plot active vs idle vs idle-in-transaction
+// over time. Answers "is the spike PG connection-count vs PG query-rate?"
+// — pairs naturally with the pg_latency panel.
+//
+// Output:
+//   {"ts": <ms>, "active": N, "idle": M, "idle_in_xact": K, "total": T}
+
+import { MinikubeProvisioner } from "./k8s_provisioner.ts";
+
+export type PgConnPoller = {
+  cont: { value: boolean };
+  done: Promise<void>;
+};
+
+export function startPgConnPoller(
+  prov: MinikubeProvisioner,
+  outPath: string,
+  opts: { intervalMs?: number; namespace?: string; pgPodSelector?: string } = {},
+): PgConnPoller {
+  const intervalMs = opts.intervalMs ?? 1000;
+  const namespace = opts.namespace ?? "default";
+  const selector = opts.pgPodSelector ?? "app=windmill-postgresql-demo-app";
+  const cont = { value: true };
+  const f = Deno.openSync(outPath, { write: true, create: true, truncate: true });
+  const enc = new TextEncoder();
+
+  const done = (async () => {
+    let podName = "";
+    try {
+      const r = await prov.kubectl([
+        "-n", namespace, "get", "pods", "-l", selector,
+        "-o", "jsonpath={.items[0].metadata.name}",
+      ]);
+      if (r.code === 0) podName = r.stdout.trim();
+    } catch (_e) { /* retry in loop */ }
+
+    while (cont.value) {
+      const startMs = Date.now();
+      let row: Record<string, unknown> = { ts: startMs };
+      try {
+        if (!podName) {
+          const r = await prov.kubectl([
+            "-n", namespace, "get", "pods", "-l", selector,
+            "-o", "jsonpath={.items[0].metadata.name}",
+          ]);
+          if (r.code === 0) podName = r.stdout.trim();
+        }
+        if (podName) {
+          const r = await prov.kubectl([
+            "-n", namespace, "exec", podName, "--",
+            "psql", "-U", "postgres", "-d", "windmill", "-tAc",
+            // Statement timeout so a wedged PG can't stall the poller.
+            "SET statement_timeout=2000; SELECT coalesce(state,'unknown') || ':' || count(*) FROM pg_stat_activity GROUP BY state",
+          ]);
+          if (r.code === 0) {
+            let active = 0, idle = 0, idle_in_xact = 0, unknown = 0;
+            for (const line of r.stdout.split("\n")) {
+              if (!line.includes(":")) continue;
+              const [state, countStr] = line.split(":");
+              const n = parseInt(countStr.trim());
+              if (!Number.isFinite(n)) continue;
+              if (state === "active") active = n;
+              else if (state === "idle") idle = n;
+              else if (state === "idle in transaction") idle_in_xact = n;
+              else unknown += n;
+            }
+            row = { ts: startMs, active, idle, idle_in_xact, unknown, total: active + idle + idle_in_xact + unknown };
+          } else {
+            row = { ts: startMs, err: (r.stderr || r.stdout).slice(0, 200) };
+          }
+        }
+      } catch (e) {
+        row = { ts: startMs, err: (e as Error).message };
+      }
+      f.writeSync(enc.encode(JSON.stringify(row) + "\n"));
+      const elapsed = Date.now() - startMs;
+      if (cont.value && elapsed < intervalMs) {
+        await new Promise((r) => setTimeout(r, intervalMs - elapsed));
+      }
+    }
+    f.close();
+  })();
+
+  return { cont, done };
+}

benchmarks/sim/pg_latency_poller.ts

@@ -0,0 +1,105 @@
+// PG response-time poller. Every interval (default 1s), runs a trivial
+// `SELECT 1` against the bundled PG via `kubectl exec ... psql` and records
+// how long it took. The point is to make PG's responsiveness visible OVER
+// TIME — when the DB is healthy this is a flat line near 0; when PG gets
+// contended (heavy bench load, connection storm, autovacuum stalling, etc.)
+// the latency rises and you can correlate it with the throughput dip on the
+// same x-axis.
+//
+// Output: JSONL, one line per poll
+//   {"ts": <ms>, "latency_ms": <number>, "ok": true|false, "err"?: "..."}
+//
+// Why `SELECT 1` (vs a real queue query): we want PG's *infrastructure*
+// latency, not query cost. A trivial query exposes connection-pool /
+// backend-fork / network round-trip cost, which is exactly what bench
+// pressure inflates. Switching to a heavier query muddies "is PG fast" with
+// "is this query fast".
+
+import { MinikubeProvisioner } from "./k8s_provisioner.ts";
+
+export type PgLatencyPoller = {
+  cont: { value: boolean };
+  done: Promise<void>;
+};
+
+export function startPgLatencyPoller(
+  prov: MinikubeProvisioner,
+  outPath: string,
+  opts: { intervalMs?: number; namespace?: string; pgPodSelector?: string } = {},
+): PgLatencyPoller {
+  const intervalMs = opts.intervalMs ?? 250;
+  const namespace = opts.namespace ?? "default";
+  const selector = opts.pgPodSelector ?? "app=windmill-postgresql-demo-app";
+  const cont = { value: true };
+  const f = Deno.openSync(outPath, { write: true, create: true, truncate: true });
+  const enc = new TextEncoder();
+
+  // Resolve the PG pod name once at startup. If PG restarts mid-bench the
+  // name shouldn't change (StatefulSet), so caching is safe.
+  const done = (async () => {
+    let podName = "";
+    try {
+      const r = await prov.kubectl([
+        "-n", namespace, "get", "pods", "-l", selector,
+        "-o", "jsonpath={.items[0].metadata.name}",
+      ]);
+      if (r.code === 0) podName = r.stdout.trim();
+    } catch (_e) { /* fall through; loop will retry */ }
+
+    while (cont.value) {
+      const startMs = Date.now();
+      let ok = false;
+      let err: string | undefined;
+      let pgQueryMs: number | undefined;
+      try {
+        if (!podName) {
+          // Retry pod lookup if the initial resolve failed.
+          const r = await prov.kubectl([
+            "-n", namespace, "get", "pods", "-l", selector,
+            "-o", "jsonpath={.items[0].metadata.name}",
+          ]);
+          if (r.code === 0) podName = r.stdout.trim();
+        }
+        if (podName) {
+          // Run with `\timing on` so we get TWO signals:
+          //   1. Wall-clock around the whole kubectl exec → "kubectl
+          //      roundtrip" (~200ms baseline = API hop + containerd exec +
+          //      fork psql + open new pg conn). Useful as a cluster-control-
+          //      plane health signal, not as a PG signal.
+          //   2. The `Time: X.XXX ms` line psql emits → pure PG query
+          //      latency (sub-millisecond when PG is happy; rises only
+          //      under contention/lock waits).
+          // Both are emitted as separate `kind` series so the chart shows
+          // both lines on the same axis.
+          const r = await prov.kubectl([
+            "-n", namespace, "exec", podName, "--",
+            "psql", "-U", "postgres", "-d", "windmill",
+            "-c", "\\timing on",
+            "-c", "SELECT 1",
+          ]);
+          ok = r.code === 0;
+          if (!ok) err = (r.stderr || r.stdout).slice(0, 200);
+          // Parse psql's "Time: 0.420 ms" line for the pure PG query time.
+          const m = r.stdout.match(/Time:\s*([\d.]+)\s*ms/);
+          if (m) {
+            pgQueryMs = parseFloat(m[1]);
+          }
+        } else {
+          err = "no PG pod";
+        }
+      } catch (e) {
+        err = (e as Error).message;
+      }
+      const latency_ms = Date.now() - startMs;
+      const row = JSON.stringify({ ts: startMs, latency_ms, pg_query_ms: pgQueryMs, ok, err });
+      f.writeSync(enc.encode(row + "\n"));
+      const elapsed = Date.now() - startMs;
+      if (cont.value && elapsed < intervalMs) {
+        await new Promise((r) => setTimeout(r, intervalMs - elapsed));
+      }
+    }
+    f.close();
+  })();
+
+  return { cont, done };
+}

benchmarks/sim/pg_logging.ts

@@ -0,0 +1,210 @@
+// k8s glue around the bundled PG for pgBadger-grade reporting.
+//
+// The Windmill helm chart's `postgresql.enabled` deploys a vanilla `postgres:18`
+// Deployment (called `windmill-postgresql-demo-app` by default — confusingly,
+// despite the chart's comment claiming cloudnative-pg). We need it to emit
+// every statement + connection so pgBadger can render a real report, which
+// means setting PostgreSQL `-c` flags. The chart doesn't expose those, so we
+// patch the Deployment after `helm install`.
+//
+// At end of bench we `kubectl logs` the PG pod and run pgBadger on it. The
+// `runPgbadger` helper lives in pgbadger.ts so it's reusable.
+
+import { MinikubeProvisioner } from "./k8s_provisioner.ts";
+import { PGBADGER_PG_SETTINGS } from "./pgbadger.ts";
+
+const DEFAULT_PG_DEPLOYMENT = "windmill-postgresql-demo-app";
+const DEFAULT_NAMESPACE = "default";
+const DEFAULT_PG_USER = "postgres";
+const DEFAULT_PG_DB = "windmill";
+const PG_POD_LABEL = "app=windmill-postgresql-demo-app";
+
+// The chart switches kind/name based on `postgresql.persistence.enabled`:
+//   persistence=false  -> Deployment/windmill-postgresql-demo-app
+//   persistence=true   -> StatefulSet/windmill-postgresql
+// Both use the same pod label, so exec/logs can take the label selector path
+// — but `kubectl patch` and `rollout status` need the right kind/name. Probe.
+async function findPgResource(
+  prov: MinikubeProvisioner,
+  namespace: string,
+): Promise<{ kind: "statefulset" | "deployment"; name: string }> {
+  const ss = await prov.kubectl([
+    "-n", namespace, "get", "statefulset", "windmill-postgresql",
+    "--ignore-not-found", "-o", "name",
+  ]);
+  if (ss.code === 0 && ss.stdout.trim()) {
+    return { kind: "statefulset", name: "windmill-postgresql" };
+  }
+  const dep = await prov.kubectl([
+    "-n", namespace, "get", "deployment", DEFAULT_PG_DEPLOYMENT,
+    "--ignore-not-found", "-o", "name",
+  ]);
+  if (dep.code === 0 && dep.stdout.trim()) {
+    return { kind: "deployment", name: DEFAULT_PG_DEPLOYMENT };
+  }
+  throw new Error(`PG not found in ns/${namespace}: neither StatefulSet nor Deployment present`);
+}
+
+// Apply pgBadger-grade settings to the running PG via `ALTER SYSTEM SET ... +
+// pg_reload_conf()`. All PGBADGER_PG_SETTINGS are SIGHUP-able, so no restart
+// is needed — which is critical: the chart's PG uses an emptyDir (persistence
+// disabled by default), so a container restart wipes PGDATA and destroys the
+// migrations Windmill just ran. ALTER SYSTEM writes to postgresql.auto.conf
+// inside PGDATA and is picked up via SIGHUP without dropping connections.
+export async function enableVerbosePgLogging(
+  prov: MinikubeProvisioner,
+  opts: { namespace?: string; user?: string; db?: string } = {},
+): Promise<void> {
+  const namespace = opts.namespace ?? DEFAULT_NAMESPACE;
+  const user = opts.user ?? DEFAULT_PG_USER;
+  const db = opts.db ?? DEFAULT_PG_DB;
+
+  // ALTER SYSTEM SET cannot run inside a transaction block, so each statement
+  // must go in its own psql `-c`. With multiple `-c`, psql commits each
+  // separately.
+  const cFlags: string[] = [];
+  for (const kv of PGBADGER_PG_SETTINGS) {
+    const eq = kv.indexOf("=");
+    const name = kv.slice(0, eq);
+    const value = kv.slice(eq + 1).replace(/'/g, "''");
+    cFlags.push("-c", `ALTER SYSTEM SET ${name} = '${value}';`);
+  }
+  cFlags.push("-c", "SELECT pg_reload_conf();");
+
+  console.log(`[pg-logging] applying pgBadger settings via ALTER SYSTEM + SIGHUP (no restart)`);
+  // `kubectl exec` doesn't take --selector — resolve the pod name from the
+  // label first so this works for both Deployment and StatefulSet layouts.
+  const podRes = await prov.kubectl([
+    "-n", namespace, "get", "pods", "-l", PG_POD_LABEL,
+    "-o", "jsonpath={.items[0].metadata.name}",
+  ]);
+  const podName = podRes.stdout.trim();
+  if (!podName) {
+    throw new Error(`[pg-logging] no pod found for selector ${PG_POD_LABEL}`);
+  }
+  const res = await prov.kubectl([
+    "-n", namespace, "exec", podName,
+    "--",
+    "psql", "-U", user, "-d", db, "-v", "ON_ERROR_STOP=1", ...cFlags,
+  ]);
+  if (res.code !== 0) {
+    throw new Error(
+      `[pg-logging] psql failed (code ${res.code}): ${res.stderr || res.stdout || "(no output)"}`,
+    );
+  }
+  console.log(`[pg-logging] settings reloaded — verbose logging active without restart`);
+}
+
+// Bump max_connections on the bundled PG. The chart doesn't expose a way to
+// pass postmaster args, so we kubectl-patch the Deployment to inject
+// `args: ["postgres", "-c", "max_connections=N"]`.
+//
+// PG persistence is off in the smoke values, so any PG restart wipes PGDATA.
+// When the patch is a real change, the new PG pod comes up with an empty DB
+// and windmill-app's stale connections fail — so we also rollout-restart
+// windmill-app and wait for it to re-run migrations.
+//
+// On reuse (Deployment already has these args) we detect the no-op via the
+// generation counter and skip the windmill-app restart entirely.
+export async function patchPgMaxConnections(
+  prov: MinikubeProvisioner,
+  maxConnections: number,
+  opts: { namespace?: string; appDeployment?: string } = {},
+): Promise<void> {
+  const namespace = opts.namespace ?? DEFAULT_NAMESPACE;
+  const appDeployment = opts.appDeployment ?? "windmill-app";
+  const pg = await findPgResource(prov, namespace);
+  const ref = `${pg.kind}/${pg.name}`;
+
+  const genRes = await prov.kubectl([
+    "-n", namespace, "get", pg.kind, pg.name,
+    "-o", "jsonpath={.metadata.generation}",
+  ]);
+  const beforeGen = parseInt(genRes.stdout.trim()) || 0;
+
+  console.log(`[pg-logging] patching PG (${ref}) to max_connections=${maxConnections}`);
+  const patch = JSON.stringify({
+    spec: { template: { spec: { containers: [{
+      name: "postgres",
+      args: ["postgres", "-c", `max_connections=${maxConnections}`],
+    }] } } },
+  });
+  const r = await prov.kubectl([
+    "-n", namespace, "patch", pg.kind, pg.name,
+    "--type=strategic", "-p", patch,
+  ]);
+  if (r.code !== 0) {
+    throw new Error(
+      `[pg-logging] PG patch failed (code ${r.code}): ${r.stderr || r.stdout || "(no output)"}`,
+    );
+  }
+
+  const genRes2 = await prov.kubectl([
+    "-n", namespace, "get", pg.kind, pg.name,
+    "-o", "jsonpath={.metadata.generation}",
+  ]);
+  const afterGen = parseInt(genRes2.stdout.trim()) || 0;
+  if (afterGen === beforeGen) {
+    console.log(`[pg-logging] PG patch was a no-op (max_connections already set)`);
+    return;
+  }
+
+  // Real rollout — wait for PG. With persistence enabled, the rollout
+  // preserves PGDATA so windmill-app doesn't need to re-migrate; without
+  // persistence we'd also need a windmill-app restart, but that path is no
+  // longer used (smoke.yaml enables postgresql.persistence).
+  const w = await prov.kubectl([
+    "-n", namespace, "rollout", "status", ref, "--timeout=180s",
+  ]);
+  if (w.code !== 0) {
+    throw new Error(
+      `[pg-logging] PG rollout did not complete: ${w.stderr || w.stdout || "(no output)"}`,
+    );
+  }
+  // Bounce windmill-app to drop stale connections that point at the old PG pod
+  // — fast (the new pod reuses the persistent migrations).
+  console.log(`[pg-logging] PG restarted — bouncing ${appDeployment} to reconnect`);
+  await prov.kubectl([
+    "-n", namespace, "rollout", "restart", `deployment/${appDeployment}`,
+  ]);
+  const aw = await prov.kubectl([
+    "-n", namespace, "rollout", "status", `deployment/${appDeployment}`,
+    "--timeout=300s",
+  ]);
+  if (aw.code !== 0) {
+    throw new Error(
+      `[pg-logging] ${appDeployment} rollout did not complete: ${aw.stderr || aw.stdout || "(no output)"}`,
+    );
+  }
+}
+
+// Capture the PG pod's stdout/stderr (kubectl logs) to a file.
+//
+// k8s/containerd has its own log-rotation gotcha similar to the journald drop
+// we hit earlier: the default `containerLogMaxSize` is 10Mi, so very-busy PG
+// logs can get rotated and the `connection received` burst at fleet-boot
+// disappears. For a smoke / short run that's fine; long benches need to raise
+// that on the kubelet or write PG to a PVC file (TODO #51 follow-on).
+export async function capturePgLog(
+  prov: MinikubeProvisioner,
+  outPath: string,
+  opts: { namespace?: string; sinceTime?: string } = {},
+): Promise<void> {
+  const namespace = opts.namespace ?? DEFAULT_NAMESPACE;
+  console.log(`[pg-logging] capturing PG logs (selector ${PG_POD_LABEL}) -> ${outPath}`);
+  // Label selector works for both StatefulSet and Deployment PG layouts.
+  const args = [
+    "-n", namespace,
+    "logs", "-l", PG_POD_LABEL,
+    "--all-containers=true",
+    "--tail=-1",
+  ];
+  if (opts.sinceTime) args.push(`--since-time=${opts.sinceTime}`);
+  const res = await prov.kubectl(args);
+  if (res.code !== 0) {
+    throw new Error(
+      `[pg-logging] kubectl logs failed: ${res.stderr || res.stdout || "(no output)"}`,
+    );
+  }
+  await Deno.writeTextFile(outPath, res.stdout);
+}

benchmarks/sim/pgbadger.ts

@@ -0,0 +1,64 @@
+// Reusable pgBadger integration: turn a PostgreSQL stderr log into a rich HTML
+// report (top queries by time + call counts, lock waits, temp files,
+// checkpoints, connection/session timeline, query-type distribution).
+//
+// Used two ways:
+//   1. The sim (`sim.ts --pgbadger`) sets these PG log settings on the
+//      provisioned Postgres, captures its log after the run, and renders a
+//      per-run `pgbadger.html` into the results folder.
+//   2. Standalone, against ANY PostgreSQL log (e.g. from a manual benchmark
+//      run against an external Windmill): enable PGBADGER_PG_SETTINGS on that
+//      PG, capture its log, then:
+//          deno run -A pgbadger.ts <pg.log> <out.html>
+//
+// pgBadger must be on PATH (it's in the flake devshell / `nix run nixpkgs#pgbadger`).
+
+// Must stay in sync with PGBADGER_PG_SETTINGS' log_line_prefix below.
+export const PGBADGER_LOG_PREFIX = "%t [%p]: user=%u,db=%d,app=%a,client=%h ";
+
+// PostgreSQL settings that make the log pgBadger-parseable and rich. Pass each
+// as a `-c name=value` to postgres. log_min_duration_statement=0 logs EVERY
+// query with its duration (verbose — only enable when you want the report).
+export const PGBADGER_PG_SETTINGS: string[] = [
+  "log_min_duration_statement=0",
+  "log_checkpoints=on",
+  "log_connections=on",
+  "log_disconnections=on",
+  "log_lock_waits=on",
+  "log_temp_files=0",
+  "log_autovacuum_min_duration=0",
+  "lc_messages=C", // pgBadger needs English log messages to parse them
+  `log_line_prefix=${PGBADGER_LOG_PREFIX}`,
+];
+
+// Render a pgBadger HTML report from a PostgreSQL log file. Best-effort: logs a
+// warning and returns false on failure rather than throwing.
+export async function runPgbadger(logPath: string, outHtml: string): Promise<boolean> {
+  try {
+    const p = new Deno.Command("pgbadger", {
+      args: ["--prefix", PGBADGER_LOG_PREFIX, "-f", "stderr", "-o", outHtml, logPath],
+      stdout: "piped",
+      stderr: "piped",
+    });
+    const { code, stderr } = await p.output();
+    if (code !== 0) {
+      console.warn(`[pgbadger] failed (${code}): ${new TextDecoder().decode(stderr).slice(0, 500)}`);
+      return false;
+    }
+    return true;
+  } catch (e) {
+    console.warn(`[pgbadger] could not run (is it on PATH?): ${(e as Error).message}`);
+    return false;
+  }
+}
+
+if (import.meta.main) {
+  const [log, out] = Deno.args;
+  if (!log || !out) {
+    console.error("usage: deno run -A pgbadger.ts <pg.log> <out.html>");
+    Deno.exit(2);
+  }
+  const ok = await runPgbadger(log, out);
+  if (ok) console.log(`pgBadger report written to ${out}`);
+  Deno.exit(ok ? 0 : 1);
+}

benchmarks/sim/pod_inventory.ts

@@ -0,0 +1,86 @@
+// Snapshot of pods alive on the cluster at teardown — written as JSON so the
+// report renderer can map pod UIDs (the only thing the cgroup-based CPU
+// sampler can extract) to human-meaningful names + nodes + the worker-group
+// label.
+//
+// Shape (one entry per pod):
+//   { uid, name, namespace, node, labels: {...} }
+
+import type { MinikubeProvisioner } from "./k8s_provisioner.ts";
+
+export type PodEntry = {
+  uid: string;
+  name: string;
+  namespace: string;
+  node: string;
+  labels: Record<string, string>;
+  // Captured so the renderer can show "what died during the bench" without
+  // relying on the event-based oom_events.json (which misses containerd's
+  // mislabeled "Error" exit 137 kills, dmesg-source flakiness, etc.).
+  containers?: Array<{
+    name: string;
+    restartCount: number;
+    lastState?: {
+      terminated?: {
+        reason?: string;
+        exitCode?: number;
+        finishedAt?: string;
+      };
+    };
+  }>;
+};
+
+export async function capturePodInventory(
+  prov: MinikubeProvisioner,
+  outPath: string,
+): Promise<PodEntry[]> {
+  const res = await prov.kubectl([
+    "get", "pods", "-A",
+    "-o", "json",
+  ]);
+  if (res.code !== 0) {
+    console.warn(`[inventory] kubectl get pods failed`);
+    return [];
+  }
+  let parsed: { items?: Array<Record<string, unknown>> };
+  try {
+    parsed = JSON.parse(res.stdout);
+  } catch (e) {
+    console.warn(`[inventory] JSON parse failed: ${(e as Error).message}`);
+    return [];
+  }
+  const entries: PodEntry[] = (parsed.items ?? []).map((p) => {
+    const meta = (p.metadata as Record<string, unknown> | undefined) ?? {};
+    const spec = (p.spec as Record<string, unknown> | undefined) ?? {};
+    const status = (p.status as Record<string, unknown> | undefined) ?? {};
+    const rawCs = (status.containerStatuses as Array<Record<string, unknown>> | undefined) ?? [];
+    const containers = rawCs.map((cs) => {
+      const ls = (cs.lastState as Record<string, unknown> | undefined) ?? {};
+      const term = (ls.terminated as Record<string, unknown> | undefined);
+      return {
+        name: String(cs.name ?? ""),
+        restartCount: Number(cs.restartCount ?? 0),
+        lastState: term
+          ? {
+            terminated: {
+              reason: term.reason as string | undefined,
+              exitCode: term.exitCode as number | undefined,
+              finishedAt: term.finishedAt as string | undefined,
+            },
+          }
+          : undefined,
+      };
+    });
+    return {
+      uid: String(meta.uid ?? ""),
+      name: String(meta.name ?? ""),
+      namespace: String(meta.namespace ?? ""),
+      node: String(spec.nodeName ?? ""),
+      labels: (meta.labels as Record<string, string> | undefined) ?? {},
+      containers,
+    };
+  });
+  await Deno.writeTextFile(outPath, JSON.stringify(entries, null, 2));
+  console.log(`[inventory] ${entries.length} pods captured -> ${outPath}`);
+  return entries;
+}

benchmarks/sim/pod_timeline.ts

@@ -0,0 +1,87 @@
+// kubectl-based per-second pod timeline. Replaces the cgroup-derived
+// workers-per-node series from the sampler (which had two known bugs: stale
+// pods.json captured only at end of bench, and the sampler's 5s rescan
+// interval missing short-lived pods). Polling kubectl every 1s gives us the
+// authoritative pod inventory at each tick — slower resolution but accurate.
+//
+// Output: JSONL, one line per poll, each line:
+//   {"ts": <ms>, "pods": [{"name":"...","node":"...","phase":"..."}, ...]}
+//
+// Stopping: caller flips the `cont` ref to false; the loop exits within ~1s.
+
+import { MinikubeProvisioner } from "./k8s_provisioner.ts";
+
+export type PodTimelinePoller = {
+  // Mutate this to false to stop the loop. The polling promise resolves when
+  // the in-flight kubectl call finishes after the flip.
+  cont: { value: boolean };
+  done: Promise<void>;
+};
+
+export function startPodTimeline(
+  prov: MinikubeProvisioner,
+  outPath: string,
+  opts: { intervalMs?: number } = {},
+): PodTimelinePoller {
+  const intervalMs = opts.intervalMs ?? 1000;
+  const cont = { value: true };
+  const f = Deno.openSync(outPath, { write: true, create: true, truncate: true });
+  const enc = new TextEncoder();
+
+  const done = (async () => {
+    while (cont.value) {
+      const startMs = Date.now();
+      try {
+        // Capture name|node|phase|ready per pod. `ready` is the first
+        // container's readiness gate — that's what kubelet uses for Service
+        // routing decisions, and it's the truthy "this worker is actually
+        // serving traffic" signal. Excluding the cgroup-still-around-but-
+        // CrashLooping case that was making the old chart misleading.
+        // CRITICAL: the newline delimiter MUST be the jsonpath string form
+        // `{"\n"}` — not literal `\n`. Bare \n means "two-char string '\n'"
+        // when emitted by kubectl, so JS split("\n") collapses everything to
+        // one mangled row. AND in TypeScript source, the inside-string `\n`
+        // must be written as `\\n` so the compiled runtime string preserves
+        // the literal `\n` for kubectl to parse — passing a TS template with
+        // raw `\n` would put a real newline INSIDE the quoted string,
+        // breaking jsonpath's quoted-string parser. Use `\\n` here.
+        const res = await prov.kubectl([
+          "get", "pods", "-A",
+          "-o", "jsonpath={range .items[*]}{.metadata.name}|{.spec.nodeName}|{.status.phase}|{.status.containerStatuses[0].ready}|{.status.containerStatuses[0].restartCount}{\"\\n\"}{end}",
+        ]);
+        if (res.code === 0) {
+          const pods: Array<{ name: string; node: string; phase: string; ready: boolean; restarts: number }> = [];
+          for (const line of res.stdout.split("\n")) {
+            if (!line.trim()) continue;
+            const [name, node, phase, ready, restartCount] = line.split("|");
+            if (name && node) {
+              pods.push({
+                name,
+                node,
+                phase: phase ?? "",
+                ready: ready === "true",
+                // restartCount lets the renderer detect "pod was restarted
+                // mid-bench" events (catches preemption, OOMKill, liveness
+                // probe failure, helm rollouts — anything that increments
+                // the kubelet's restart counter on the container).
+                restarts: Number.isFinite(parseInt(restartCount)) ? parseInt(restartCount) : 0,
+              });
+            }
+          }
+          const row = JSON.stringify({ ts: startMs, pods });
+          f.writeSync(enc.encode(row + "\n"));
+        }
+      } catch (_e) {
+        // Transient kubectl errors are tolerated — bench should not die because
+        // one poll failed. Next iteration retries.
+      }
+      const elapsed = Date.now() - startMs;
+      if (cont.value && elapsed < intervalMs) {
+        await new Promise((r) => setTimeout(r, intervalMs - elapsed));
+      }
+    }
+    f.close();
+  })();
+
+  return { cont, done };
+}

benchmarks/sim/readiness.ts

@@ -0,0 +1,284 @@
+// Pre-bench readiness check. Catches the "fire bench on broken cluster" trap
+// that wasted multiple runs this session — pods Pending, samplers in
+// CrashLoopBackOff, queue full of leftover jobs from a prior bench, PG not
+// responding fast enough. Each of those silently produced a useless report.
+//
+// Usage from main.ts:
+//   const r = await checkReadiness(prov);
+//   if (!r.ready) {
+//     console.error("[bench] cluster NOT ready:");
+//     for (const issue of r.issues) console.error("  - " + issue);
+//     Deno.exit(2);
+//   }
+//
+// Optional `waitForReady` polls every 5s up to a timeout, printing transient
+// issues until they clear. Use it when the cluster might be mid-reconcile
+// (e.g. right after a helm upgrade).
+
+import { MinikubeProvisioner } from "./k8s_provisioner.ts";
+
+export type ReadinessReport = {
+  ready: boolean;
+  issues: string[];
+  details: {
+    samplers_running: number;
+    samplers_total: number;
+    workers_ready: number;
+    workers_total: number;
+    pg_phase: string;
+    pg_responsive: boolean;
+    queue_depth: number;
+    toxiproxy_ready: boolean;
+    app_ready: boolean;
+  };
+};
+
+type PodSnapshot = {
+  metadata?: { name?: string };
+  spec?: { nodeName?: string };
+  status?: {
+    phase?: string;
+    containerStatuses?: Array<{
+      ready?: boolean;
+      restartCount?: number;
+      state?: { running?: unknown; waiting?: { reason?: string }; terminated?: unknown };
+    }>;
+  };
+};
+
+async function listPods(
+  prov: MinikubeProvisioner,
+  namespace: string,
+  selector: string,
+): Promise<PodSnapshot[]> {
+  const res = await prov.kubectl([
+    "-n", namespace, "get", "pods", "-l", selector, "-o", "json",
+  ]);
+  if (res.code !== 0) return [];
+  try {
+    return (JSON.parse(res.stdout) as { items?: PodSnapshot[] }).items ?? [];
+  } catch {
+    return [];
+  }
+}
+
+function countReady(pods: PodSnapshot[]): number {
+  return pods.filter((p) => p.status?.containerStatuses?.[0]?.ready === true).length;
+}
+
+function countRunning(pods: PodSnapshot[]): number {
+  return pods.filter((p) => p.status?.phase === "Running").length;
+}
+
+export async function checkReadiness(
+  prov: MinikubeProvisioner,
+  opts: {
+    expectSamplers?: number;
+    expectWorkers?: number;
+    namespace?: string;
+    samplerNamespace?: string;
+    requireEmptyQueue?: boolean;
+  } = {},
+): Promise<ReadinessReport> {
+  const namespace = opts.namespace ?? "default";
+  const samplerNs = opts.samplerNamespace ?? "kube-system";
+  const expectSamplers = opts.expectSamplers ?? 4;
+  const requireEmptyQueue = opts.requireEmptyQueue ?? true;
+
+  const issues: string[] = [];
+
+  // --- Samplers ---
+  // Not just "Running" — must have been Running long enough for kubelet's
+  // log buffer to have stable data covering the start of the bench. A
+  // sampler that was just-deployed 5s ago is technically Running, but its
+  // stdout buffer hasn't reached the kubelet log file yet, so a kubectl-
+  // logs --since-time at bench-end gets nothing for the first ~30s of the
+  // bench window. Require startTime to be at least MIN_STABLE_S in the past.
+  const MIN_STABLE_S = 30;
+  const samplerPods = await listPods(prov, samplerNs, "app=wm-sim-cpu-sampler");
+  const samplersRunning = countRunning(samplerPods);
+  if (samplersRunning < expectSamplers) {
+    const bad = samplerPods
+      .filter((p) => p.status?.phase !== "Running")
+      .map((p) => {
+        const reason = (p.status as { containerStatuses?: Array<{ state?: { waiting?: { reason?: string } } }> } | undefined)
+          ?.containerStatuses?.[0]?.state?.waiting?.reason;
+        return `${p.metadata?.name}${reason ? `(${reason})` : ""}`;
+      })
+      .join(", ");
+    issues.push(
+      `samplers ${samplersRunning}/${expectSamplers} Running — not Running: ${bad || "(none listed)"}`,
+    );
+  } else {
+    const nowMs = Date.now();
+    const tooYoung = samplerPods.filter((p) => {
+      const startStr = (p.status as { startTime?: string } | undefined)?.startTime;
+      if (!startStr) return true;
+      const ageS = (nowMs - Date.parse(startStr)) / 1000;
+      return ageS < MIN_STABLE_S;
+    });
+    if (tooYoung.length > 0) {
+      const tags = tooYoung.map((p) => {
+        const startStr = (p.status as { startTime?: string } | undefined)?.startTime;
+        const ageS = startStr ? Math.floor((nowMs - Date.parse(startStr)) / 1000) : -1;
+        return `${p.metadata?.name}(age=${ageS}s)`;
+      }).join(", ");
+      issues.push(
+        `samplers Running but not yet stable (need ≥ ${MIN_STABLE_S}s uptime): ${tags}`,
+      );
+    }
+  }
+
+  // --- Workers ---
+  const workerPods = await listPods(prov, namespace, "app=windmill-workers");
+  const workersReady = countReady(workerPods);
+  const workersTotal = workerPods.length;
+  if (opts.expectWorkers !== undefined) {
+    if (workersReady < opts.expectWorkers) {
+      issues.push(`workers ${workersReady}/${opts.expectWorkers} ready (${workersTotal} pods total)`);
+    }
+  } else if (workersReady < workersTotal) {
+    issues.push(`workers ${workersReady}/${workersTotal} ready`);
+  }
+
+  // --- Worker Deployments: rollout must be fully complete ---
+  // Pod-level readiness above passes during rolling-updates (200 ready while
+  // 50 old pods still being torn down). That worker churn floods the kernel's
+  // cgroup_mutex and starves the CPU sampler — m04 had a 96s data gap last
+  // bench because of exactly this. Bench MUST wait until each deployment's
+  // status reflects: observedGeneration == metadata.generation AND
+  // updatedReplicas == spec.replicas AND availableReplicas == spec.replicas
+  // AND no leftover replicas from the old ReplicaSet.
+  const deplRes = await prov.kubectl([
+    "-n", namespace, "get", "deploy", "-l", "app=windmill-workers", "-o", "json",
+  ]);
+  if (deplRes.code === 0) {
+    try {
+      const list = JSON.parse(deplRes.stdout) as {
+        items?: Array<{
+          metadata?: { name?: string; generation?: number };
+          spec?: { replicas?: number };
+          status?: {
+            observedGeneration?: number;
+            updatedReplicas?: number;
+            availableReplicas?: number;
+            replicas?: number;
+          };
+        }>;
+      };
+      for (const d of list.items ?? []) {
+        const name = d.metadata?.name ?? "?";
+        const gen = d.metadata?.generation ?? 0;
+        const obsGen = d.status?.observedGeneration ?? -1;
+        const desired = d.spec?.replicas ?? 0;
+        const updated = d.status?.updatedReplicas ?? 0;
+        const avail = d.status?.availableReplicas ?? 0;
+        const total = d.status?.replicas ?? 0;
+        if (obsGen < gen) {
+          issues.push(`deploy/${name} controller behind: observedGen=${obsGen} < gen=${gen}`);
+          continue;
+        }
+        if (updated < desired) {
+          issues.push(`deploy/${name} rollout incomplete: updated=${updated}/${desired}`);
+        }
+        if (avail < desired) {
+          issues.push(`deploy/${name} rollout incomplete: available=${avail}/${desired}`);
+        }
+        if (total > desired) {
+          issues.push(`deploy/${name} old replicas not yet terminated: total=${total} > desired=${desired}`);
+        }
+      }
+    } catch (e) {
+      issues.push(`worker deploy parse failed: ${(e as Error).message}`);
+    }
+  } else {
+    issues.push(`worker deploy lookup failed (code ${deplRes.code})`);
+  }
+
+  // --- PG pod + responsiveness ---
+  const pgPods = await listPods(prov, namespace, "app=windmill-postgresql-demo-app");
+  const pgPhase = pgPods[0]?.status?.phase ?? "absent";
+  if (pgPhase !== "Running") {
+    issues.push(`PG phase=${pgPhase}`);
+  }
+  let pgResponsive = false;
+  let queueDepth = -1;
+  if (pgPods.length > 0 && pgPhase === "Running") {
+    // Force a fast statement_timeout so a wedged PG doesn't hang the check.
+    const r = await prov.kubectl([
+      "-n", namespace, "exec", pgPods[0].metadata!.name!, "--",
+      "psql", "-U", "postgres", "-d", "windmill", "-tAc",
+      "SET statement_timeout=3000; SELECT count(*) FROM v2_job_queue;",
+    ]);
+    if (r.code === 0) {
+      const last = r.stdout.trim().split("\n").pop() ?? "";
+      const n = parseInt(last);
+      if (Number.isFinite(n)) {
+        pgResponsive = true;
+        queueDepth = n;
+        if (requireEmptyQueue && n > 0) {
+          issues.push(`queue has ${n} leftover job(s) — drain or DELETE FROM v2_job_queue before benching`);
+        }
+      } else {
+        issues.push(`PG returned non-numeric queue count: ${last.slice(0, 60)}`);
+      }
+    } else {
+      issues.push(`PG psql failed (statement_timeout=3s): ${(r.stderr || r.stdout).slice(0, 100)}`);
+    }
+  }
+
+  // --- toxiproxy + app ---
+  const toxPods = await listPods(prov, namespace, "app=toxiproxy");
+  const toxReady = countReady(toxPods) > 0;
+  if (!toxReady) issues.push(`toxiproxy not Ready (${toxPods.length} pod(s))`);
+
+  const appPods = await listPods(prov, namespace, "app=windmill-app");
+  const appReady = countReady(appPods) > 0;
+  if (!appReady) issues.push(`windmill-app not Ready (${appPods.length} pod(s))`);
+
+  return {
+    ready: issues.length === 0,
+    issues,
+    details: {
+      samplers_running: samplersRunning,
+      samplers_total: samplerPods.length,
+      workers_ready: workersReady,
+      workers_total: workersTotal,
+      pg_phase: pgPhase,
+      pg_responsive: pgResponsive,
+      queue_depth: queueDepth,
+      toxiproxy_ready: toxReady,
+      app_ready: appReady,
+    },
+  };
+}
+
+// Poll checkReadiness every N seconds until ready or timeout. Transient
+// issues that clear on a subsequent tick aren't treated as failures.
+export async function waitForReady(
+  prov: MinikubeProvisioner,
+  opts: {
+    timeoutMs?: number;
+    pollMs?: number;
+    expectSamplers?: number;
+    expectWorkers?: number;
+    requireEmptyQueue?: boolean;
+  } = {},
+): Promise<ReadinessReport> {
+  const timeoutMs = opts.timeoutMs ?? 180_000;
+  const pollMs = opts.pollMs ?? 5_000;
+  const start = Date.now();
+  let last: ReadinessReport | undefined;
+  while (true) {
+    last = await checkReadiness(prov, {
+      expectSamplers: opts.expectSamplers,
+      expectWorkers: opts.expectWorkers,
+      requireEmptyQueue: opts.requireEmptyQueue,
+    });
+    if (last.ready) return last;
+    if (Date.now() - start >= timeoutMs) return last;
+    console.log(`[readiness] not ready (${last.issues.length} issue(s)) — polling again in ${pollMs / 1000}s`);
+    for (const issue of last.issues) console.log(`  - ${issue}`);
+    await new Promise((r) => setTimeout(r, pollMs));
+  }
+}

benchmarks/sim/sim.ts

@@ -0,0 +1,225 @@
+// Windmill benchmark sim — CLI entry, provisioning ONLY (no benchmarking).
+//
+// Provisions a minikube cluster from a topology and deploys Windmill onto it
+// via helm (bundled PG). Leaves the cluster running until Ctrl-C, then tears
+// it down. Benchmarking is a separate concern — point the bench at the URL
+// this prints (`benchmarks/main.ts --host <url> ...` / `wm-bench`).
+//
+// Usage (wired via the `wm_sim` flake helper):
+//   wm_sim --topology sim/topologies/k8s_smoke.json
+//   wm_sim --topology <t>.json --helm <local-chart-path>
+//   wm_sim --topology <t>.json --helm <chart> --helm-values values.yaml
+//   wm_sim --topology <t>.json --helm-values v.yaml \
+//          --helm-set-override windmill.workerGroups[0].replicas=250
+//
+// The helm chart's `values.yaml` is the primary source of deployment config
+// (replicas, resources, image). `--helm-set-override` is for surgical CLI
+// tweaks on top — naming reflects "override, not primary source". The values
+// files are archived under results/<stamp>_<topology>/ so each run records
+// exactly what was deployed.
+//
+// Requires minikube + the kvm2 driver on PATH and libvirtd reachable. The
+// `wm_sim` wrapper sets SIM_KVM2_DRIVER_DIR / SIM_LIBVIRT_LIB_DIR.
+
+import { Command } from "https://deno.land/x/cliffy@v0.25.7/command/mod.ts";
+import { loadTopology, validateTopology } from "./topology.ts";
+import { MinikubeProvisioner } from "./k8s_provisioner.ts";
+import {
+  helmDeployWindmill,
+  portForwardApi,
+  waitPgReady,
+  waitWindmillAppReady,
+  type ApiEndpoint,
+} from "./helm_deploy.ts";
+import { buildToxiproxyManifests } from "./toxiproxy_k8s.ts";
+import { enableVerbosePgLogging } from "./pg_logging.ts";
+import { applyCpuSampler } from "./cpu_sampler_k8s.ts";
+import { loadCachedImages, saveImagesToCache } from "./image_cache.ts";
+
+// minikube profile names allow only alphanumerics + dashes; topology names
+// often contain underscores, so sanitise.
+function profileFor(prefix: string, name: string): string {
+  const slug = name.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "");
+  return `${prefix}-${slug || "topology"}`;
+}
+
+async function loadValidTopology(path: string) {
+  const topology = await loadTopology(path);
+  const issues = await validateTopology(topology);
+  for (const w of issues.filter((i) => i.severity === "warn")) {
+    console.warn(`[topology] WARN: ${w.message}`);
+  }
+  const errors = issues.filter((i) => i.severity === "error");
+  if (errors.length > 0) {
+    for (const e of errors) console.error(`[topology] ERROR: ${e.message}`);
+    Deno.exit(1);
+  }
+  return topology;
+}
+
+type DeployArgs = {
+  helm?: string;
+  helmValues?: string[];
+  helmSetOverride?: string[];
+};
+
+async function deployWindmill(prov: MinikubeProvisioner, args: DeployArgs): Promise<ApiEndpoint> {
+  await helmDeployWindmill({
+    profile: prov.profile,
+    chart: args.helm,
+    valuesFiles: args.helmValues,
+    set: args.helmSetOverride,
+  });
+  // helm runs without --wait — gate PG + app readiness ourselves so port-forward
+  // doesn't race startup. PG max_connections is templated into the chart now
+  // (postgresql.maxConnections value), no post-install patching needed.
+  await waitPgReady(prov.profile);
+  await waitWindmillAppReady(prov.profile);
+  return await portForwardApi(prov.profile);
+}
+
+type UpOpts = DeployArgs & { topology: string };
+
+async function upMain(opts: UpOpts) {
+  const topology = await loadValidTopology(opts.topology);
+
+  // Per-run reports dir — archives the inputs (topology + values files) so
+  // each cluster bring-up records exactly what was deployed.
+  const isoStamp = new Date().toISOString().replace(/[:.]/g, "-").replace(/Z$/, "");
+  const outDir = `reports/${isoStamp}_${topology.name}`;
+  await Deno.mkdir(outDir, { recursive: true });
+  await Deno.copyFile(opts.topology, `${outDir}/topology.json`);
+  for (const v of opts.helmValues ?? []) {
+    const base = v.split("/").pop() || "values.yaml";
+    await Deno.copyFile(v, `${outDir}/${base}`);
+  }
+  console.log(`[sim] inputs archived to ${outDir}/`);
+
+  const prov = new MinikubeProvisioner({ profile: profileFor("wm-sim", topology.name) });
+  // Clean any leftover state (crashed prior run, leaked libvirt domains)
+  // before provisioning. teardown() now sweeps both minikube + libvirt.
+  console.log(`[sim] cleaning stale state for profile=${prov.profile}...`);
+  await prov.teardown();
+  await prov.provision(topology);
+  try {
+    await loadCachedImages(prov.profile);
+  } catch (e) {
+    console.warn(`[sim] image cache preload failed: ${(e as Error).message}`);
+  }
+
+  // Toxiproxy: single Deployment + Service, no nodeSelector. Worker count +
+  // resources + DATABASE_URL live in the user's values.yaml.
+  const toxManifestPath = `${outDir}/toxiproxy.yaml`;
+  await Deno.writeTextFile(toxManifestPath, buildToxiproxyManifests(topology));
+  console.log(`[sim] applying toxiproxy...`);
+  const applyRes = await prov.kubectl(["apply", "-f", toxManifestPath]);
+  if (applyRes.code !== 0) {
+    console.warn(`[sim] toxiproxy apply non-zero: ${applyRes.stdout}`);
+  }
+
+  // helm: user values are the only source of truth (workerGroups +
+  // DATABASE_URL routing live there now).
+  const endpoint = await deployWindmill(prov, opts);
+
+  // PG max_connections bump + readiness gates are inside deployWindmill now.
+
+  // Enable verbose PG logging so a pgBadger report can be generated on
+  // teardown — best-effort; if the chart's PG layout differs, log and
+  // continue (the cluster is still usable for ad-hoc work).
+  try {
+    await enableVerbosePgLogging(prov);
+  } catch (e) {
+    console.warn(`[sim] could not enable verbose PG logging: ${(e as Error).message}`);
+  }
+
+  // Apply the sampler DaemonSet once per provision — it runs continuously
+  // between bench runs; wm-bench scopes its output via `--since-time` so each
+  // report only sees rows from its own window. No rollout-restart on reuse.
+  try {
+    await applyCpuSampler(prov, outDir);
+  } catch (e) {
+    console.warn(`[sim] CPU sampler not applied: ${(e as Error).message}`);
+  }
+
+  await Deno.writeTextFile(
+    `${outDir}/meta.json`,
+    JSON.stringify(
+      {
+        topology: topology.name,
+        profile: prov.profile,
+        api_host: endpoint.host,
+        helm_chart: opts.helm ?? "windmill/windmill",
+        helm_values_files: opts.helmValues ?? [],
+        helm_set_overrides: opts.helmSetOverride ?? [],
+        started_at: new Date().toISOString(),
+      },
+      null,
+      2,
+    ),
+  );
+
+  console.log("\n" + "=".repeat(56));
+  console.log(`  Topology "${topology.name}" is up.`);
+  console.log(`  Windmill API:     ${endpoint.host}`);
+  console.log(`  minikube profile: ${prov.profile}`);
+  console.log(`  Login:            admin@windmill.dev / changeme`);
+  console.log("");
+  console.log(`  To bench against this cluster:`);
+  console.log(`    wm-bench --host ${endpoint.host} --minikube-profile ${prov.profile} ...`);
+  console.log("");
+  console.log(`  Ctrl-C to tear down the cluster.`);
+  console.log("=".repeat(56));
+  await new Promise<void>((resolve) => {
+    const handler = () => {
+      Deno.removeSignalListener("SIGINT", handler);
+      Deno.removeSignalListener("SIGTERM", handler);
+      resolve();
+    };
+    Deno.addSignalListener("SIGINT", handler);
+    Deno.addSignalListener("SIGTERM", handler);
+  });
+
+  // Save images BEFORE minikube delete — they're gone with the VM otherwise.
+  try {
+    await saveImagesToCache(prov.profile, (args) => prov.kubectl(args));
+  } catch (e) {
+    console.warn(`[sim] image cache save failed: ${(e as Error).message}`);
+  }
+  console.log("\nTearing down...");
+  endpoint.stop();
+  await prov.teardown();
+  Deno.exit(0);
+}
+
+if (import.meta.main) {
+  // `up` subcommand mirrors the root action — both forms work:
+  //   wm_sim --topology X        (root action)
+  //   wm_sim up --topology X     (explicit subcommand)
+  const upSub = new Command()
+    .description("Provision the cluster + deploy Windmill, leave running (Ctrl-C tears down).")
+    .option("--topology <path:string>", "Topology JSON.", { required: true })
+    .option(
+      "--helm <chart:string>",
+      "Helm chart for Windmill: local path (preferred) or remote 'repo/name'. Default 'windmill/windmill'.",
+    )
+    .option(
+      "--helm-values <path:string>",
+      "Helm values.yaml — primary source of deployment config (replicas/resources/image). Repeatable; later files take precedence. Copied into results/ as the run record.",
+      { collect: true },
+    )
+    .option(
+      "--helm-set-override <kv:string>",
+      "Inline key=value override on top of --helm-values (repeatable). For surgical CLI tweaks; the values file is the source of truth.",
+      { collect: true },
+    )
+    .action(upMain);
+
+  await new Command()
+    .name("wm_sim")
+    .description(
+      "Windmill benchmark sim: provision a minikube cluster from a topology and deploy Windmill on it. Provisioning only — no benchmarking.",
+    )
+    .command("up", upSub)
+    .default("up")
+    .parse();
+}

benchmarks/sim/svg_to_pdf.ts

@@ -0,0 +1,46 @@
+// SVG → PDF rendering. Uses jspdf + svg2pdf.js via npm: specifiers (Deno).
+// Used by render_report.ts to also emit a dashboard.pdf alongside dashboard.svg
+// so the report is readable in PDF viewers / printable without losing the
+// vector format.
+//
+// Implementation note: svg2pdf.js needs a DOM SVGElement, which Deno doesn't
+// have natively. We use jsdom (already used by the chart code) to parse the
+// SVG string into a real Element, then pass it to svg2pdf.
+
+import { JSDOM } from "https://jspm.dev/jsdom";
+import { jsPDF } from "npm:jspdf@2.5.1";
+import "npm:svg2pdf.js@2.2.4";
+
+// jsPDF augments its prototype when svg2pdf.js is imported.
+type JsPDFWithSvg = jsPDF & {
+  svg(element: Element, opts?: Record<string, unknown>): Promise<jsPDF>;
+};
+
+export async function renderSvgToPdf(
+  svgPath: string,
+  pdfPath: string,
+): Promise<void> {
+  const svgText = await Deno.readTextFile(svgPath);
+
+  // Parse the SVG into a real DOM Element via jsdom.
+  // deno-lint-ignore no-explicit-any
+  const dom = new (JSDOM as any)(`<!DOCTYPE html><body>${svgText}</body>`);
+  const svgEl = dom.window.document.querySelector("svg");
+  if (!svgEl) throw new Error(`no <svg> in ${svgPath}`);
+
+  // Read intrinsic dimensions from the SVG; default to A3 landscape if missing.
+  const w = parseFloat(svgEl.getAttribute("width") ?? "1200");
+  const h = parseFloat(svgEl.getAttribute("height") ?? "850");
+
+  // 1 SVG unit = 1 pt in jsPDF. Page sized exactly to SVG so nothing crops.
+  const pdf = new jsPDF({
+    orientation: w >= h ? "landscape" : "portrait",
+    unit: "pt",
+    format: [w, h],
+  }) as JsPDFWithSvg;
+
+  await pdf.svg(svgEl, { x: 0, y: 0, width: w, height: h });
+  const bytes = pdf.output("arraybuffer");
+  await Deno.writeFile(pdfPath, new Uint8Array(bytes));
+  console.log(`[pdf] ${pdfPath}`);
+}

benchmarks/sim/topologies/k8s-4node.json

@@ -0,0 +1,38 @@
+{
+  "name": "k8s_4node",
+  "postgres": {
+    "cpu": 1,
+    "memory": "2G"
+  },
+  "nodes": [
+    {
+      "id": "n0",
+      "mode": "native",
+      "standalone": true,
+      "cpu": 2,
+      "memory": "4G",
+      "db_latency_ms": 5
+    },
+    {
+      "id": "n1",
+      "mode": "native",
+      "cpu": 4,
+      "memory": "20G",
+      "db_latency_ms": 5
+    },
+    {
+      "id": "n2",
+      "mode": "native",
+      "cpu": 4,
+      "memory": "20G",
+      "db_latency_ms": 5
+    },
+    {
+      "id": "n3",
+      "mode": "native",
+      "cpu": 4,
+      "memory": "20G",
+      "db_latency_ms": 5
+    }
+  ]
+}

benchmarks/sim/topology.ts

@@ -0,0 +1,324 @@
+// Topology definition: PG + N Windmill nodes, each with configurable
+// CPU/memory and optional per-node DB latency. Loaded from a standalone
+// JSON file so suites can reference topologies by path.
+//
+// Every node runs exactly ONE Windmill worker. This is a hard rule. To scale
+// up, spawn more nodes; do NOT stack workers in one process.
+//
+// A node runs in one of two modes, configurable per node:
+//
+//   - "native"           1 native worker connecting to PG directly via SQL.
+//                        Maps to Windmill's MODE=worker by default — just a
+//                        worker, no API server, no host port.
+//                        Set `standalone: true` to colocate an API server in
+//                        the same process (Windmill's MODE=standalone). That
+//                        node gets a host port and becomes the user-facing
+//                        entrypoint. Docker-based topologies need exactly one
+//                        such node.
+//
+//   - "k8s"              A k3d-managed cluster. Helm decides server placement
+//                        and replica count. Helm chart is selected via CLI
+//                        flags.
+//
+// Windmill's MODE=agent (HTTP-only worker, no PG access) is a separate shape
+// we don't model yet — deferred.
+
+export type NativeMode = {
+  mode: "native";
+  // Defaults to false → MODE=worker (worker-only, no API, no host port,
+  // connects to PG via SQL). When true → MODE=standalone (server + worker
+  // in one process, exposed on a host port). Exactly one node per
+  // docker-based topology must have this true.
+  standalone?: boolean;
+  // Only meaningful with `standalone: true`. When true, the node runs
+  // MODE=server (API only, NO embedded worker) instead of MODE=standalone.
+  // Use this when separate worker nodes do all the work and you don't want the
+  // server's own worker competing for jobs (a standalone server's worker
+  // ignores WORKER_TAGS restrictions and listens on the full default tag set).
+  server_only?: boolean;
+  tag?: string;
+  // Caps the PG pool for this node via the DATABASE_CONNECTIONS env var.
+  // Default Windmill behaviour is 5 per worker / 50 for a server; at high
+  // worker counts that exhausts PG's max_connections, so large topologies
+  // set this to 2-3 per worker.
+  db_connections?: number;
+};
+
+// k8s mode is provisioned by the k3d provisioner (M5). The k3d cluster runs
+// inside docker containers; the actual Windmill workload is deployed onto it
+// via helm. `pods` and `workers_per_pod` are convenience shortcuts that the
+// k3d provisioner maps onto helm values.
+//
+// Helm chart selection (upstream vs local, repo URL, version, value overrides)
+// is *not* in the topology — it lives on the CLI (--helm-chart, --helm-repo,
+// --helm-set, --helm-values-file). Topology describes the cluster shape;
+// CLI describes what to deploy onto it.
+export type K8sMode = {
+  mode: "k8s";
+  cluster_agents?: number;  // k3d cluster agent count (k3s nodes); defaults to 1.
+                            // This shapes the underlying cluster, NOT the
+                            // workload. The Windmill deployment scale (worker
+                            // replica count, HPA, etc.) is owned by helm —
+                            // pass it via --helm-set on the sim CLI.
+  tag?: string;
+};
+
+// Helm config supplied at the CLI level for k8s-mode topologies.
+export type HelmConfig = {
+  chart?: string;          // local path (relative to cwd) OR chart name like "windmill/windmill"
+  repo?: string;           // helm repo URL, used when chart is a name (not a path)
+  version?: string;        // pin a specific chart version
+  values?: Record<string, unknown>; // inline overrides from --helm-set
+  values_files?: string[]; // paths from --helm-values-file
+};
+
+export type NodeMode = NativeMode | K8sMode;
+
+export type NodeSpec = {
+  id: string; // unique identifier, used for container naming and metrics keys
+  cpu: number; // --cpus value (count of CPUs, not a fraction)
+  memory: string; // --memory value (e.g. "4G", "512M")
+  db_latency_ms?: number; // injected via toxiproxy; 0 or omitted means no proxy
+} & NodeMode;
+
+// Worker count per "node" in the topology. Hard rule for native/agent:
+// NUM_WORKERS=1 per process. For k8s the worker count is whatever helm
+// decides (configurable via --helm-set), so we don't know it at the topology
+// layer — return null to signal "ask helm".
+export function workerCount(n: NodeSpec): number | null {
+  if (n.mode === "k8s") return null;
+  return 1;
+}
+
+export type PostgresSpec = {
+  cpu: number;
+  memory: string;
+  // PG max_connections. Default PG ships 100; large worker counts need more
+  // (sum of per-node pools). When set, also bump `memory` since each backend
+  // reserves a few MB.
+  max_connections?: number;
+  // shared_buffers etc. could go here later; defaults are fine for v1
+};
+
+export type ReservedHostResources = {
+  cpus: number; // cores left untouched for harness + toxiproxy + host OS
+  memory: string;
+};
+
+export type Topology = {
+  name: string;
+  pin_cpus?: boolean; // when true, provisioner assigns non-overlapping cpusets
+  host_resources?: {
+    cpus: number;
+    memory: string;
+    reserved?: ReservedHostResources;
+  };
+  postgres: PostgresSpec;
+  nodes: NodeSpec[];
+  // Note: container runtime (docker vs podman) is selected at runtime via the
+  // SIM_CONTAINER_CMD env var, not declared in the topology. Topology is
+  // workload-shape, not infrastructure-runtime.
+};
+
+// Parse a memory string like "4G" / "512M" / "1024K" / "1048576" into bytes.
+// Docker accepts the same shorthand; we just need it for validation arithmetic.
+export function parseMemory(value: string): number {
+  const m = value.trim().match(/^(\d+(?:\.\d+)?)\s*([KMGT]?)([Bi]*)?$/i);
+  if (!m) throw new Error(`Invalid memory value: ${value}`);
+  const n = parseFloat(m[1]);
+  const unit = m[2].toUpperCase();
+  const mult: Record<string, number> = { "": 1, K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 };
+  if (!(unit in mult)) throw new Error(`Invalid memory unit in ${value}`);
+  return n * mult[unit];
+}
+
+export function formatBytes(bytes: number): string {
+  const units = ["B", "K", "M", "G", "T"];
+  let i = 0;
+  let n = bytes;
+  while (n >= 1024 && i < units.length - 1) { n /= 1024; i++; }
+  return `${n.toFixed(2)}${units[i]}`;
+}
+
+export type ValidationIssue = { severity: "error" | "warn"; message: string };
+
+// Detect the number of CPUs available to the harness process. On Linux,
+// navigator.hardwareConcurrency reports logical cores. This is the same view
+// the host kernel exposes; it does not respect cgroup limits, which is fine
+// because the harness itself should not be containerised.
+export function hostCores(): number {
+  return navigator.hardwareConcurrency;
+}
+
+// Read total host memory from /proc/meminfo. Falls back to 0 if unavailable
+// (e.g. on macOS dev machines); the validator treats 0 as "unknown" and only
+// warns rather than blocks.
+export async function hostMemoryBytes(): Promise<number> {
+  try {
+    const text = await Deno.readTextFile("/proc/meminfo");
+    const m = text.match(/^MemTotal:\s+(\d+)\s+kB/m);
+    if (!m) return 0;
+    return parseInt(m[1], 10) * 1024;
+  } catch (_) {
+    return 0;
+  }
+}
+
+// Sum requested CPU and memory across PG + all nodes, compare to host minus
+// reserved buffer. Fail closed: any overcommit returns an error so the sim
+// refuses to run rather than silently degrade benchmark accuracy.
+export async function validateTopology(t: Topology): Promise<ValidationIssue[]> {
+  const issues: ValidationIssue[] = [];
+
+  const reserved = t.host_resources?.reserved ?? { cpus: 2, memory: "2G" };
+  const declaredCpus = t.host_resources?.cpus ?? hostCores();
+  const declaredMem = t.host_resources?.memory
+    ? parseMemory(t.host_resources.memory)
+    : await hostMemoryBytes();
+
+  const totalCpu = t.postgres.cpu + t.nodes.reduce((s, n) => s + n.cpu, 0);
+  const totalMem = parseMemory(t.postgres.memory) +
+    t.nodes.reduce((s, n) => s + parseMemory(n.memory), 0);
+
+  const cpuBudget = declaredCpus - reserved.cpus;
+  if (totalCpu > cpuBudget) {
+    issues.push({
+      severity: "error",
+      message: `CPU oversubscription: topology requests ${totalCpu} cores, ` +
+        `host has ${declaredCpus} cores with ${reserved.cpus} reserved ` +
+        `(budget ${cpuBudget}). Reduce node.cpu values or run on a larger host.`,
+    });
+  }
+
+  if (declaredMem > 0) {
+    const memBudget = declaredMem - parseMemory(reserved.memory);
+    if (totalMem > memBudget) {
+      issues.push({
+        severity: "error",
+        message: `Memory oversubscription: topology requests ${formatBytes(totalMem)}, ` +
+          `host has ${formatBytes(declaredMem)} with ${reserved.memory} reserved ` +
+          `(budget ${formatBytes(memBudget)}).`,
+      });
+    }
+  } else {
+    issues.push({
+      severity: "warn",
+      message: "Could not read host memory; skipping memory budget check. " +
+        "Set host_resources.memory in the topology to enforce it.",
+    });
+  }
+
+  const ids = new Set<string>();
+  let nativeCount = 0;
+  let standaloneCount = 0; // native nodes with standalone: true
+  let k8sCount = 0;
+  for (const n of t.nodes) {
+    if (ids.has(n.id)) {
+      issues.push({ severity: "error", message: `Duplicate node id: ${n.id}` });
+    }
+    ids.add(n.id);
+    if (n.cpu <= 0) {
+      issues.push({ severity: "error", message: `Node ${n.id} has non-positive cpu` });
+    }
+    // Use a typed-as-unknown probe because TS narrows away nodes that don't
+    // match any union branch, leaving `never` and breaking property access.
+    const probe = n as { id: string; mode?: string };
+    if (!probe.mode) {
+      issues.push({
+        severity: "error",
+        message: `Node ${probe.id} is missing a "mode" field (must be one of "native", "k8s")`,
+      });
+      continue;
+    }
+    if (probe.mode === "native") {
+      nativeCount++;
+      if ((n as NativeMode).standalone === true) standaloneCount++;
+    } else if (probe.mode === "k8s") {
+      k8sCount++;
+    } else if (probe.mode === "agent") {
+      issues.push({
+        severity: "error",
+        message: `Node ${probe.id}: mode "agent" is not supported yet. ` +
+          `Use mode "native" (the default is worker-only); set ` +
+          `\`standalone: true\` on the one node that should run an API server.`,
+      });
+    } else {
+      issues.push({
+        severity: "error",
+        message: `Node ${probe.id} has unknown mode "${probe.mode}"`,
+      });
+    }
+
+    // Reject the now-removed knobs explicitly so older configs fail loudly
+    // instead of silently shipping a multi-worker setup the sim isn't built
+    // to measure.
+    const raw = n as unknown as Record<string, unknown>;
+    if ("num_workers" in raw) {
+      issues.push({
+        severity: "error",
+        message: `Node ${probe.id}: num_workers is no longer configurable. Every node runs exactly one worker.`,
+      });
+    }
+    if (n.mode === "k8s") {
+      if ("pods" in raw) {
+        issues.push({
+          severity: "error",
+          message: `Node ${probe.id}: pods is no longer configurable in the topology. Helm owns workload scale — pass via --helm-set worker.replicaCount=N.`,
+        });
+      }
+      if ("workers_per_pod" in raw) {
+        issues.push({
+          severity: "error",
+          message: `Node ${probe.id}: workers_per_pod is no longer configurable. Helm owns workload scale.`,
+        });
+      }
+    }
+  }
+
+  // Docker-based topologies must have exactly one standalone node — that's
+  // the user-facing API entrypoint. Zero means there's no API; more than one
+  // means the harness has no clear push target. k8s topologies are exempt
+  // (helm provides its own service).
+  if (k8sCount === 0) {
+    if (standaloneCount === 0) {
+      issues.push({
+        severity: "error",
+        message: "Topology has no `standalone: true` node. Exactly one " +
+          "native node must set `standalone: true` to host the API server.",
+      });
+    } else if (standaloneCount > 1) {
+      issues.push({
+        severity: "error",
+        message: `Topology has ${standaloneCount} nodes with \`standalone: true\`. ` +
+          `Only one node should host the API server.`,
+      });
+    }
+  }
+
+  if (k8sCount > 0 && nativeCount > 0) {
+    issues.push({
+      severity: "error",
+      message: "Mixed k8s and non-k8s nodes in one topology is not supported. " +
+        "All nodes must share a provisioner (docker or k3d).",
+    });
+  }
+
+  if (t.pin_cpus && totalCpu + reserved.cpus > declaredCpus) {
+    issues.push({
+      severity: "error",
+      message: "pin_cpus requires sum(cpu) + reserved.cpus <= host cores so " +
+        "every container gets a disjoint cpuset.",
+    });
+  }
+
+  return issues;
+}
+
+export async function loadTopology(path: string): Promise<Topology> {
+  const text = await Deno.readTextFile(path);
+  const t = JSON.parse(text) as Topology;
+  if (!t.name || !t.nodes || !t.postgres) {
+    throw new Error(`Topology at ${path} is missing required fields (name, nodes, postgres)`);
+  }
+  return t;
+}

benchmarks/sim/topology_test.ts

@@ -0,0 +1,240 @@
+import { assertEquals, assertStringIncludes } from "https://deno.land/std@0.224.0/assert/mod.ts";
+
+import {
+  formatBytes,
+  parseMemory,
+  validateTopology,
+  workerCount,
+  type NodeSpec,
+  type Topology,
+} from "./topology.ts";
+
+const standaloneNode = (id: string, cpu = 2): NodeSpec => ({
+  id, mode: "native", standalone: true, cpu, memory: "4G",
+});
+const workerNode = (id: string, cpu = 2): NodeSpec => ({
+  id, mode: "native", cpu, memory: "4G",
+});
+
+const okTopo = (): Topology => ({
+  name: "ok",
+  host_resources: {
+    cpus: 8,
+    memory: "16G",
+    reserved: { cpus: 2, memory: "2G" },
+  },
+  postgres: { cpu: 2, memory: "4G" },
+  nodes: [standaloneNode("server")],
+});
+
+Deno.test("parseMemory: integer values across units", () => {
+  assertEquals(parseMemory("1024"), 1024);
+  assertEquals(parseMemory("1K"), 1024);
+  assertEquals(parseMemory("1M"), 1024 ** 2);
+  assertEquals(parseMemory("1G"), 1024 ** 3);
+  assertEquals(parseMemory("2T"), 2 * 1024 ** 4);
+});
+
+Deno.test("parseMemory: decimal multipliers", () => {
+  assertEquals(parseMemory("1.5G"), 1.5 * 1024 ** 3);
+  assertEquals(parseMemory("0.5M"), 0.5 * 1024 ** 2);
+});
+
+Deno.test("parseMemory: case-insensitive units", () => {
+  assertEquals(parseMemory("4g"), 4 * 1024 ** 3);
+  assertEquals(parseMemory("512m"), 512 * 1024 ** 2);
+});
+
+Deno.test("parseMemory: rejects garbage", () => {
+  let threw = false;
+  try {
+    parseMemory("twelve");
+  } catch (_) {
+    threw = true;
+  }
+  assertEquals(threw, true);
+});
+
+Deno.test("formatBytes: round-trips through parseMemory", () => {
+  for (const bytes of [1024, 1024 ** 2, 4 * 1024 ** 3, 1.5 * 1024 ** 3]) {
+    const formatted = formatBytes(bytes);
+    // formatBytes uses 2dp so the parse may have a tiny rounding delta; just
+    // confirm the unit landed in the right ballpark.
+    const reparsed = parseMemory(formatted);
+    const delta = Math.abs(reparsed - bytes) / bytes;
+    assertEquals(delta < 0.02, true, `${bytes} -> ${formatted} -> ${reparsed}`);
+  }
+});
+
+Deno.test("validateTopology: healthy topology produces no errors", async () => {
+  const issues = await validateTopology(okTopo());
+  assertEquals(issues.filter((i) => i.severity === "error").length, 0);
+});
+
+Deno.test("validateTopology: catches CPU oversubscription", async () => {
+  const t = okTopo();
+  t.host_resources!.cpus = 4; // 4 - 2 reserved = 2 budget, requesting 4
+  const issues = await validateTopology(t);
+  const errors = issues.filter((i) => i.severity === "error");
+  assertEquals(errors.length >= 1, true);
+  assertStringIncludes(errors[0].message, "CPU oversubscription");
+});
+
+Deno.test("validateTopology: catches memory oversubscription", async () => {
+  const t = okTopo();
+  t.host_resources!.memory = "8G"; // 8G - 2G reserved = 6G budget, requesting 8G
+  const issues = await validateTopology(t);
+  const errors = issues.filter((i) => i.severity === "error");
+  assertEquals(
+    errors.some((e) => e.message.includes("Memory oversubscription")),
+    true,
+  );
+});
+
+Deno.test("validateTopology: catches duplicate node ids", async () => {
+  const t = okTopo();
+  t.nodes.push(workerNode("server"));
+  // Bump host so CPU/mem don't also error out and obscure the dup detection.
+  t.host_resources!.cpus = 16;
+  t.host_resources!.memory = "32G";
+  const issues = await validateTopology(t);
+  assertEquals(
+    issues.some((i) => i.severity === "error" && i.message.includes("Duplicate node id")),
+    true,
+  );
+});
+
+Deno.test("validateTopology: catches non-positive cpu on a node", async () => {
+  const t = okTopo();
+  t.nodes[0].cpu = 0;
+  const issues = await validateTopology(t);
+  assertEquals(
+    issues.some((i) => i.severity === "error" && i.message.includes("non-positive cpu")),
+    true,
+  );
+});
+
+Deno.test("validateTopology: rejects num_workers field (no longer configurable)", async () => {
+  const t = okTopo();
+  (t.nodes[0] as unknown as Record<string, unknown>).num_workers = 2;
+  const issues = await validateTopology(t);
+  assertEquals(
+    issues.some((i) => i.severity === "error" && i.message.includes("no longer configurable")),
+    true,
+  );
+});
+
+Deno.test("validateTopology: rejects pods on k8s mode (helm owns scale)", async () => {
+  const t = okTopo();
+  t.nodes = [{ id: "c", mode: "k8s", cpu: 4, memory: "8G" } as NodeSpec];
+  (t.nodes[0] as unknown as Record<string, unknown>).pods = 3;
+  t.host_resources!.cpus = 8;
+  const issues = await validateTopology(t);
+  assertEquals(
+    issues.some((i) => i.severity === "error" && i.message.includes("pods is no longer configurable")),
+    true,
+  );
+});
+
+Deno.test("validateTopology: rejects workers_per_pod on k8s mode", async () => {
+  const t = okTopo();
+  t.nodes = [{ id: "c", mode: "k8s", cpu: 4, memory: "8G" } as NodeSpec];
+  (t.nodes[0] as unknown as Record<string, unknown>).workers_per_pod = 2;
+  t.host_resources!.cpus = 8;
+  const issues = await validateTopology(t);
+  assertEquals(
+    issues.some((i) => i.severity === "error" && i.message.includes("workers_per_pod is no longer configurable")),
+    true,
+  );
+});
+
+Deno.test("validateTopology: no standalone node fails", async () => {
+  const t = okTopo();
+  t.nodes = [workerNode("a"), workerNode("b")];
+  const issues = await validateTopology(t);
+  assertEquals(
+    issues.some((i) => i.severity === "error" && i.message.includes("`standalone: true` node")),
+    true,
+  );
+});
+
+Deno.test("validateTopology: more than one standalone node fails", async () => {
+  const t = okTopo();
+  t.nodes = [standaloneNode("a"), standaloneNode("b")];
+  t.host_resources!.cpus = 16;
+  const issues = await validateTopology(t);
+  assertEquals(
+    issues.some((i) => i.severity === "error" && i.message.includes("nodes with `standalone: true`")),
+    true,
+  );
+});
+
+Deno.test("validateTopology: standalone + worker mix is valid", async () => {
+  const t = okTopo();
+  t.nodes = [standaloneNode("server"), workerNode("worker1"), workerNode("worker2")];
+  t.host_resources!.cpus = 16;
+  t.host_resources!.memory = "32G";
+  const issues = await validateTopology(t);
+  assertEquals(issues.filter((i) => i.severity === "error").length, 0);
+});
+
+Deno.test("validateTopology: legacy mode=agent rejected with helpful message", async () => {
+  const t = okTopo();
+  t.nodes = [standaloneNode("server"), { id: "a", mode: "agent" as "native", cpu: 1, memory: "2G" }];
+  t.host_resources!.cpus = 16;
+  const issues = await validateTopology(t);
+  assertEquals(
+    issues.some((i) => i.severity === "error" && i.message.includes("\"agent\" is not supported yet")),
+    true,
+  );
+});
+
+Deno.test("validateTopology: mixed k8s and non-k8s fails", async () => {
+  const t = okTopo();
+  t.nodes = [
+    standaloneNode("server"),
+    { id: "k8s", mode: "k8s", cpu: 4, memory: "8G" } as NodeSpec,
+  ];
+  t.host_resources!.cpus = 16;
+  t.host_resources!.memory = "32G";
+  const issues = await validateTopology(t);
+  assertEquals(
+    issues.some((i) => i.message.includes("Mixed k8s and non-k8s")),
+    true,
+  );
+});
+
+Deno.test("workerCount: native/agent always 1", () => {
+  assertEquals(workerCount({ id: "x", mode: "native", cpu: 4, memory: "1G" }), 1);
+  assertEquals(workerCount({ id: "x", mode: "agent" as "native",  cpu: 4, memory: "1G" }), 1);
+});
+
+Deno.test("workerCount: k8s returns null (helm owns scale)", () => {
+  assertEquals(workerCount({ id: "x", mode: "k8s", cpu: 4, memory: "1G" }), null);
+});
+
+Deno.test("validateTopology: pin_cpus requires enough cores", async () => {
+  const t = okTopo();
+  t.pin_cpus = true;
+  t.host_resources!.cpus = 4; // budget 2 for postgres+nodes, sum is 4
+  const issues = await validateTopology(t);
+  // First it errors on oversubscription, then again on the pin_cpus constraint.
+  assertEquals(
+    issues.some((i) => i.message.includes("pin_cpus")),
+    true,
+  );
+});
+
+Deno.test("validateTopology: drops host_resources entirely → falls back to live host", async () => {
+  // When no host_resources block is declared, hostCores() / hostMemoryBytes()
+  // are used as the budget. We don't assert exact behavior here (host-dependent);
+  // we just confirm the validator doesn't crash and doesn't spuriously flag CPU
+  // oversubscription for the healthy topo.
+  const t = okTopo();
+  delete (t as { host_resources?: unknown }).host_resources;
+  const issues = await validateTopology(t);
+  assertEquals(
+    issues.some((i) => i.severity === "error" && i.message.includes("CPU oversubscription")),
+    false,
+  );
+});

benchmarks/sim/toxiproxy.ts

@@ -0,0 +1,113 @@
+// Toxiproxy admin client + per-node proxy planning.
+//
+// One toxiproxy container is run for the whole sim. For every node that
+// declares db_latency_ms > 0, we create a proxy with a unique listener inside
+// the toxiproxy container; the node's DATABASE_URL points at
+// `toxiproxy:<port>` instead of `postgres:5432`. The provisioner is
+// responsible for starting the container; this module only configures it.
+//
+// Wire format docs: https://github.com/Shopify/toxiproxy#http-api
+
+export type ProxyPlan = {
+  proxyName: string;     // toxiproxy proxy id, e.g. "node-far"
+  listenPort: number;    // port inside the toxiproxy container
+  upstream: string;      // e.g. "pg:5432"
+  latencyMs: number;     // 0 means no toxic, plain pass-through
+};
+
+// Allocate ports starting at this offset so they don't collide with toxiproxy's
+// admin port (8474) or anything else common. 15400 is unused in well-known
+// service registries.
+export const FIRST_PROXY_PORT = 15400;
+
+export function buildProxyPlans(
+  nodes: { id: string; db_latency_ms?: number }[],
+  upstream: string,
+): ProxyPlan[] {
+  const plans: ProxyPlan[] = [];
+  for (let i = 0; i < nodes.length; i++) {
+    const n = nodes[i];
+    if (!n.db_latency_ms || n.db_latency_ms <= 0) continue;
+    plans.push({
+      proxyName: `node-${n.id}`,
+      listenPort: FIRST_PROXY_PORT + i,
+      upstream,
+      latencyMs: n.db_latency_ms,
+    });
+  }
+  return plans;
+}
+
+export class ToxiproxyClient {
+  constructor(public adminUrl: string) {}
+
+  async waitReady(timeoutMs = 30_000): Promise<void> {
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+      try {
+        const r = await fetch(`${this.adminUrl}/version`);
+        if (r.ok) {
+          await r.body?.cancel();
+          return;
+        }
+        await r.body?.cancel();
+      } catch (_) { /* not ready yet */ }
+      await new Promise((r) => setTimeout(r, 250));
+    }
+    throw new Error(`toxiproxy admin API not reachable at ${this.adminUrl}`);
+  }
+
+  async reset(): Promise<void> {
+    const r = await fetch(`${this.adminUrl}/reset`, { method: "POST" });
+    if (!r.ok) throw new Error(`toxiproxy reset failed: ${r.status} ${await r.text()}`);
+    await r.body?.cancel();
+  }
+
+  async createProxy(plan: ProxyPlan): Promise<void> {
+    const r = await fetch(`${this.adminUrl}/proxies`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        name: plan.proxyName,
+        listen: `0.0.0.0:${plan.listenPort}`,
+        upstream: plan.upstream,
+        enabled: true,
+      }),
+    });
+    if (!r.ok) {
+      const body = await r.text();
+      throw new Error(`createProxy(${plan.proxyName}) failed: ${r.status} ${body}`);
+    }
+    await r.body?.cancel();
+  }
+
+  async addLatencyToxic(proxyName: string, latencyMs: number): Promise<void> {
+    const r = await fetch(`${this.adminUrl}/proxies/${proxyName}/toxics`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        // The default downstream direction adds latency to traffic going from
+        // upstream (PG) to client (worker), which is what matters for query RTT.
+        type: "latency",
+        attributes: { latency: latencyMs, jitter: 0 },
+      }),
+    });
+    if (!r.ok) {
+      const body = await r.text();
+      throw new Error(`addLatencyToxic(${proxyName}) failed: ${r.status} ${body}`);
+    }
+    await r.body?.cancel();
+  }
+
+  // Convenience: clear all proxies and recreate the plan. Idempotent across
+  // sim runs that reuse a long-lived toxiproxy container.
+  async apply(plans: ProxyPlan[]): Promise<void> {
+    await this.reset();
+    for (const p of plans) {
+      await this.createProxy(p);
+      if (p.latencyMs > 0) {
+        await this.addLatencyToxic(p.proxyName, p.latencyMs);
+      }
+    }
+  }
+}

benchmarks/sim/toxiproxy_k8s.ts

@@ -0,0 +1,186 @@
+// Toxiproxy for the k8s sim path.
+//
+// Architecture: ONE toxiproxy Deployment (no nodeSelector, k8s schedules it
+// wherever), one Service in front of it, and ONE worker group named "default"
+// with N replicas — also no nodeSelector, so the scheduler spreads workers
+// across nodes naturally. All workers route through the single toxiproxy
+// Service to PG, with the topology's RTT applied as a latency toxic.
+//
+// Caveat: per-node `db_latency_ms` variance from the topology is collapsed —
+// we use the first node's value as the shared latency. Heterogeneous-RTT
+// topologies need a per-node-host toxiproxy DaemonSet + downward-API NODE_IP;
+// defer until a topology actually needs it.
+
+import type { Topology } from "./topology.ts";
+import { stringify as yamlStringify } from "https://deno.land/std@0.224.0/yaml/mod.ts";
+
+const TOXIPROXY_IMAGE = "ghcr.io/shopify/toxiproxy:2.9.0";
+const PROXY_LISTEN_PORT = 15400;
+const TOXIPROXY_ADMIN_PORT = 8474;
+const PG_SERVICE = "windmill-postgresql";
+const PG_PORT = 5432;
+const TOXIPROXY_NAME = "toxiproxy";
+
+// Build the toxiproxy Deployment+Service+ConfigMap only. Worker count,
+// resources, and the DATABASE_URL that routes through toxiproxy all live in
+// the user's values.yaml — the chart is the source of truth for windmill
+// config.
+//
+// Service URL the values.yaml should target:
+//   postgres://postgres:windmill@toxiproxy.default.svc:15400/windmill?sslmode=disable
+export function buildToxiproxyManifests(
+  topology: Topology,
+  namespace = "default",
+): string {
+  const latencyMs = topology.nodes[0]?.db_latency_ms ?? 0;
+  return [
+    buildConfigMap({ name: TOXIPROXY_NAME, namespace, proxyName: "pg", latencyMs }),
+    buildDeployment({ name: TOXIPROXY_NAME, namespace }),
+    buildService({ name: TOXIPROXY_NAME, namespace }),
+  ].join("---\n");
+}
+
+// toxiproxy-server `-config` JSON: pre-populates proxies + toxics at startup,
+// no shell or admin-API call needed. The shopify/toxiproxy image is distroless
+// (no /bin/sh), so this is the way.
+function buildConfigMap(args: {
+  name: string;
+  namespace: string;
+  proxyName: string;
+  latencyMs: number;
+}): string {
+  const proxy: Record<string, unknown> = {
+    name: args.proxyName,
+    listen: `0.0.0.0:${PROXY_LISTEN_PORT}`,
+    upstream: `${PG_SERVICE}:${PG_PORT}`,
+    enabled: true,
+  };
+  if (args.latencyMs > 0) {
+    proxy.toxics = [{
+      name: "latency",
+      type: "latency",
+      stream: "downstream",
+      attributes: { latency: args.latencyMs, jitter: 0 },
+    }];
+  }
+  const config = JSON.stringify([proxy], null, 2);
+  return yamlStringify({
+    apiVersion: "v1",
+    kind: "ConfigMap",
+    metadata: { name: args.name, namespace: args.namespace },
+    data: { "config.json": config },
+  });
+}
+
+function buildDeployment(args: {
+  name: string;
+  namespace: string;
+}): string {
+  // OOM-immunity for toxiproxy:
+  //   1. priorityClassName=wm-critical — kubelet L1 eviction sorts pods by
+  //      priority, so toxiproxy gets evicted after worker pods.
+  //   2. A busybox sidecar (`oom-immune`) writes -999 to the toxiproxy
+  //      process's /proc/<pid>/oom_score_adj for kernel-OOM (L2) protection.
+  //      The shopify/toxiproxy image is distroless (no shell), so we can't
+  //      use a postStart `exec` hook on it directly — `shareProcessNamespace:
+  //      true` lets the sidecar see toxiproxy's PID, and CAP_SYS_RESOURCE on
+  //      the sidecar allows lowering oom_score_adj below 0.
+  return yamlStringify({
+    apiVersion: "apps/v1",
+    kind: "Deployment",
+    metadata: { name: args.name, namespace: args.namespace, labels: { app: args.name } },
+    spec: {
+      replicas: 1,
+      selector: { matchLabels: { app: args.name } },
+      template: {
+        metadata: { labels: { app: args.name } },
+        spec: {
+          priorityClassName: "wm-critical",
+          shareProcessNamespace: true,
+          containers: [
+            {
+              name: "toxiproxy",
+              image: TOXIPROXY_IMAGE,
+              // toxiproxy-server is the image's ENTRYPOINT; we just pass flags.
+              args: [
+                "-host", "0.0.0.0",
+                "-port", String(TOXIPROXY_ADMIN_PORT),
+                "-config", "/etc/toxiproxy/config.json",
+              ],
+              ports: [
+                { name: "proxy", containerPort: PROXY_LISTEN_PORT },
+                { name: "admin", containerPort: TOXIPROXY_ADMIN_PORT },
+              ],
+              // No memory limit — even 1280Mi was being tripped on busy
+              // nodes (toxiproxy went CrashLoopBackOff multiple times this
+              // session). Without a cgroup ceiling, only the node-level
+              // kernel OOM can take it down, and the wm-critical priority
+              // class + oom_score_adj=-999 sidecar make it last-to-pick.
+              resources: {
+                requests: { cpu: "1000m", memory: "320Mi" },
+              },
+              readinessProbe: {
+                httpGet: { path: "/proxies", port: TOXIPROXY_ADMIN_PORT },
+                initialDelaySeconds: 1,
+                periodSeconds: 2,
+              },
+              volumeMounts: [
+                { name: "config", mountPath: "/etc/toxiproxy", readOnly: true },
+              ],
+            },
+            {
+              name: "oom-immune",
+              image: "busybox:1.36",
+              securityContext: { capabilities: { add: ["SYS_RESOURCE"] } },
+              // With shareProcessNamespace=true the sidecar sees toxiproxy's
+              // PID. Retry up to 60s for the process to come up, then write
+              // -999 and sleep forever (sidecar must stay alive — exit would
+              // trigger pod restart logic).
+              command: ["/bin/sh", "-c"],
+              args: [
+                // The shopify/toxiproxy image's binary is `/toxiproxy` (not
+                // `toxiproxy-server` — that's the older name). pidof matches
+                // on basename, so `pidof toxiproxy` is what we need.
+                [
+                  "set -e",
+                  "for i in $(seq 1 60); do",
+                  "  pid=$(pidof toxiproxy || true)",
+                  "  if [ -n \"$pid\" ]; then",
+                  "    for p in $pid; do",
+                  "      echo -999 > /proc/$p/oom_score_adj && \\",
+                  "        echo \"[oom-immune] pid=$p adj=$(cat /proc/$p/oom_score_adj)\" || \\",
+                  "        echo \"[oom-immune] failed on pid=$p\"",
+                  "    done",
+                  "    break",
+                  "  fi",
+                  "  sleep 1",
+                  "done",
+                  "exec sleep infinity",
+                ].join("\n"),
+              ],
+            },
+          ],
+          volumes: [
+            { name: "config", configMap: { name: args.name } },
+          ],
+        },
+      },
+    },
+  });
+}
+
+function buildService(args: { name: string; namespace: string }): string {
+  return yamlStringify({
+    apiVersion: "v1",
+    kind: "Service",
+    metadata: { name: args.name, namespace: args.namespace, labels: { app: args.name } },
+    spec: {
+      type: "ClusterIP",
+      selector: { app: args.name },
+      ports: [
+        { name: "proxy", port: PROXY_LISTEN_PORT, targetPort: PROXY_LISTEN_PORT },
+        { name: "admin", port: TOXIPROXY_ADMIN_PORT, targetPort: TOXIPROXY_ADMIN_PORT },
+      ],
+    },
+  });
+}

benchmarks/sim/toxiproxy_test.ts

@@ -0,0 +1,68 @@
+import { assertEquals } from "https://deno.land/std@0.224.0/assert/mod.ts";
+
+import { buildProxyPlans, FIRST_PROXY_PORT } from "./toxiproxy.ts";
+
+Deno.test("buildProxyPlans: skips nodes with no latency", () => {
+  const plans = buildProxyPlans(
+    [
+      { id: "a" }, // no latency field
+      { id: "b", db_latency_ms: 0 },
+      { id: "c", db_latency_ms: 50 },
+    ],
+    "pg:5432",
+  );
+  assertEquals(plans.length, 1);
+  assertEquals(plans[0].proxyName, "node-c");
+  assertEquals(plans[0].latencyMs, 50);
+  assertEquals(plans[0].upstream, "pg:5432");
+});
+
+Deno.test("buildProxyPlans: assigns unique ports starting at FIRST_PROXY_PORT", () => {
+  const plans = buildProxyPlans(
+    [
+      { id: "x", db_latency_ms: 10 },
+      { id: "y", db_latency_ms: 20 },
+      { id: "z", db_latency_ms: 30 },
+    ],
+    "pg:5432",
+  );
+  assertEquals(plans.length, 3);
+  const ports = plans.map((p) => p.listenPort);
+  const unique = new Set(ports);
+  assertEquals(unique.size, 3, "ports must be unique");
+  assertEquals(ports[0] >= FIRST_PROXY_PORT, true);
+});
+
+Deno.test("buildProxyPlans: port index follows node array position, not skip index", () => {
+  // Edge case: node[1] is skipped (no latency), but node[2] gets a port
+  // derived from its position. This is important because the provisioner
+  // maps proxies back to nodes via the "node-<id>" naming, not via index.
+  const plans = buildProxyPlans(
+    [
+      { id: "a" },
+      { id: "b" },
+      { id: "c", db_latency_ms: 50 },
+    ],
+    "pg:5432",
+  );
+  assertEquals(plans.length, 1);
+  // c is at index 2 in the input → port FIRST + 2
+  assertEquals(plans[0].listenPort, FIRST_PROXY_PORT + 2);
+});
+
+Deno.test("buildProxyPlans: empty input yields empty plans", () => {
+  assertEquals(buildProxyPlans([], "pg:5432").length, 0);
+});
+
+Deno.test("buildProxyPlans: all-zero latency yields empty plans (no toxiproxy needed)", () => {
+  assertEquals(
+    buildProxyPlans(
+      [
+        { id: "a", db_latency_ms: 0 },
+        { id: "b", db_latency_ms: 0 },
+      ],
+      "pg:5432",
+    ).length,
+    0,
+  );
+});

benchmarks/sim/types.ts

@@ -0,0 +1,25 @@
+// Shared types that don't depend on the existing benchmark harness. Pulling
+// these out of workload.ts keeps test modules (report_test.ts, sim_test.ts)
+// from transitively importing benchmark_oneoff.ts → lib.ts, which has a
+// pre-existing type error unrelated to the sim.
+
+export type WorkloadResult = {
+  kind: string;
+  jobs: number;
+  throughput: number;
+  ts: number;
+};
+
+// Provisioner output: a handle the rest of the sim can use to run the
+// workload + collect metrics. Provisioned by SimProvisioner.provision().
+export type NodeHandle = {
+  id: string;
+  api_host: string;       // base URL the harness pushes at, empty for agent nodes
+  worker_token?: string;
+};
+
+export type Provisioned = {
+  postgres_url: string;          // for the harness's collector connection
+  postgres_internal_url: string; // for cross-container references
+  nodes: NodeHandle[];
+};

benchmarks/sim/util_metrics.ts

@@ -0,0 +1,26 @@
+// Pure helpers for the util-group panel math. Kept separate from
+// render_report.ts so they can be unit-tested without dragging in d3/jsdom
+// (drawGraphMulti pulls a ~10MB dep tree, far too heavy for a formula test).
+
+// Oversaturation percent: how many more runnable processes than CPUs, as a
+// percentage of ncpu. >0 means there's queued work waiting for CPU.
+//
+//   procs_running = 4, ncpu = 4   →   0%   (exactly utilized)
+//   procs_running = 8, ncpu = 4   →   100% (one extra core's worth queued)
+//   procs_running = 4, ncpu = 8   →   0%   (clamped — never negative)
+//
+// Prefers `procs_running` from /proc/stat (runnable only — excludes D-state
+// processes waiting on disk/network). Falls back to loadavg-1min when the
+// poller didn't capture procs_running (older bench reports). loadavg
+// overcounts because it includes D-state, which is why we switched.
+export function computeOversatPct(
+  ncpu: number,
+  opts: { procs_running?: number | null; load1?: number | null },
+): number {
+  if (!Number.isFinite(ncpu) || ncpu < 1) return 0;
+  const runnable = (typeof opts.procs_running === "number" && Number.isFinite(opts.procs_running))
+    ? opts.procs_running
+    : (typeof opts.load1 === "number" && Number.isFinite(opts.load1) ? opts.load1 : NaN);
+  if (!Number.isFinite(runnable)) return 0;
+  return Math.max(0, ((runnable - ncpu) / ncpu) * 100);
+}

benchmarks/sim/util_metrics_test.ts

@@ -0,0 +1,41 @@
+import { assertEquals } from "https://deno.land/std@0.224.0/assert/mod.ts";
+import { computeOversatPct } from "./util_metrics.ts";
+
+Deno.test("computeOversatPct: at capacity returns 0", () => {
+  assertEquals(computeOversatPct(4, { procs_running: 4 }), 0);
+  assertEquals(computeOversatPct(8, { procs_running: 8 }), 0);
+});
+
+Deno.test("computeOversatPct: idle returns 0 (clamped, not negative)", () => {
+  assertEquals(computeOversatPct(4, { procs_running: 1 }), 0);
+  assertEquals(computeOversatPct(4, { procs_running: 0 }), 0);
+});
+
+Deno.test("computeOversatPct: one extra core's worth → 100%", () => {
+  assertEquals(computeOversatPct(4, { procs_running: 8 }), 100);
+});
+
+Deno.test("computeOversatPct: 10x runnable on 4 CPUs → 900%", () => {
+  assertEquals(computeOversatPct(4, { procs_running: 40 }), 900);
+});
+
+Deno.test("computeOversatPct: prefers procs_running over load1", () => {
+  // load1 says oversat, procs_running says fine — should report fine.
+  assertEquals(computeOversatPct(4, { procs_running: 4, load1: 100 }), 0);
+});
+
+Deno.test("computeOversatPct: falls back to load1 when procs_running missing", () => {
+  assertEquals(computeOversatPct(4, { procs_running: null, load1: 8 }), 100);
+  assertEquals(computeOversatPct(4, { load1: 12 }), 200);
+});
+
+Deno.test("computeOversatPct: returns 0 when ncpu is invalid", () => {
+  assertEquals(computeOversatPct(0, { procs_running: 100 }), 0);
+  assertEquals(computeOversatPct(NaN, { procs_running: 100 }), 0);
+  assertEquals(computeOversatPct(-1, { procs_running: 100 }), 0);
+});
+
+Deno.test("computeOversatPct: returns 0 when neither metric present", () => {
+  assertEquals(computeOversatPct(4, {}), 0);
+  assertEquals(computeOversatPct(4, { procs_running: null, load1: null }), 0);
+});

benchmarks/sim/util_panel_snapshot_test.ts

@@ -0,0 +1,146 @@
+// Snapshot-ish test for the util panel render path. We don't snapshot the
+// full SVG (too brittle — d3 formatting varies between releases) but we
+// assert the load-bearing markers are present:
+//   - blue CPU-util area fill (#377eb8)
+//   - orange oversaturation area fill (#ff8c00)
+//   - 100% horizontal reference line
+//   - relative-time x-axis ticks ("0s", "30s", ...) — NOT wall-clock HH:MM
+//   - phase-boundary dashed verticals
+//
+// This catches the regressions we've already paid for: cap-at-100 leaking
+// onto oversaturation, color override not wired, wall-clock leaking back in,
+// area-fill ordering inverted.
+
+import {
+  assert,
+  assertStringIncludes,
+} from "https://deno.land/std@0.224.0/assert/mod.ts";
+import { drawGraphMulti, type DataPointMulti } from "../graph.ts";
+
+function buildUtilSeries(originMs: number): DataPointMulti[] {
+  const out: DataPointMulti[] = [];
+  // 120 samples at 1Hz — enough for the d3 axis to emit several ticks.
+  for (let i = 0; i < 120; i++) {
+    const date = new Date(originMs + i * 1000);
+    // CPU util ramps 0 → 100%, capped.
+    out.push({ value: Math.min(100, i), date, kind: "cpu util" });
+    // Oversaturation kicks in after t=30s, climbs uncapped to 400%.
+    const oversat = i < 30 ? 0 : (i - 30) * 4;
+    out.push({ value: oversat, date, kind: "oversaturation" });
+  }
+  return out;
+}
+
+Deno.test("util panel: orange + blue area fills both present, oversat in front of cpu", () => {
+  const t0 = Date.parse("2026-06-08T10:00:00Z");
+  const svg = drawGraphMulti(
+    buildUtilSeries(t0),
+    "m04 — CPU + oversat",
+    "[%]",
+    undefined, // no yMax — oversat must be allowed >100
+    [new Date(t0 + 60_000)], // phase boundary at 60s
+    [{ y: 100, label: "100% (full VM)" }],
+    undefined,
+    undefined,
+    [
+      { kind: "oversaturation", color: "#ff8c00", opacity: 0.55 },
+      { kind: "cpu util", color: "#377eb8", opacity: 1.0 },
+    ],
+    { "oversaturation": "#ff8c00", "cpu util": "#377eb8" },
+    t0,
+  );
+
+  // Both fill colors must appear.
+  assertStringIncludes(svg, "#ff8c00");
+  assertStringIncludes(svg, "#377eb8");
+  // Backmost-first: orange must appear in the SVG document before blue (DOM
+  // order = paint order — later siblings paint on top).
+  const orangeIdx = svg.indexOf('fill="#ff8c00"');
+  const blueIdx = svg.indexOf('fill="#377eb8"');
+  assert(orangeIdx >= 0 && blueIdx > orangeIdx,
+    `expected oversaturation fill before cpu-util fill (orange=${orangeIdx}, blue=${blueIdx})`);
+});
+
+Deno.test("util panel: 100% horizontal reference line is rendered", () => {
+  const t0 = Date.parse("2026-06-08T10:00:00Z");
+  const svg = drawGraphMulti(
+    buildUtilSeries(t0),
+    "m04 — CPU + oversat",
+    "[%]",
+    undefined,
+    undefined,
+    [{ y: 100, label: "100% (full VM)" }],
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    t0,
+  );
+  assertStringIncludes(svg, "100% (full VM)");
+});
+
+Deno.test("util panel: x-axis uses relative seconds, NOT wall-clock HH:MM", () => {
+  const t0 = Date.parse("2026-06-08T10:00:00Z");
+  const svg = drawGraphMulti(
+    buildUtilSeries(t0),
+    "m04 — CPU + oversat",
+    "[%]",
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    t0, // shared origin = the bench start, same as series origin → ticks count from 0s
+  );
+  // d3 picks ~5 ticks across 120s → expect "0s" and a higher second mark.
+  assertStringIncludes(svg, ">0s<");
+  // Wall-clock format must NOT appear when relative formatter is active.
+  assert(!/>1?[0-9]:[0-5][0-9]</.test(svg),
+    `wall-clock-style HH:MM tick label leaked into relative-time axis svg`);
+});
+
+Deno.test("util panel: phase boundary dashed verticals are emitted", () => {
+  const t0 = Date.parse("2026-06-08T10:00:00Z");
+  const svg = drawGraphMulti(
+    buildUtilSeries(t0),
+    "m04 — CPU + oversat",
+    "[%]",
+    undefined,
+    { dates: [new Date(t0 + 60_000)], hideLabels: true },
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    t0,
+  );
+  assertStringIncludes(svg, 'class="phase-boundary"');
+  assertStringIncludes(svg, 'stroke-dasharray="3 3"');
+  // hideLabels: true → no `P1>P2` text overlay.
+  assert(!svg.includes("phase-boundary-label"),
+    "phase-boundary-label leaked despite hideLabels: true");
+});
+
+Deno.test("util panel: shared origin overrides per-chart origin", () => {
+  const seriesT0 = Date.parse("2026-06-08T10:00:30Z"); // 30s LATER than shared origin
+  const sharedT0 = Date.parse("2026-06-08T10:00:00Z");
+  const svg = drawGraphMulti(
+    buildUtilSeries(seriesT0),
+    "m04 — CPU + oversat",
+    "[%]",
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    sharedT0,
+  );
+  // With shared origin, the first tick label should be "30s" (or larger),
+  // not "0s" — series starts 30s after the shared origin.
+  assert(!svg.includes(">0s<"),
+    `expected first tick to NOT be "0s" when series starts after shared origin`);
+});

benchmarks/sim/values/local.example.yaml

@@ -0,0 +1,32 @@
+# Local helm values overlay for wm_sim — secrets + per-developer tweaks.
+# Copy this file to `local.yaml` (gitignored) and fill in the values you want.
+#
+# Pass alongside smoke.yaml so it overrides:
+#   wm_sim up --topology <t>.json \
+#     --helm ~/Dev/windmill-helm-charts/charts/windmill \
+#     --helm-values benchmarks/sim/values/smoke.yaml \
+#     --helm-values benchmarks/sim/values/local.yaml
+#
+# Helm merges -f files in order; the later file wins on conflicts, so anything
+# you put here trumps the matching key in smoke.yaml.
+
+# Enterprise license — set to enable EE-gated features (autoscaling, distributed
+# workers, readiness probes, etc). With enterprise.enabled=true the chart also
+# automatically swaps to the EE image (`ghcr.io/windmill-labs/windmill-ee`) and
+# wires the license env var — no other changes needed.
+enterprise:
+  enabled: true
+  licenseKey: "PASTE_YOUR_EE_KEY_HERE"
+  # Or, to read from an existing Secret instead of inlining the key:
+  # licenseKeySecretName: windmill-ee-license
+  # licenseKeySecretKey:  license
+  # Autoscaling RBAC (Role + RoleBinding for `patch deployments`). Without
+  # this the EE health check fails with "Insufficient permissions to access
+  # verb `patch`" — see https://www.windmill.dev/docs/core_concepts/autoscaling#kubernetes
+  createKubernetesAutoscalingRolesAndBindings: true
+
+# Optional: pin a specific image/tag. With enterprise.enabled=true the default
+# repo is `ghcr.io/windmill-labs/windmill-ee`; uncomment to override.
+# windmill:
+#   image: ghcr.io/windmill-labs/windmill-ee
+#   tag:   "1.711.0"

benchmarks/sim/values/smoke.yaml

@@ -0,0 +1,154 @@
+# Helm values for the k8s_4node bench topology (1 small CP + 3 × 4vCPU/20GiB workers).
+#
+# Critical-pod safety pattern:
+#   1. Chart creates a `wm-critical` PriorityClass (value 2_000_000_000) — this
+#      hits kubelet's IsCriticalPodBasedOnPriority special case, giving any pod
+#      assigned to it oom_score_adj=-997. Burstable QoS, but kernel-OOM-immune.
+#   2. App and PG both reference wm-critical via priorityClassName.
+#   3. App and PG have NO `limits.memory` — combined with the priority class,
+#      they cannot cgroup-OOM (no own ceiling) and the kernel won't pick them.
+#      Workers are the designated OOM victims (adj~984) under node pressure.
+#   4. App has a podAntiAffinity preference vs PG so they land on different
+#      worker nodes when capacity allows.
+#
+# Pass at the wm_sim invocation:
+#   wm_sim --topology benchmarks/sim/topologies/k8s-4node.json \
+#          --helm-values  benchmarks/sim/values/smoke.yaml
+
+# EE / enterprise settings live in `local.yaml` (gitignored). Layer that file
+# alongside smoke.yaml with `-f smoke.yaml -f local.yaml` to enable EE + the
+# license key. This file stays committable; license keys must never land here.
+
+windmill:
+  appReplicas: 1
+  extraReplicas: 0
+  indexer:
+    enabled: false
+
+  # Declare wm-critical, which is referenced by app.priorityClassName and
+  # postgresql.priorityClassName below.
+  #
+  # NOTE: k8s caps user-defined PriorityClass `value` at 1_000_000_000. The
+  # kubelet's IsCriticalPodBasedOnPriority shortcut (which would set
+  # oom_score_adj = -997) requires value >= 2_000_000_000, which is reserved
+  # for the built-in system-cluster-critical/system-node-critical classes
+  # (kube-system namespace only). So we use the max user-allowed value: that
+  # still gives us scheduling priority + eviction protection. For OOM ranking,
+  # we lean on big memory requests below to pull the Burstable-formula adj
+  # well below the workers'.
+  priorityClasses:
+    enabled: true
+    classes:
+      - name: wm-critical
+        value: 1000000000
+        description: "Windmill critical pods (app, PG) — scheduled/evicted last."
+
+  app:
+    priorityClassName: wm-critical
+    # postStart hook writes -999 to /proc/1/oom_score_adj — kernel cannot
+    # pick this pod for OOM. The k8s PriorityClass route to -997 is blocked
+    # for non-kube-system pods, so we do it directly inside the container.
+    oomImmune: true
+    resources:
+      requests:
+        cpu: 1                # 1024 CFS shares — ~20× any worker's 50m
+        # 1 GiB request leaves room for workers on the same node; oom_score_adj
+        # = -999 (via postStart hook) keeps app immune to OOM regardless of
+        # request size.
+        memory: 1Gi
+      limits: {}              # no cpu cap, no memory cap. App takes what it
+                              # needs; workers on the same node lose CFS share
+                              # → process fewer jobs → less load on app → balance.
+    # Spread app away from PG when possible. PG carries its PVC and stays
+    # pinned to its first-scheduled node; app should land elsewhere.
+    affinity:
+      podAntiAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              topologyKey: kubernetes.io/hostname
+              labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - windmill-postgresql-demo-app
+
+  workerGroups:
+    - name: default
+      replicas: 200
+      mode: worker
+      # No priorityClassName — workers are intentionally the disposable
+      # layer that absorbs OOM pressure on a node where PG or app is heavy.
+      resources:
+        requests:
+          # Tiny CPU request — workers are mostly idle (poll PG, run sleep
+          # jobs) so 50m was over-reserving and starving m02 (where PG sits)
+          # of worker capacity. 5m means ~80 workers can fit anywhere CPU-
+          # budget-wise; the scheduler spreads them evenly across nodes.
+          # Actual CPU burst is bounded by the host's CFS rather than this
+          # floor.
+          cpu: 5m
+          memory: 128Mi
+        limits:
+          # No cpu limit — workers can burst into idle cores.
+          # Memory ceiling at 8 GiB (was 2 GiB): the ops_day etl_storm phase
+          # has lognormal ram_mb cap=4096 MiB, so individual deno subprocesses
+          # legitimately need >2 GiB and were being kernel-cgroup-OOM-killed
+          # (visible as L2 node-kernel "deno" entries via dmesg parsing).
+          # 8 GiB swallows the tail without over-committing 20 GiB nodes too
+          # aggressively — realistic concurrent peak is well under 3 workers.
+          memory: 8Gi
+      extraEnv:
+        - name: DATABASE_URL
+          value: postgres://postgres:windmill@toxiproxy.default.svc:15400/windmill?sslmode=disable
+        - name: DATABASE_CONNECTIONS
+          value: "1"
+    # Agent workers — connect to the app via HTTP+JWT instead of directly to
+    # PG. Useful for measuring how much of the throughput cap is PG contention
+    # vs. the app's HTTP-mediated path. Token lives in K8s Secret
+    # `windmill-agent-token` (not in this file), created via kubectl.
+    - name: agent
+      replicas: 5
+      mode: agent
+      resources:
+        requests:
+          cpu: 5m
+          memory: 128Mi
+        limits:
+          memory: 1Gi
+      extraEnv:
+        - name: MODE
+          value: agent
+        - name: BASE_INTERNAL_URL
+          value: http://windmill-app.default.svc:8000
+        - name: AGENT_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: windmill-agent-token
+              key: token
+
+postgresql:
+  maxConnections: 2000
+  priorityClassName: wm-critical
+  # postStart hook → /proc/1/oom_score_adj = -999. See app.oomImmune note.
+  oomImmune: true
+  resources:
+    requests:
+      cpu: 3                  # 3 cores — fits on a worker node where ~33
+                              # workers reserve 1.65 cores leaving ~2.35
+                              # free (cpu=4 was Pending forever). PG still
+                              # bursts to all 4 cores when workers are idle
+                              # (limits empty). Under full contention CFS
+                              # gives PG 3072/(3072+1683) ≈ 65% of cores.
+      # No memory request/limit. PG is free to grow as large as the node
+      # has memory for. OOM-immunity comes from priorityClassName +
+      # postStart oom_score_adj=-999 (not from request-driven Burstable
+      # ranking). The previous 12Gi request was just a scheduling fence
+      # the cluster doesn't need — PG should land on whichever worker
+      # node has the most free RAM at deploy time.
+    limits: {}                # uncapped — PG takes whatever it needs; only
+                              # the node's physical memory ceiling applies.
+  persistence:
+    enabled: true
+    size: 10Gi

benchmarks/worker.ts

@@ -5,6 +5,7 @@ import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 import { Action, evaluate } from "./action.ts";
 import { getFlowPayload } from "./lib.ts";
+import { sampleJobParams, isPusherActive, type WorkloadConfig } from "./workloads/distribution.ts";
 
 async function getQueueCount() {
   return (
@@ -29,6 +30,9 @@ const promise = new Promise<{
   server: string;
   token: string;
   hideProgress: boolean;
+  workloadConfig?: WorkloadConfig;
+  i: number; // 0-based pusher index — used by phased workloads to gate
+             // which pushers are active in each phase (others idle).
 }>((resolve, _reject) => {
   self.onmessage = (evt) => {
     const sharedConfig = evt.data;
@@ -45,6 +49,8 @@ const promise = new Promise<{
       server: sharedConfig.server,
       token: sharedConfig.token,
       hideProgress: sharedConfig.hideProgress,
+      workloadConfig: sharedConfig.workloadConfig,
+      i: sharedConfig.i,
     };
     self.name = "Worker " + sharedConfig.i;
     resolve(config);
@@ -70,6 +76,19 @@ const updateStatusInterval = setInterval(() => {
 
 while (cont) {
   try {
+    // Phased workloads cap the number of active pushers per phase. Inactive
+    // pushers wait without sending jobs — this is how "low load" phases
+    // (warmup, cooldown) produce real low load even though all N workers
+    // were spawned at bench start.
+    const elapsed_s = (Date.now() - start_time) / 1000;
+    if (
+      config.workloadConfig &&
+      !isPusherActive(config.workloadConfig, config.i, elapsed_s)
+    ) {
+      await sleep(0.5);
+      continue;
+    }
+
     const queue_length = await getQueueCount();
     if (queue_length > 2500) {
       console.log(
@@ -116,10 +135,16 @@ while (cont) {
             },
           });
         } else {
+          // `random` pattern: sample per-job args from the workload config so
+          // each push carries its own (ram_mb, duration_ms, mode). Phased
+          // configs sample from the currently-active phase's distributions.
+          const args = (config.scriptPattern === "random" && config.workloadConfig)
+            ? sampleJobParams(config.workloadConfig, { elapsed_s })
+            : undefined;
           uuid = await windmill.JobService.runScriptByPath({
             workspace: config.workspace_id,
             path: "f/benchmarks/" + (config.scriptPattern || "deno"),
-            requestBody: {},
+            requestBody: args ?? {},
           });
         }
       } catch (e) {

benchmarks/workloads/burst.json

@@ -0,0 +1,37 @@
+{
+  "_comment": "Four-phase burst pattern: light IO baseline -> mixed mid -> CPU-heavy burst -> cooldown. Demonstrates phase transitions in mode mix, pusher count, and job heaviness. Total push window = 100s. Designed for max 50 pushers so it fits the PG cgroup ceiling (we OOM'd PG at 100 workers in earlier benches).",
+  "phases": [
+    {
+      "name": "light_io",
+      "duration_s": 20,
+      "pushers": 8,
+      "ram_mb":      { "dist": "lognormal", "median": 100, "shape": 0.5, "cap": 512,  "floor": 1 },
+      "duration_ms": { "dist": "lognormal", "median": 200, "shape": 0.8, "cap": 2000, "floor": 1 },
+      "mode":        { "dist": "categorical", "weights": { "sleep": 1.0, "busy": 0.0 } }
+    },
+    {
+      "name": "mixed",
+      "duration_s": 30,
+      "pushers": 25,
+      "ram_mb":      { "dist": "lognormal", "median": 400, "shape": 1.0, "cap": 2048, "floor": 1 },
+      "duration_ms": { "dist": "lognormal", "median": 600, "shape": 1.2, "cap": 5000, "floor": 1 },
+      "mode":        { "dist": "categorical", "weights": { "sleep": 0.5, "busy": 0.5 } }
+    },
+    {
+      "name": "cpu_burst",
+      "duration_s": 30,
+      "pushers": 50,
+      "ram_mb":      { "dist": "lognormal", "median": 800, "shape": 1.5, "cap": 4096, "floor": 1 },
+      "duration_ms": { "dist": "lognormal", "median": 1500, "shape": 1.5, "cap": 8000, "floor": 1 },
+      "mode":        { "dist": "categorical", "weights": { "sleep": 0.0, "busy": 1.0 } }
+    },
+    {
+      "name": "cooldown",
+      "duration_s": 20,
+      "pushers": 8,
+      "ram_mb":      { "dist": "lognormal", "median": 100, "shape": 0.5, "cap": 512,  "floor": 1 },
+      "duration_ms": { "dist": "lognormal", "median": 200, "shape": 0.8, "cap": 2000, "floor": 1 },
+      "mode":        { "dist": "categorical", "weights": { "sleep": 1.0, "busy": 0.0 } }
+    }
+  ]
+}

benchmarks/workloads/distribution.ts

@@ -0,0 +1,199 @@
+// Per-job parameter distributions for the synthetic `random` script pattern.
+//
+// Three supported distribution types — picked because they cover the common
+// shapes for a job-scheduler workload:
+//   uniform     — flat range, predictable for stress runs
+//   lognormal   — heavy-tail; matches real durations (lots of small + a few big)
+//   categorical — discrete weighted choice (e.g. sleep vs busy mode)
+//
+// Add more (pareto, normal, etc.) here if a workload needs them.
+
+// Optional clamp on the sampled value. `cap` and `floor` are post-sample —
+// any value above `cap` becomes `cap`, below `floor` becomes `floor`. Use
+// these to prevent lognormal's long tail from sampling impossible values
+// (e.g. 50 GB RAM, 60 s durations).
+type ClampOpts = { cap?: number; floor?: number };
+
+export type DistSpec =
+  | ({ dist: "uniform"; min: number; max: number } & ClampOpts)
+  | ({ dist: "normal"; mean: number; stddev: number } & ClampOpts)
+  | ({ dist: "lognormal"; median: number; shape: number } & ClampOpts)
+  | { dist: "categorical"; weights: Record<string, number> }
+  // Weighted mix of N sub-distributions. With 2 components this is the
+  // classic bimodal shape (e.g. 70% fast CRUD jobs + 30% slow ETL).
+  // Weights need not sum to 1 — they're relative.
+  | { dist: "mixture"; components: { weight: number; spec: DistSpec }[] };
+
+// Static (legacy) workload — one set of distributions for the whole bench.
+export type StaticWorkloadConfig = {
+  ram_mb: DistSpec;
+  duration_ms: DistSpec;
+  mode: DistSpec; // expected to be categorical with "sleep" / "busy"
+};
+
+// Phased workload — time-bounded blocks, each with its own distributions
+// and an active-pusher count. Useful for simulating diurnal patterns or
+// step-function load changes (warmup → peak → cooldown).
+export type WorkloadPhase = StaticWorkloadConfig & {
+  name?: string;
+  duration_s: number;
+  pushers: number; // max active pushers in this phase (others sleep)
+};
+
+export type PhasedWorkloadConfig = {
+  phases: WorkloadPhase[];
+};
+
+export type WorkloadConfig = StaticWorkloadConfig | PhasedWorkloadConfig;
+
+export function isPhasedConfig(cfg: WorkloadConfig): cfg is PhasedWorkloadConfig {
+  return (cfg as PhasedWorkloadConfig).phases !== undefined;
+}
+
+// Total bench wall-time covered by a phased config (sum of phase durations).
+// Static configs return undefined — caller uses --seconds for their window.
+export function totalPhasedDurationS(cfg: WorkloadConfig): number | undefined {
+  if (!isPhasedConfig(cfg)) return undefined;
+  return cfg.phases.reduce((a, p) => a + p.duration_s, 0);
+}
+
+// Find which phase covers a given elapsed time. Past the last phase end,
+// returns the last phase (caller decides whether to keep pushing or stop).
+export function findActivePhase(
+  cfg: PhasedWorkloadConfig,
+  elapsed_s: number,
+): { phase: WorkloadPhase; index: number; phase_elapsed_s: number } {
+  let acc = 0;
+  for (let i = 0; i < cfg.phases.length; i++) {
+    const p = cfg.phases[i];
+    if (elapsed_s < acc + p.duration_s) {
+      return { phase: p, index: i, phase_elapsed_s: elapsed_s - acc };
+    }
+    acc += p.duration_s;
+  }
+  const last = cfg.phases.length - 1;
+  return { phase: cfg.phases[last], index: last, phase_elapsed_s: elapsed_s - acc };
+}
+
+export type JobParams = {
+  ram_mb: number;
+  duration_ms: number;
+  mode: "sleep" | "busy";
+};
+
+// Box-Muller — one normal sample per call, plenty for per-job use.
+function gauss(): number {
+  let u1 = Math.random();
+  while (u1 === 0) u1 = Math.random();
+  const u2 = Math.random();
+  return Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
+}
+
+function clamp(v: number, spec: { cap?: number; floor?: number }): number {
+  if (spec.cap !== undefined && v > spec.cap) v = spec.cap;
+  if (spec.floor !== undefined && v < spec.floor) v = spec.floor;
+  return v;
+}
+
+function sample(spec: DistSpec): number | string {
+  switch (spec.dist) {
+    case "uniform":
+      return clamp(spec.min + Math.random() * (spec.max - spec.min), spec);
+    case "normal":
+      return clamp(spec.mean + spec.stddev * gauss(), spec);
+    case "lognormal":
+      // median = e^μ, shape = σ.  log(median) gives μ, then add σ·gauss().
+      return clamp(Math.exp(Math.log(spec.median) + spec.shape * gauss()), spec);
+    case "categorical": {
+      const total = Object.values(spec.weights).reduce((a, b) => a + b, 0);
+      let r = Math.random() * total;
+      for (const [k, w] of Object.entries(spec.weights)) {
+        r -= w;
+        if (r <= 0) return k;
+      }
+      // fallthrough on rounding — return the last key
+      const keys = Object.keys(spec.weights);
+      return keys[keys.length - 1];
+    }
+    case "mixture": {
+      const total = spec.components.reduce((a, c) => a + c.weight, 0);
+      let r = Math.random() * total;
+      for (const c of spec.components) {
+        r -= c.weight;
+        if (r <= 0) return sample(c.spec);
+      }
+      // fallthrough on rounding — sample the last component
+      return sample(spec.components[spec.components.length - 1].spec);
+    }
+  }
+}
+
+function asNumber(v: number | string, ctx: string): number {
+  if (typeof v !== "number" || !Number.isFinite(v)) {
+    throw new Error(`workload[${ctx}]: expected numeric distribution, got ${typeof v}`);
+  }
+  return v;
+}
+
+export function sampleJobParams(
+  cfg: WorkloadConfig,
+  opts: { elapsed_s?: number } = {},
+): JobParams {
+  const spec: StaticWorkloadConfig = isPhasedConfig(cfg)
+    ? findActivePhase(cfg, opts.elapsed_s ?? 0).phase
+    : cfg;
+  const mode = sample(spec.mode);
+  if (mode !== "sleep" && mode !== "busy") {
+    throw new Error(`workload[mode]: expected "sleep" or "busy", got ${JSON.stringify(mode)}`);
+  }
+  return {
+    ram_mb: Math.max(0, Math.round(asNumber(sample(spec.ram_mb), "ram_mb"))),
+    duration_ms: Math.max(0, Math.round(asNumber(sample(spec.duration_ms), "duration_ms"))),
+    mode,
+  };
+}
+
+// Whether pusher index `i` should be sending jobs at this moment, given the
+// phased config's per-phase `pushers` cap. Static configs: always active.
+export function isPusherActive(
+  cfg: WorkloadConfig,
+  workerIndex: number,
+  elapsed_s: number,
+): boolean {
+  if (!isPhasedConfig(cfg)) return true;
+  return workerIndex < findActivePhase(cfg, elapsed_s).phase.pushers;
+}
+
+function validatePhase(p: unknown, ctx: string): void {
+  const obj = p as Record<string, unknown>;
+  for (const k of ["ram_mb", "duration_ms", "mode"] as const) {
+    const v = obj[k] as { dist?: unknown } | undefined;
+    if (!v || typeof v.dist !== "string") {
+      throw new Error(`workload ${ctx} missing or malformed field "${k}"`);
+    }
+  }
+  if (typeof obj.duration_s !== "number" || obj.duration_s <= 0) {
+    throw new Error(`workload ${ctx}: "duration_s" must be a positive number`);
+  }
+  if (typeof obj.pushers !== "number" || obj.pushers < 0) {
+    throw new Error(`workload ${ctx}: "pushers" must be a non-negative number`);
+  }
+}
+
+export async function loadWorkloadConfig(path: string): Promise<WorkloadConfig> {
+  const raw = await Deno.readTextFile(path);
+  const parsed = JSON.parse(raw);
+  if (Array.isArray(parsed.phases)) {
+    if (parsed.phases.length === 0) {
+      throw new Error("workload config: phases array is empty");
+    }
+    parsed.phases.forEach((p: unknown, i: number) => validatePhase(p, `phase[${i}]`));
+    return parsed as PhasedWorkloadConfig;
+  }
+  for (const k of ["ram_mb", "duration_ms", "mode"] as const) {
+    if (!parsed[k] || typeof parsed[k].dist !== "string") {
+      throw new Error(`workload config missing or malformed field "${k}"`);
+    }
+  }
+  return parsed as WorkloadConfig;
+}

flake.lock

@@ -48,6 +48,21 @@
         "type": "indirect"
       }
     },
+    "nixpkgs-sim": {
+      "locked": {
+        "lastModified": 1779560665,
+        "narHash": "sha256-tpyBcxPpcQb8ukyNF7DoCwfSY3VPsxHoYwj00Cayv5o=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "64c08a7ca051951c8eae34e3e3cb1e202fe36786",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "rev": "64c08a7ca051951c8eae34e3e3cb1e202fe36786",
+        "type": "indirect"
+      }
+    },
     "nixpkgs_2": {
       "locked": {
         "lastModified": 1744536153,
@@ -69,6 +84,7 @@
         "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs",
         "nixpkgs-oapi-gen": "nixpkgs-oapi-gen",
+        "nixpkgs-sim": "nixpkgs-sim",
         "rust-overlay": "rust-overlay"
       }
     },

flake.nix

@@ -5,9 +5,15 @@
     rust-overlay.url = "github:oxalica/rust-overlay";
     # Pin openapi-generator-cli to 7.10.0
     nixpkgs-oapi-gen.url = "nixpkgs/2d068ae5c6516b2d04562de50a58c682540de9bf";
+    # Fresh nixos-unstable for the k8s sim tools (minikube/libvirt/kvm2-driver).
+    # The main `nixpkgs` lock can lag behind the host system, but the kvm2
+    # driver MUST link against system-compatible glibc/libvirt — otherwise it
+    # crashes loading the system libvirt's transitive libs at runtime. Verified
+    # at this rev: minikube 1.38.1, libvirt 12.2.0 (matches NixOS 26.05 hosts).
+    nixpkgs-sim.url = "nixpkgs/64c08a7ca051951c8eae34e3e3cb1e202fe36786";
   };
 
-  outputs = { self, nixpkgs, flake-utils, rust-overlay, nixpkgs-oapi-gen }:
+  outputs = { self, nixpkgs, flake-utils, rust-overlay, nixpkgs-oapi-gen, nixpkgs-sim }:
     flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = import nixpkgs {
@@ -16,6 +22,14 @@
           overlays = [ (import rust-overlay) ];
         };
 
+        # Fresh nixos-unstable just for the k8s sim tools — keeps the kvm2
+        # driver / minikube / helm aligned with the host system's libvirtd.
+        # See `inputs.nixpkgs-sim` above for the why.
+        pkgsSim = import nixpkgs-sim {
+          inherit system;
+          config.allowUnfree = true;
+        };
+
         lib = pkgs.lib;
         stdenv = pkgs.stdenv;
 
@@ -287,6 +301,33 @@
         # Helper scripts — base set (default + full)
         # ---------------------------------------------------------------
 
+        # minikube kvm2 driver for the k8s benchmark sim (not in nixpkgs).
+        # Built against pkgsSim so glibc/libvirt match the host system.
+        kvm2Driver = import ./benchmarks/sim/nix/kvm2-driver.nix { pkgs = pkgsSim; };
+
+        # `wm_sim` — k8s sim entry, factored out so wasm/cli devShells can pull
+        # just it without inheriting the rest of helperScriptsBase.
+        wmSimWrapper = pkgs.writeShellScriptBin "wm_sim" ''
+          export SIM_KVM2_DRIVER_DIR="${kvm2Driver}/bin"
+          # minikube + helm from pkgsSim so they share the kvm2 driver's libc.
+          export SIM_MINIKUBE_BIN="${pkgsSim.minikube}/bin/minikube"
+          export SIM_HELM_BIN="${pkgsSim.kubernetes-helm}/bin/helm"
+          # System libvirt — matches the system virsh that minikube's kvm2
+          # preflight invokes. Falls back to pkgsSim's libvirt (also 12.2 in
+          # nixos-unstable) if the system path can't be resolved.
+          sysvirsh="$(command -v virsh 2>/dev/null || echo /run/current-system/sw/bin/virsh)"
+          syslib="$(ldd "$sysvirsh" 2>/dev/null | awk '/libvirt\.so\.0/{print $3}' | head -1)"
+          if [ -n "$syslib" ]; then
+            export SIM_LIBVIRT_LIB_DIR="$(dirname "$syslib")"
+          else
+            export SIM_LIBVIRT_LIB_DIR="${pkgsSim.libvirt}/lib"
+          fi
+          # virsh path for the provisioner's pre-start sweep of leaked domains.
+          export SIM_VIRSH_BIN="$sysvirsh"
+          root="$(git rev-parse --show-toplevel 2>/dev/null || echo .)"
+          exec ${pkgs.deno}/bin/deno run --no-check -A "$root/benchmarks/sim/sim.ts" "$@"
+        '';
+
         helperScriptsBase = [
           (pkgs.writeScriptBin "wm" ''
             cd ./frontend
@@ -344,6 +385,25 @@
             echo "bucket: wmill"
             echo "endpoint: http://localhost:9000"
           '')
+          # k8s sim entry — provisioning only, no bench. See `wmSimWrapper`.
+          wmSimWrapper
+          # Bench entry — same env shape as wm_sim so `--topology` (sim-driven
+          # provisioning) works. For plain `--host` benches the env is harmless.
+          (pkgs.writeShellScriptBin "wm-bench" ''
+            export SIM_KVM2_DRIVER_DIR="${kvm2Driver}/bin"
+            export SIM_MINIKUBE_BIN="${pkgsSim.minikube}/bin/minikube"
+            export SIM_HELM_BIN="${pkgsSim.kubernetes-helm}/bin/helm"
+            sysvirsh="$(command -v virsh 2>/dev/null || echo /run/current-system/sw/bin/virsh)"
+            syslib="$(ldd "$sysvirsh" 2>/dev/null | awk '/libvirt\.so\.0/{print $3}' | head -1)"
+            if [ -n "$syslib" ]; then
+              export SIM_LIBVIRT_LIB_DIR="$(dirname "$syslib")"
+            else
+              export SIM_LIBVIRT_LIB_DIR="${pkgsSim.libvirt}/lib"
+            fi
+            export SIM_VIRSH_BIN="$sysvirsh"
+            root="$(git rev-parse --show-toplevel 2>/dev/null || echo .)"
+            exec ${pkgs.deno}/bin/deno run --no-check -A "$root/benchmarks/main.ts" -e admin@windmill.dev -p changeme "$@"
+          '')
         ];
 
         # ---------------------------------------------------------------
@@ -363,9 +423,6 @@
             wm-caddy
             wm-migrate
           '')
-          (pkgs.writeScriptBin "wm-bench" ''
-            deno run -A benchmarks/main.ts -e admin@windmill.dev -p changeme "$@"
-          '')
         ];
 
         # ---------------------------------------------------------------
@@ -385,6 +442,21 @@
             gh
             asciinema
             mermaid-cli
+
+            # Network shaping / topology benches
+            toxiproxy
+            pgbadger
+
+            # Local k8s for sim topologies. Single-node uses the built-in qemu2
+            # driver (no libvirt, no sudo). Multi-node (--nodes=N) needs the
+            # kvm2 driver — the docker-machine-driver-kvm2 binary isn't in
+            # nixpkgs and minikube's auto-download won't run on NixOS, so kvm2
+            # requires a custom overlay + host-level libvirtd. qemu is shared by
+            # both. VM-based nodes sidestep the rootless-cgroup wall (k3d/kind).
+            minikube
+            kubectl
+            kubernetes-helm
+            qemu
           ]);
 
         # Playwright: use Nix-provided browsers (version-matched to playwright-driver)
@@ -536,6 +608,7 @@
             nodejs
             glibc_multi
           ]);
+          packages = [ wmSimWrapper ];
         });
 
         # =============================================================
@@ -555,6 +628,7 @@
           buildInputs = with pkgs; [ bun nodejs git ];
 
           packages = [
+            wmSimWrapper
             (pkgs.writeScriptBin "wm-cli" ''
               bun run $FLAKE_ROOT/cli/src/main.ts "$@"
             '')