// Windmill benchmark sim — CLI entry, provisioning ONLY (no benchmarking). // // Provisions a minikube cluster from a topology and deploys Windmill onto it // via helm (bundled PG). Leaves the cluster running until Ctrl-C, then tears // it down. Benchmarking is a separate concern — point the bench at the URL // this prints (`benchmarks/main.ts --host ...` / `wm-bench`). // // Usage (wired via the `wm_sim` flake helper): // wm_sim --topology sim/topologies/k8s_smoke.json // wm_sim --topology .json --helm // wm_sim --topology .json --helm --helm-values values.yaml // wm_sim --topology .json --helm-values v.yaml \ // --helm-set-override windmill.workerGroups[0].replicas=250 // // The helm chart's `values.yaml` is the primary source of deployment config // (replicas, resources, image). `--helm-set-override` is for surgical CLI // tweaks on top — naming reflects "override, not primary source". The values // files are archived under results/_/ so each run records // exactly what was deployed. // // Requires minikube + the kvm2 driver on PATH and libvirtd reachable. The // `wm_sim` wrapper sets SIM_KVM2_DRIVER_DIR / SIM_LIBVIRT_LIB_DIR. import { Command } from "https://deno.land/x/cliffy@v0.25.7/command/mod.ts"; import { loadTopology, validateTopology } from "./topology.ts"; import { MinikubeProvisioner } from "./k8s_provisioner.ts"; import { helmDeployWindmill, portForwardApi, waitPgReady, waitWindmillAppReady, type ApiEndpoint, } from "./helm_deploy.ts"; import { buildToxiproxyManifests } from "./toxiproxy_k8s.ts"; import { enableVerbosePgLogging } from "./pg_logging.ts"; import { applyCpuSampler } from "./cpu_sampler_k8s.ts"; import { loadCachedImages, saveImagesToCache } from "./image_cache.ts"; // minikube profile names allow only alphanumerics + dashes; topology names // often contain underscores, so sanitise. function profileFor(prefix: string, name: string): string { const slug = name.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, ""); return `${prefix}-${slug || "topology"}`; } async function loadValidTopology(path: string) { const topology = await loadTopology(path); const issues = await validateTopology(topology); for (const w of issues.filter((i) => i.severity === "warn")) { console.warn(`[topology] WARN: ${w.message}`); } const errors = issues.filter((i) => i.severity === "error"); if (errors.length > 0) { for (const e of errors) console.error(`[topology] ERROR: ${e.message}`); Deno.exit(1); } return topology; } type DeployArgs = { helm?: string; helmValues?: string[]; helmSetOverride?: string[]; }; async function deployWindmill(prov: MinikubeProvisioner, args: DeployArgs): Promise { await helmDeployWindmill({ profile: prov.profile, chart: args.helm, valuesFiles: args.helmValues, set: args.helmSetOverride, }); // helm runs without --wait — gate PG + app readiness ourselves so port-forward // doesn't race startup. PG max_connections is templated into the chart now // (postgresql.maxConnections value), no post-install patching needed. await waitPgReady(prov.profile); await waitWindmillAppReady(prov.profile); return await portForwardApi(prov.profile); } type UpOpts = DeployArgs & { topology: string }; async function upMain(opts: UpOpts) { const topology = await loadValidTopology(opts.topology); // Per-run reports dir — archives the inputs (topology + values files) so // each cluster bring-up records exactly what was deployed. const isoStamp = new Date().toISOString().replace(/[:.]/g, "-").replace(/Z$/, ""); const outDir = `reports/${isoStamp}_${topology.name}`; await Deno.mkdir(outDir, { recursive: true }); await Deno.copyFile(opts.topology, `${outDir}/topology.json`); for (const v of opts.helmValues ?? []) { const base = v.split("/").pop() || "values.yaml"; await Deno.copyFile(v, `${outDir}/${base}`); } console.log(`[sim] inputs archived to ${outDir}/`); const prov = new MinikubeProvisioner({ profile: profileFor("wm-sim", topology.name) }); // Clean any leftover state (crashed prior run, leaked libvirt domains) // before provisioning. teardown() now sweeps both minikube + libvirt. console.log(`[sim] cleaning stale state for profile=${prov.profile}...`); await prov.teardown(); await prov.provision(topology); try { await loadCachedImages(prov.profile); } catch (e) { console.warn(`[sim] image cache preload failed: ${(e as Error).message}`); } // Toxiproxy: single Deployment + Service, no nodeSelector. Worker count + // resources + DATABASE_URL live in the user's values.yaml. const toxManifestPath = `${outDir}/toxiproxy.yaml`; await Deno.writeTextFile(toxManifestPath, buildToxiproxyManifests(topology)); console.log(`[sim] applying toxiproxy...`); const applyRes = await prov.kubectl(["apply", "-f", toxManifestPath]); if (applyRes.code !== 0) { console.warn(`[sim] toxiproxy apply non-zero: ${applyRes.stdout}`); } // helm: user values are the only source of truth (workerGroups + // DATABASE_URL routing live there now). const endpoint = await deployWindmill(prov, opts); // PG max_connections bump + readiness gates are inside deployWindmill now. // Enable verbose PG logging so a pgBadger report can be generated on // teardown — best-effort; if the chart's PG layout differs, log and // continue (the cluster is still usable for ad-hoc work). try { await enableVerbosePgLogging(prov); } catch (e) { console.warn(`[sim] could not enable verbose PG logging: ${(e as Error).message}`); } // Apply the sampler DaemonSet once per provision — it runs continuously // between bench runs; wm-bench scopes its output via `--since-time` so each // report only sees rows from its own window. No rollout-restart on reuse. try { await applyCpuSampler(prov, outDir); } catch (e) { console.warn(`[sim] CPU sampler not applied: ${(e as Error).message}`); } await Deno.writeTextFile( `${outDir}/meta.json`, JSON.stringify( { topology: topology.name, profile: prov.profile, api_host: endpoint.host, helm_chart: opts.helm ?? "windmill/windmill", helm_values_files: opts.helmValues ?? [], helm_set_overrides: opts.helmSetOverride ?? [], started_at: new Date().toISOString(), }, null, 2, ), ); console.log("\n" + "=".repeat(56)); console.log(` Topology "${topology.name}" is up.`); console.log(` Windmill API: ${endpoint.host}`); console.log(` minikube profile: ${prov.profile}`); console.log(` Login: admin@windmill.dev / changeme`); console.log(""); console.log(` To bench against this cluster:`); console.log(` wm-bench --host ${endpoint.host} --minikube-profile ${prov.profile} ...`); console.log(""); console.log(` Ctrl-C to tear down the cluster.`); console.log("=".repeat(56)); await new Promise((resolve) => { const handler = () => { Deno.removeSignalListener("SIGINT", handler); Deno.removeSignalListener("SIGTERM", handler); resolve(); }; Deno.addSignalListener("SIGINT", handler); Deno.addSignalListener("SIGTERM", handler); }); // Save images BEFORE minikube delete — they're gone with the VM otherwise. try { await saveImagesToCache(prov.profile, (args) => prov.kubectl(args)); } catch (e) { console.warn(`[sim] image cache save failed: ${(e as Error).message}`); } console.log("\nTearing down..."); endpoint.stop(); await prov.teardown(); Deno.exit(0); } if (import.meta.main) { // `up` subcommand mirrors the root action — both forms work: // wm_sim --topology X (root action) // wm_sim up --topology X (explicit subcommand) const upSub = new Command() .description("Provision the cluster + deploy Windmill, leave running (Ctrl-C tears down).") .option("--topology ", "Topology JSON.", { required: true }) .option( "--helm ", "Helm chart for Windmill: local path (preferred) or remote 'repo/name'. Default 'windmill/windmill'.", ) .option( "--helm-values ", "Helm values.yaml — primary source of deployment config (replicas/resources/image). Repeatable; later files take precedence. Copied into results/ as the run record.", { collect: true }, ) .option( "--helm-set-override ", "Inline key=value override on top of --helm-values (repeatable). For surgical CLI tweaks; the values file is the source of truth.", { collect: true }, ) .action(upMain); await new Command() .name("wm_sim") .description( "Windmill benchmark sim: provision a minikube cluster from a topology and deploy Windmill on it. Provisioning only — no benchmarking.", ) .command("up", upSub) .default("up") .parse(); }