// Helm deploy of Windmill onto a provisioned minikube cluster, + API URL // resolution via port-forward. Part of the k8s sim path (task #49). // // Chart source (`chart`): // - omitted -> the public remote chart "windmill/windmill" // (repo auto-added) // - a local path -> used directly (e.g. ../windmill-helm-charts/charts/windmill) // - "windmill/" -> remote; the windmill repo is auto-added // - any other "repo/x" -> used as-is (assumes the repo is already configured) // // PG: the chart bundles Postgres (postgresql.enabled, via cloudnative-pg). API: // the chart exposes a ClusterIP Service `windmill-app` on port 8000; we // port-forward it to localhost so the existing bench can hit it over http. import { parse as yamlParse } from "https://deno.land/std@0.224.0/yaml/mod.ts"; const HELM = Deno.env.get("SIM_HELM_BIN") ?? "helm"; const MINIKUBE = Deno.env.get("SIM_MINIKUBE_BIN") ?? "minikube"; const WINDMILL_REPO = "https://windmill-labs.github.io/windmill-helm-charts/"; const APP_SERVICE = "windmill-app"; const APP_PORT = 8000; export type HelmDeployOptions = { profile: string; // minikube profile == kube context chart?: string; // local path or "repo/name"; default = vendored chart at // benchmarks/sim/charts/windmill (forked from upstream // 4.0.169 with `priorityClassName`, PG `maxConnections`, // and `priorityClasses.classes[]` exposed via values). release?: string; // default "wm" namespace?: string; // default "default" set?: string[]; // extra --set key=value valuesFiles?: string[]; // extra -f values.yaml timeoutSec?: number; // helm --wait timeout; default 600 }; // minikube needs the kvm2 driver dir on PATH + LD_LIBRARY_PATH for libvirt // (see k8s_provisioner.ts). `minikube kubectl` inherits it; helm doesn't need it. function minikubeEnv(): Record { const base = Deno.env.toObject(); const driverDir = Deno.env.get("SIM_KVM2_DRIVER_DIR"); const libDir = Deno.env.get("SIM_LIBVIRT_LIB_DIR"); if (driverDir) base.PATH = `${driverDir}:${base.PATH ?? ""}`; if (libDir) base.LD_LIBRARY_PATH = base.LD_LIBRARY_PATH ? `${libDir}:${base.LD_LIBRARY_PATH}` : libDir; return base; } async function run( bin: string, args: string[], { check = true, env }: { check?: boolean; env?: Record } = {}, ): Promise<{ stdout: string; stderr: string; code: number }> { const p = new Deno.Command(bin, { args, env, stdout: "piped", stderr: "piped" }); const { code, stdout, stderr } = await p.output(); const out = new TextDecoder().decode(stdout); const err = new TextDecoder().decode(stderr); if (check && code !== 0) { throw new Error(`${bin} ${args.join(" ")} failed (code ${code}):\n${err || out}`); } return { stdout: out, stderr: err, code }; } async function pathExists(p: string): Promise { try { await Deno.stat(p); return true; } catch { return false; } } async function pickFreePort(): Promise { const l = Deno.listen({ port: 0 }); const port = (l.addr as Deno.NetAddr).port; l.close(); return port; } // Parse a local chart's Chart.yaml, register each dep's helm repo. Idempotent // (`helm repo add --force-update` overwrites). Skips oci:// URLs (helm fetches // those directly without `repo add`). async function registerLocalChartDepRepos(chartDir: string): Promise { let raw: string; try { raw = await Deno.readTextFile(`${chartDir}/Chart.yaml`); } catch (e) { console.warn(`[helm] could not read ${chartDir}/Chart.yaml: ${(e as Error).message}`); return; } let parsed: unknown; try { parsed = yamlParse(raw); } catch (e) { console.warn(`[helm] Chart.yaml parse failed: ${(e as Error).message}`); return; } const deps = (parsed as { dependencies?: Array<{ name?: string; repository?: string }> }) .dependencies ?? []; const seen = new Set(); for (const d of deps) { const url = d.repository; if (!url) continue; if (!/^https?:\/\//i.test(url)) continue; if (seen.has(url)) continue; seen.add(url); // Use the dep's chart name as the repo alias. Helm allows any alias. const alias = d.name ?? new URL(url).hostname.replace(/[^a-z0-9]/g, ""); console.log(`[helm] repo add ${alias} ${url}`); await run(HELM, ["repo", "add", alias, url, "--force-update"], { check: false }); } await run(HELM, ["repo", "update"], { check: false }); } // helm install Windmill (bundled PG on). Waits for the release to be ready. export async function helmDeployWindmill(opts: HelmDeployOptions): Promise { // Default to the in-repo vendored chart so sim runs are reproducible without // depending on upstream chart version drift (we hit 4.0.165 → 4.0.166 silent // breakage during this project; the vendored copy is our pinned baseline). const chart = opts.chart ?? new URL("./charts/windmill", import.meta.url).pathname; const release = opts.release ?? "wm"; const namespace = opts.namespace ?? "default"; const isLocalPath = chart.startsWith(".") || chart.startsWith("/") || (await pathExists(chart)); if (!isLocalPath && chart.startsWith("windmill/")) { // Idempotent: `repo add` errors if it exists with a different URL; --force-update avoids that. await run(HELM, ["repo", "add", "windmill", WINDMILL_REPO, "--force-update"], { check: false }); await run(HELM, ["repo", "update", "windmill"], { check: false }); } if (isLocalPath) { // The chart has subchart dependencies declared in Chart.yaml. For a remote // chart helm fetches them at pull/install; for a LOCAL chart path we must // (a) register each dependency's repository with helm (it errors otherwise // with "no repository definition for ..."), and (b) `helm dependency build` // to vendor the subcharts into the local chart's ./charts/ dir. console.log(`[helm] resolving chart dependencies for local chart at ${chart}`); await registerLocalChartDepRepos(chart); await run(HELM, ["dependency", "build", chart]); } // `upgrade --install`: install when absent, upgrade (reconcile) when // present. Required for the cluster-reuse path — bare `install` errors with // "cannot re-use a name that is still in use" on a second run. // // No `--wait`. With many workers the chart's default --wait blocks until // EVERY deployment is Ready, which deadlocks if workers crashloop on the // default PG max_connections (we bump that AFTER helm in main.ts). The // caller waits for PG + windmill-app explicitly via kubectl wait, which is // both faster and avoids the worker-readiness lock-step. const args = [ "--kube-context", opts.profile, "upgrade", "--install", release, chart, "--namespace", namespace, "--create-namespace", "--set", "postgresql.enabled=true", "--timeout", `${opts.timeoutSec ?? 600}s`, ]; for (const s of opts.set ?? []) args.push("--set", s); for (const f of opts.valuesFiles ?? []) args.push("-f", f); console.log(`[helm] upgrade --install Windmill (release=${release}, chart=${chart}, ns=${namespace})`); await run(HELM, args); console.log(`[helm] manifests applied — caller waits for PG + app readiness`); } // Wait for the bundled PG (StatefulSet or Deployment) to report Ready. Used // after `helmDeployWindmill` so the PG-bound steps that follow (max_conn // patch, verbose logging) can run. export async function waitPgReady( profile: string, namespace = "default", timeoutSec = 300, ): Promise { // Probe both — only one of these will exist depending on persistence flag. for (const ref of ["statefulset/windmill-postgresql", "deployment/windmill-postgresql-demo-app"]) { const r = await run( MINIKUBE, [ "kubectl", "-p", profile, "--", "-n", namespace, "rollout", "status", ref, `--timeout=${timeoutSec}s`, ], { check: false }, ); if (r.code === 0) { console.log(`[helm] PG ready (${ref})`); return; } } throw new Error("[helm] PG not found or not ready (neither statefulset nor deployment)"); } // Wait for windmill-app Deployment to report at least 1 Ready replica. export async function waitWindmillAppReady( profile: string, namespace = "default", timeoutSec = 300, ): Promise { const r = await run( MINIKUBE, [ "kubectl", "-p", profile, "--", "-n", namespace, "rollout", "status", "deployment/windmill-app", `--timeout=${timeoutSec}s`, ], { check: false }, ); if (r.code !== 0) { throw new Error(`[helm] windmill-app not ready after ${timeoutSec}s`); } console.log(`[helm] windmill-app ready`); } export type ApiEndpoint = { host: string; stop: () => void }; // Port-forward svc/windmill-app:8000 to a free localhost port and wait until // the API answers. Returns the URL + a stop() to kill the forward. export async function portForwardApi(profile: string, namespace = "default"): Promise { const localPort = await pickFreePort(); const child = new Deno.Command(MINIKUBE, { args: [ "kubectl", "-p", profile, "--", "port-forward", "-n", namespace, `svc/${APP_SERVICE}`, `${localPort}:${APP_PORT}`, ], env: minikubeEnv(), stdout: "piped", stderr: "piped", }).spawn(); const host = `http://127.0.0.1:${localPort}`; const deadline = Date.now() + 60_000; let lastErr = ""; while (Date.now() < deadline) { try { const r = await fetch(`${host}/api/version`, { signal: AbortSignal.timeout(2000) }); if (r.ok) { await r.body?.cancel(); console.log(`[helm] API reachable at ${host} (port-forward svc/${APP_SERVICE})`); return { host, stop: () => { try { child.kill("SIGTERM"); } catch { /* already gone */ } } }; } } catch (e) { lastErr = (e as Error).message; } await new Promise((r) => setTimeout(r, 1000)); } try { child.kill("SIGTERM"); } catch { /* ignore */ } throw new Error(`[helm] port-forward to svc/${APP_SERVICE} never became reachable (last: ${lastErr})`); }