import { randomUUID } from "node:crypto"; import type { WindmillBackendSettings } from "../../core/windmillBackendSettings"; const tokenCache = new Map>(); const sharedWorkspaceQueue = new Map>(); const DEFAULT_WORKSPACE_PREFIX = "ai-evals"; export class WindmillBackendClient { constructor(private readonly settings: WindmillBackendSettings) {} async withWorkspace( caseId: string, attempt: number, body: (workspaceId: string) => Promise, ): Promise { const workspaceId = this.settings.workspaceOverride ?? buildWorkspaceId(caseId, attempt); const run = async () => { await this.ensureWorkspace(workspaceId); try { return await body(workspaceId); } finally { if (!this.settings.workspaceOverride) { await this.deleteWorkspace(workspaceId).catch(() => undefined); } } }; if (this.settings.workspaceOverride) { return await withSharedWorkspaceLock(workspaceId, run); } return await run(); } async request(path: string, init?: RequestInit): Promise { const token = await this.getToken(); return await fetch(`${this.settings.baseUrl}/api${path}`, { ...init, headers: { Authorization: `Bearer ${token}`, ...(init?.headers ?? {}), }, }); } async getToken(): Promise { const cacheKey = `${this.settings.baseUrl}|${this.settings.email}`; let tokenPromise = tokenCache.get(cacheKey); if (!tokenPromise) { tokenPromise = this.login().catch((error) => { if (tokenCache.get(cacheKey) === tokenPromise) { tokenCache.delete(cacheKey); } throw error; }); tokenCache.set(cacheKey, tokenPromise); } return await tokenPromise; } async upsertResource(input: { workspaceId: string; path: string; resourceType: string; value: Record; }): Promise { const response = await this.request( `/w/${encodeURIComponent(input.workspaceId)}/resources/create?update_if_exists=true`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ path: input.path, resource_type: input.resourceType, value: input.value, }), }, ); await expectOk(response, `upsert resource ${input.path}`); } private async ensureWorkspace(workspaceId: string): Promise { const existsResponse = await this.request("/workspaces/exists", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ id: workspaceId }), }); await expectOk(existsResponse, `check workspace ${workspaceId}`); if ((await existsResponse.text()).trim() === "true") { return; } const createResponse = await this.request("/workspaces/create", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ id: workspaceId, name: workspaceId }), }); try { await expectOk(createResponse, `create workspace ${workspaceId}`); } catch (error) { const message = error instanceof Error ? error.message : String(error); if (message.includes("maximum number of workspaces")) { throw new Error( `${message}. Reuse an existing workspace with WMILL_AI_EVAL_BACKEND_WORKSPACE=.`, ); } throw error; } } private async deleteWorkspace(workspaceId: string): Promise { const response = await this.request( `/workspaces/delete/${encodeURIComponent(workspaceId)}`, { method: "DELETE", }, ); await expectOk(response, `delete workspace ${workspaceId}`); } private async login(): Promise { const response = await fetch(`${this.settings.baseUrl}/api/auth/login`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ email: this.settings.email, password: this.settings.password, }), }); await expectOk(response, "login to Windmill backend"); return (await response.text()).trim(); } } export async function assertWindmillBackendReachable( settings: WindmillBackendSettings, ): Promise { try { await new WindmillBackendClient(settings).getToken(); } catch (error) { const details = error instanceof Error ? error.message : String(error); throw new Error( [ `Could not initialize the Windmill backend for AI eval proxy at ${settings.baseUrl}.`, "Start a Windmill backend at that URL, or set WMILL_AI_EVAL_BACKEND_URL=.", `Using login ${settings.email}; if authentication failed, set WMILL_AI_EVAL_BACKEND_EMAIL and WMILL_AI_EVAL_BACKEND_PASSWORD.`, `Details: ${details}`, ].join("\n"), ); } } async function withSharedWorkspaceLock( workspaceId: string, body: () => Promise, ): Promise { const previous = sharedWorkspaceQueue.get(workspaceId) ?? Promise.resolve(); let releaseCurrent: (() => void) | undefined; const current = new Promise((resolve) => { releaseCurrent = resolve; }); const tail = previous.catch(() => undefined).then(() => current); sharedWorkspaceQueue.set(workspaceId, tail); await previous.catch(() => undefined); try { return await body(); } finally { releaseCurrent?.(); if (sharedWorkspaceQueue.get(workspaceId) === tail) { sharedWorkspaceQueue.delete(workspaceId); } } } function buildWorkspaceId(caseId: string, attempt: number): string { const caseSlug = caseId .toLowerCase() .replace(/[^a-z0-9-]+/g, "-") .replace(/^-+|-+$/g, "") .slice(0, 30); const suffix = randomUUID().slice(0, 8); return `${DEFAULT_WORKSPACE_PREFIX}-${caseSlug || "case"}-a${attempt}-${suffix}`; } async function expectOk(response: Response, context: string): Promise { if (response.ok) { return; } throw new Error( `${context} failed: ${response.status} ${response.statusText} - ${await response.text()}`, ); }