fix(ai_evals): adapt global eval harness to DB-backed user drafts (#9641)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

ai_evals/adapters/frontend/core/global/globalEvalRunner.ts

@@ -9,6 +9,7 @@ import {
 } from "../../../../../frontend/src/lib/components/copilot/chat/global/core";
 import {
   clearGlobalDrafts,
+  getGlobalDraft,
   listGlobalDrafts,
 } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter";
 import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
@@ -18,6 +19,7 @@ import type { GlobalDraftState } from "../../../../core/validators";
 import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
 import {
   registerBenchmarkWorkspaceRunnables,
+  seedBenchmarkDraft,
   unregisterBenchmarkWorkspaceRunnables,
   type BenchmarkWorkspaceRunnables,
 } from "../../mockBackend";
@@ -94,7 +96,7 @@ export async function runGlobalEval(
       tools: getGlobalEvalTools(),
       helpers: {},
       apiKey,
-      getOutput: () => ({ drafts: listGlobalDrafts(workspaceRoot) }),
+      getOutput: () => collectGlobalDraftState(workspaceRoot),
       onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
       onAssistantToken: options.runContext?.onAssistantChunk,
       onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
@@ -130,6 +132,32 @@ export async function runGlobalEval(
   }
 }
 
+// Build the harness output from the DB-backed drafts. `listGlobalDrafts` returns
+// metadata-only rows for backend drafts (the model's `write_script` etc. persist
+// straight to the backend with no in-tab editor cell), so re-read each such row
+// with `getGlobalDraft` to attach the full value the validators assert on. A row
+// that already carries a value (the production in-tab cell overlay) is kept as-is.
+async function collectGlobalDraftState(
+  workspace: string,
+): Promise<GlobalDraftState> {
+  const items = await listGlobalDrafts(workspace);
+  const drafts = await Promise.all(
+    items.map(async (item) => {
+      if (item.value !== undefined) {
+        return item;
+      }
+      const full = await getGlobalDraft(
+        workspace,
+        item.type,
+        item.path,
+        item.triggerKind,
+      );
+      return full ?? item;
+    }),
+  );
+  return { drafts: drafts as GlobalDraftState["drafts"] };
+}
+
 function seedLiveEditorDrafts(
   workspace: string,
   fixtures: GlobalLiveEditorDraftFixture[],
@@ -138,7 +166,9 @@ function seedLiveEditorDrafts(
     const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type];
     const storagePath = fixture.storagePath ?? fixture.effectivePath ?? "";
     if (fixture.value !== undefined) {
-      UserDraft.save(itemKind, storagePath, fixture.value, { workspace });
+      // Seed as a backend draft row, not an in-tab cell: a cell would shadow the
+      // model's DB-backed edit when the output is read back via listGlobalDrafts.
+      seedBenchmarkDraft(workspace, itemKind, storagePath, fixture.value);
     }
     UserDraft.setLiveEditorDraft({
       workspace,

ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts

@@ -38,8 +38,9 @@ export interface RunEvalParams<THelpers, TOutput> {
   helpers: THelpers;
   /** API key for the provider */
   apiKey: string;
-  /** Function to get the current output state */
-  getOutput: () => TOutput;
+  /** Function to get the current output state. May be async — global mode reads
+   * DB-backed drafts back through the (mocked) backend to build its output. */
+  getOutput: () => TOutput | Promise<TOutput>;
   /** Model and Windmill backend configuration */
   options: EvalRunnerOptions;
   onAssistantMessageStart?: () => void;
@@ -154,7 +155,7 @@ export async function runEval<THelpers, TOutput>(
       if (result.hitMaxIterations) {
         return {
           success: false,
-          output: getOutput(),
+          output: (await getOutput()) as TOutput,
           error: `Reached max turns (${maxIterations})`,
           tokenUsage: result.tokenUsage,
           toolCallsCount,
@@ -170,7 +171,7 @@ export async function runEval<THelpers, TOutput>(
 
       return {
         success: true,
-        output: getOutput(),
+        output: (await getOutput()) as TOutput,
         tokenUsage: result.tokenUsage,
         toolCallsCount,
         toolsCalled,
@@ -191,7 +192,7 @@ export async function runEval<THelpers, TOutput>(
 
       return {
         success: false,
-        output: getOutput(),
+        output: (await getOutput()) as TOutput,
         error: errorMessage,
         tokenUsage: { prompt: 0, completion: 0, total: 0 },
         toolCallsCount,

ai_evals/adapters/frontend/mockBackend.ts

@@ -3,7 +3,11 @@ import type { CompletedJob, Flow, Job, Script } from '../../../frontend/src/lib/
 import type {
 	DataTableTables,
 	DataTableTableSchema,
-	ScriptLang
+	GetDraftForUserResponse,
+	ListDraftsResponse,
+	ScriptLang,
+	UpdateDraftResponse,
+	UserDraftItemKind
 } from '../../../frontend/src/lib/gen/types.gen'
 import { buildScriptLintResult } from './core/script/preview'
 import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine'
@@ -63,6 +67,7 @@ export function resetBenchmarkMockBackend(): void {
 	benchmarkWorkspaces.clear()
 	benchmarkWorkspaceRunnables.clear()
 	benchmarkJobs.clear()
+	benchmarkDrafts.clear()
 }
 
 export function registerBenchmarkWorkspace(workspace: string): void {
@@ -74,6 +79,8 @@ export function registerBenchmarkWorkspaceRunnables(
 	runnables: BenchmarkWorkspaceRunnables
 ): void {
 	benchmarkWorkspaces.add(workspace)
+	// Fresh case: drop any drafts left from a prior run on this workspace id.
+	clearBenchmarkDrafts(workspace)
 	// Datatables are mutated in place by exec_datatable_sql (a write must be visible
 	// to later reads), so store an isolated deep copy — never mutate the caller's seed.
 	benchmarkWorkspaceRunnables.set(workspace, {
@@ -98,6 +105,7 @@ export function registerBenchmarkWorkspaceRunnables(
 export function unregisterBenchmarkWorkspace(workspace: string): void {
 	benchmarkWorkspaces.delete(workspace)
 	benchmarkWorkspaceRunnables.delete(workspace)
+	clearBenchmarkDrafts(workspace)
 	for (const [jobId, entry] of benchmarkJobs.entries()) {
 		if (entry.workspace === workspace) {
 			benchmarkJobs.delete(jobId)
@@ -238,6 +246,110 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string {
 	return job.logs ?? ''
 }
 
+// ============= Drafts (per-user, DB-backed in production) =============
+
+/**
+ * In-memory stand-in for the per-user draft backend (`DraftService`). The global
+ * AI chat now persists and reads drafts through the backend DB instead of an
+ * in-tab `UserDraft` cell, so the eval mocks the three draft endpoints it
+ * exercises (`updateDraft` / `getDraftForUser` / `listDrafts`) and keeps the
+ * saved values here, keyed by workspace + draft kind + storage path. Mirrors the
+ * semantics of the production unit test's mock in
+ * `frontend/src/lib/components/copilot/chat/global/core.test.ts`.
+ */
+const benchmarkDrafts = new Map<
+	string,
+	{ workspace: string; kind: UserDraftItemKind; path: string; value: unknown }
+>()
+
+// Fixed timestamp so artifacts stay deterministic. No eval simulates a
+// concurrent writer, so every save is accepted and the conflict branch is
+// never taken — the syncer just records this as its `last_sync` baseline.
+const BENCHMARK_DRAFT_TIMESTAMP = '1970-01-01T00:00:00.000Z'
+
+function benchmarkDraftKey(workspace: string, kind: string, path: string): string {
+	return `${workspace}::${kind}::${path}`
+}
+
+export function clearBenchmarkDrafts(workspace: string): void {
+	for (const [key, entry] of benchmarkDrafts.entries()) {
+		if (entry.workspace === workspace) {
+			benchmarkDrafts.delete(key)
+		}
+	}
+}
+
+/**
+ * Seed a draft straight into the store — used by the eval's live-editor draft
+ * fixtures, which model "the user already has this draft open/saved". Writing it
+ * here (instead of through `UserDraft.save`) keeps it a backend draft row with no
+ * shadowing in-tab cell, so a model edit that persists to the backend is what the
+ * output read-back captures — not the stale seed.
+ */
+export function seedBenchmarkDraft(
+	workspace: string,
+	kind: UserDraftItemKind,
+	path: string,
+	value: unknown
+): void {
+	benchmarkDrafts.set(benchmarkDraftKey(workspace, kind, path), {
+		workspace,
+		kind,
+		path,
+		value
+	})
+}
+
+/** Mirror `DraftService.updateDraft`: a `null`/omitted value deletes the row. */
+export function updateBenchmarkDraft(input: {
+	workspace: string
+	kind: UserDraftItemKind
+	path: string
+	requestBody?: { value?: unknown }
+}): UpdateDraftResponse {
+	const key = benchmarkDraftKey(input.workspace, input.kind, input.path)
+	const value = input.requestBody?.value
+	if (value == null) {
+		benchmarkDrafts.delete(key)
+	} else {
+		benchmarkDrafts.set(key, {
+			workspace: input.workspace,
+			kind: input.kind,
+			path: input.path,
+			value
+		})
+	}
+	return { status: 'saved', current_timestamp: BENCHMARK_DRAFT_TIMESTAMP }
+}
+
+/** Mirror `DraftService.getDraftForUser`: 404-shaped throw when absent so the
+ * adapter's narrowed catch treats it as "no draft" instead of re-throwing. */
+export function getBenchmarkDraftForUser(input: {
+	workspace: string
+	kind: UserDraftItemKind
+	path: string
+}): GetDraftForUserResponse {
+	const entry = benchmarkDrafts.get(benchmarkDraftKey(input.workspace, input.kind, input.path))
+	if (!entry) {
+		throw Object.assign(new Error(`no draft for "${input.path}"`), { status: 404 })
+	}
+	return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP }
+}
+
+/** Mirror `DraftService.listDrafts`: metadata rows (no value) for a workspace. */
+export function listBenchmarkDrafts(workspace: string): ListDraftsResponse {
+	return [...benchmarkDrafts.values()]
+		.filter((entry) => entry.workspace === workspace)
+		.map((entry) => ({
+			kind: entry.kind,
+			path: entry.path,
+			summary: (entry.value as { summary?: string } | null)?.summary,
+			draft_only: true,
+			legacy_draft: false,
+			created_at: BENCHMARK_DRAFT_TIMESTAMP
+		}))
+}
+
 // ============= Datatables (best-effort in-memory SQL) =============
 
 /**

ai_evals/adapters/frontend/mockBackendDrafts.test.ts

@@ -0,0 +1,94 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
+import {
+	clearBenchmarkDrafts,
+	getBenchmarkDraftForUser,
+	listBenchmarkDrafts,
+	resetBenchmarkMockBackend,
+	seedBenchmarkDraft,
+	updateBenchmarkDraft
+} from './mockBackend'
+
+const WORKSPACE = 'benchmark-drafts-ws'
+
+// Drives the in-memory stand-in for the per-user draft backend (`DraftService`)
+// that the global AI-chat eval round-trips its drafts through. Mirrors the
+// production-unit-test mock in
+// `frontend/src/lib/components/copilot/chat/global/core.test.ts`.
+describe('mockBackend drafts', () => {
+	beforeEach(() => resetBenchmarkMockBackend())
+	afterEach(() => resetBenchmarkMockBackend())
+
+	it('round-trips a saved draft through update / get / list', () => {
+		const value = { summary: 'Greet a user', content: 'export async function main() {}' }
+		const res = updateBenchmarkDraft({
+			workspace: WORKSPACE,
+			kind: 'script',
+			path: 'f/evals/greet',
+			requestBody: { value }
+		})
+		expect(res.status).toBe('saved')
+
+		expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/greet' }).value).toEqual(
+			value
+		)
+
+		const rows = listBenchmarkDrafts(WORKSPACE)
+		expect(rows).toHaveLength(1)
+		expect(rows[0]).toMatchObject({ kind: 'script', path: 'f/evals/greet', summary: 'Greet a user', draft_only: true })
+	})
+
+	it('treats a null value as a delete', () => {
+		updateBenchmarkDraft({
+			workspace: WORKSPACE,
+			kind: 'variable',
+			path: 'f/evals/token',
+			requestBody: { value: { summary: 'token' } }
+		})
+		updateBenchmarkDraft({
+			workspace: WORKSPACE,
+			kind: 'variable',
+			path: 'f/evals/token',
+			requestBody: { value: null }
+		})
+
+		expect(listBenchmarkDrafts(WORKSPACE)).toHaveLength(0)
+		expect(() => getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'variable', path: 'f/evals/token' })).toThrow()
+	})
+
+	it('throws a 404-shaped error when no draft exists', () => {
+		try {
+			getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/missing' })
+			throw new Error('expected a throw')
+		} catch (e) {
+			expect((e as { status?: number }).status).toBe(404)
+		}
+	})
+
+	it('seeds a draft as a backend row that a later edit overwrites', () => {
+		seedBenchmarkDraft(WORKSPACE, 'script', 'f/evals/current', { content: 'seed' })
+		expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/current' }).value).toEqual({
+			content: 'seed'
+		})
+
+		// A model edit persists the same path and must win over the seed.
+		updateBenchmarkDraft({
+			workspace: WORKSPACE,
+			kind: 'script',
+			path: 'f/evals/current',
+			requestBody: { value: { content: 'edited' } }
+		})
+		expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/current' }).value).toEqual({
+			content: 'edited'
+		})
+	})
+
+	it('clears only the targeted workspace', () => {
+		seedBenchmarkDraft(WORKSPACE, 'script', 'f/a', { content: 'a' })
+		seedBenchmarkDraft('other-ws', 'script', 'f/b', { content: 'b' })
+
+		clearBenchmarkDrafts(WORKSPACE)
+
+		expect(listBenchmarkDrafts(WORKSPACE)).toHaveLength(0)
+		expect(listBenchmarkDrafts('other-ws')).toHaveLength(1)
+	})
+})

ai_evals/adapters/frontend/vitestAdapter.test.ts

@@ -36,12 +36,14 @@ vi.mock('$lib/gen', async () => {
 		getBenchmarkCompletedJob,
 		getBenchmarkCompletedJobResultMaybe,
 		getBenchmarkDatatableSchema,
+		getBenchmarkDraftForUser,
 		getBenchmarkFlowByPath,
 		getBenchmarkJobLogs,
 		getBenchmarkScriptByHash,
 		getBenchmarkScriptByPath,
 		hasBenchmarkWorkspace,
 		listBenchmarkDatatables,
+		listBenchmarkDrafts,
 		listBenchmarkFlows,
 		listBenchmarkJobs,
 		listBenchmarkScripts,
@@ -50,7 +52,8 @@ vi.mock('$lib/gen', async () => {
 		previewBenchmarkSchedule,
 		runBenchmarkDatatableSql,
 		runBenchmarkFlowByPath,
-		runBenchmarkScriptPreview
+		runBenchmarkScriptPreview,
+		updateBenchmarkDraft
 	} = await import('./mockBackend')
 
 	function wrapService<T extends object>(target: T, overrides: Record<string, unknown>): T {
@@ -66,6 +69,25 @@ vi.mock('$lib/gen', async () => {
 
 	return {
 		...actual,
+		DraftService: wrapService(actual.DraftService, {
+			updateDraft: async (data: {
+				workspace: string
+				kind: any
+				path: string
+				requestBody?: { value?: unknown }
+			}) =>
+				hasBenchmarkWorkspace(data.workspace)
+					? updateBenchmarkDraft(data)
+					: actual.DraftService.updateDraft(data),
+			getDraftForUser: async (data: { workspace: string; kind: any; path: string }) =>
+				hasBenchmarkWorkspace(data.workspace)
+					? getBenchmarkDraftForUser(data)
+					: actual.DraftService.getDraftForUser(data),
+			listDrafts: async (data: { workspace: string }) =>
+				hasBenchmarkWorkspace(data.workspace)
+					? listBenchmarkDrafts(data.workspace)
+					: actual.DraftService.listDrafts(data)
+		}),
 		ScriptService: wrapService(actual.ScriptService, {
 			listScripts: async (data: { workspace: string }) =>
 				hasBenchmarkWorkspace(data.workspace)