From e87ff79ecf6a6e0958916ed1b3756fb3addf719f Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:53:21 +0200 Subject: [PATCH] fix(ai_evals): adapt global eval harness to DB-backed user drafts (#9641) Co-authored-by: Claude Opus 4.8 (1M context) --- .../frontend/core/global/globalEvalRunner.ts | 34 +++++- .../frontend/core/shared/baseEvalRunner.ts | 11 +- ai_evals/adapters/frontend/mockBackend.ts | 114 +++++++++++++++++- .../frontend/mockBackendDrafts.test.ts | 94 +++++++++++++++ .../adapters/frontend/vitestAdapter.test.ts | 24 +++- 5 files changed, 268 insertions(+), 9 deletions(-) create mode 100644 ai_evals/adapters/frontend/mockBackendDrafts.test.ts diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index 058adc3644..226fba7c0a 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -9,6 +9,7 @@ import { } from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; import { clearGlobalDrafts, + getGlobalDraft, listGlobalDrafts, } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; @@ -18,6 +19,7 @@ import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; import { registerBenchmarkWorkspaceRunnables, + seedBenchmarkDraft, unregisterBenchmarkWorkspaceRunnables, type BenchmarkWorkspaceRunnables, } from "../../mockBackend"; @@ -94,7 +96,7 @@ export async function runGlobalEval( tools: getGlobalEvalTools(), helpers: {}, apiKey, - getOutput: () => ({ drafts: listGlobalDrafts(workspaceRoot) }), + getOutput: () => collectGlobalDraftState(workspaceRoot), onAssistantMessageStart: options.runContext?.onAssistantMessageStart, onAssistantToken: options.runContext?.onAssistantChunk, onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd, @@ -130,6 +132,32 @@ export async function runGlobalEval( } } +// Build the harness output from the DB-backed drafts. `listGlobalDrafts` returns +// metadata-only rows for backend drafts (the model's `write_script` etc. persist +// straight to the backend with no in-tab editor cell), so re-read each such row +// with `getGlobalDraft` to attach the full value the validators assert on. A row +// that already carries a value (the production in-tab cell overlay) is kept as-is. +async function collectGlobalDraftState( + workspace: string, +): Promise { + const items = await listGlobalDrafts(workspace); + const drafts = await Promise.all( + items.map(async (item) => { + if (item.value !== undefined) { + return item; + } + const full = await getGlobalDraft( + workspace, + item.type, + item.path, + item.triggerKind, + ); + return full ?? item; + }), + ); + return { drafts: drafts as GlobalDraftState["drafts"] }; +} + function seedLiveEditorDrafts( workspace: string, fixtures: GlobalLiveEditorDraftFixture[], @@ -138,7 +166,9 @@ function seedLiveEditorDrafts( const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type]; const storagePath = fixture.storagePath ?? fixture.effectivePath ?? ""; if (fixture.value !== undefined) { - UserDraft.save(itemKind, storagePath, fixture.value, { workspace }); + // Seed as a backend draft row, not an in-tab cell: a cell would shadow the + // model's DB-backed edit when the output is read back via listGlobalDrafts. + seedBenchmarkDraft(workspace, itemKind, storagePath, fixture.value); } UserDraft.setLiveEditorDraft({ workspace, diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 5f4ea2c307..faec6c51ed 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -38,8 +38,9 @@ export interface RunEvalParams { helpers: THelpers; /** API key for the provider */ apiKey: string; - /** Function to get the current output state */ - getOutput: () => TOutput; + /** Function to get the current output state. May be async — global mode reads + * DB-backed drafts back through the (mocked) backend to build its output. */ + getOutput: () => TOutput | Promise; /** Model and Windmill backend configuration */ options: EvalRunnerOptions; onAssistantMessageStart?: () => void; @@ -154,7 +155,7 @@ export async function runEval( if (result.hitMaxIterations) { return { success: false, - output: getOutput(), + output: (await getOutput()) as TOutput, error: `Reached max turns (${maxIterations})`, tokenUsage: result.tokenUsage, toolCallsCount, @@ -170,7 +171,7 @@ export async function runEval( return { success: true, - output: getOutput(), + output: (await getOutput()) as TOutput, tokenUsage: result.tokenUsage, toolCallsCount, toolsCalled, @@ -191,7 +192,7 @@ export async function runEval( return { success: false, - output: getOutput(), + output: (await getOutput()) as TOutput, error: errorMessage, tokenUsage: { prompt: 0, completion: 0, total: 0 }, toolCallsCount, diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 622c79bf46..74fa812883 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -3,7 +3,11 @@ import type { CompletedJob, Flow, Job, Script } from '../../../frontend/src/lib/ import type { DataTableTables, DataTableTableSchema, - ScriptLang + GetDraftForUserResponse, + ListDraftsResponse, + ScriptLang, + UpdateDraftResponse, + UserDraftItemKind } from '../../../frontend/src/lib/gen/types.gen' import { buildScriptLintResult } from './core/script/preview' import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine' @@ -63,6 +67,7 @@ export function resetBenchmarkMockBackend(): void { benchmarkWorkspaces.clear() benchmarkWorkspaceRunnables.clear() benchmarkJobs.clear() + benchmarkDrafts.clear() } export function registerBenchmarkWorkspace(workspace: string): void { @@ -74,6 +79,8 @@ export function registerBenchmarkWorkspaceRunnables( runnables: BenchmarkWorkspaceRunnables ): void { benchmarkWorkspaces.add(workspace) + // Fresh case: drop any drafts left from a prior run on this workspace id. + clearBenchmarkDrafts(workspace) // Datatables are mutated in place by exec_datatable_sql (a write must be visible // to later reads), so store an isolated deep copy — never mutate the caller's seed. benchmarkWorkspaceRunnables.set(workspace, { @@ -98,6 +105,7 @@ export function registerBenchmarkWorkspaceRunnables( export function unregisterBenchmarkWorkspace(workspace: string): void { benchmarkWorkspaces.delete(workspace) benchmarkWorkspaceRunnables.delete(workspace) + clearBenchmarkDrafts(workspace) for (const [jobId, entry] of benchmarkJobs.entries()) { if (entry.workspace === workspace) { benchmarkJobs.delete(jobId) @@ -238,6 +246,110 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string { return job.logs ?? '' } +// ============= Drafts (per-user, DB-backed in production) ============= + +/** + * In-memory stand-in for the per-user draft backend (`DraftService`). The global + * AI chat now persists and reads drafts through the backend DB instead of an + * in-tab `UserDraft` cell, so the eval mocks the three draft endpoints it + * exercises (`updateDraft` / `getDraftForUser` / `listDrafts`) and keeps the + * saved values here, keyed by workspace + draft kind + storage path. Mirrors the + * semantics of the production unit test's mock in + * `frontend/src/lib/components/copilot/chat/global/core.test.ts`. + */ +const benchmarkDrafts = new Map< + string, + { workspace: string; kind: UserDraftItemKind; path: string; value: unknown } +>() + +// Fixed timestamp so artifacts stay deterministic. No eval simulates a +// concurrent writer, so every save is accepted and the conflict branch is +// never taken — the syncer just records this as its `last_sync` baseline. +const BENCHMARK_DRAFT_TIMESTAMP = '1970-01-01T00:00:00.000Z' + +function benchmarkDraftKey(workspace: string, kind: string, path: string): string { + return `${workspace}::${kind}::${path}` +} + +export function clearBenchmarkDrafts(workspace: string): void { + for (const [key, entry] of benchmarkDrafts.entries()) { + if (entry.workspace === workspace) { + benchmarkDrafts.delete(key) + } + } +} + +/** + * Seed a draft straight into the store — used by the eval's live-editor draft + * fixtures, which model "the user already has this draft open/saved". Writing it + * here (instead of through `UserDraft.save`) keeps it a backend draft row with no + * shadowing in-tab cell, so a model edit that persists to the backend is what the + * output read-back captures — not the stale seed. + */ +export function seedBenchmarkDraft( + workspace: string, + kind: UserDraftItemKind, + path: string, + value: unknown +): void { + benchmarkDrafts.set(benchmarkDraftKey(workspace, kind, path), { + workspace, + kind, + path, + value + }) +} + +/** Mirror `DraftService.updateDraft`: a `null`/omitted value deletes the row. */ +export function updateBenchmarkDraft(input: { + workspace: string + kind: UserDraftItemKind + path: string + requestBody?: { value?: unknown } +}): UpdateDraftResponse { + const key = benchmarkDraftKey(input.workspace, input.kind, input.path) + const value = input.requestBody?.value + if (value == null) { + benchmarkDrafts.delete(key) + } else { + benchmarkDrafts.set(key, { + workspace: input.workspace, + kind: input.kind, + path: input.path, + value + }) + } + return { status: 'saved', current_timestamp: BENCHMARK_DRAFT_TIMESTAMP } +} + +/** Mirror `DraftService.getDraftForUser`: 404-shaped throw when absent so the + * adapter's narrowed catch treats it as "no draft" instead of re-throwing. */ +export function getBenchmarkDraftForUser(input: { + workspace: string + kind: UserDraftItemKind + path: string +}): GetDraftForUserResponse { + const entry = benchmarkDrafts.get(benchmarkDraftKey(input.workspace, input.kind, input.path)) + if (!entry) { + throw Object.assign(new Error(`no draft for "${input.path}"`), { status: 404 }) + } + return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP } +} + +/** Mirror `DraftService.listDrafts`: metadata rows (no value) for a workspace. */ +export function listBenchmarkDrafts(workspace: string): ListDraftsResponse { + return [...benchmarkDrafts.values()] + .filter((entry) => entry.workspace === workspace) + .map((entry) => ({ + kind: entry.kind, + path: entry.path, + summary: (entry.value as { summary?: string } | null)?.summary, + draft_only: true, + legacy_draft: false, + created_at: BENCHMARK_DRAFT_TIMESTAMP + })) +} + // ============= Datatables (best-effort in-memory SQL) ============= /** diff --git a/ai_evals/adapters/frontend/mockBackendDrafts.test.ts b/ai_evals/adapters/frontend/mockBackendDrafts.test.ts new file mode 100644 index 0000000000..a720de5e43 --- /dev/null +++ b/ai_evals/adapters/frontend/mockBackendDrafts.test.ts @@ -0,0 +1,94 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { + clearBenchmarkDrafts, + getBenchmarkDraftForUser, + listBenchmarkDrafts, + resetBenchmarkMockBackend, + seedBenchmarkDraft, + updateBenchmarkDraft +} from './mockBackend' + +const WORKSPACE = 'benchmark-drafts-ws' + +// Drives the in-memory stand-in for the per-user draft backend (`DraftService`) +// that the global AI-chat eval round-trips its drafts through. Mirrors the +// production-unit-test mock in +// `frontend/src/lib/components/copilot/chat/global/core.test.ts`. +describe('mockBackend drafts', () => { + beforeEach(() => resetBenchmarkMockBackend()) + afterEach(() => resetBenchmarkMockBackend()) + + it('round-trips a saved draft through update / get / list', () => { + const value = { summary: 'Greet a user', content: 'export async function main() {}' } + const res = updateBenchmarkDraft({ + workspace: WORKSPACE, + kind: 'script', + path: 'f/evals/greet', + requestBody: { value } + }) + expect(res.status).toBe('saved') + + expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/greet' }).value).toEqual( + value + ) + + const rows = listBenchmarkDrafts(WORKSPACE) + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ kind: 'script', path: 'f/evals/greet', summary: 'Greet a user', draft_only: true }) + }) + + it('treats a null value as a delete', () => { + updateBenchmarkDraft({ + workspace: WORKSPACE, + kind: 'variable', + path: 'f/evals/token', + requestBody: { value: { summary: 'token' } } + }) + updateBenchmarkDraft({ + workspace: WORKSPACE, + kind: 'variable', + path: 'f/evals/token', + requestBody: { value: null } + }) + + expect(listBenchmarkDrafts(WORKSPACE)).toHaveLength(0) + expect(() => getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'variable', path: 'f/evals/token' })).toThrow() + }) + + it('throws a 404-shaped error when no draft exists', () => { + try { + getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/missing' }) + throw new Error('expected a throw') + } catch (e) { + expect((e as { status?: number }).status).toBe(404) + } + }) + + it('seeds a draft as a backend row that a later edit overwrites', () => { + seedBenchmarkDraft(WORKSPACE, 'script', 'f/evals/current', { content: 'seed' }) + expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/current' }).value).toEqual({ + content: 'seed' + }) + + // A model edit persists the same path and must win over the seed. + updateBenchmarkDraft({ + workspace: WORKSPACE, + kind: 'script', + path: 'f/evals/current', + requestBody: { value: { content: 'edited' } } + }) + expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/current' }).value).toEqual({ + content: 'edited' + }) + }) + + it('clears only the targeted workspace', () => { + seedBenchmarkDraft(WORKSPACE, 'script', 'f/a', { content: 'a' }) + seedBenchmarkDraft('other-ws', 'script', 'f/b', { content: 'b' }) + + clearBenchmarkDrafts(WORKSPACE) + + expect(listBenchmarkDrafts(WORKSPACE)).toHaveLength(0) + expect(listBenchmarkDrafts('other-ws')).toHaveLength(1) + }) +}) diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 2739eedce2..657f9aca6b 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -36,12 +36,14 @@ vi.mock('$lib/gen', async () => { getBenchmarkCompletedJob, getBenchmarkCompletedJobResultMaybe, getBenchmarkDatatableSchema, + getBenchmarkDraftForUser, getBenchmarkFlowByPath, getBenchmarkJobLogs, getBenchmarkScriptByHash, getBenchmarkScriptByPath, hasBenchmarkWorkspace, listBenchmarkDatatables, + listBenchmarkDrafts, listBenchmarkFlows, listBenchmarkJobs, listBenchmarkScripts, @@ -50,7 +52,8 @@ vi.mock('$lib/gen', async () => { previewBenchmarkSchedule, runBenchmarkDatatableSql, runBenchmarkFlowByPath, - runBenchmarkScriptPreview + runBenchmarkScriptPreview, + updateBenchmarkDraft } = await import('./mockBackend') function wrapService(target: T, overrides: Record): T { @@ -66,6 +69,25 @@ vi.mock('$lib/gen', async () => { return { ...actual, + DraftService: wrapService(actual.DraftService, { + updateDraft: async (data: { + workspace: string + kind: any + path: string + requestBody?: { value?: unknown } + }) => + hasBenchmarkWorkspace(data.workspace) + ? updateBenchmarkDraft(data) + : actual.DraftService.updateDraft(data), + getDraftForUser: async (data: { workspace: string; kind: any; path: string }) => + hasBenchmarkWorkspace(data.workspace) + ? getBenchmarkDraftForUser(data) + : actual.DraftService.getDraftForUser(data), + listDrafts: async (data: { workspace: string }) => + hasBenchmarkWorkspace(data.workspace) + ? listBenchmarkDrafts(data.workspace) + : actual.DraftService.listDrafts(data) + }), ScriptService: wrapService(actual.ScriptService, { listScripts: async (data: { workspace: string }) => hasBenchmarkWorkspace(data.workspace)