From e29dfbaa877c7b911727e060f408a134ad6b4f76 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 27 May 2026 14:19:57 +0200 Subject: [PATCH] test: add global chat eval coverage (#9320) * test: improve global chat eval parity * test: add human-style global chat evals --- .../frontend/core/global/globalEvalRunner.ts | 11 +- .../adapters/frontend/vitestAdapter.test.ts | 30 +++ ai_evals/cases/global.yaml | 254 ++++++++++++++++++ ai_evals/core/types.ts | 4 +- ai_evals/core/validators.test.ts | 63 +++++ ai_evals/core/validators.ts | 115 ++++++-- .../global/initial/process_invoice_flow.json | 40 +++ 7 files changed, 497 insertions(+), 20 deletions(-) create mode 100644 ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index 5e00dd6f34..8541348c51 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -7,7 +7,10 @@ import { prepareGlobalSystemMessage, prepareGlobalUserMessage, } from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; -import { globalDraftStore } from "../../../../../frontend/src/lib/components/copilot/chat/global/draftStore.svelte"; +import { + clearGlobalDrafts, + listGlobalDrafts, +} from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; import type { ModeRunContext } from "../../../../core/types"; import type { GlobalDraftState } from "../../../../core/validators"; @@ -55,7 +58,7 @@ export async function runGlobalEval( options.workspaceRoot ?? (await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-"))); - globalDraftStore.clearDrafts(workspaceRoot); + clearGlobalDrafts(workspaceRoot); registerBenchmarkWorkspaceRunnables(workspaceRoot, options.workspaceFixtures ?? {}); try { @@ -67,7 +70,7 @@ export async function runGlobalEval( tools: getGlobalEvalTools(), helpers: {}, apiKey, - getOutput: () => ({ drafts: globalDraftStore.listDrafts(workspaceRoot) }), + getOutput: () => ({ drafts: listGlobalDrafts(workspaceRoot) }), onAssistantMessageStart: options.runContext?.onAssistantMessageStart, onAssistantToken: options.runContext?.onAssistantChunk, onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd, @@ -94,7 +97,7 @@ export async function runGlobalEval( tokenUsage: rawResult.tokenUsage, }; } finally { - globalDraftStore.clearDrafts(workspaceRoot); + clearGlobalDrafts(workspaceRoot); unregisterBenchmarkWorkspaceRunnables(workspaceRoot); if (!options.workspaceRoot) { await rm(workspaceRoot, { recursive: true, force: true }); diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 1275acf0b4..92c43aab06 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -79,6 +79,16 @@ vi.mock('$lib/gen', async () => { } return actual.ScriptService.getScriptByPath(data) }, + getScriptByPathWithDraft: async (data: { workspace: string; path: string }) => { + if (hasBenchmarkWorkspace(data.workspace)) { + const script = getBenchmarkScriptByPath(data.workspace, data.path) + if (!script) { + throw new Error(`Script "${data.path}" not found in benchmark workspace`) + } + return script + } + return actual.ScriptService.getScriptByPathWithDraft(data) + }, getScriptByHash: async (data: { workspace: string; hash: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { const script = getBenchmarkScriptByHash(data.workspace, data.hash) @@ -108,6 +118,26 @@ vi.mock('$lib/gen', async () => { return flow } return actual.FlowService.getFlowByPath(data) + }, + getFlowByPathWithDraft: async (data: { workspace: string; path: string }) => { + if (hasBenchmarkWorkspace(data.workspace)) { + const flow = getBenchmarkFlowByPath(data.workspace, data.path) + if (!flow) { + throw new Error(`Flow "${data.path}" not found in benchmark workspace`) + } + return flow + } + return actual.FlowService.getFlowByPathWithDraft(data) + }, + getFlowLatestVersion: async (data: { workspace: string; path: string }) => { + if (hasBenchmarkWorkspace(data.workspace)) { + const flow = getBenchmarkFlowByPath(data.workspace, data.path) + if (!flow) { + throw new Error(`Flow "${data.path}" not found in benchmark workspace`) + } + return { id: 1 } + } + return actual.FlowService.getFlowLatestVersion(data) } }), JobService: wrapService(actual.JobService, { diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index b4526f8a85..373408a23d 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -87,3 +87,257 @@ - the flow accepts numeric inputs a and b - the flow returns the sum of a and b - the result stays as an AI draft and is not deployed or saved to the workspace + +- id: global-test4-multi-artifact-notification-job + prompt: |- + Set up a draft stale-trial notification job. + Create a Bun script at `f/evals/global/check_stale_trials` that accepts `max_age_days`, uses mocked inline trial account data, and returns the stale trial account IDs. + Also create a weekday 09:00 UTC schedule at `f/evals/global/check_stale_trials_weekday` for that script with `max_age_days` set to 14. + Add an HTTP POST trigger at `f/evals/global/check_stale_trials_manual` with route path `evals/check-stale-trials` that runs the same script manually. + Leave everything as AI drafts only; do not deploy or save anything to the workspace. + runtime: + maxTurns: 12 + validate: + draftCountExactly: 3 + requiredDrafts: + - type: script + path: f/evals/global/check_stale_trials + language: bun + valueIncludes: + - max_age_days + - trial + - type: schedule + path: f/evals/global/check_stale_trials_weekday + valueIncludes: + - f/evals/global/check_stale_trials + - UTC + - "14" + - type: trigger + triggerKind: http + path: f/evals/global/check_stale_trials_manual + valueIncludes: + - evals/check-stale-trials + - f/evals/global/check_stale_trials + toolExpect: + requiredToolsUsed: + - write_script + - write_schedule + - write_trigger + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - creates a Bun script draft for stale trial accounts + - creates a weekday 09:00 UTC schedule draft for the script with max_age_days set to 14 + - creates an HTTP POST trigger draft with route path evals/check-stale-trials for the same script + - leaves all artifacts as drafts only and does not deploy + +- id: global-test5-existing-flow-inline-code-edit + prompt: |- + Update the existing flow at `f/evals/global/process_invoice`. + Only change the `calculate_total` inline code so it applies 8% tax and returns an object containing `subtotal`, `tax`, and `total`. + Leave the updated flow as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + path: f/evals/global/process_invoice + valueIncludes: + - calculate_total + - tax + - total + toolExpect: + requiredToolsUsed: + - read_workspace_item + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - reads the existing process_invoice flow before editing it + - updates the calculate_total inline code to apply 8% tax + - returns subtotal, tax, and total from the updated flow logic + - leaves the result as an AI draft only + +- id: global-test6-secret-variable-draft + prompt: |- + Create a secret variable draft at `f/evals/global/slack_bot_token`. + Use the placeholder value `xoxb-redacted-test-token` and description `Slack bot token for eval notifications`. + Do not create any resource or deploy anything. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: variable + path: f/evals/global/slack_bot_token + valueIncludes: + - Slack bot token + - "true" + forbiddenDrafts: + - type: resource + path: f/evals/global/slack_bot_token + toolExpect: + requiredToolsUsed: + - write_variable + forbiddenToolsUsed: + - write_resource + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: write_variable + field: value + stringStartsWithAnyOf: + - xoxb-redacted-test-token + skipJudge: true + judgeChecklist: + - creates exactly one secret variable draft at f/evals/global/slack_bot_token + - uses the requested placeholder value and description + - does not create a resource or deploy anything + +- id: global-test7-ambiguous-app-asks-question + prompt: |- + Create a new raw app for triaging support tickets. + runtime: + maxTurns: 4 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - askUserQuestion + forbiddenToolsUsed: + - init_app + - write_app_file + - write_app_runnable + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + +- id: global-test8-human-script-infer-path-language + prompt: |- + I need a small helper that formats a customer-facing welcome line. + It should take a person's name and return "Welcome aboard, !". + Please just stage it as a draft for now. + runtime: + maxTurns: 8 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + valueIncludes: + - Welcome aboard + - name + toolExpect: + requiredToolsUsed: + - write_script + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - creates a single script draft for a welcome-line helper + - accepts a person's name as input + - returns a message containing Welcome aboard, the provided name, and an exclamation mark + - chooses a reasonable workspace path and script language without needing the user to specify them + - leaves the result as an AI draft only + +- id: global-test9-human-weekday-trial-job + prompt: |- + Can you set up a draft daily job that checks a few hard-coded trial accounts and returns the ones whose trial has ended? + It should run every weekday morning around 9 in UTC with a 30 day cutoff. + Keep it as draft work only. + runtime: + maxTurns: 10 + validate: + draftCountExactly: 2 + requiredDrafts: + - type: script + pathIncludes: + - trial + valueIncludes: + - trial + - "30" + - type: schedule + pathIncludes: + - trial + valueIncludes: + - UTC + toolExpect: + requiredToolsUsed: + - write_script + - write_schedule + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - creates a script draft that checks hard-coded trial accounts + - returns the accounts whose trial has ended based on a 30 day cutoff + - creates a schedule draft for weekday mornings around 09:00 UTC + - links the schedule to the generated script + - leaves both artifacts as drafts only + +- id: global-test10-human-secret-variable + prompt: |- + I need a placeholder Slack bot token stored securely for future notification work. + Use xoxb-redacted-test-token and note that it is for eval notifications. + Only prepare a draft. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: variable + pathIncludes: + - slack + valueIncludes: + - eval notifications + - "true" + toolExpect: + requiredToolsUsed: + - write_variable + forbiddenToolsUsed: + - write_resource + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: write_variable + field: value + stringStartsWithAnyOf: + - xoxb-redacted-test-token + skipJudge: true + judgeChecklist: + - creates a single secret variable draft for the Slack bot token placeholder + - uses the requested placeholder value + - includes a note or description that it is for eval notifications + - does not create a resource or deploy anything + +- id: global-test11-human-existing-flow-informal-edit + prompt: |- + There is an invoice processing flow in this workspace. + Can you adjust its total calculation so it adds 8% tax and returns subtotal, tax, and total? + Keep the change as a draft. + initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + pathIncludes: + - invoice + valueIncludes: + - calculate_total + - tax + - total + toolExpect: + requiredToolsUsed: + - read_workspace_item + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - finds and edits the existing invoice processing flow without the user providing its exact path + - updates the total calculation to apply 8% tax + - returns subtotal, tax, and total from the updated flow logic + - leaves the result as an AI draft only diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 2b42a0dfc5..19c97bce11 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -110,7 +110,9 @@ export interface AppValidationSpec { export interface GlobalDraftRequirement { type: string; - path: string; + path?: string; + pathIncludes?: string[]; + pathStartsWith?: string; triggerKind?: string; language?: string; summaryIncludes?: string[]; diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index d2a6e954bb..6010f6351a 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -195,6 +195,69 @@ describe("validateGlobalState", () => { }); }); + it("accepts a required script draft without an exact path", () => { + const checks = validateGlobalState({ + actual: { + drafts: [ + { + type: "script", + path: "f/team_tools/friendly_greeting", + language: "bun", + summary: "Friendly greeting helper", + value: + "export async function main(name: string) {\n return `Hello, ${name}!`\n}\n", + isDraft: true, + }, + ], + }, + validate: { + draftCountExactly: 1, + requiredDrafts: [ + { + type: "script", + pathIncludes: ["greeting"], + language: "bun", + summaryIncludes: ["Friendly"], + valueIncludes: ["Hello"], + }, + ], + }, + }); + + expect(checks.every((check) => check.passed)).toBe(true); + }); + + it("reports flexible global draft path filters when no draft matches", () => { + const checks = validateGlobalState({ + actual: { + drafts: [ + { + type: "script", + path: "f/team_tools/friendly_greeting", + language: "bun", + value: + "export async function main(name: string) {\n return `Hello, ${name}!`\n}\n", + isDraft: true, + }, + ], + }, + validate: { + requiredDrafts: [ + { + type: "script", + pathIncludes: ["invoice"], + }, + ], + }, + }); + + expect(checks).toContainEqual({ + name: "global includes script draft (path includes invoice)", + passed: false, + details: "drafts: script:f/team_tools/friendly_greeting", + }); + }); + it("does not require a TypeScript entrypoint for non-TypeScript script drafts", () => { const checks = validateGlobalState({ actual: { diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 4f59368113..693d34a013 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -315,10 +315,11 @@ export function validateGlobalState(input: { } for (const required of validate.requiredDrafts ?? []) { - const draft = findGlobalDraft(drafts, required.type, required.path, required.triggerKind); + const requirementLabel = formatGlobalDraftRequirement(required); + const draft = findGlobalDraft(drafts, required); checks.push( check( - `global includes ${required.type} draft ${required.path}`, + `global includes ${requirementLabel}`, Boolean(draft), summarizeGlobalDrafts(drafts) ) @@ -330,7 +331,7 @@ export function validateGlobalState(input: { if (required.language !== undefined) { checks.push( check( - `${required.type} draft ${required.path} uses ${required.language}`, + `${requirementLabel} uses ${required.language}`, draft.language === required.language, `language=${draft.language ?? "(none)"}` ) @@ -340,7 +341,7 @@ export function validateGlobalState(input: { for (const snippet of required.summaryIncludes ?? []) { checks.push( check( - `${required.type} draft ${required.path} summary includes '${snippet}'`, + `${requirementLabel} summary includes '${snippet}'`, normalizeText(draft.summary ?? "").includes(normalizeText(snippet)), `summary=${draft.summary ?? ""}` ) @@ -351,7 +352,7 @@ export function validateGlobalState(input: { for (const snippet of required.valueIncludes ?? []) { checks.push( check( - `${required.type} draft ${required.path} value includes '${snippet}'`, + `${requirementLabel} value includes '${snippet}'`, normalizeText(valueText).includes(normalizeText(snippet)), truncateForDetails(valueText) ) @@ -361,7 +362,7 @@ export function validateGlobalState(input: { for (const snippet of required.valueExcludes ?? []) { checks.push( check( - `${required.type} draft ${required.path} value excludes '${snippet}'`, + `${requirementLabel} value excludes '${snippet}'`, !normalizeText(valueText).includes(normalizeText(snippet)), truncateForDetails(valueText) ) @@ -373,7 +374,7 @@ export function validateGlobalState(input: { checks.push( check( `global does not include ${forbidden.type} draft ${forbidden.path}`, - !findGlobalDraft(drafts, forbidden.type, forbidden.path, forbidden.triggerKind), + !findGlobalDraft(drafts, forbidden), summarizeGlobalDrafts(drafts) ) ); @@ -615,16 +616,100 @@ function summarizeProblems(problems: string[], limit = 5): string | undefined { function findGlobalDraft( drafts: GlobalDraft[], - type: string, - path: string, - triggerKind?: string + requirement: { + type: string; + path?: string; + pathIncludes?: string[]; + pathStartsWith?: string; + triggerKind?: string; + summaryIncludes?: string[]; + valueIncludes?: string[]; + valueExcludes?: string[]; + } ): GlobalDraft | undefined { - return drafts.find( - (draft) => - draft.type === type && - draft.path === path && - (triggerKind === undefined || draft.triggerKind === triggerKind) + const candidates = drafts.filter((draft) => + globalDraftMatchesLocator(draft, requirement) ); + return ( + candidates.find((draft) => globalDraftMatchesContent(draft, requirement)) ?? + candidates[0] + ); +} + +function globalDraftMatchesLocator( + draft: GlobalDraft, + requirement: { + type: string; + path?: string; + pathIncludes?: string[]; + pathStartsWith?: string; + triggerKind?: string; + } +): boolean { + return ( + draft.type === requirement.type && + (requirement.path === undefined || draft.path === requirement.path) && + (requirement.pathStartsWith === undefined || + draft.path.startsWith(requirement.pathStartsWith)) && + (requirement.pathIncludes ?? []).every((snippet) => + normalizeText(draft.path).includes(normalizeText(snippet)) + ) && + (requirement.triggerKind === undefined || + draft.triggerKind === requirement.triggerKind) + ); +} + +function globalDraftMatchesContent( + draft: GlobalDraft, + requirement: { + summaryIncludes?: string[]; + valueIncludes?: string[]; + valueExcludes?: string[]; + } +): boolean { + const summary = normalizeText(draft.summary ?? ""); + const value = normalizeText(stringifyGlobalDraftValue(draft.value)); + return ( + (requirement.summaryIncludes ?? []).every((snippet) => + summary.includes(normalizeText(snippet)) + ) && + (requirement.valueIncludes ?? []).every((snippet) => + value.includes(normalizeText(snippet)) + ) && + (requirement.valueExcludes ?? []).every( + (snippet) => !value.includes(normalizeText(snippet)) + ) + ); +} + +function formatGlobalDraftRequirement( + requirement: { + type: string; + path?: string; + pathIncludes?: string[]; + pathStartsWith?: string; + triggerKind?: string; + } +): string { + const typeLabel = + requirement.triggerKind === undefined + ? requirement.type + : `${requirement.triggerKind} ${requirement.type}`; + if (requirement.path !== undefined) { + return `${typeLabel} draft ${requirement.path}`; + } + + const filters = [ + ...(requirement.pathStartsWith === undefined + ? [] + : [`path starts with ${requirement.pathStartsWith}`]), + ...(requirement.pathIncludes ?? []).map( + (snippet) => `path includes ${snippet}` + ), + ]; + return filters.length === 0 + ? `${typeLabel} draft` + : `${typeLabel} draft (${filters.join(", ")})`; } function summarizeGlobalDrafts(drafts: GlobalDraft[]): string { diff --git a/ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json b/ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json new file mode 100644 index 0000000000..b9b4c675c5 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json @@ -0,0 +1,40 @@ +{ + "workspace": { + "flows": [ + { + "path": "f/evals/global/process_invoice", + "summary": "Process an invoice subtotal", + "description": "Calculates invoice totals from a subtotal.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "subtotal": { + "type": "number" + } + }, + "required": ["subtotal"] + }, + "value": { + "modules": [ + { + "id": "calculate_total", + "summary": "Calculate total from subtotal", + "value": { + "type": "rawscript", + "language": "bun", + "content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n", + "input_transforms": { + "subtotal": { + "type": "javascript", + "expr": "flow_input.subtotal" + } + } + } + } + ] + } + } + ] + } +}