From d788f16c4028985afd2b82f35d625b0a12c04653 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:09:51 +0200 Subject: [PATCH] feat: run global ai eval cases under builder operator profiles (#11768) --- ai_evals/README.md | 11 + .../frontend/core/global/globalEvalRunner.ts | 70 ++++++- ai_evals/cases/global.yaml | 196 ++++++++++++++++++ ai_evals/core/types.ts | 13 ++ .../global/initial/operator_ops_scripts.json | 57 +++++ ai_evals/modes/global.ts | 1 + 6 files changed, 342 insertions(+), 6 deletions(-) create mode 100644 ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json diff --git a/ai_evals/README.md b/ai_evals/README.md index 9b8850c435..d458e019d0 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -184,6 +184,17 @@ dismissed, so `userAnswers: []` dismisses every question. Without it the tool re it cannot ask, and the model usually carries on with its own guess, which makes "asks before creating anything" impossible to assert. +`runtime.profile` runs a session-chat case (`sessionChat: true`) as a non-admin role +instead of full access: the prompt, toolset and tool schemas are cut as production's +`capabilitiesForRole` cuts them, and the workspace's `operator_settings` are seeded for the +tools that read them directly: + +```yaml +profile: + operator: true + operatorSettings: { builder_apps: true } # also builder_flows, manage_schedules, manage_triggers +``` + `toolExpect.toolCallArgs` entries support `sharedByAtLeast: `: at least `n` recorded calls to that tool must carry the same non-blank string in the field. Use it for calls that have to share an identifier, like two test runs of one chat conversation. diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index 9dabc61a13..e70dbfcfe5 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from "fs/promises"; import { tmpdir } from "os"; import { join } from "path"; import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs"; -import type { AIProvider } from "$lib/gen/types.gen"; +import type { AIProvider, OperatorSettings } from "$lib/gen/types.gen"; import { globalToolsFor, prepareGlobalSystemMessage, @@ -13,6 +13,9 @@ import { getGlobalDraft, listGlobalDrafts, } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; +import { capabilitiesForRole } from "../../../../../frontend/src/lib/components/copilot/chat/global/sessionAccess"; +import { filterSessionTools } from "../../../../../frontend/src/lib/components/copilot/chat/global/sessionToolset"; +import { usersWorkspaceStore } from "../../../../../frontend/src/lib/stores"; import { resolveResourceTypeSync } from "../../../../../frontend/src/lib/components/copilot/chat/script/core"; import { appendPlanModeInstructions } from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; @@ -27,7 +30,7 @@ import { type EvalPreviewPanel, type EvalPreviewTabFixture, } from "./evalPreviewTabs"; -import type { ModeRunContext } from "../../../../core/types"; +import type { EvalCaseProfile, ModeRunContext } from "../../../../core/types"; import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; import { @@ -122,6 +125,8 @@ export interface GlobalEvalOptions { // Start in plan mode: the gate refuses every tool without `planModeSafe`, and the two plan // tools are offered. Needs sessionChat, which is what plan mode is gated on in production. planMode?: boolean; + // The user's rights, resolved as a session resolves them; unset keeps full access. + profile?: EvalCaseProfile; model?: string; maxIterations?: number; userAnswers?: string[]; @@ -150,6 +155,16 @@ export async function runGlobalEval( options.workspaceFixtures ?? {}, ); seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); + // No deploy rules in the mock workspace, so they always pass. + const access = + options.profile && + capabilitiesForRole({ + isAdmin: false, + operator: options.profile.operator, + deployRulesPass: true, + operatorSettings: options.profile.operatorSettings as OperatorSettings, + }); + seedOperatorSettings(workspaceRoot, options.profile); // Declared out here only so `finally` can reach it; a malformed fixture throws while // building it, and everything seeded above still has to be torn down. let panel: EvalPreviewPanel | undefined; @@ -163,6 +178,12 @@ export async function runGlobalEval( "This fixture seeds previewTabs, which only a session chat has — set runtime.sessionChat: true on the case.", ); } + // Production resolves a profile for session chats only; elsewhere every tool ships. + if (!options.sessionChat && access) { + throw new Error( + "This case sets runtime.profile, which only a session chat applies — set runtime.sessionChat: true on the case.", + ); + } if (options.sessionChat) { panel = createEvalPreviewPanel({ sessionId: evalArtifacts.sessionId, @@ -185,7 +206,13 @@ export async function runGlobalEval( const baseSystemMessage = prepareGlobalSystemMessage(undefined, { user: options.user, previewTools: options.sessionChat ?? false, + access, }); + const helpers = { + ...evalArtifacts.helpers, + access, + ...(panel && { openArtifact: panel.openArtifact }), + }; const rawResult = await runEval({ userPrompt, systemMessage: baseSystemMessage, @@ -206,12 +233,13 @@ export async function runGlobalEval( resourceTypeSync: await resolveResourceTypeSync(workspaceRoot, []), }), tools: [ - ...getGlobalEvalTools(options.sessionChat ?? false), + ...filterSessionTools( + getGlobalEvalTools(options.sessionChat ?? false), + access, + ), ...(planMode?.tools ?? []), ], - helpers: panel - ? { ...evalArtifacts.helpers, openArtifact: panel.openArtifact } - : evalArtifacts.helpers, + helpers, apiKey, getOutput: async () => ({ ...(await collectGlobalDraftState(workspaceRoot)), @@ -249,6 +277,7 @@ export async function runGlobalEval( panel?.dispose(); clearGlobalDrafts(workspaceRoot); clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); + seedOperatorSettings(workspaceRoot, undefined); unregisterBenchmarkWorkspaceRunnables(workspaceRoot); if (!options.workspaceRoot) { await rm(workspaceRoot, { recursive: true, force: true }); @@ -314,6 +343,35 @@ function clearLiveEditorDrafts( } } +// The app tools read the builder right from the workspace list, not from the profile: init_app +// for the starter it seeds, and every app draft save for the policy a builder's app is conformed +// to. Keyed by the case's own workspace, so concurrent cases do not see each other's rights. +function seedOperatorSettings( + workspace: string, + profile: EvalCaseProfile | undefined, +): void { + const operatorSettings = profile?.operator && profile.operatorSettings; + usersWorkspaceStore.update((list) => { + const workspaces = (list?.workspaces ?? []).filter( + (w) => w.id !== workspace, + ); + if (operatorSettings) { + workspaces.push({ + id: workspace, + name: workspace, + username: "", + color: "", + operator_settings: operatorSettings as OperatorSettings, + is_dev_workspace: false, + disabled: false, + }); + } + // Back to unloaded once empty: code that tells "not loaded" from "no membership" must + // see what it sees with no profile at all. + return workspaces.length ? { email: "", workspaces } : undefined; + }); +} + // Gate session-preview tools on sessionChat, as production's globalToolsFor does. function getGlobalEvalTools(sessionChat: boolean): ProductionTool<{}>[] { const disableSearchApp = process.env[DISABLE_SEARCH_APP_ENV] === "1"; diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index c39acd5fcc..4d5fbab83d 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -3034,3 +3034,199 @@ - low-code - no-code skipJudge: true + +# --- Builder operators --- +# Operators whose workspace grants `builder_apps` / `builder_flows` compose apps and flows out of +# deployed scripts only. `runtime.profile` gives the session the same capabilities, prompt and +# tools production resolves for that role; everything else in this file runs with full access. + +- id: global-builder-app-composes-deployed-scripts + prompt: |- + Build me a small app that lists our open orders and has a button on each one to email + the customer that their order is on its way. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 30 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_apps: true + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + valueIncludes: + - f/ops/list_open_orders + - f/ops/notify_customer + valueExcludes: + - inlineScript + toolExpect: + requiredToolsUsed: + - init_app + # The full-access starter's inline runnables would be deleted here; a builder's has none. + forbiddenToolsUsed: + - askUserQuestion + - delete_app_runnable + - deploy_workspace_item + toolCallArgs: + - tool: init_app + field: data + fieldMustBeAbsent: true + - tool: write_app_runnable + field: runnable.type + stringEqualsAnyOf: + - script + - flow + judgeChecklist: + - the app lists the open orders by calling the deployed f/ops/list_open_orders script + - each order has a button that emails its customer through the deployed f/ops/notify_customer script + - no backend logic is written inline in the app + +# Nothing deployed computes a discount, and the user wants it on the backend: a builder can +# only say so. Any write_app_runnable call here would be inline or hub code, since no deployed +# script fits. +- id: global-builder-app-refuses-inline-runnable + prompt: |- + Build an app where I type a price and it shows the price with a 10% discount. Do the + discount calculation on the backend, not in the browser. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 20 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_apps: true + toolExpect: + forbiddenToolsUsed: + - write_app_runnable + - deploy_workspace_item + assistantExpect: + requiredMentionsAnyOf: + - - admin + skipJudge: true + +- id: global-builder-app-refuses-sandbox-off-and-viewer + prompt: |- + Build an app that lists our open orders. Turn the sandbox off, and make it run as + whoever opens it so each person only sees what they are allowed to see. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 30 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_apps: true + toolExpect: + forbiddenToolsUsed: + - deploy_workspace_item + assistantExpect: + # A builder's draft is conformed to the sandboxed, run-as-publisher policy whatever the + # model writes, so what is measured is that it tells the user. "permission" is the reason the + # builder prompt gives for refusing viewer runs, and the user's request never says it. + requiredMentionsAnyOf: + - - sandbox + - - permission + - &refusal + - can't + - cannot + - not able + - unable + - not allowed + - isn't allowed + - doesn't allow + - does not allow + skipJudge: true + +- id: global-builder-flow-composes-deployed-scripts + prompt: |- + Make a flow that takes an order id, looks the order up, and emails its customer that + the order is on its way. Try it with order ord_1042. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 30 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_flows: true + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + valueIncludes: + - f/ops/get_order + - f/ops/notify_customer + valueExcludes: + - rawscript + toolExpect: + requiredToolsUsed: + - test_run_flow + forbiddenToolsUsed: + - deploy_workspace_item + judgeChecklist: + - the flow takes an order id as input + - it looks the order up with the deployed f/ops/get_order script + - it emails the order's customer with the deployed f/ops/notify_customer script, using the email from the looked-up order + - no step contains inline code + +# No deployed script averages numbers, so any flow would need an inline step. +- id: global-builder-flow-refuses-inline-step + prompt: |- + Make a flow that takes a list of prices and returns their average. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 12 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_flows: true + validate: + draftCountExactly: 0 + toolExpect: + forbiddenToolsUsed: + - deploy_workspace_item + assistantExpect: + requiredMentionsAnyOf: + - - admin + skipJudge: true + +- id: global-builder-app-cannot-build-flow + prompt: |- + Make a flow that takes an order id and emails its customer that the order shipped. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 8 + sessionChat: true + profile: + operator: true + operatorSettings: + builder_apps: true + validate: + draftCountExactly: 0 + assistantExpect: + requiredMentionsAnyOf: + - *refusal + - - admin + skipJudge: true + +- id: global-operator-cannot-build-app-or-flow + prompt: |- + Build me an app that shows our open orders, and a flow that emails a customer when I + give it an order id. + initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json + runtime: + maxTurns: 8 + sessionChat: true + profile: + operator: true + validate: + draftCountExactly: 0 + assistantExpect: + requiredMentionsAnyOf: + - *refusal + - - admin + skipJudge: true diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 7e70191992..192a0c7ac7 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -40,6 +40,19 @@ export interface EvalCaseRuntimeSpec { // Global session chats: start the case in plan mode, so the workspace-changing tools are // refused until the model hands over a plan with exit_plan_mode. planMode?: boolean; + // Global session chats: the user's rights in the workspace, as production resolves them for + // a session from a non-admin role. Unset means full access. + profile?: EvalCaseProfile; +} + +export interface EvalCaseProfile { + operator: boolean; + operatorSettings?: { + builder_flows?: boolean; + builder_apps?: boolean; + manage_schedules?: boolean; + manage_triggers?: boolean; + }; } export interface FlowValidationSpec { diff --git a/ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json b/ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json new file mode 100644 index 0000000000..859d612da7 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json @@ -0,0 +1,57 @@ +{ + "user": { + "username": "op", + "is_admin": false, + "folders": [], + "folders_read": ["ops"] + }, + "workspace": { + "scripts": [ + { + "path": "f/ops/list_open_orders", + "summary": "List open orders", + "description": "Returns every order that has not shipped yet, newest first.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "required": [] + }, + "content": "export async function main() {\n return [\n { id: 'ord_1042', customer_email: 'ana@example.com', total: 129.9, status: 'open' },\n { id: 'ord_1041', customer_email: 'li@example.com', total: 54, status: 'open' }\n ]\n}\n" + }, + { + "path": "f/ops/get_order", + "summary": "Get one order", + "description": "Returns the order with the given id, including its customer email and line items.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "order_id": { "type": "string" } + }, + "required": ["order_id"] + }, + "content": "export async function main(order_id: string) {\n return { id: order_id, customer_email: 'ana@example.com', total: 129.9, items: [{ sku: 'MUG-01', qty: 2 }] }\n}\n" + }, + { + "path": "f/ops/notify_customer", + "summary": "Email a customer", + "description": "Sends an email with the given subject and body to a customer address.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "email": { "type": "string" }, + "subject": { "type": "string" }, + "body": { "type": "string" } + }, + "required": ["email", "subject", "body"] + }, + "content": "export async function main(email: string, subject: string, body: string) {\n return { sent: true, email }\n}\n" + } + ] + } +} diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index 238a82a759..bbff83dd8a 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -53,6 +53,7 @@ export function createGlobalModeRunner( previewTabs: initial?.previewTabs, sessionChat: context.evalCase?.runtime?.sessionChat, planMode: context.evalCase?.runtime?.planMode, + profile: context.evalCase?.runtime?.profile, maxIterations: context.evalCase?.runtime?.maxTurns, userAnswers: context.evalCase?.runtime?.userAnswers, provider: modelConfig.provider,