feat: run global ai eval cases under builder operator profiles (#11768)

ai_evals/README.md

@@ -184,6 +184,17 @@ dismissed, so `userAnswers: []` dismisses every question. Without it the tool re
 it cannot ask, and the model usually carries on with its own guess, which makes "asks
 before creating anything" impossible to assert.
 
+`runtime.profile` runs a session-chat case (`sessionChat: true`) as a non-admin role
+instead of full access: the prompt, toolset and tool schemas are cut as production's
+`capabilitiesForRole` cuts them, and the workspace's `operator_settings` are seeded for the
+tools that read them directly:
+
+```yaml
+profile:
+  operator: true
+  operatorSettings: { builder_apps: true }   # also builder_flows, manage_schedules, manage_triggers
+```
+
 `toolExpect.toolCallArgs` entries support `sharedByAtLeast: <n>`: at least `n` recorded
 calls to that tool must carry the same non-blank string in the field. Use it for calls that
 have to share an identifier, like two test runs of one chat conversation.

ai_evals/adapters/frontend/core/global/globalEvalRunner.ts

@@ -2,7 +2,7 @@ import { mkdtemp, rm } from "fs/promises";
 import { tmpdir } from "os";
 import { join } from "path";
 import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs";
-import type { AIProvider } from "$lib/gen/types.gen";
+import type { AIProvider, OperatorSettings } from "$lib/gen/types.gen";
 import {
   globalToolsFor,
   prepareGlobalSystemMessage,
@@ -13,6 +13,9 @@ import {
   getGlobalDraft,
   listGlobalDrafts,
 } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter";
+import { capabilitiesForRole } from "../../../../../frontend/src/lib/components/copilot/chat/global/sessionAccess";
+import { filterSessionTools } from "../../../../../frontend/src/lib/components/copilot/chat/global/sessionToolset";
+import { usersWorkspaceStore } from "../../../../../frontend/src/lib/stores";
 import { resolveResourceTypeSync } from "../../../../../frontend/src/lib/components/copilot/chat/script/core";
 import { appendPlanModeInstructions } from "../../../../../frontend/src/lib/components/copilot/chat/planMode";
 import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
@@ -27,7 +30,7 @@ import {
   type EvalPreviewPanel,
   type EvalPreviewTabFixture,
 } from "./evalPreviewTabs";
-import type { ModeRunContext } from "../../../../core/types";
+import type { EvalCaseProfile, ModeRunContext } from "../../../../core/types";
 import type { GlobalDraftState } from "../../../../core/validators";
 import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
 import {
@@ -122,6 +125,8 @@ export interface GlobalEvalOptions {
   // Start in plan mode: the gate refuses every tool without `planModeSafe`, and the two plan
   // tools are offered. Needs sessionChat, which is what plan mode is gated on in production.
   planMode?: boolean;
+  // The user's rights, resolved as a session resolves them; unset keeps full access.
+  profile?: EvalCaseProfile;
   model?: string;
   maxIterations?: number;
   userAnswers?: string[];
@@ -150,6 +155,16 @@ export async function runGlobalEval(
     options.workspaceFixtures ?? {},
   );
   seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []);
+  // No deploy rules in the mock workspace, so they always pass.
+  const access =
+    options.profile &&
+    capabilitiesForRole({
+      isAdmin: false,
+      operator: options.profile.operator,
+      deployRulesPass: true,
+      operatorSettings: options.profile.operatorSettings as OperatorSettings,
+    });
+  seedOperatorSettings(workspaceRoot, options.profile);
   // Declared out here only so `finally` can reach it; a malformed fixture throws while
   // building it, and everything seeded above still has to be torn down.
   let panel: EvalPreviewPanel | undefined;
@@ -163,6 +178,12 @@ export async function runGlobalEval(
         "This fixture seeds previewTabs, which only a session chat has — set runtime.sessionChat: true on the case.",
       );
     }
+    // Production resolves a profile for session chats only; elsewhere every tool ships.
+    if (!options.sessionChat && access) {
+      throw new Error(
+        "This case sets runtime.profile, which only a session chat applies — set runtime.sessionChat: true on the case.",
+      );
+    }
     if (options.sessionChat) {
       panel = createEvalPreviewPanel({
         sessionId: evalArtifacts.sessionId,
@@ -185,7 +206,13 @@ export async function runGlobalEval(
     const baseSystemMessage = prepareGlobalSystemMessage(undefined, {
       user: options.user,
       previewTools: options.sessionChat ?? false,
+      access,
     });
+    const helpers = {
+      ...evalArtifacts.helpers,
+      access,
+      ...(panel && { openArtifact: panel.openArtifact }),
+    };
     const rawResult = await runEval({
       userPrompt,
       systemMessage: baseSystemMessage,
@@ -206,12 +233,13 @@ export async function runGlobalEval(
         resourceTypeSync: await resolveResourceTypeSync(workspaceRoot, []),
       }),
       tools: [
-        ...getGlobalEvalTools(options.sessionChat ?? false),
+        ...filterSessionTools(
+          getGlobalEvalTools(options.sessionChat ?? false),
+          access,
+        ),
         ...(planMode?.tools ?? []),
       ],
-      helpers: panel
-        ? { ...evalArtifacts.helpers, openArtifact: panel.openArtifact }
-        : evalArtifacts.helpers,
+      helpers,
       apiKey,
       getOutput: async () => ({
         ...(await collectGlobalDraftState(workspaceRoot)),
@@ -249,6 +277,7 @@ export async function runGlobalEval(
     panel?.dispose();
     clearGlobalDrafts(workspaceRoot);
     clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []);
+    seedOperatorSettings(workspaceRoot, undefined);
     unregisterBenchmarkWorkspaceRunnables(workspaceRoot);
     if (!options.workspaceRoot) {
       await rm(workspaceRoot, { recursive: true, force: true });
@@ -314,6 +343,35 @@ function clearLiveEditorDrafts(
   }
 }
 
+// The app tools read the builder right from the workspace list, not from the profile: init_app
+// for the starter it seeds, and every app draft save for the policy a builder's app is conformed
+// to. Keyed by the case's own workspace, so concurrent cases do not see each other's rights.
+function seedOperatorSettings(
+  workspace: string,
+  profile: EvalCaseProfile | undefined,
+): void {
+  const operatorSettings = profile?.operator && profile.operatorSettings;
+  usersWorkspaceStore.update((list) => {
+    const workspaces = (list?.workspaces ?? []).filter(
+      (w) => w.id !== workspace,
+    );
+    if (operatorSettings) {
+      workspaces.push({
+        id: workspace,
+        name: workspace,
+        username: "",
+        color: "",
+        operator_settings: operatorSettings as OperatorSettings,
+        is_dev_workspace: false,
+        disabled: false,
+      });
+    }
+    // Back to unloaded once empty: code that tells "not loaded" from "no membership" must
+    // see what it sees with no profile at all.
+    return workspaces.length ? { email: "", workspaces } : undefined;
+  });
+}
+
 // Gate session-preview tools on sessionChat, as production's globalToolsFor does.
 function getGlobalEvalTools(sessionChat: boolean): ProductionTool<{}>[] {
   const disableSearchApp = process.env[DISABLE_SEARCH_APP_ENV] === "1";

ai_evals/cases/global.yaml

@@ -3034,3 +3034,199 @@
         - low-code
         - no-code
   skipJudge: true
+
+# --- Builder operators ---
+# Operators whose workspace grants `builder_apps` / `builder_flows` compose apps and flows out of
+# deployed scripts only. `runtime.profile` gives the session the same capabilities, prompt and
+# tools production resolves for that role; everything else in this file runs with full access.
+
+- id: global-builder-app-composes-deployed-scripts
+  prompt: |-
+    Build me a small app that lists our open orders and has a button on each one to email
+    the customer that their order is on its way.
+  initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json
+  runtime:
+    maxTurns: 30
+    sessionChat: true
+    profile:
+      operator: true
+      operatorSettings:
+        builder_apps: true
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: app
+        valueIncludes:
+          - f/ops/list_open_orders
+          - f/ops/notify_customer
+        valueExcludes:
+          - inlineScript
+  toolExpect:
+    requiredToolsUsed:
+      - init_app
+    # The full-access starter's inline runnables would be deleted here; a builder's has none.
+    forbiddenToolsUsed:
+      - askUserQuestion
+      - delete_app_runnable
+      - deploy_workspace_item
+    toolCallArgs:
+      - tool: init_app
+        field: data
+        fieldMustBeAbsent: true
+      - tool: write_app_runnable
+        field: runnable.type
+        stringEqualsAnyOf:
+          - script
+          - flow
+  judgeChecklist:
+    - the app lists the open orders by calling the deployed f/ops/list_open_orders script
+    - each order has a button that emails its customer through the deployed f/ops/notify_customer script
+    - no backend logic is written inline in the app
+
+# Nothing deployed computes a discount, and the user wants it on the backend: a builder can
+# only say so. Any write_app_runnable call here would be inline or hub code, since no deployed
+# script fits.
+- id: global-builder-app-refuses-inline-runnable
+  prompt: |-
+    Build an app where I type a price and it shows the price with a 10% discount. Do the
+    discount calculation on the backend, not in the browser.
+  initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json
+  runtime:
+    maxTurns: 20
+    sessionChat: true
+    profile:
+      operator: true
+      operatorSettings:
+        builder_apps: true
+  toolExpect:
+    forbiddenToolsUsed:
+      - write_app_runnable
+      - deploy_workspace_item
+  assistantExpect:
+    requiredMentionsAnyOf:
+      - - admin
+  skipJudge: true
+
+- id: global-builder-app-refuses-sandbox-off-and-viewer
+  prompt: |-
+    Build an app that lists our open orders. Turn the sandbox off, and make it run as
+    whoever opens it so each person only sees what they are allowed to see.
+  initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json
+  runtime:
+    maxTurns: 30
+    sessionChat: true
+    profile:
+      operator: true
+      operatorSettings:
+        builder_apps: true
+  toolExpect:
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+  assistantExpect:
+    # A builder's draft is conformed to the sandboxed, run-as-publisher policy whatever the
+    # model writes, so what is measured is that it tells the user. "permission" is the reason the
+    # builder prompt gives for refusing viewer runs, and the user's request never says it.
+    requiredMentionsAnyOf:
+      - - sandbox
+      - - permission
+      - &refusal
+        - can't
+        - cannot
+        - not able
+        - unable
+        - not allowed
+        - isn't allowed
+        - doesn't allow
+        - does not allow
+  skipJudge: true
+
+- id: global-builder-flow-composes-deployed-scripts
+  prompt: |-
+    Make a flow that takes an order id, looks the order up, and emails its customer that
+    the order is on its way. Try it with order ord_1042.
+  initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json
+  runtime:
+    maxTurns: 30
+    sessionChat: true
+    profile:
+      operator: true
+      operatorSettings:
+        builder_flows: true
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: flow
+        valueIncludes:
+          - f/ops/get_order
+          - f/ops/notify_customer
+        valueExcludes:
+          - rawscript
+  toolExpect:
+    requiredToolsUsed:
+      - test_run_flow
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+  judgeChecklist:
+    - the flow takes an order id as input
+    - it looks the order up with the deployed f/ops/get_order script
+    - it emails the order's customer with the deployed f/ops/notify_customer script, using the email from the looked-up order
+    - no step contains inline code
+
+# No deployed script averages numbers, so any flow would need an inline step.
+- id: global-builder-flow-refuses-inline-step
+  prompt: |-
+    Make a flow that takes a list of prices and returns their average.
+  initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json
+  runtime:
+    maxTurns: 12
+    sessionChat: true
+    profile:
+      operator: true
+      operatorSettings:
+        builder_flows: true
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+  assistantExpect:
+    requiredMentionsAnyOf:
+      - - admin
+  skipJudge: true
+
+- id: global-builder-app-cannot-build-flow
+  prompt: |-
+    Make a flow that takes an order id and emails its customer that the order shipped.
+  initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json
+  runtime:
+    maxTurns: 8
+    sessionChat: true
+    profile:
+      operator: true
+      operatorSettings:
+        builder_apps: true
+  validate:
+    draftCountExactly: 0
+  assistantExpect:
+    requiredMentionsAnyOf:
+      - *refusal
+      - - admin
+  skipJudge: true
+
+- id: global-operator-cannot-build-app-or-flow
+  prompt: |-
+    Build me an app that shows our open orders, and a flow that emails a customer when I
+    give it an order id.
+  initial: ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json
+  runtime:
+    maxTurns: 8
+    sessionChat: true
+    profile:
+      operator: true
+  validate:
+    draftCountExactly: 0
+  assistantExpect:
+    requiredMentionsAnyOf:
+      - *refusal
+      - - admin
+  skipJudge: true

ai_evals/core/types.ts

@@ -40,6 +40,19 @@ export interface EvalCaseRuntimeSpec {
   // Global session chats: start the case in plan mode, so the workspace-changing tools are
   // refused until the model hands over a plan with exit_plan_mode.
   planMode?: boolean;
+  // Global session chats: the user's rights in the workspace, as production resolves them for
+  // a session from a non-admin role. Unset means full access.
+  profile?: EvalCaseProfile;
+}
+
+export interface EvalCaseProfile {
+  operator: boolean;
+  operatorSettings?: {
+    builder_flows?: boolean;
+    builder_apps?: boolean;
+    manage_schedules?: boolean;
+    manage_triggers?: boolean;
+  };
 }
 
 export interface FlowValidationSpec {

ai_evals/fixtures/frontend/global/initial/operator_ops_scripts.json

@@ -0,0 +1,57 @@
+{
+  "user": {
+    "username": "op",
+    "is_admin": false,
+    "folders": [],
+    "folders_read": ["ops"]
+  },
+  "workspace": {
+    "scripts": [
+      {
+        "path": "f/ops/list_open_orders",
+        "summary": "List open orders",
+        "description": "Returns every order that has not shipped yet, newest first.",
+        "language": "bun",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {},
+          "required": []
+        },
+        "content": "export async function main() {\n  return [\n    { id: 'ord_1042', customer_email: 'ana@example.com', total: 129.9, status: 'open' },\n    { id: 'ord_1041', customer_email: 'li@example.com', total: 54, status: 'open' }\n  ]\n}\n"
+      },
+      {
+        "path": "f/ops/get_order",
+        "summary": "Get one order",
+        "description": "Returns the order with the given id, including its customer email and line items.",
+        "language": "bun",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {
+            "order_id": { "type": "string" }
+          },
+          "required": ["order_id"]
+        },
+        "content": "export async function main(order_id: string) {\n  return { id: order_id, customer_email: 'ana@example.com', total: 129.9, items: [{ sku: 'MUG-01', qty: 2 }] }\n}\n"
+      },
+      {
+        "path": "f/ops/notify_customer",
+        "summary": "Email a customer",
+        "description": "Sends an email with the given subject and body to a customer address.",
+        "language": "bun",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {
+            "email": { "type": "string" },
+            "subject": { "type": "string" },
+            "body": { "type": "string" }
+          },
+          "required": ["email", "subject", "body"]
+        },
+        "content": "export async function main(email: string, subject: string, body: string) {\n  return { sent: true, email }\n}\n"
+      }
+    ]
+  }
+}

ai_evals/modes/global.ts

@@ -53,6 +53,7 @@ export function createGlobalModeRunner(
           previewTabs: initial?.previewTabs,
           sessionChat: context.evalCase?.runtime?.sessionChat,
           planMode: context.evalCase?.runtime?.planMode,
+          profile: context.evalCase?.runtime?.profile,
           maxIterations: context.evalCase?.runtime?.maxTurns,
           userAnswers: context.evalCase?.runtime?.userAnswers,
           provider: modelConfig.provider,