diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index f2ff64248f..bf376b66e8 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1050,6 +1050,38 @@ - opens the Kafka triggers page - does not write, deploy, or delete anything +- id: global-openpage7-compare-review + prompt: |- + Create a TypeScript script draft at f/evals/global/compare_review_demo that returns the string "ok" (no need to test it), then open the review page so I can look over the pending change and deploy it myself. + runtime: + maxTurns: 8 + validate: + draftCountExactly: 1 + toolExpect: + requiredToolsUsed: + - write_script + - open_page + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: open_page + field: page + stringIncludesAnyOf: + - compare + # The eval chat is untracked (no modified-items mask), so the model must scope + # the review by passing the item it changed explicitly — an omitted mask would + # preselect every pending change in the workspace. + - tool: open_page + field: items + stringIncludesAnyOf: + - f/evals/global/compare_review_demo + skipJudge: true + judgeChecklist: + - creates the script draft, then opens the Compare & Deploy review page instead of deploying itself + - preselects only the created script on the review page + - does not deploy or delete anything + - id: global-closepage1-close-runs-tab prompt: |- You just opened the runs page for me in the side panel. Close that tab, I'm done looking at it. diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index 5183a20226..d603dae626 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -174,6 +174,40 @@ describe("validateToolExpectations", () => { expect(checks.every((check) => check.passed)).toBe(true); }); + it("accepts a stringIncludesAnyOf substring inside an array-valued field", () => { + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["open_page"], + toolCallDetails: [ + { + name: "open_page", + arguments: { + page: "compare", + items: ["script:f/evals/global/compare_review_demo"], + }, + }, + ], + skillsInvoked: [], + }, + toolExpect: { + requiredToolsUsed: ["open_page"], + toolCallArgs: [ + { + tool: "open_page", + field: "items", + stringIncludesAnyOf: ["f/evals/global/compare_review_demo"], + }, + ], + }, + }); + + expect(checks.every((check) => check.passed)).toBe(true); + }); + it("accepts stringIncludesAnyOf when only one of several calls matches", () => { // Existential: a mutation mixed with verification SELECTs still passes. const checks = validateToolExpectations({ diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 7570ccd12d..32ff225fc1 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -239,9 +239,15 @@ export function validateToolExpectations(input: { // model mixes the requested statement (e.g. an UPDATE) with verification // SELECTs that would otherwise fail an "all calls" check. const needles = rule.stringIncludesAnyOf.map((needle) => needle.toLowerCase()); - const hasMatch = values.some( - (value) => - typeof value === "string" && needles.some((needle) => value.toLowerCase().includes(needle)) + // Array-valued fields (e.g. open_page.items) match on any element. + const haystacks = (value: unknown): string[] => + typeof value === "string" + ? [value] + : Array.isArray(value) + ? value.filter((v): v is string => typeof v === "string") + : []; + const hasMatch = values.some((value) => + haystacks(value).some((hay) => needles.some((needle) => hay.toLowerCase().includes(needle))) ); checks.push( check( diff --git a/frontend/src/lib/components/CompareDrafts.svelte b/frontend/src/lib/components/CompareDrafts.svelte index b37ba4fcac..b7970c90e1 100644 --- a/frontend/src/lib/components/CompareDrafts.svelte +++ b/frontend/src/lib/components/CompareDrafts.svelte @@ -1,5 +1,6 @@