From 358d0cb1d3c56bce37e39756b41c65c1ff880169 Mon Sep 17 00:00:00 2001
From: Guilhem Lemouel <guilhemlemouel@gmail.com>
Date: Thu, 10 Sep 2026 20:31:26 +0200
Subject: [PATCH] test: pin the MCP eval case to the one schema-valid orderBy

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JgzuxyafKNF2uaEeL35XQw
---
 ai_evals/adapters/frontend/mockBackend.ts | 28 +++++++++++++++++++----
 ai_evals/cases/global.yaml                | 11 +++++----
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts
index 00ddab58b0..da98af05c3 100644
--- a/ai_evals/adapters/frontend/mockBackend.ts
+++ b/ai_evals/adapters/frontend/mockBackend.ts
@@ -1366,10 +1366,30 @@ export function listBenchmarkMcpServerTools(path: string) {
 }
 
 const BENCHMARK_MCP_ISSUES = [
-	{ identifier: 'ENG-412', title: 'Retry webhook deliveries', updatedAt: '2026-01-14T09:12:00Z' },
-	{ identifier: 'ENG-408', title: 'Flaky worker restart', updatedAt: '2026-01-13T17:40:00Z' },
-	{ identifier: 'ENG-401', title: 'Paginate the runs table', updatedAt: '2026-01-12T08:05:00Z' },
-	{ identifier: 'ENG-399', title: 'Audit log filters', updatedAt: '2026-01-11T15:22:00Z' }
+	{
+		identifier: 'ENG-412',
+		title: 'Retry webhook deliveries',
+		createdAt: '2026-01-09T11:03:00Z',
+		updatedAt: '2026-01-14T09:12:00Z'
+	},
+	{
+		identifier: 'ENG-408',
+		title: 'Flaky worker restart',
+		createdAt: '2026-01-08T14:27:00Z',
+		updatedAt: '2026-01-13T17:40:00Z'
+	},
+	{
+		identifier: 'ENG-401',
+		title: 'Paginate the runs table',
+		createdAt: '2026-01-06T09:41:00Z',
+		updatedAt: '2026-01-12T08:05:00Z'
+	},
+	{
+		identifier: 'ENG-399',
+		title: 'Audit log filters',
+		createdAt: '2026-01-05T16:18:00Z',
+		updatedAt: '2026-01-11T15:22:00Z'
+	}
 ]
 
 /** The result of calling one of the served tools. `list_issues` answers with issues so a
diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml
index f5875ed0a2..2510cb3bd4 100644
--- a/ai_evals/cases/global.yaml
+++ b/ai_evals/cases/global.yaml
@@ -2406,7 +2406,7 @@
 
 - id: global-test41-mcp-tool-argument-from-schema
   prompt: |-
-    Show me my 3 most recently updated Linear issues.
+    Show me the 3 Linear issues I created most recently.
   initial: ai_evals/fixtures/frontend/global/initial/linear_mcp_server.json
   runtime:
     maxTurns: 8
@@ -2424,13 +2424,14 @@
       - write_script
       - deploy_workspace_item
     toolCallArgs:
-      # `orderBy` is an enum the server declares and the model cannot infer: the
-      # reported bug was inventing a plausible value ("priority") for it. Universal
-      # over calls, so a wrong first guess fails even if a retry recovers.
+      # `orderBy` is an enum the server declares and the model cannot infer: the reported
+      # bug was inventing a plausible value ("priority") for it. The prompt asks for
+      # creation order, which the schema's `updatedAt` default does not give, so the one
+      # correct value is `createdAt` — omitting the field is a failure too. Universal over
+      # calls, so a wrong first guess fails even if a retry recovers.
       - tool: mcp_f_evals_global_linear_mcp__list_issues
         field: orderBy
         stringEqualsAnyOf:
-          - updatedAt
           - createdAt
   # A read-only question produces no draft, and the judge sees only drafts — the
   # deliverable is the argument shape, checked deterministically above.