starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-21 00:02:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: pin the MCP eval case to the one schema-valid orderBy

Browse Source 
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JgzuxyafKNF2uaEeL35XQw
This commit is contained in:
Guilhem Lemouel
2026-09-10 20:31:26 +02:00
co-authored by Claude Opus 5
parent 28e793db89
commit 358d0cb1d3
2 changed files with 30 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai_evals/adapters/frontend/mockBackend.ts
+24 -4
View File
@@ -1366,10 +1366,30 @@ export function listBenchmarkMcpServerTools(path: string) {
}
const BENCHMARK_MCP_ISSUES = [
{ identifier: 'ENG-412', title: 'Retry webhook deliveries', updatedAt: '2026-01-14T09:12:00Z' },
{ identifier: 'ENG-408', title: 'Flaky worker restart', updatedAt: '2026-01-13T17:40:00Z' },
{ identifier: 'ENG-401', title: 'Paginate the runs table', updatedAt: '2026-01-12T08:05:00Z' },
{ identifier: 'ENG-399', title: 'Audit log filters', updatedAt: '2026-01-11T15:22:00Z' }
{
identifier: 'ENG-412',
title: 'Retry webhook deliveries',
createdAt: '2026-01-09T11:03:00Z',
updatedAt: '2026-01-14T09:12:00Z'
},
{
identifier: 'ENG-408',
title: 'Flaky worker restart',
createdAt: '2026-01-08T14:27:00Z',
updatedAt: '2026-01-13T17:40:00Z'
},
{
identifier: 'ENG-401',
title: 'Paginate the runs table',
createdAt: '2026-01-06T09:41:00Z',
updatedAt: '2026-01-12T08:05:00Z'
},
{
identifier: 'ENG-399',
title: 'Audit log filters',
createdAt: '2026-01-05T16:18:00Z',
updatedAt: '2026-01-11T15:22:00Z'
}
]
/** The result of calling one of the served tools. `list_issues` answers with issues so a
ai_evals/cases/global.yaml
+6 -5
View File
@@ -2406,7 +2406,7 @@
- id: global-test41-mcp-tool-argument-from-schema
prompt: |-
Show me my 3 most recently updated Linear issues.
Show me the 3 Linear issues I created most recently.
initial: ai_evals/fixtures/frontend/global/initial/linear_mcp_server.json
runtime:
maxTurns: 8
@@ -2424,13 +2424,14 @@
- write_script
- deploy_workspace_item
toolCallArgs:
# `orderBy` is an enum the server declares and the model cannot infer: the
# reported bug was inventing a plausible value ("priority") for it. Universal
# over calls, so a wrong first guess fails even if a retry recovers.
# `orderBy` is an enum the server declares and the model cannot infer: the reported
# bug was inventing a plausible value ("priority") for it. The prompt asks for
# creation order, which the schema's `updatedAt` default does not give, so the one
# correct value is `createdAt` — omitting the field is a failure too. Universal over
# calls, so a wrong first guess fails even if a retry recovers.
- tool: mcp_f_evals_global_linear_mcp__list_issues
field: orderBy
stringEqualsAnyOf:
- updatedAt
- createdAt
# A read-only question produces no draft, and the judge sees only drafts — the
# deliverable is the argument shape, checked deterministically above.