From be7023500b9af19fb97c775cd0e266a2293bcbaa Mon Sep 17 00:00:00 2001 From: Guilhem Lemouel Date: Thu, 10 Sep 2026 20:17:22 +0200 Subject: [PATCH] test: cover MCP tool argument correctness in the global evals Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JgzuxyafKNF2uaEeL35XQw --- .../frontend/core/global/globalEvalRunner.ts | 20 +++++ .../frontend/core/shared/baseEvalRunner.ts | 18 +++- ai_evals/adapters/frontend/mockBackend.ts | 84 +++++++++++++++++++ .../adapters/frontend/vitestAdapter.test.ts | 31 ++++++- ai_evals/cases/global.yaml | 40 +++++++++ .../global/initial/linear_mcp_server.json | 15 ++++ 6 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 ai_evals/fixtures/frontend/global/initial/linear_mcp_server.json diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index be8c1d74cc..9e30acc4d9 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -13,6 +13,12 @@ import { getGlobalDraft, listGlobalDrafts, } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; +import { + createMcpTools, + forgetLoadedMcpTools, + loadedMcpTools, + loadMcpServers, +} from "../../../../../frontend/src/lib/components/copilot/chat/global/mcpTools"; import { appendPlanModeInstructions } from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; import { createEvalPlanTools } from "./planModeTools"; @@ -142,6 +148,10 @@ export async function runGlobalEval( options.workspaceRoot ?? (await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-"))); + // The MCP tool registry is keyed by owner so concurrent cases never see each + // other's registrations, as concurrent chat sessions don't in production. + const mcpOwnerId = `eval:${workspaceRoot}`; + clearGlobalDrafts(workspaceRoot); registerBenchmarkWorkspaceRunnables( workspaceRoot, @@ -178,11 +188,15 @@ export async function runGlobalEval( chatId: evalArtifacts.helpers.getChatId(), }) : undefined; + // The `mcp` resources this case seeded; a case that seeds none gets no MCP tools + // and no MCP section in the prompt, exactly as a workspace with nothing connected. + const mcpServers = await loadMcpServers(workspaceRoot); // Pass the seeded identity straight to the prompt builder rather than mutating // the process-global `userStore`, so concurrent cases never race on it. const baseSystemMessage = prepareGlobalSystemMessage(undefined, { user: options.user, previewTools: options.sessionChat ?? false, + mcpServers, }); const rawResult = await runEval({ userPrompt, @@ -205,7 +219,12 @@ export async function runGlobalEval( tools: [ ...getGlobalEvalTools(options.sessionChat ?? false), ...(planMode?.tools ?? []), + ...createMcpTools(mcpOwnerId, mcpServers), ], + // A tool search registers the tools it matched under this owner; they are offered + // from the next request on, which is the behaviour the case measures. + getDynamicTools: () => + loadedMcpTools(mcpOwnerId) as ProductionTool[], helpers: panel ? { ...evalArtifacts.helpers, openArtifact: panel.openArtifact } : evalArtifacts.helpers, @@ -243,6 +262,7 @@ export async function runGlobalEval( }; } finally { panel?.dispose(); + forgetLoadedMcpTools(mcpOwnerId); clearGlobalDrafts(workspaceRoot); clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); unregisterBenchmarkWorkspaceRunnables(workspaceRoot); diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 4501f24810..0c856db35a 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -48,6 +48,10 @@ export interface RunEvalParams { isPlanModeActive?: () => boolean; /** Which of `tools` the model is offered on this request. Absent offers all of them. */ isToolAvailable?: (name: string) => boolean; + /** Tools a previous call registered, re-read before every request as production's + * tools getter is: an MCP search registers the tools it matched, and the model has to + * be offered them on the very next request or the names it was just handed are dead. */ + getDynamicTools?: () => ProductionTool[]; /** Re-read before every request, as production's systemMessage getter is. Needed when a * tool changes what the prompt should say — plan mode's instructions have to come back * out once the plan is approved. Falls back to the fixed `systemMessage`. */ @@ -80,6 +84,7 @@ export async function runEval( isPlanModeActive, isToolAvailable, getSystemMessage, + getDynamicTools, } = params; let shouldEmitMessageStart = true; @@ -98,7 +103,7 @@ export async function runEval( // Wrap tools to intercept fn calls for tracking. // Cast to ProductionTool since the eval Tool has a narrower toolCallbacks type // but the actual callbacks passed at runtime will satisfy both interfaces. - const wrappedTools = tools.map((tool) => ({ + const trackCalls = (tool: ProductionTool) => ({ ...tool, fn: async (p: any) => { toolCallsCount++; @@ -122,7 +127,9 @@ export async function runEval( }); return tool.fn(p); }, - })); + }); + + const wrappedTools = tools.map(trackCalls); // No-op callbacks for eval const callbacks: ToolCallbacks & { @@ -160,9 +167,12 @@ export async function runEval( // must leave the schema too, or the model keeps being offered a call the run has // moved past — and the token counts a case reports include a tool it cannot use. get tools() { - return isToolAvailable - ? wrappedTools.filter((t) => isToolAvailable(t.def.function.name)) + const all = getDynamicTools + ? [...wrappedTools, ...getDynamicTools().map(trackCalls)] : wrappedTools; + return isToolAvailable + ? all.filter((t) => isToolAvailable(t.def.function.name)) + : all; }, helpers, abortController, diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 6d80f19197..00ddab58b0 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -1318,3 +1318,87 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon } return Response.json({ error: `no benchmark handler for ${path}` }, { status: 404 }) } + +// ============= Connected MCP servers (ResourceService.getMcpTools / callMcpTool) ============= +// The chat reaches a connected MCP server through the backend, which opens a real +// connection to a third party — no meaning here. One server is served instead, shaped +// like a real listing: `list_issues` carries the constrained `orderBy` the model used to +// guess at, which is what the eval measures. + +export const BENCHMARK_MCP_SERVER_PATH = 'f/evals/global/linear_mcp' + +const BENCHMARK_MCP_SERVER_TOOLS = [ + { + name: 'list_issues', + description: 'List issues in the user\'s Linear workspace. For my issues, use "me" as the assignee.', + inputSchema: { + type: 'object', + properties: { + assignee: { type: 'string', description: 'User ID, name, email, or "me"' }, + orderBy: { + type: 'string', + enum: ['createdAt', 'updatedAt'], + default: 'updatedAt', + description: 'Sort: createdAt | updatedAt' + }, + limit: { type: 'number', default: 50, maximum: 250, description: 'Max results' } + } + }, + annotations: { readOnlyHint: true } + }, + { + name: 'create_issue', + description: 'Create a new issue in Linear.', + inputSchema: { + type: 'object', + properties: { title: { type: 'string' }, team: { type: 'string' } }, + required: ['title'] + }, + annotations: { readOnlyHint: false } + } +] + +export function listBenchmarkMcpServerTools(path: string) { + if (path !== BENCHMARK_MCP_SERVER_PATH) { + throw new Error(`No benchmark MCP server at "${path}"`) + } + return BENCHMARK_MCP_SERVER_TOOLS +} + +const BENCHMARK_MCP_ISSUES = [ + { identifier: 'ENG-412', title: 'Retry webhook deliveries', updatedAt: '2026-01-14T09:12:00Z' }, + { identifier: 'ENG-408', title: 'Flaky worker restart', updatedAt: '2026-01-13T17:40:00Z' }, + { identifier: 'ENG-401', title: 'Paginate the runs table', updatedAt: '2026-01-12T08:05:00Z' }, + { identifier: 'ENG-399', title: 'Audit log filters', updatedAt: '2026-01-11T15:22:00Z' } +] + +/** The result of calling one of the served tools. `list_issues` answers with issues so a + * model that asked correctly can report them instead of retrying through the wrapper; + * arguments are echoed so a case can assert on what it actually sent. */ +export function callBenchmarkMcpServerTool(path: string, tool: string, args: unknown) { + const known = listBenchmarkMcpServerTools(path).find((t) => t.name === tool) + if (!known) { + throw new Error(`Unknown tool "${tool}" on ${path}`) + } + const payload: Record = { ok: true, tool, arguments: args ?? {} } + if (tool === 'list_issues') { + // The served schema constrains `orderBy`, and the real server refuses a value + // outside it — which is the failure the registered-schema path exists to avoid, + // so the fixture has to refuse it too rather than accept anything. + const orderBy = (args as { orderBy?: unknown } | undefined)?.orderBy + if (orderBy !== undefined && !['createdAt', 'updatedAt'].includes(orderBy as string)) { + return { + isError: true, + content: [ + { + type: 'text', + text: `Invalid value "${String(orderBy)}" for orderBy. Expected createdAt or updatedAt.` + } + ] + } + } + const limit = (args as { limit?: unknown } | undefined)?.limit + payload.issues = BENCHMARK_MCP_ISSUES.slice(0, typeof limit === 'number' ? limit : undefined) + } + return { content: [{ type: 'text', text: JSON.stringify(payload) }] } +} diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 338ed8504c..086d42873f 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -3,7 +3,13 @@ import { expect, it, vi } from 'vitest' import { mkdir, writeFile } from 'fs/promises' // @ts-ignore - Node.js path import { dirname, resolve } from 'path' -import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' +import { + BENCHMARK_MCP_SERVER_PATH, + callBenchmarkMcpServerTool, + handleBenchmarkApiFetch, + hasBenchmarkApiHandler, + listBenchmarkMcpServerTools +} from './mockBackend' // The API catalog executor issues relative fetch('/api/...') calls, which have // no meaning in the vitest environment — serve the ones the benchmark handles. @@ -27,6 +33,15 @@ globalThis.fetch = (async (input: unknown, init?: RequestInit) => { return ORIGINAL_FETCH(input as Parameters[0], init) }) as typeof fetch +// Which MCP servers a chat may act through is a per-browser preference, and node has no +// localStorage — so nothing is ever enabled and the MCP tools never register. The served +// server stands in as the enabled set. +vi.mock('$lib/components/mcp/enabledServers', () => ({ + enabledMcpPaths: () => [BENCHMARK_MCP_SERVER_PATH], + isMcpEnabled: (_ws: string, path: string) => path === BENCHMARK_MCP_SERVER_PATH, + setMcpEnabled: () => true +})) + vi.mock('monaco-editor', () => ({ editor: {}, languages: {}, @@ -393,7 +408,19 @@ vi.mock('$lib/gen', async () => { return value }, queryResourceTypes: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data) + hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data), + getMcpTools: async (data: { workspace: string; path: string }) => + hasBenchmarkWorkspace(data.workspace) + ? listBenchmarkMcpServerTools(data.path) + : actual.ResourceService.getMcpTools(data), + callMcpTool: async (data: { + workspace: string + path: string + requestBody: { tool: string; arguments?: unknown } + }) => + hasBenchmarkWorkspace(data.workspace) + ? callBenchmarkMcpServerTool(data.path, data.requestBody.tool, data.requestBody.arguments) + : actual.ResourceService.callMcpTool(data) }), McpService: wrapService(actual.McpService, { listMcpTools: async (data: { workspace: string }) => diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 94ecb5c029..f5875ed0a2 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -2399,3 +2399,43 @@ judgeChecklist: - runs the existing script rather than rewriting it - passes the GitHub resource as the bare string $res:f/evals/global/github_main + +# --- Connected MCP servers (search_mcp_tools + the tools it registers) --- +# The harness serves one MCP server itself (mock getMcpTools/callMcpTool in +# adapters/frontend), so this does not need a reachable third party. + +- id: global-test41-mcp-tool-argument-from-schema + prompt: |- + Show me my 3 most recently updated Linear issues. + initial: ai_evals/fixtures/frontend/global/initial/linear_mcp_server.json + runtime: + maxTurns: 8 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - search_mcp_tools + - mcp_f_evals_global_linear_mcp__list_issues + forbiddenToolsUsed: + # The free-form wrapper is the fallback for a tool that was not registered. + # Reaching for it here means the model is guessing arguments it could have read. + - call_mcp_read_tool + - call_mcp_write_tool + - write_script + - deploy_workspace_item + toolCallArgs: + # `orderBy` is an enum the server declares and the model cannot infer: the + # reported bug was inventing a plausible value ("priority") for it. Universal + # over calls, so a wrong first guess fails even if a retry recovers. + - tool: mcp_f_evals_global_linear_mcp__list_issues + field: orderBy + stringEqualsAnyOf: + - updatedAt + - createdAt + # A read-only question produces no draft, and the judge sees only drafts — the + # deliverable is the argument shape, checked deterministically above. + skipJudge: true + judgeChecklist: + - finds the Linear issue tool through search_mcp_tools + - calls it with arguments the server's schema accepts, first time + - reports the issues instead of writing a script to fetch them diff --git a/ai_evals/fixtures/frontend/global/initial/linear_mcp_server.json b/ai_evals/fixtures/frontend/global/initial/linear_mcp_server.json new file mode 100644 index 0000000000..e2fab59e31 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/linear_mcp_server.json @@ -0,0 +1,15 @@ +{ + "workspace": { + "resources": [ + { + "path": "f/evals/global/linear_mcp", + "resource_type": "mcp", + "description": "Linear MCP server", + "value": { + "name": "linear", + "url": "https://mcp.linear.app/mcp" + } + } + ] + } +}