serve getJob in the ai evals benchmark api catalog (#10511)

* fix: serve getJob in the ai evals benchmark api catalog

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* chore: scope the frontend format hook to the frontend dir

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: answer the run-by-path endpoints and mirror the real getJob entry

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: gate the format hook on a repo-root frontend, not the project dir

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

.claude/hooks/format-frontend.sh

@@ -10,8 +10,12 @@ if [ -z "$FILE_PATH" ]; then
     exit 0
 fi
 
-# Check if the file is in the frontend directory
-if [[ "$FILE_PATH" == *"/frontend/"* ]]; then
+# Only the frontend app itself, i.e. a "frontend" directory sitting at a repo root.
+# A bare */frontend/* substring also matches ai_evals/adapters/frontend and the
+# ai_evals app fixtures, which no prettier config governs — prettier then falls back
+# to its defaults and rewrites the whole file. Anchoring to $CLAUDE_PROJECT_DIR
+# instead would skip worktrees edited from a session rooted elsewhere.
+if [[ "$FILE_PATH" == *"/frontend/"* ]] && [[ -e "${FILE_PATH%%/frontend/*}/.git" ]]; then
     # Check if it's a formattable file type
     if [[ "$FILE_PATH" =~ \.(ts|js|svelte|json|css|html|md)$ ]]; then
         cd "$CLAUDE_PROJECT_DIR/frontend" || exit 0

.github/workflows/ai-evals-test.yml

@@ -118,6 +118,12 @@ jobs:
           npm ci
           npm run generate-backend-client
 
+      - name: Run harness unit tests
+        working-directory: ./ai_evals
+        run: |
+          bun install
+          bun test adapters/
+
       - name: Run global AI evals
         timeout-minutes: 20
         working-directory: ./ai_evals

ai_evals/adapters/frontend/mockBackend.ts

@@ -561,6 +561,35 @@ export function runBenchmarkScriptPreview(input: {
 	})
 }
 
+export function runBenchmarkScriptByPath(input: {
+	workspace: string
+	path: string
+	args?: Record<string, unknown>
+}): string {
+	const script = getBenchmarkScriptByPath(input.workspace, input.path)
+	return createBenchmarkCompletedJob({
+		workspace: input.workspace,
+		jobKind: 'script',
+		success: script !== null,
+		scriptPath: input.path,
+		args: input.args,
+		result:
+			script !== null
+				? {
+						path: input.path,
+						args: input.args ?? {},
+						mocked: true
+					}
+				: {
+						error: `Script "${input.path}" not found in benchmark workspace`
+					},
+		logs:
+			script !== null
+				? 'Mock benchmark script run completed successfully.'
+				: `Script "${input.path}" not found in benchmark workspace.`
+	})
+}
+
 export function runBenchmarkFlowByPath(input: {
 	workspace: string
 	path: string
@@ -747,6 +776,27 @@ const BENCHMARK_MCP_TOOLS: EndpointTool[] = [
 			required: ['workspace']
 		}
 	},
+	{
+		name: 'getJob',
+		description: 'get job',
+		instructions: '',
+		path: '/w/{workspace}/jobs_u/get/{id}',
+		method: 'GET',
+		path_params_schema: {
+			type: 'object',
+			properties: { workspace: { type: 'string' }, id: { type: 'string', format: 'uuid' } },
+			required: ['workspace', 'id']
+		},
+		query_params_schema: {
+			type: 'object',
+			properties: {
+				no_logs: { type: 'boolean' },
+				no_code: { type: 'boolean' },
+				approval_token: { type: 'string' }
+			},
+			required: []
+		}
+	},
 	{
 		name: 'runScriptByPath',
 		description: 'Run the deployed version of a script by path',
@@ -968,6 +1018,26 @@ const BENCHMARK_WORKERS = [
 	}
 ]
 
+const BENCHMARK_JOB_GET_PATH = /^\/api\/w\/([^/]+)\/jobs_u\/get\/([^/]+)$/
+const BENCHMARK_RUN_BY_PATH = /^\/api\/w\/([^/]+)\/jobs\/run\/(p|f)\/([^/]+)$/
+
+/** `executeEndpoint` sends a JSON string; anything else means no args were supplied. */
+function parseBenchmarkRequestBody(
+	body: BodyInit | null | undefined
+): Record<string, unknown> | undefined {
+	if (typeof body !== 'string') {
+		return undefined
+	}
+	try {
+		const parsed = JSON.parse(body)
+		return typeof parsed === 'object' && parsed !== null
+			? (parsed as Record<string, unknown>)
+			: undefined
+	} catch {
+		return undefined
+	}
+}
+
 /** True when `handleBenchmarkApiFetch` has an answer for this `/api/...` url.
  * Any other relative fetch must keep its normal (non-benchmark) behavior —
  * intercepting it with a synthetic 404 sends the model into retry loops. */
@@ -975,6 +1045,8 @@ export function hasBenchmarkApiHandler(url: string): boolean {
 	const path = url.split('?')[0]
 	return (
 		path === '/api/workers/list' ||
+		BENCHMARK_JOB_GET_PATH.test(path) ||
+		BENCHMARK_RUN_BY_PATH.test(path) ||
 		/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) ||
 		path === '/api/embeddings/query_hub_scripts' ||
 		path.startsWith('/api/scripts/hub/get_full/')
@@ -983,7 +1055,7 @@ export function hasBenchmarkApiHandler(url: string): boolean {
 
 /** Answer a relative `/api/...` fetch — from the API catalog executor, or from the
  * chat's hub tools. */
-export function handleBenchmarkApiFetch(url: string): Response {
+export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Response {
 	const path = url.split('?')[0]
 	if (path === '/api/workers/list') {
 		return Response.json(BENCHMARK_WORKERS)
@@ -991,6 +1063,36 @@ export function handleBenchmarkApiFetch(url: string): Response {
 	if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) {
 		return Response.json([])
 	}
+	const jobGet = BENCHMARK_JOB_GET_PATH.exec(path)
+	if (jobGet) {
+		const id = decodeURIComponent(jobGet[2])
+		const job = getBenchmarkCompletedJob(decodeURIComponent(jobGet[1]), id)
+		if (!job) {
+			return Response.json({ error: `Job not found for "${id}"` }, { status: 404 })
+		}
+		// The real endpoint lets a caller drop the bulky fields. Ignoring that here would
+		// size the model's context off a payload it explicitly asked to shrink.
+		const query = new URLSearchParams(url.split('?')[1] ?? '')
+		if (query.get('no_logs') === 'true') {
+			delete job.logs
+		}
+		if (query.get('no_code') === 'true') {
+			delete job.raw_code
+		}
+		return Response.json(job)
+	}
+	const runByPath = BENCHMARK_RUN_BY_PATH.exec(path)
+	if (runByPath) {
+		const workspace = decodeURIComponent(runByPath[1])
+		const runnablePath = decodeURIComponent(runByPath[3])
+		const args = parseBenchmarkRequestBody(init?.body)
+		// The real endpoint answers with the bare job id as text, not JSON.
+		return new Response(
+			runByPath[2] === 'f'
+				? runBenchmarkFlowByPath({ workspace, path: runnablePath, args })
+				: runBenchmarkScriptByPath({ workspace, path: runnablePath, args })
+		)
+	}
 	if (path === '/api/embeddings/query_hub_scripts') {
 		const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? ''
 		return Response.json(searchBenchmarkHubScripts(text))

ai_evals/adapters/frontend/mockBackendApi.test.ts

@@ -0,0 +1,83 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
+import {
+	createBenchmarkCompletedJob,
+	getBenchmarkCompletedJob,
+	handleBenchmarkApiFetch,
+	hasBenchmarkApiHandler,
+	listBenchmarkMcpTools,
+	resetBenchmarkMockBackend,
+	registerBenchmarkWorkspaceRunnables
+} from './mockBackend'
+
+const WORKSPACE = 'benchmark-api-ws'
+
+// A catalog entry with no fetch handler is a dead end: the catalog executor builds a
+// relative `/api/...` url, the stub declines it, and node's fetch throws on the relative
+// url instead of returning a result the model can act on. Mutating entries are reachable
+// too — the eval runners define no `requestConfirmation`, so `call_api_endpoint` executes
+// unconfirmed.
+describe('benchmark API catalog', () => {
+	beforeEach(() => resetBenchmarkMockBackend())
+	afterEach(() => resetBenchmarkMockBackend())
+
+	it('answers every endpoint it advertises', () => {
+		const unanswered = listBenchmarkMcpTools()
+			.map((tool) =>
+				`/api${tool.path.replace('{workspace}', WORKSPACE)}`.replace(/\{[^}]+\}/g, 'x')
+			)
+			.filter((url) => !hasBenchmarkApiHandler(url))
+
+		// The draft-covered entries are refused by name before any fetch, so they are
+		// advertised without a handler on purpose.
+		expect(unanswered).toEqual([
+			`/api/w/${WORKSPACE}/scripts/get/p/x`,
+			`/api/w/${WORKSPACE}/flows/create`,
+			`/api/w/${WORKSPACE}/schedules/delete/x`,
+			`/api/w/${WORKSPACE}/variables/get/x`
+		])
+	})
+
+	it('runs a deployed script by path, the way call_api_endpoint reaches it', async () => {
+		registerBenchmarkWorkspaceRunnables(WORKSPACE, {
+			scripts: [
+				{
+					path: 'f/evals/greet',
+					summary: 'Greet',
+					language: 'bun',
+					content: 'export async function main() {}'
+				}
+			]
+		})
+
+		const res = handleBenchmarkApiFetch(
+			`/api/w/${WORKSPACE}/jobs/run/p/${encodeURIComponent('f/evals/greet')}`,
+			{ method: 'POST', body: JSON.stringify({ name: 'ada' }) }
+		)
+
+		expect(res.status).toBe(200)
+		const job = getBenchmarkCompletedJob(WORKSPACE, (await res.text()).trim())
+		expect(job).toMatchObject({ success: true, args: { name: 'ada' } })
+	})
+
+	it('serves a recorded job so a model can check the run it just started', async () => {
+		const id = createBenchmarkCompletedJob({
+			workspace: WORKSPACE,
+			jobKind: 'preview',
+			result: 'Hello, World!'
+		})
+
+		const res = handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/${id}`)
+
+		expect(res.status).toBe(200)
+		expect(await res.json()).toMatchObject({
+			id,
+			success: true,
+			result: 'Hello, World!'
+		})
+	})
+
+	it('404s an unknown job id instead of letting the fetch fall through', () => {
+		expect(hasBenchmarkApiHandler(`/api/w/${WORKSPACE}/jobs_u/get/missing`)).toBe(true)
+		expect(handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/missing`).status).toBe(404)
+	})
+})

ai_evals/adapters/frontend/vitestAdapter.test.ts

@@ -13,7 +13,7 @@ const ORIGINAL_FETCH = globalThis.fetch
 globalThis.fetch = (async (input: unknown, init?: RequestInit) => {
 	const url = typeof input === 'string' ? input : ((input as Request | URL | null)?.url ?? '')
 	if (typeof url === 'string' && hasBenchmarkApiHandler(url)) {
-		return handleBenchmarkApiFetch(url)
+		return handleBenchmarkApiFetch(url, init)
 	}
 	return ORIGINAL_FETCH(input as Parameters<typeof fetch>[0], init)
 }) as typeof fetch