From 505705fd3d894ac74ba7f787729dfd66a2aa7a41 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 4 Aug 2026 14:51:48 +0000 Subject: [PATCH] serve getJob in the ai evals benchmark api catalog (#10511) * fix: serve getJob in the ai evals benchmark api catalog Co-Authored-By: Claude Opus 5 (1M context) * chore: scope the frontend format hook to the frontend dir Co-Authored-By: Claude Opus 5 (1M context) * fix: answer the run-by-path endpoints and mirror the real getJob entry Co-Authored-By: Claude Opus 5 (1M context) * fix: gate the format hook on a repo-root frontend, not the project dir Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .claude/hooks/format-frontend.sh | 8 +- .github/workflows/ai-evals-test.yml | 6 + ai_evals/adapters/frontend/mockBackend.ts | 104 +++++++++++++++++- .../adapters/frontend/mockBackendApi.test.ts | 83 ++++++++++++++ .../adapters/frontend/vitestAdapter.test.ts | 2 +- 5 files changed, 199 insertions(+), 4 deletions(-) create mode 100644 ai_evals/adapters/frontend/mockBackendApi.test.ts diff --git a/.claude/hooks/format-frontend.sh b/.claude/hooks/format-frontend.sh index 37c3f8d4ec..14b2c2d97c 100755 --- a/.claude/hooks/format-frontend.sh +++ b/.claude/hooks/format-frontend.sh @@ -10,8 +10,12 @@ if [ -z "$FILE_PATH" ]; then exit 0 fi -# Check if the file is in the frontend directory -if [[ "$FILE_PATH" == *"/frontend/"* ]]; then +# Only the frontend app itself, i.e. a "frontend" directory sitting at a repo root. +# A bare */frontend/* substring also matches ai_evals/adapters/frontend and the +# ai_evals app fixtures, which no prettier config governs — prettier then falls back +# to its defaults and rewrites the whole file. Anchoring to $CLAUDE_PROJECT_DIR +# instead would skip worktrees edited from a session rooted elsewhere. +if [[ "$FILE_PATH" == *"/frontend/"* ]] && [[ -e "${FILE_PATH%%/frontend/*}/.git" ]]; then # Check if it's a formattable file type if [[ "$FILE_PATH" =~ \.(ts|js|svelte|json|css|html|md)$ ]]; then cd "$CLAUDE_PROJECT_DIR/frontend" || exit 0 diff --git a/.github/workflows/ai-evals-test.yml b/.github/workflows/ai-evals-test.yml index 04179346ef..3ee7aed876 100644 --- a/.github/workflows/ai-evals-test.yml +++ b/.github/workflows/ai-evals-test.yml @@ -118,6 +118,12 @@ jobs: npm ci npm run generate-backend-client + - name: Run harness unit tests + working-directory: ./ai_evals + run: | + bun install + bun test adapters/ + - name: Run global AI evals timeout-minutes: 20 working-directory: ./ai_evals diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 40a6dfa4ad..b0b5bbabe2 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -561,6 +561,35 @@ export function runBenchmarkScriptPreview(input: { }) } +export function runBenchmarkScriptByPath(input: { + workspace: string + path: string + args?: Record +}): string { + const script = getBenchmarkScriptByPath(input.workspace, input.path) + return createBenchmarkCompletedJob({ + workspace: input.workspace, + jobKind: 'script', + success: script !== null, + scriptPath: input.path, + args: input.args, + result: + script !== null + ? { + path: input.path, + args: input.args ?? {}, + mocked: true + } + : { + error: `Script "${input.path}" not found in benchmark workspace` + }, + logs: + script !== null + ? 'Mock benchmark script run completed successfully.' + : `Script "${input.path}" not found in benchmark workspace.` + }) +} + export function runBenchmarkFlowByPath(input: { workspace: string path: string @@ -747,6 +776,27 @@ const BENCHMARK_MCP_TOOLS: EndpointTool[] = [ required: ['workspace'] } }, + { + name: 'getJob', + description: 'get job', + instructions: '', + path: '/w/{workspace}/jobs_u/get/{id}', + method: 'GET', + path_params_schema: { + type: 'object', + properties: { workspace: { type: 'string' }, id: { type: 'string', format: 'uuid' } }, + required: ['workspace', 'id'] + }, + query_params_schema: { + type: 'object', + properties: { + no_logs: { type: 'boolean' }, + no_code: { type: 'boolean' }, + approval_token: { type: 'string' } + }, + required: [] + } + }, { name: 'runScriptByPath', description: 'Run the deployed version of a script by path', @@ -968,6 +1018,26 @@ const BENCHMARK_WORKERS = [ } ] +const BENCHMARK_JOB_GET_PATH = /^\/api\/w\/([^/]+)\/jobs_u\/get\/([^/]+)$/ +const BENCHMARK_RUN_BY_PATH = /^\/api\/w\/([^/]+)\/jobs\/run\/(p|f)\/([^/]+)$/ + +/** `executeEndpoint` sends a JSON string; anything else means no args were supplied. */ +function parseBenchmarkRequestBody( + body: BodyInit | null | undefined +): Record | undefined { + if (typeof body !== 'string') { + return undefined + } + try { + const parsed = JSON.parse(body) + return typeof parsed === 'object' && parsed !== null + ? (parsed as Record) + : undefined + } catch { + return undefined + } +} + /** True when `handleBenchmarkApiFetch` has an answer for this `/api/...` url. * Any other relative fetch must keep its normal (non-benchmark) behavior — * intercepting it with a synthetic 404 sends the model into retry loops. */ @@ -975,6 +1045,8 @@ export function hasBenchmarkApiHandler(url: string): boolean { const path = url.split('?')[0] return ( path === '/api/workers/list' || + BENCHMARK_JOB_GET_PATH.test(path) || + BENCHMARK_RUN_BY_PATH.test(path) || /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) || path === '/api/embeddings/query_hub_scripts' || path.startsWith('/api/scripts/hub/get_full/') @@ -983,7 +1055,7 @@ export function hasBenchmarkApiHandler(url: string): boolean { /** Answer a relative `/api/...` fetch — from the API catalog executor, or from the * chat's hub tools. */ -export function handleBenchmarkApiFetch(url: string): Response { +export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Response { const path = url.split('?')[0] if (path === '/api/workers/list') { return Response.json(BENCHMARK_WORKERS) @@ -991,6 +1063,36 @@ export function handleBenchmarkApiFetch(url: string): Response { if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) { return Response.json([]) } + const jobGet = BENCHMARK_JOB_GET_PATH.exec(path) + if (jobGet) { + const id = decodeURIComponent(jobGet[2]) + const job = getBenchmarkCompletedJob(decodeURIComponent(jobGet[1]), id) + if (!job) { + return Response.json({ error: `Job not found for "${id}"` }, { status: 404 }) + } + // The real endpoint lets a caller drop the bulky fields. Ignoring that here would + // size the model's context off a payload it explicitly asked to shrink. + const query = new URLSearchParams(url.split('?')[1] ?? '') + if (query.get('no_logs') === 'true') { + delete job.logs + } + if (query.get('no_code') === 'true') { + delete job.raw_code + } + return Response.json(job) + } + const runByPath = BENCHMARK_RUN_BY_PATH.exec(path) + if (runByPath) { + const workspace = decodeURIComponent(runByPath[1]) + const runnablePath = decodeURIComponent(runByPath[3]) + const args = parseBenchmarkRequestBody(init?.body) + // The real endpoint answers with the bare job id as text, not JSON. + return new Response( + runByPath[2] === 'f' + ? runBenchmarkFlowByPath({ workspace, path: runnablePath, args }) + : runBenchmarkScriptByPath({ workspace, path: runnablePath, args }) + ) + } if (path === '/api/embeddings/query_hub_scripts') { const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? '' return Response.json(searchBenchmarkHubScripts(text)) diff --git a/ai_evals/adapters/frontend/mockBackendApi.test.ts b/ai_evals/adapters/frontend/mockBackendApi.test.ts new file mode 100644 index 0000000000..d9cfa08bf4 --- /dev/null +++ b/ai_evals/adapters/frontend/mockBackendApi.test.ts @@ -0,0 +1,83 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { + createBenchmarkCompletedJob, + getBenchmarkCompletedJob, + handleBenchmarkApiFetch, + hasBenchmarkApiHandler, + listBenchmarkMcpTools, + resetBenchmarkMockBackend, + registerBenchmarkWorkspaceRunnables +} from './mockBackend' + +const WORKSPACE = 'benchmark-api-ws' + +// A catalog entry with no fetch handler is a dead end: the catalog executor builds a +// relative `/api/...` url, the stub declines it, and node's fetch throws on the relative +// url instead of returning a result the model can act on. Mutating entries are reachable +// too — the eval runners define no `requestConfirmation`, so `call_api_endpoint` executes +// unconfirmed. +describe('benchmark API catalog', () => { + beforeEach(() => resetBenchmarkMockBackend()) + afterEach(() => resetBenchmarkMockBackend()) + + it('answers every endpoint it advertises', () => { + const unanswered = listBenchmarkMcpTools() + .map((tool) => + `/api${tool.path.replace('{workspace}', WORKSPACE)}`.replace(/\{[^}]+\}/g, 'x') + ) + .filter((url) => !hasBenchmarkApiHandler(url)) + + // The draft-covered entries are refused by name before any fetch, so they are + // advertised without a handler on purpose. + expect(unanswered).toEqual([ + `/api/w/${WORKSPACE}/scripts/get/p/x`, + `/api/w/${WORKSPACE}/flows/create`, + `/api/w/${WORKSPACE}/schedules/delete/x`, + `/api/w/${WORKSPACE}/variables/get/x` + ]) + }) + + it('runs a deployed script by path, the way call_api_endpoint reaches it', async () => { + registerBenchmarkWorkspaceRunnables(WORKSPACE, { + scripts: [ + { + path: 'f/evals/greet', + summary: 'Greet', + language: 'bun', + content: 'export async function main() {}' + } + ] + }) + + const res = handleBenchmarkApiFetch( + `/api/w/${WORKSPACE}/jobs/run/p/${encodeURIComponent('f/evals/greet')}`, + { method: 'POST', body: JSON.stringify({ name: 'ada' }) } + ) + + expect(res.status).toBe(200) + const job = getBenchmarkCompletedJob(WORKSPACE, (await res.text()).trim()) + expect(job).toMatchObject({ success: true, args: { name: 'ada' } }) + }) + + it('serves a recorded job so a model can check the run it just started', async () => { + const id = createBenchmarkCompletedJob({ + workspace: WORKSPACE, + jobKind: 'preview', + result: 'Hello, World!' + }) + + const res = handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/${id}`) + + expect(res.status).toBe(200) + expect(await res.json()).toMatchObject({ + id, + success: true, + result: 'Hello, World!' + }) + }) + + it('404s an unknown job id instead of letting the fetch fall through', () => { + expect(hasBenchmarkApiHandler(`/api/w/${WORKSPACE}/jobs_u/get/missing`)).toBe(true) + expect(handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/missing`).status).toBe(404) + }) +}) diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index c07d104f47..e9c33b5632 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -13,7 +13,7 @@ const ORIGINAL_FETCH = globalThis.fetch globalThis.fetch = (async (input: unknown, init?: RequestInit) => { const url = typeof input === 'string' ? input : ((input as Request | URL | null)?.url ?? '') if (typeof url === 'string' && hasBenchmarkApiHandler(url)) { - return handleBenchmarkApiFetch(url) + return handleBenchmarkApiFetch(url, init) } return ORIGINAL_FETCH(input as Parameters[0], init) }) as typeof fetch