diff --git a/.github/workflows/ai-evals-test.yml b/.github/workflows/ai-evals-test.yml index 3ee7aed876..71e79395f3 100644 --- a/.github/workflows/ai-evals-test.yml +++ b/.github/workflows/ai-evals-test.yml @@ -22,6 +22,7 @@ on: - "frontend/src/lib/userDraft.svelte.ts" - "frontend/src/lib/userDraftDbSyncer.svelte.ts" - "frontend/src/lib/infer.ts" + - "frontend/src/lib/components/sessions/**" - ".github/workflows/ai-evals-test.yml" pull_request: types: [opened, reopened, ready_for_review] @@ -35,6 +36,7 @@ on: - "frontend/src/lib/userDraft.svelte.ts" - "frontend/src/lib/userDraftDbSyncer.svelte.ts" - "frontend/src/lib/infer.ts" + - "frontend/src/lib/components/sessions/**" - ".github/workflows/ai-evals-test.yml" concurrency: @@ -124,6 +126,11 @@ jobs: bun install bun test adapters/ + # Harness code that reaches into the frontend module graph; bun cannot load it. + - name: Run harness unit tests (frontend graph) + working-directory: ./ai_evals + run: bun run test:frontend-graph + - name: Run global AI evals timeout-minutes: 20 working-directory: ./ai_evals diff --git a/ai_evals/README.md b/ai_evals/README.md index 08725e8aa8..ee8f730fc3 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -150,6 +150,21 @@ Global initial fixtures can also seed `liveEditorDrafts` with `type`, currently open script, flow, or raw app editor so cases can test prompts that refer to "this" or the "current" item. +Global initial fixtures can seed the session's `artifacts` — `{ name, versions: [{ content, +note? }], role?, approvedVersion? }`, oldest version first, so the artifact starts with the +history `list_artifact_versions` reports — and the `previewTabs` open in its side panel, for +cases that run with `runtime.sessionChat: true`. A tab entry names one destination and may +be the `active` one: + +```json +"previewTabs": [{ "artifact": { "name": "Onboarding plan", "version": 2 }, "active": true }] +``` + +`page` (`{ href, label }`) and `item` (`{ kind, path }`) tabs work the same way. Tabs are +driven by the production tab model, so `open_preview`, `get_preview_status` and +`close_page` really open, report and close them, and a `version` is the pin a reader +chose in the artifact's version picker — which only `get_preview_status` reports. + Global initial fixtures can seed `workspace.variables` with `{ path, value, is_secret, description?, labels?, ws_specific? }` entries so cases can read and edit variables that already exist in the workspace. The mock mirrors the real @@ -265,6 +280,10 @@ Typical artifacts by mode: - `history/`: optional tracked pass-rate history written by `run --record`, one JSONL file per mode - `results/`: local benchmark output and artifacts +Harness unit tests run in two lanes: `bun test adapters/` for plain TypeScript, and +`bun run test:frontend-graph` for `*.vitest.ts` files, which exercise adapters built on +frontend code (Svelte runes, SvelteKit aliases) that bun cannot load. + ## Notes - Frontend modes reuse the production frontend chat code through the Vitest bridge. diff --git a/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts b/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts index 95b67743e1..207238cc02 100644 --- a/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts +++ b/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts @@ -1,6 +1,8 @@ // SessionArtifactsStore can't run here (bun has no IndexedDB, nor the compiled $state runes), // so mirror only the shape the artifact tools call, not its scoping or race handling. -export const EVAL_SESSION_ID = "eval-session"; +// Cases run concurrently in one process and the preview handlers are registered +// process-wide, keyed by session id — so each run needs its own. +let sessionSeq = 0; /** An artifact the session already holds when the case starts: history has to predate the * run, since one prompt cannot both build a past and reason about it. */ @@ -15,17 +17,28 @@ export interface SeededArtifact { } export function createEvalArtifactHelpers(seed: SeededArtifact[] = []) { + const sessionId = `eval-session-${sessionSeq++}`; const items = new Map>(); // Snapshots per artifact id, oldest first — the version tools read history from here. const history = new Map>>(); + // How a preview-tab fixture names the artifact its tab shows. + const seededIds = new Map(); let seq = 0; for (const entry of seed) { const id = `eval-artifact-${seq++}`; const current = entry.versions.at(-1); if (!current) continue; + // A preview tab names the artifact it shows, so a shared name would open whichever + // one happened to be seeded last. + if (seededIds.has(entry.name)) { + throw new Error( + `Two seeded artifacts are named "${entry.name}" — a preview tab fixture could not tell them apart`, + ); + } + seededIds.set(entry.name, id); items.set(id, { id, - sessionId: EVAL_SESSION_ID, + sessionId, chatId: "eval-chat", kind: "md", name: entry.name, @@ -150,10 +163,12 @@ export function createEvalArtifactHelpers(seed: SeededArtifact[] = []) { return { helpers: { artifacts: store, - sessionId: EVAL_SESSION_ID, + sessionId, getChatId: () => "eval-chat", - openArtifact: () => {}, + openArtifact: (_id: string, _name: string) => {}, }, + sessionId, + seededIds, snapshot: () => [...items.values()], }; } diff --git a/ai_evals/adapters/frontend/core/global/evalPreviewTabs.ts b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.ts new file mode 100644 index 0000000000..f32bf1e22e --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.ts @@ -0,0 +1,188 @@ +import { + setClosePreviewTabsHandler, + setGetPreviewStatusHandler, + setOpenPagePreviewHandler, + setOpenPreviewHandler, +} from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; +import type { GlobalActivePreviewContext } from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; +import { + describePreview, + previewTargetForSessionTarget, + selectPreviewTabsToClose, + SessionPreviewTabs, + whereIs, +} from "../../../../../frontend/src/lib/components/sessions/sessionPreviewTabs.svelte"; +import { + previewLocationContext, + previewLocationLabel, + promptSafe, + resolvePreviewTab, +} from "../../../../../frontend/src/lib/components/sessions/previewRouter"; +import type { ArtifactVersionTarget } from "../../../../../frontend/src/lib/components/sessions/previewRouter"; +import type { SessionTarget } from "../../../../../frontend/src/lib/components/sessions/sessionState.svelte"; + +// The side panel a session chat talks to, driven by the production tab model rather than +// by canned tool results — so a case measures what the real open_preview / get_preview_status +// / close_page report about the tabs the reader has. sessionRuntime.svelte.ts (the production +// owner of these handlers) can't run here: it reaches for IndexedDB, stores and live editors. + +export interface EvalPreviewTabFixture { + /** Artifact tab, named by the artifact fixture it shows. `version` pins it, as a reader does. */ + artifact?: { name: string; version?: number }; + /** Workspace page tab, e.g. `{ href: "/runs", label: "Runs" }`. */ + page?: { href: string; label: string }; + /** Editor tab for a workspace item. */ + item?: { kind: SessionTarget["kind"]; path: string }; + /** Tab the reader is looking at. Defaults to the last seeded one. */ + active?: boolean; +} + +// Registered once for the whole process, as production does at module load, and dispatched +// by session id: global cases run concurrently, so a per-run registration would have every +// case answering out of whichever run registered last. +const panels = new Map(); + +const NO_SESSION = "No active session; the preview panel is unavailable."; + +function panelFor(sessionId: string | undefined): SessionPreviewTabs | undefined { + return sessionId ? panels.get(sessionId) : undefined; +} + +setGetPreviewStatusHandler((sessionId) => { + const owner = panelFor(sessionId); + if (!owner) return NO_SESSION; + return describePreview(owner.tabs, owner.activeId, !!owner.displayedTab); +}); + +setOpenPreviewHandler(async ({ sessionId, kind, path }) => { + const owner = panelFor(sessionId); + if (!owner) return "Error: no active session to open the preview in."; + const target = previewTargetForSessionTarget(kind, path); + if (!target) { + return `Error: ${kind} targets cannot be shown in the preview panel.`; + } + // The pipeline branch of the production handler waits on an editor that only exists once + // a canvas mounts, which never happens here — a pipeline preview reports as any other. + const result = owner.open(target); + return result.status === "focused" + ? `A preview tab is already showing ${kind} "${path}" — focused it.` + : `Opened ${kind} preview for ${path} in a new tab in the side panel.`; +}); + +setOpenPagePreviewHandler(({ sessionId, href, label, newTab }) => { + const owner = panelFor(sessionId); + if (!owner) return undefined; + const result = owner.open({ type: "page", href, label }, { forceNewTab: newTab }); + if (result.status === "focused") { + return `A preview tab is already showing ${label} — focused it.`; + } + if (result.status === "retargeted") { + return `Updated the ${label} preview tab with the requested view.`; + } + return `Opened ${label} in a new preview tab in the side panel.`; +}); + +setClosePreviewTabsHandler(({ sessionId, all, match }) => { + const owner = panelFor(sessionId); + if (!owner) return NO_SESSION; + if (owner.tabs.length === 0) return "The preview panel has no open tabs."; + const labelFor = (t: (typeof owner.tabs)[number]) => + promptSafe(previewLocationLabel(whereIs(t))); + const doomed = selectPreviewTabsToClose(owner.tabs, { all, match }); + if (doomed.length === 0) { + return `No open tab matched "${match}". Open tabs: ${owner.tabs.map(labelFor).join(", ")}.`; + } + const closedLabels = doomed.map(labelFor); + for (const t of doomed) owner.close(t.id); + return `Closed ${closedLabels.length} preview tab${closedLabels.length === 1 ? "" : "s"} (${closedLabels.join(", ")}).`; +}); + +export interface EvalPreviewPanel { + /** Mirrors production: a written artifact is shown in the panel. `version` carries the + * caller's intent for the version picker — `latest` drops a pin the reader had set. */ + openArtifact: (id: string, name: string, version?: ArtifactVersionTarget) => void; + /** What the user message stamps as ACTIVE PREVIEW, as sessionRuntime's resolver reads it. */ + activePreview: () => GlobalActivePreviewContext | undefined; + dispose: () => void; +} + +export function createEvalPreviewPanel(input: { + sessionId: string; + tabs: EvalPreviewTabFixture[]; + /** Artifact ids by name, from the artifact fixture seeding. */ + artifactIds: Map; +}): EvalPreviewPanel { + // Nothing durable to write back to, and no debounce worth waiting on. + const owner = new SessionPreviewTabs( + { tabs: [], activeId: "", collapsed: false }, + { persist: () => {} }, + 0, + ); + // Opening a tab makes it the active one, so the fixture's pick can only be applied once + // every tab is seeded — selecting inside the loop would lose to the next open. + let requestedActive: string | undefined; + for (const fixture of input.tabs) { + const opened = seedTab(owner, fixture, input.artifactIds); + if (opened && fixture.active) requestedActive = opened; + } + if (requestedActive) owner.select(requestedActive); + // Registered last: seeding throws on a malformed fixture, and this map outlives the run. + panels.set(input.sessionId, owner); + + return { + openArtifact: (id, name, version) => { + owner.open({ type: "artifact", id, name, version }); + }, + activePreview: () => { + const tab = owner.displayedTab; + if (!tab) return undefined; + // Artifact and editor tabs are not iframes: they carry no page location, and an + // artifact's pinned version reaches the chat only through get_preview_status. + if (resolvePreviewTab(tab.url).kind !== "iframe") return undefined; + return previewLocationContext(whereIs(tab)); + }, + dispose: () => { + panels.delete(input.sessionId); + }, + }; +} + +// Seeds one tab through the production open path and returns its id, so a fixture cannot +// describe a tab the panel could not have reached on its own. +function seedTab( + owner: SessionPreviewTabs, + fixture: EvalPreviewTabFixture, + artifactIds: Map, +): string | undefined { + // A tab shows one destination; the branches below would silently keep the first. + const named = [fixture.artifact, fixture.page, fixture.item].filter(Boolean); + if (named.length > 1) { + throw new Error( + "Preview tab fixture sets more than one of artifact, page and item — a tab shows one of them", + ); + } + if (fixture.artifact) { + const id = artifactIds.get(fixture.artifact.name); + if (!id) { + throw new Error( + `Preview tab fixture references artifact "${fixture.artifact.name}", which no artifact fixture seeds`, + ); + } + owner.open({ type: "artifact", id, name: fixture.artifact.name }); + // A pin is the reader's own pick in the version picker, never a side effect of opening. + if (fixture.artifact.version !== undefined) { + owner.pinArtifactVersion(id, fixture.artifact.version); + } + } else if (fixture.page) { + owner.open({ type: "page", href: fixture.page.href, label: fixture.page.label }); + } else if (fixture.item) { + const target = previewTargetForSessionTarget(fixture.item.kind, fixture.item.path); + if (!target) { + throw new Error(`Preview tab fixture has an unpreviewable item kind: ${fixture.item.kind}`); + } + owner.open(target); + } else { + throw new Error("Preview tab fixture must set one of artifact, page or item"); + } + return owner.activeId; +} diff --git a/ai_evals/adapters/frontend/core/global/evalPreviewTabs.vitest.ts b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.vitest.ts new file mode 100644 index 0000000000..b9a2990a0a --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.vitest.ts @@ -0,0 +1,37 @@ +import { expect, it, vi } from 'vitest' + +// The panel pulls in the global tool module, which reaches the editor stack it never uses here. +vi.mock('monaco-editor', () => ({ + editor: {}, + languages: {}, + KeyCode: {}, + Uri: { parse: (value: string) => ({ toString: () => value }) }, + MarkerSeverity: { Error: 8, Warning: 4, Info: 2, Hint: 1 } +})) +vi.mock('@codingame/monaco-vscode-standalone-typescript-language-features', () => ({ + getTypeScriptWorker: async () => async () => ({}), + typescriptVersion: 'test' +})) +vi.mock('@codingame/monaco-vscode-languages-service-override', () => ({ default: () => ({}) })) +vi.mock('$lib/components/vscode', () => ({})) + +const { createEvalPreviewPanel } = await import('./evalPreviewTabs') + +// Every open makes its own tab active, so a fixture's `active` flag only means anything if +// it survives the tabs seeded after it. Lose that and a case still runs — against a panel +// state its author never described. +it('keeps the tab a fixture marks active, not the last one seeded', () => { + const panel = createEvalPreviewPanel({ + sessionId: 'eval-preview-tabs-unit-test', + tabs: [ + { page: { href: '/runs', label: 'Runs' }, active: true }, + { artifact: { name: 'Onboarding plan' } } + ], + artifactIds: new Map([['Onboarding plan', 'eval-artifact-0']]) + }) + try { + expect(panel.activePreview()?.location).toBe('/runs') + } finally { + panel.dispose() + } +}) diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index f1b33e9d13..d35d874f71 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -20,6 +20,11 @@ import { createEvalArtifactHelpers, type SeededArtifact, } from "./evalArtifactStore"; +import { + createEvalPreviewPanel, + type EvalPreviewPanel, + type EvalPreviewTabFixture, +} from "./evalPreviewTabs"; import type { ModeRunContext } from "../../../../core/types"; import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; @@ -98,6 +103,8 @@ export interface GlobalEvalOptions { workspaceRoot?: string; // Artifacts the session already holds when the run starts. artifacts?: SeededArtifact[]; + /** Tabs already open in the side panel, including any artifact version the reader pinned. */ + previewTabs?: EvalPreviewTabFixture[]; runContext?: ModeRunContext; } @@ -116,14 +123,29 @@ export async function runGlobalEval( options.workspaceFixtures ?? {}, ); seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); + // Declared out here only so `finally` can reach it; a malformed fixture throws while + // building it, and everything seeded above still has to be torn down. + let panel: EvalPreviewPanel | undefined; try { + const evalArtifacts = createEvalArtifactHelpers(options.artifacts); + // Only a session chat has a side panel, so only it gets one here. Seeded tabs would + // otherwise vanish without a word, and the case would measure an empty panel. + if (!options.sessionChat && options.previewTabs?.length) { + throw new Error( + "This fixture seeds previewTabs, which only a session chat has — set runtime.sessionChat: true on the case.", + ); + } + if (options.sessionChat) { + panel = createEvalPreviewPanel({ + sessionId: evalArtifacts.sessionId, + tabs: options.previewTabs ?? [], + artifactIds: evalArtifacts.seededIds, + }); + } const model = options.model ?? "claude-haiku-4-5-20251001"; const injectActiveEditorContext = process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1"; - // Pass the seeded identity straight to the prompt builder rather than mutating - // the process-global `userStore`, so concurrent cases never race on it. - const evalArtifacts = createEvalArtifactHelpers(options.artifacts); const planMode = options.planMode ? createEvalPlanTools({ create: evalArtifacts.helpers.artifacts.create, @@ -131,6 +153,8 @@ export async function runGlobalEval( chatId: evalArtifacts.helpers.getChatId(), }) : undefined; + // Pass the seeded identity straight to the prompt builder rather than mutating + // the process-global `userStore`, so concurrent cases never race on it. const baseSystemMessage = prepareGlobalSystemMessage(undefined, { user: options.user, previewTools: options.sessionChat ?? false, @@ -149,16 +173,17 @@ export async function runGlobalEval( : undefined, isPlanModeActive: planMode?.isPlanModeActive, isToolAvailable: planMode?.isToolAvailable, - userMessage: prepareGlobalUserMessage( - userPrompt, - [], - injectActiveEditorContext ? { workspace: workspaceRoot } : {}, - ), + userMessage: prepareGlobalUserMessage(userPrompt, [], { + ...(injectActiveEditorContext ? { workspace: workspaceRoot } : {}), + activePreview: panel?.activePreview(), + }), tools: [ ...getGlobalEvalTools(options.sessionChat ?? false), ...(planMode?.tools ?? []), ], - helpers: evalArtifacts.helpers, + helpers: panel + ? { ...evalArtifacts.helpers, openArtifact: panel.openArtifact } + : evalArtifacts.helpers, apiKey, getOutput: async () => ({ ...(await collectGlobalDraftState(workspaceRoot)), @@ -191,6 +216,7 @@ export async function runGlobalEval( finalContextTokens: rawResult.finalContextTokens, }; } finally { + panel?.dispose(); clearGlobalDrafts(workspaceRoot); clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); unregisterBenchmarkWorkspaceRunnables(workspaceRoot); diff --git a/ai_evals/adapters/frontend/vitest.unit.config.ts b/ai_evals/adapters/frontend/vitest.unit.config.ts new file mode 100644 index 0000000000..7c5a794508 --- /dev/null +++ b/ai_evals/adapters/frontend/vitest.unit.config.ts @@ -0,0 +1,31 @@ +import { fileURLToPath } from 'node:url' +import frontendConfig from '../../../frontend/vite.config.js' + +// Harness unit tests that reach into the frontend module graph. They can't run under +// `bun test` (Svelte runes and the SvelteKit aliases both need this build), so they are +// named `*.vitest.ts` — bun's `*.test.ts` sweep skips them and this config claims them. +const FRONTEND_VITE_CONFIG_PATH = fileURLToPath(new URL('../../../frontend/vite.config.js', import.meta.url)) +const FRONTEND_TEST_SETUP_PATH = fileURLToPath( + new URL('../../../frontend/src/lib/test-setup.ts', import.meta.url) +) +const UNIT_TESTS = fileURLToPath(new URL('./**/*.vitest.ts', import.meta.url)) + +const config = { + ...frontendConfig, + test: { + ...frontendConfig.test, + projects: [ + { + extends: FRONTEND_VITE_CONFIG_PATH, + test: { + name: 'server', + environment: 'node', + include: [UNIT_TESTS], + setupFiles: [FRONTEND_TEST_SETUP_PATH] + } + } + ] + } +} + +export default config diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 2269f9d727..5fb8a1f2c5 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1180,6 +1180,7 @@ - id: global-closepage1-close-runs-tab prompt: |- You just opened the runs page for me in the side panel. Close that tab, I'm done looking at it. + initial: ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json runtime: maxTurns: 6 sessionChat: true @@ -1233,6 +1234,32 @@ - creates one artifact and revises it rather than creating a second artifact - each revision carries a short description of what changed +# A reader who pins an older version in the artifact's picker is looking at something the +# artifact tools never report: an artifact tab carries no ACTIVE PREVIEW section, so the pin +# reaches the chat through get_preview_status alone. Asked what is on screen, the model has +# to read the panel instead of answering from the artifact's own history. + +- id: global-artifact-pinned-version-question + prompt: |- + Which version of the onboarding plan am I looking at right now? + initial: ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json + runtime: + maxTurns: 6 + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - get_preview_status + forbiddenToolsUsed: + - create_artifact + - update_artifact + - deploy_workspace_item + skipJudge: true + judgeChecklist: + - answers that the panel is showing version 2, not the latest version 5 + - does not edit or re-create the artifact + # --- Documentation search (search_docs) --- # Pure product-knowledge questions: the assistant should consult the docs via # search_docs and answer conversationally, not draft or mutate anything. No diff --git a/ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json b/ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json new file mode 100644 index 0000000000..dd4ab37873 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json @@ -0,0 +1,38 @@ +{ + "user": { + "username": "admin", + "is_admin": true + }, + "artifacts": [ + { + "name": "Onboarding plan", + "versions": [ + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Create the customer record\n- Send the welcome email\n" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record\n- Send the welcome email\n", + "note": "Added domain verification" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record\n- Send the welcome email\n- Schedule the 7-day check-in\n", + "note": "Added the 7-day check-in" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record in the CRM\n- Send the welcome email\n- Schedule the 7-day check-in\n", + "note": "Named the CRM as the record store" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record in the CRM\n- Send the welcome email\n- Schedule the 7-day check-in\n- Hand over to the account manager\n", + "note": "Added the account-manager handover" + } + ] + } + ], + "previewTabs": [ + { + "artifact": { "name": "Onboarding plan", "version": 2 }, + "active": true + } + ] +} diff --git a/ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json b/ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json new file mode 100644 index 0000000000..565e5ede37 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json @@ -0,0 +1,12 @@ +{ + "user": { + "username": "admin", + "is_admin": true + }, + "previewTabs": [ + { + "page": { "href": "/runs", "label": "Runs" }, + "active": true + } + ] +} diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index e7a36039a2..ad93b09a55 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -7,6 +7,7 @@ import { type GlobalUserFixture, } from "../adapters/frontend/core/global/globalEvalRunner"; import type { SeededArtifact } from "../adapters/frontend/core/global/evalArtifactStore"; +import type { EvalPreviewTabFixture } from "../adapters/frontend/core/global/evalPreviewTabs"; import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend"; import type { FrontendEvalModelConfig } from "../core/models"; import type { @@ -23,6 +24,7 @@ export interface GlobalInitialFixture { liveEditorDrafts?: GlobalLiveEditorDraftFixture[]; user?: GlobalUserFixture; artifacts?: SeededArtifact[]; + previewTabs?: EvalPreviewTabFixture[]; } export function createGlobalModeRunner( @@ -48,6 +50,7 @@ export function createGlobalModeRunner( liveEditorDrafts: initial?.liveEditorDrafts, user: initial?.user, artifacts: initial?.artifacts, + previewTabs: initial?.previewTabs, sessionChat: context.evalCase?.runtime?.sessionChat, planMode: context.evalCase?.runtime?.planMode, maxIterations: context.evalCase?.runtime?.maxTurns, @@ -122,6 +125,7 @@ async function loadGlobalInitialFixture( liveEditorDrafts: parsed.liveEditorDrafts ?? [], user: parsed.user, artifacts: parsed.artifacts, + previewTabs: parsed.previewTabs ?? [], }; } diff --git a/ai_evals/package.json b/ai_evals/package.json index b7fbeaa1a5..6720c910cb 100644 --- a/ai_evals/package.json +++ b/ai_evals/package.json @@ -4,7 +4,8 @@ "type": "module", "scripts": { "cli": "bun cli/index.ts", - "typecheck": "tsc -p tsconfig.json" + "typecheck": "tsc -p tsconfig.json", + "test:frontend-graph": "cd ../frontend && node_modules/.bin/vitest run --project server --config ../ai_evals/adapters/frontend/vitest.unit.config.ts" }, "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.25", diff --git a/ai_evals/tsconfig.json b/ai_evals/tsconfig.json index 7b06d5788a..50b2b1110d 100644 --- a/ai_evals/tsconfig.json +++ b/ai_evals/tsconfig.json @@ -14,6 +14,8 @@ ], "exclude": [ "./**/*.test.ts", - "./adapters/frontend/vitest.config.ts" + "./**/*.vitest.ts", + "./adapters/frontend/vitest.config.ts", + "./adapters/frontend/vitest.unit.config.ts" ] } diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactTools.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactTools.ts index 7683a5a603..61f411f2b2 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactTools.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactTools.ts @@ -150,7 +150,7 @@ export const artifactTools: Tool<{}>[] = [ def: createToolDef( listArtifactsSchema, 'list_artifacts', - "List the current session's artifacts (id, name, kind, version, role, approvedVersion). `role` is `plan` on the session's one plan document and on nothing else. On that one, `approvedVersion` is the version the user signed off: below `version` means the current text is a proposal they have not agreed to, and absent means nothing here was ever approved." + "List the current session's artifacts (id, name, kind, version, role, approvedVersion). `version` is the latest one saved, which is not necessarily the one the user is reading — get_preview_status reports the version they have pinned. `role` is `plan` on the session's one plan document and on nothing else. On that one, `approvedVersion` is the version the user signed off: below `version` means the current text is a proposal they have not agreed to, and absent means nothing here was ever approved." ), planModeSafe: true, fn: async ({ toolId, toolCallbacks, helpers }) => { diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 2802e3efa6..e4d9528ec1 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -3794,7 +3794,7 @@ export const globalTools: Tool<{}>[] = [ def: createToolDef( getPreviewStatusSchema, 'get_preview_status', - 'Check whether the side-panel preview is open in this AI session and which item (kind + path) it is showing. Call this before offering or calling open_preview so you do not re-open a preview that is already showing the item you just edited. Only meaningful inside a session.' + 'Check whether the side-panel preview is open in this AI session and which item (kind + path) it is showing — for an artifact tab, also the version the user has pinned in its version picker, which may be older than the latest. Call this before offering or calling open_preview so you do not re-open a preview that is already showing the item you just edited, and whenever the user asks what they are looking at with no ACTIVE PREVIEW section to answer from. Only meaningful inside a session.' ), planModeSafe: true, fn: async (ctx) => getSessionPreviewStatus(sessionIdFromCtx(ctx))