From 10c68196a877fb6b3636e4192bb1ef3e4e013ee8 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 11 Aug 2026 15:24:52 +0200 Subject: [PATCH] feat: show a dataset's cases as a table, and fix round-4 review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The case list showed one case at a time with no overview. It is now a table with the case, where it was captured from, and its last run — the last-run column is a single jobs query on the path stamp rather than a request per row. Review fixes in the same file: - keep the edit baseline on the selected case rather than looking it up in the loaded page, so a case beyond page 1 is not treated as unedited and run stale - release the loading state when a superseded case load returns early - reload every loaded page after a write instead of collapsing to page 1 - last remaining 'resolved to' wording in the version tooltip Co-Authored-By: Claude Opus 5 (1M context) --- .../components/aiEvals/AgentEvalDrawer.svelte | 186 ++++++++++++++---- 1 file changed, 147 insertions(+), 39 deletions(-) diff --git a/frontend/src/lib/components/aiEvals/AgentEvalDrawer.svelte b/frontend/src/lib/components/aiEvals/AgentEvalDrawer.svelte index 10d6b392ec..e280a882ca 100644 --- a/frontend/src/lib/components/aiEvals/AgentEvalDrawer.svelte +++ b/frontend/src/lib/components/aiEvals/AgentEvalDrawer.svelte @@ -13,12 +13,16 @@ type EvalCase, type EvalCaseDraft, type EvalDataset, - type Job + type Job, + JobService } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { untrack } from 'svelte' import { sendUserToast } from '$lib/toast' - import { Plus, Trash2, Bot, FlaskConical } from 'lucide-svelte' + import { Plus, Trash2, Bot, FlaskConical, MessagesSquare } from 'lucide-svelte' + import DataTable from '$lib/components/table/DataTable.svelte' + import Head from '$lib/components/table/Head.svelte' + import Cell from '$lib/components/table/Cell.svelte' import type { AgentTool } from '$lib/components/flows/agentToolUtils' import EvalCaseEditor from './EvalCaseEditor.svelte' import EvalRunResult from './EvalRunResult.svelte' @@ -88,6 +92,7 @@ if (!ws || !path) { cases = [] totalCases = 0 + loadingCases = false return } loadingCases = true @@ -170,14 +175,21 @@ }) }) - function setDraft(next: CaseDraft) { + // The stored form of the selected case, kept independently of the loaded pages: looking the + // baseline up in `cases` made an off-page case read as unedited, so Run sent it by reference + // and evaluated the persisted inputs instead of the visible ones. + let draftBaseline = $state(undefined) + + function setDraft(next: CaseDraft, baseline?: unknown) { + draftBaseline = baseline draft = next draftGeneration += 1 job = undefined } function selectCase(c: EvalCase) { - setDraft(fromStoredCase(c)) + const stored = fromStoredCase(c) + setDraft(stored, comparableCase(stored)) } function newCase() { @@ -219,6 +231,7 @@ path: selectedDataset, requestBody: { id: draft.id, ...body } }) + draftBaseline = comparableCase(draft) sendUserToast('Case updated') } else { const id = await AiEvalsService.addEvalCase({ @@ -227,9 +240,19 @@ requestBody: body }) draft = { ...draft, id } + draftBaseline = comparableCase(draft) sendUserToast('Case saved to dataset') } - await loadCases(selectedDataset) + await reloadCases() + } + + // Re-fetch as many pages as were loaded, so a write does not silently collapse the list back + // to its first page. + async function reloadCases() { + const pages = Math.max(1, Math.ceil(cases.length / CASE_PAGE_SIZE)) + for (let page = 1; page <= pages; page++) { + await loadCases(selectedDataset, page) + } } async function deleteCase(c: EvalCase) { @@ -240,7 +263,7 @@ requestBody: { id: c.id } }) if (draft.id === c.id) newCase() - await loadCases(selectedDataset) + await reloadCases() } async function run() { @@ -253,10 +276,10 @@ // A saved case is run by reference so the job records which case it executed. Unsaved edits // have to go inline instead — running the stored case while the editor shows something else // would silently test the wrong thing — and such a run has no case to trace back to. - const savedCase = cases.find((c) => c.id === draft.id) - const edited = - savedCase != undefined && - !deepEqual(comparableCase(fromStoredCase(savedCase)), comparableCase(draft)) + // No baseline for a case that claims an id means we cannot prove it is unchanged, so treat + // it as edited: running the persisted case while the editor shows something else is the + // silent failure, and an inline run is merely unstamped. + const edited = draft.id != undefined && !deepEqual(draftBaseline, comparableCase(draft)) const stored = draft.id && selectedDataset && !edited if (edited) { sendUserToast('Running the unsaved edits; save the case to record the run against it') @@ -295,6 +318,45 @@ ) } + // One query for the whole table: every run of this dataset's cases is stamped with a path under + // `//`, so the last run per case is a group-by on that rather than a request + // per row. The child agent job shares the prefix with a `/a` suffix, so only exact case + // segments count. + let lastRunByCase = $state>({}) + let runsGeneration = 0 + async function loadLastRuns(dataset: string | undefined, agent: string | undefined) { + const generation = ++runsGeneration + lastRunByCase = {} + if (!ws || !dataset || !agent) return + const prefix = `${agent}/${dataset}/` + try { + const jobs = await JobService.listJobs({ + workspace: ws, + scriptPathStart: prefix, + perPage: 200 + }) + if (generation !== runsGeneration) return + const byCase: Record = {} + for (const job of jobs) { + const caseId = job.script_path?.slice(prefix.length) + // listJobs is newest first, so the first hit per case is its latest run. + if (caseId && !caseId.includes('/') && !byCase[caseId]) byCase[caseId] = job + } + lastRunByCase = byCase + } catch { + // A missing run history must not empty the table. + } + } + $effect(() => { + if (isOpen) loadLastRuns(selectedDataset, agentPath) + }) + + function caseSource(c: EvalCase): string { + if (c.source?.job_id) return 'run' + if (c.source?.conversation_id) return 'conversation' + return 'manual' + } + let historyPath = $derived( draft.id && selectedDataset && agentPath ? caseRunPath(agentPath, selectedDataset, draft.id) @@ -318,7 +380,7 @@ Delete “{deleting ? caseLabel(deleting) : ''}” from this dataset? - + v{agentVersion} - The version this agent is at now. A run records the version it resolved to, so a - result stays attributable to a prompt state. A version captures the resource only: a - `$var:` it references can change underneath two identical versions. + The version this agent is at now. A run records the version at the moment it is + enqueued, so a result stays attributable to a prompt state — a run that waits in + the queue while the agent is edited executes a newer value than the one recorded. + A version captures the resource only: a `$var:` it references can change underneath + two identical versions. {/if} @@ -350,7 +414,7 @@ {/snippet} - +
@@ -403,28 +467,72 @@ here.
{:else} -
- {#each cases as c (c.id)} -
- -
- {/each} -
+ + + + Case + From + Last run + + + + + {#each cases as c (c.id)} + {@const lastRun = lastRunByCase[c.id]} + selectCase(c)} + > + +
+ {caseLabel(c)} + {#if c.input?.messages?.length} + + {/if} +
+ + + {caseSource(c)} + + + {#if lastRun} + + {lastRun.type === 'CompletedJob' + ? lastRun.success + ? 'success' + : 'failure' + : 'running'} + + {:else} + never + {/if} + + +
- +
{#key draftGeneration} - +