From 1462f17643302127b4bd76bacfde80cc03f9d606 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Mon, 31 Aug 2026 19:30:26 +0200 Subject: [PATCH 1/3] feat: rework the evals dataset drawer and run navigation (#10884) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: create eval datasets from the run dialog, not the empty table Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * feat: paginate the evals dialog and show live run progress Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * feat: rework the evals dataset drawer and run navigation Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: take the write lock on the empty-state add-a-case action Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: paginate the cases editor and add keyboard page navigation Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: wire cases pagination and select runs from the keyboard Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: one highlight for pointer and keyboard, and guard keys on the topmost overlay Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: restore page navigation and answer arrows outside the pages Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: scope eval keyboard navigation to the active topmost surface Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: leave Enter to focused controls and declare topmost from drawers too Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: highlight with the hover surface and open the highlighted run Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: restore the run's dataset on the arrow-right fallback Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: keep the chosen comparison when reopening the same run Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * docs: state the Modal Enter caveat on EditableTextarea Modal handles Enter at `window` in the capture phase and stops propagation, so inside a dialog the key confirms the dialog rather than committing the edit. The docstring already carried this caveat for Escape; it now covers both keys and names `enterConfirms={false}` as the opt-out. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: render the case result with the chat's prose stack GfmMarkdown falls back to the legacy `!prose-xs` when no `prose` is given, so the case result read differently from a chat answer. Pass `sm`, the stack AssistantMessage renders with, and drop the wrapper whose `text-xs text-secondary` competed with it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * refactor: drop the narrating comment on the case result render Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: restore focus to the arriving page on keyboard navigation The restore checked `document.activeElement === document.body` one frame after the page changed, but the inert-driven reset lands after that frame: it read the element the user was about to lose and returned. Ask whether focus was inside the pages before navigating, then focus the arriving page unconditionally, which removes the race rather than re-timing it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: restore focus for any keyboard-driven page change Arming only from the arrow handler missed Enter, which opens the highlighted run from the page itself and never reaches this component. Record whether the last interaction was a key pressed with focus inside the pages — cleared on pointerdown — so any caller-driven keyboard navigation restores focus too. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: track focus position rather than the key that navigated Arming from a keydown listener cannot work for Escape: Modal handles it at `window` in the capture phase, registered before this component, and steps a level back from there — Svelte flushes this component's effects inside that handler, before our listener runs. Track whether focus sits in the pages as it moves, so the answer is already settled when the page changes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: take focus back when a navigation leaves it nowhere The trail's back button sits outside the pages and is removed as the level it returns from closes, so activating it from the keyboard left focus on a dead element. Claim the arriving page when focus was in the pages, or when it has ended up on the body — never off a control that outlives the navigation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ * fix: type the TextInput binding as the textarea it renders `TextInput` is generic over its underlying element and defaults to `'input'`, so binding the textarea instance to a bare `TextInput` failed svelte-check. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BamH3sRkmn5nP7wo9iKYPJ --------- Co-authored-by: Claude Opus 5 (1M context) --- .../components/aiEvals/AgentEvalModal.svelte | 8 +- .../components/aiEvals/EvalCasesGrid.svelte | 230 ++++---- .../aiEvals/EvalDatasetDrawer.svelte | 56 +- .../components/aiEvals/EvalRunDialog.svelte | 45 +- .../components/aiEvals/EvalRunsList.svelte | 135 ++++- .../lib/components/aiEvals/EvalScorers.svelte | 21 +- .../lib/components/aiEvals/EvalsPane.svelte | 507 +++++++++++------- .../display/table/multilineCellEditor.css | 26 - .../display/table/multilineCellEditor.ts | 108 ---- .../components/common/EditableTextarea.svelte | 173 ++++++ .../components/common/drawer/Drawer.svelte | 6 + .../common/emptyState/EmptyState.svelte | 18 +- .../lib/components/common/modal/Modal.svelte | 43 +- .../common/modal/PagedContent.svelte | 202 +++++++ .../components/common/overlayHost.svelte.ts | 20 + .../components/text_input/TextInput.svelte | 10 +- 16 files changed, 1099 insertions(+), 509 deletions(-) delete mode 100644 frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css delete mode 100644 frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts create mode 100644 frontend/src/lib/components/common/EditableTextarea.svelte create mode 100644 frontend/src/lib/components/common/modal/PagedContent.svelte diff --git a/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte b/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte index ff014f403a..61d88819e7 100644 --- a/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte +++ b/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte @@ -27,18 +27,14 @@ let trail = $derived( location ? [{ label: TITLE, onclick: location.back }, { label: location.label }] : undefined ) - let description = $derived( - location - ? undefined - : 'Each run answers a dataset of cases with this agent and scores the answers, so runs can be compared.' - ) - import { createGrid, type GridApi } from 'ag-grid-community' - import 'ag-grid-community/styles/ag-grid.css' - import 'ag-grid-community/styles/ag-theme-alpine.css' - import '$lib/components/apps/components/display/table/theme/windmill-theme.css' - import { transformColumnDefs } from '$lib/components/apps/components/display/table/utils' - import { multilineCellColDef } from '$lib/components/apps/components/display/table/multilineCellEditor' - import DarkModeObserver from '$lib/components/DarkModeObserver.svelte' + import DataTable from '$lib/components/table/DataTable.svelte' + import Head from '$lib/components/table/Head.svelte' + import Cell from '$lib/components/table/Cell.svelte' + import Row from '$lib/components/table/Row.svelte' + import EditableTextarea from '$lib/components/common/EditableTextarea.svelte' + import { Button, EmptyState } from '$lib/components/common' + import { ListPlus, Plus, Trash2 } from 'lucide-svelte' import { untrack } from 'svelte' import type { CaseDraft } from './evalUtils' let { cases = $bindable(), onRemove, - locked = false, - onEditingChange + onAdd, + focusCaseId = undefined, + locked = false }: { /** The drawer's working copy. Edits land here as they are made; the drawer writes them. */ cases: CaseDraft[] /** Asked before a row goes, since a stored case has runs that executed it. */ onRemove: (c: CaseDraft) => void + /** Adds a case. The empty table offers this itself, where the first row would be. */ + onAdd: () => void + /** A case to open for typing as soon as it appears — the one just added, so a new row is + * ready to be filled in rather than waiting to be clicked. */ + focusCaseId?: string /** The cases are being written: an edit made now is one the request already left behind. */ locked?: boolean - /** A cell opened or closed. Reported up so Save can be pressed for an edit the list does not - * hold yet: the press is what commits the cell, so it has to reach a live button. */ - onEditingChange?: (editing: boolean) => void } = $props() - type Row = { id: string; question: string; expected: string } + /** A dataset is capped at 1000 cases and every row here mounts two editors and a button, so the + * whole set at once is thousands of components for a table you read a screenful of. ag-grid + * virtualised its rows; `DataTable` paginates instead, which is the primitive already here. */ + const CASES_PER_PAGE = 25 + let currentPage = $state(1) + /** Bound to `DataTable`'s own per-page selector, so changing it there re-slices here rather + * than only relabelling the footer. */ + let perPage = $state(CASES_PER_PAGE) + let lastPage = $derived(Math.max(1, Math.ceil(cases.length / perPage))) + let pageCases = $derived(cases.slice((currentPage - 1) * perPage, currentPage * perPage)) + // A case removed from the last page leaves it empty; step back rather than show nothing. + $effect(() => { + if (currentPage > lastPage) currentPage = lastPage + }) + /** `expected` is whatever the case holds: a bare string, or a value shown as JSON. Read back + * the same way, so a case that held an object keeps holding one. */ function expectedToText(value: unknown): string { if (value == undefined) return '' return typeof value === 'string' ? value : JSON.stringify(value, null, 2) } - function toRow(c: CaseDraft): Row { - return { - id: c.id ?? '', - question: c.input?.user_message ?? '', - expected: expectedToText(c.expected) - } - } - - let api: GridApi | undefined = $state() - let eGui: HTMLDivElement | undefined = $state() - let darkMode = $state(false) - - const defaultColDef = { - flex: 1, - minWidth: 120, - ...multilineCellColDef - } - - $effect(() => eGui && untrack(() => mountGrid())) - function mountGrid() { - if (!eGui || api) return - createGrid(eGui, { - rowData: untrack(() => cases.map(toRow)), - columnDefs: columnDefs(), - defaultColDef: { ...defaultColDef, editable: untrack(() => !locked) }, - onCellValueChanged: (e) => { - const target = cases.find((c) => c.id === (e.data as Row).id) - if (!target) return - if (e.colDef.field === 'question') { - target.input = { ...target.input, user_message: e.newValue ?? '' } - } else if (e.colDef.field === 'expected') { - setExpected(target, e.newValue ?? '') - } - }, - onCellEditingStarted: () => onEditingChange?.(true), - onCellEditingStopped: () => onEditingChange?.(false), - suppressColumnMoveAnimation: true, - suppressDragLeaveHidesColumns: true, - onGridReady: (e) => (api = e.api) + /** The question editors, by case id, so a newly added row can be opened. */ + let questionEditors = $state>({}) + $effect(() => { + const id = focusCaseId + if (!id) return + untrack(() => { + // Onto the page the case is on before reaching for its editor: paginated, a case added to + // a full page is not rendered yet, and there would be nothing to open. + const at = cases.findIndex((c) => c.id === id) + if (at < 0) return + currentPage = Math.floor(at / perPage) + 1 + // After the row has been rendered and registered itself. + requestAnimationFrame(() => questionEditors[id]?.edit()) }) - } + }) - /** Text that parses as JSON is stored as JSON, so a structured answer can be written by hand. */ function setExpected(c: CaseDraft, text: string) { + // Cleared means the case has no expected answer, which is not the same as expecting the + // empty string: a scorer reads `undefined` as "nothing to measure here" and leaves the case + // out of its mean, where `''` scores it a hard zero. const trimmed = text.trim() if (!trimmed) { c.expected = undefined @@ -88,45 +82,93 @@ c.expected = text } } - - function columnDefs() { - return transformColumnDefs({ - columnDefs: [ - { field: 'question', headerName: 'Question', flex: 3 }, - { field: 'expected', headerName: 'Expected', flex: 2 } - ] as any, - onDelete: (values) => { - if (locked) return - const target = cases.find((c) => c.id === (values as Row).id) - if (target) onRemove(target) - } - }) - } - - // Keyed on which cases are in the list rather than on what is in them: pushing rows back on - // each keystroke would reset the cell being typed in. - let rowKey = $derived(cases.map((c) => c.id).join(',')) - $effect(() => { - rowKey - untrack(() => api?.updateGridOptions({ rowData: cases.map(toRow) })) - }) - - $effect(() => { - const editable = !locked - untrack(() => api?.updateGridOptions({ defaultColDef: { ...defaultColDef, editable } })) - }) - - /** Commit whatever cell is open into `cases`, and wait for it to land there: the drawer reads - * the list to save it, and a cell still being typed in is an edit that press is saving. */ - export async function flush() { - api?.stopEditing() - // The commit reaches `cases` through the grid's own event queue, a macrotask away: reading in - // the same turn, or after a microtask, reads the list as it was before the cell was typed in. - await new Promise((resolve) => setTimeout(resolve, 0)) - } - - -
+{#if cases.length === 0} + +{:else} + perPage} + bind:currentPage + bind:perPage + rowCount={cases.length} + hasMore={currentPage < lastPage} + showPrev={currentPage > 1} + on:next={() => (currentPage = Math.min(currentPage + 1, lastPage))} + on:previous={() => (currentPage = Math.max(currentPage - 1, 1))} + > + + + + + + + + Question + Expected + + + + + {#each pageCases as c (c.id)} + + + + (c.input = { ...(c.input ?? {}), user_message: v })} + /> + + + setExpected(c, v)} + /> + + + -
+
(casesEditing = v)} />
@@ -397,7 +400,7 @@ unifiedSize="md" variant="accent" loading={saving} - disabled={writing || !path || !!pathError || (nothingToSave && !casesEditing)} + disabled={writing || !path || !!pathError || nothingToSave} onclick={saveDataset} > Save @@ -408,7 +411,8 @@ variant="accent" startIcon={{ icon: Plus }} loading={creating} - disabled={creating || !path || !!pathError} + disabled={creating || !path || !!pathError || noCases} + title={noCasesTitle} onclick={createDataset} > Create dataset diff --git a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte index d4d1714fd0..63fe366628 100644 --- a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte +++ b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte @@ -47,16 +47,6 @@ let hasDraft = $derived(editedConfig !== undefined) let dataset = $state(undefined) let hoveringDataset = $state(false) - /** Set while the dialog stands aside for the dataset drawer, holding the pane's dataset as it - * was on the way out: coming back onto a different one is the detour's answer, and adopted. */ - let steppedAside = $state<{ dataset: string | undefined } | undefined>(undefined) - - /** Hands the screen to the dataset drawer, to be given back when it closes. */ - function stepAside(go: () => void) { - steppedAside = { dataset: defaultDataset } - open = false - go() - } /** The agent's versions, for pinning one. Loaded when the dialog opens rather than held: a * version list goes stale the moment the agent is saved again. */ @@ -76,21 +66,26 @@ } } + /** Whether the dialog was already open on the previous run of the effect below, which reads it + * to tell an open from the pane's dataset moving underneath. */ + let wasOpen = false + $effect(() => { - if (!open) return + const isOpen = open + const pane = defaultDataset untrack(() => { - loadVersions() - const aside = steppedAside - steppedAside = undefined - if (aside) { - // Back from the drawer: a dataset created or edited there moves the pane onto it, and - // anything else leaves the field as it was left. - if (defaultDataset !== aside.dataset) dataset = defaultDataset + if (!isOpen) { + wasOpen = false return } - // Seeded on every open: the dataset last worked in, and the state of the agent there is - // most reason to measure. - dataset = defaultDataset + // Followed while the dialog stands, not only read at open: the dataset drawer opens over + // this dialog rather than in place of it, so creating, renaming or deleting a dataset + // there moves the pane's selection with the dialog still up. Nothing else moves it then. + dataset = pane + if (wasOpen) return + wasOpen = true + loadVersions() + // The state of the agent there is most reason to measure. choice = hasDraft ? 'draft' : 'deployed' }) }) @@ -184,7 +179,7 @@ unifiedSize="sm" variant="default" startIcon={{ icon: Plus }} - onclick={() => stepAside(onNewDataset)} + onclick={onNewDataset} > New dataset @@ -215,7 +210,7 @@ title="Edit this dataset" on:click={() => { close() - stepAside(() => onEditDataset(item.value ?? '')) + onEditDataset(item.value ?? '') }} /> {/snippet} @@ -228,7 +223,7 @@ btnClasses="w-full !h-auto !justify-start !rounded-none flex items-center gap-2 px-3 py-2 text-xs !font-normal text-secondary hover:bg-surface-hover" onClick={() => { close() - stepAside(onNewDataset) + onNewDataset() }} > @@ -247,7 +242,7 @@ startIcon={{ icon: Pencil }} iconOnly title="Edit this dataset" - on:click={() => stepAside(() => onEditDataset(dataset ?? ''))} + on:click={() => onEditDataset(dataset ?? '')} /> {/if} diff --git a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte index 5d84ef88ce..b544781902 100644 --- a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte +++ b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte @@ -7,17 +7,21 @@ import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import TimeAgo from '$lib/components/TimeAgo.svelte' import { Button } from '$lib/components/common' - import { Bot, Code2, Loader2, Plus } from 'lucide-svelte' + import { Bot, ChevronRight, Code2, Loader2, Plus } from 'lucide-svelte' + import { overlayHostActive, topmostSurface } from '$lib/components/common/overlayHost.svelte' import type { EvalDataset, EvalExperiment, ExperimentScore } from '$lib/gen' import { datasetSummary, experimentName, formatScore, subjectLabel } from './evalUtils' let { experiments, datasets, + caseProgress, loaded, + active = false, deployedHash = undefined, currentVersion = undefined, onOpen, + onHighlight, onEditDataset, onNew }: { @@ -25,17 +29,87 @@ experiments: EvalExperiment[] /** Whether the list has been read: an empty table is a statement about the agent. */ loaded: boolean + /** Whether this list is the page on screen. The keyboard is only answered while it is: the + * run page keeps its own rows, and both would otherwise move on one press. */ + active?: boolean /** The workspace's datasets, for naming the one a run is of by what it is for. */ datasets: EvalDataset[] + /** How many cases each still-running run has finished, keyed by run id. A run the flow has + * not been read for yet is at none of them rather than absent: the count is on the row from + * the moment it appears, so it never arrives late and shifts the column. */ + caseProgress: Record /** What the agent hashes to as deployed, and the version it is on: they resolve a run of * edits that were later saved, so a run is labelled here as the run picker labels it. */ deployedHash?: string currentVersion?: number onOpen: (experiment: EvalExperiment) => void + /** The highlighted run, reported up so the surface can act on it — arrowing into the run + * page opens the run under the highlight rather than whichever was opened last. */ + onHighlight?: (id: string | undefined) => void onEditDataset: (dataset: string) => void onNew: () => void } = $props() + /** The highlighted run, by id. One state for both the pointer and the keyboard, as a melt menu + * does it: hovering a row moves the highlight to it, so the arrows carry on from wherever the + * pointer left off instead of running a second, invisible cursor of their own. It says where + * the highlight is, not what is chosen — a run is not opened until Enter. + * + * By id and not by index: the list is newest-first and the poll prepends to it, so an index + * would quietly come to mean a different run and Enter would open the wrong one. */ + let cursorId = $state(undefined) + let cursor = $derived( + cursorId === undefined ? -1 : experiments.findIndex((e) => e.id === cursorId) + ) + let body: HTMLTableSectionElement | undefined = $state() + + // A window listener answers keys aimed anywhere, so it has to ask two questions the DOM cannot: + // is my host the visible one — session preview tabs stay mounted when hidden — and is my surface + // still the one on top, rather than under a drawer or a dialog opened since. + const hostActive = overlayHostActive() + const onTop = topmostSurface() + const listening = () => hostActive() && onTop() + + $effect(() => { + onHighlight?.(cursorId) + }) + + // A highlight on a run that has since gone, and the highlight itself when the list is not the + // page on screen. + $effect(() => { + if (!active || (cursorId !== undefined && cursor < 0)) cursorId = undefined + }) + + function move(by: number) { + if (experiments.length === 0) return + const from = cursor < 0 ? (by > 0 ? -1 : experiments.length) : cursor + const at = Math.max(0, Math.min(experiments.length - 1, from + by)) + cursorId = experiments[at]?.id + // `nearest`, so arrowing through a long list scrolls by a row rather than jumping the table. + requestAnimationFrame(() => + body?.querySelectorAll('tr')[at]?.scrollIntoView({ block: 'nearest' }) + ) + } + + function onKeydown(event: KeyboardEvent) { + if (!active || !listening() || event.metaKey || event.ctrlKey || event.altKey) return + const el = event.target as HTMLElement | null + if (el?.closest?.('input, textarea, select, [contenteditable="true"], [role="listbox"]')) return + if (event.key === 'ArrowDown') { + event.preventDefault() + move(1) + } else if (event.key === 'ArrowUp') { + event.preventDefault() + move(-1) + } else if (event.key === 'Enter' && experiments[cursor]) { + // Enter belongs to whatever is focused if that thing does something with it. A highlighted + // row is not a reason to swallow the press on `New evaluation` or a row's dataset button. + if (el?.closest?.('button, a[href], [role="button"], summary')) return + event.preventDefault() + onOpen(experiments[cursor]) + } + } + /** The one number a column reports: a pass rate where it has a line to pass, the mean where it * does not. */ function headline(score: ExperimentScore): string | undefined { @@ -44,13 +118,16 @@ } + + - + + @@ -58,12 +135,20 @@ Dataset Cases Scores - When + When + - - {#each experiments as experiment (experiment.id)} - onOpen(experiment)}> + + {#each experiments as experiment, i (experiment.id)} + + onOpen(experiment)} + on:hover={(e) => e.detail && (cursorId = experiment.id)} + >
@@ -99,7 +184,18 @@ - {experiment.case_count} + {#if experiment.running} + + + + {caseProgress[experiment.id] ?? 0}/{experiment.case_count} + + {:else} + {experiment.case_count} + {/if}
@@ -117,8 +213,6 @@ {value} {:else if score.failed > 0} failed - {:else if experiment.running} - {:else} — {/if} @@ -126,33 +220,34 @@ {/each} {#if (experiment.scores ?? []).length === 0} - {#if experiment.running} - - - scoring - - {:else} - not scored - {/if} + + not scored {/if}
- + + + + {/each} {#if experiments.length === 0 && !loaded} - + {:else if experiments.length === 0} - +
No runs yet diff --git a/frontend/src/lib/components/aiEvals/EvalScorers.svelte b/frontend/src/lib/components/aiEvals/EvalScorers.svelte index 68287b0fbc..d00e52d77e 100644 --- a/frontend/src/lib/components/aiEvals/EvalScorers.svelte +++ b/frontend/src/lib/components/aiEvals/EvalScorers.svelte @@ -196,26 +196,31 @@
-
+
{#if scorers.length === 0} -
+
A scorer reads one run and returns a number. Every run of this dataset is measured by all of them, which is what makes two runs comparable.
{:else} -
+ +
{#each scorers as scorer (scorer.id)} -
+
{#if scorer.kind === 'agent'} - + {:else} - + {/if}
- + {scorerLabel(scorer)} - {scorer.path} + {scorer.path}
{#if scorer.pass_if != undefined} diff --git a/frontend/src/lib/components/aiEvals/EvalsPane.svelte b/frontend/src/lib/components/aiEvals/EvalsPane.svelte index d9e5b16fe4..55792ba8e2 100644 --- a/frontend/src/lib/components/aiEvals/EvalsPane.svelte +++ b/frontend/src/lib/components/aiEvals/EvalsPane.svelte @@ -9,9 +9,11 @@ import Label from '$lib/components/Label.svelte' import Popover from '$lib/components/Popover.svelte' import { Splitpanes, Pane } from 'svelte-splitpanes' + import AnimatedPane from '$lib/components/splitPanes/AnimatedPane.svelte' import { type AgentDraft, AiEvalsService, + JobService, type EvalCase, type EvalDataset, type EvalExperiment, @@ -35,9 +37,11 @@ Code2, ExternalLink } from 'lucide-svelte' + import PagedContent from '$lib/components/common/modal/PagedContent.svelte' import EvalDatasetDrawer from './EvalDatasetDrawer.svelte' import EvalRunsList from './EvalRunsList.svelte' import EvalRunDialog from './EvalRunDialog.svelte' + import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import GfmMarkdown from '$lib/components/GfmMarkdown.svelte' import { caseLabel, @@ -52,6 +56,10 @@ /** A dataset is capped at this many cases, so one page holds the whole set. */ const CASE_PAGE_SIZE = 1000 + /** The id the run's flow gives the loop over its cases (`CASES_NODE_ID` in `ai_evals/run.rs`). + * Looked up by id rather than by position: the flow has a step after the loop too. */ + const CASES_MODULE_ID = 'cases' + let { agentPath, opWorkspace = undefined, @@ -91,12 +99,56 @@ /** What the agent hashes to as deployed: a run of edits carrying it ran what was then saved. */ let deployedHash = $state(undefined) let running = $state(false) - let scorers = $derived(dataset?.scorers ?? []) + /** The run on screen belongs to a dataset still being read. Until it arrives the rows and the + * scorer columns would both be built from the *previous* dataset, so both are held back. */ + let datasetLoading = $state(false) + let scorers = $derived(datasetLoading ? [] : (dataset?.scorers ?? [])) let selectedCaseId = $state(undefined) let datasetDrawer: EvalDatasetDrawer | undefined = $state() let runDialogOpen = $state(false) - let resumeRunDialog = $state(false) + /** The run the list has highlighted, so arrowing into the run page opens that one. Without it + * the arrow could only fall back to whichever run was opened last, which on a dialog just + * opened is none at all. */ + let highlightedRunId = $state(undefined) + + /** How many cases each still-running run has finished, keyed by run id. Read from the flow + * executing the run: the list carries the case total, and counting the finished ones there + * would be a per-case query for every run listed. The flow already records it — one slot per + * case in `flow_jobs_success`, null until that case's iteration is over. */ + let caseProgress = $state>({}) + + async function readCaseProgress() { + const workspace = ws + const live = experiments.filter((e) => e.running) + if (!workspace || live.length === 0) { + if (Object.keys(caseProgress).length > 0) caseProgress = {} + return + } + const read = await Promise.all( + live.map(async (e) => { + try { + const update = await JobService.getJobUpdates({ + workspace, + id: e.run_job_id, + running: true, + noLogs: true + }) + const cases = update.flow_status?.modules?.find((m) => m.id === CASES_MODULE_ID) + if (!cases) return undefined + return [ + e.id, + (cases.flow_jobs_success ?? []).filter((s) => s != undefined).length + ] as const + } catch { + // Left out of the map, so the row reads `0/total` until a later poll answers. A + // flow that cannot be read is already the list's problem to report, not this one's. + return undefined + } + }) + ) + caseProgress = Object.fromEntries(read.filter((e) => e !== undefined)) + } let experiment = $derived(experiments.find((e) => e.id === experimentId)) @@ -120,6 +172,7 @@ runsLoadError = false try { experiments = await listSubjectExperiments() + await readCaseProgress() } catch (e) { runsLoadError = true sendUserToast(`Failed to load the runs: ${e}`, true) @@ -183,6 +236,8 @@ // Switching datasets leaves the previous request in flight; only the newest may write, or a // slow response for the dataset you just left replaces the one you are looking at. let loadGeneration = 0 + /** Which run the pane is opening; only the newest may clear `datasetLoading`. */ + let openGeneration = 0 async function loadDataset(path: string | undefined): Promise { const generation = ++loadGeneration @@ -345,6 +400,7 @@ await loadResults() } else { experiments = await listSubjectExperiments() + await readCaseProgress() } } finally { refreshing = false @@ -354,27 +410,46 @@ /** Opens a run, bringing its dataset with it and offering the run before it as the baseline. * Reading the cells is left to the effect on the selection, so every way in opens one alike. */ async function openRun(id: string) { - // Against the dataset that is loaded, not the one that is selected: skipping on the selection - // alone would leave a run open over a dataset whose cases and scorers were never read. const target = experiments.find((e) => e.id === id) - if (target && target.dataset !== dataset?.path) { - await useDataset(target.dataset) + // Only when the run itself changes: re-showing the one already open — arrowing back into it + // from the list — must keep whatever comparison the user picked. + if (id !== experimentId) { + const index = experiments.findIndex((e) => e.id === id) + // The run before it *of the same dataset*: the list spans datasets, and a run of another + // set of cases is not a baseline for this one. + baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id } - const index = experiments.findIndex((e) => e.id === id) - // The run before it *of the same dataset*: the list spans datasets, and a run of another set - // of cases is not a baseline for this one. - baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id experimentId = id - viewingRun = true selectedCaseId = undefined + // Opened first, read second: the dataset is a request, and waiting on it here is a click + // that does nothing at all until the network answers. The page carries the wait instead. + // + // Against what is *selected* as well as what is loaded. `selectedDataset` moves the moment a + // read starts, so a load still in flight for another dataset shows up here: without that + // test, opening a run of the dataset already committed would skip `useDataset` entirely and + // leave the in-flight one free to commit its cases under this run. + const needsDataset = + !!target && (target.dataset !== dataset?.path || target.dataset !== selectedDataset) + datasetLoading = needsDataset + viewingRun = true + if (needsDataset) { + // Numbered like `loadDataset`'s own read, and for the same reason: opening a second run + // while the first is still loading leaves two `finally`s racing, and the loser clearing + // the flag would uncover the table with neither dataset in hand. + const generation = ++openGeneration + try { + await useDataset(target!.dataset) + } finally { + if (generation === openGeneration) datasetLoading = false + } + } } async function runAll(runSubject: EvalSubject, path: string): Promise { if (!ws || !path) return false running = true - let id: string try { - id = await AiEvalsService.runExperiment({ + await AiEvalsService.runExperiment({ workspace: ws, requestBody: { dataset: path, subject: runSubject } }) @@ -386,9 +461,10 @@ // From here the run exists and is billing: what can still fail is reading it back, and // saying "failed to run" to that invites a second, duplicate run. try { + // Onto the list rather than into the run: a run that has just started has no answers and + // no scores, and the list already fills its row in as they land. Reading it is a click. if (path !== dataset?.path) await useDataset(path) await loadRuns() - await openRun(id) } catch (e) { sendUserToast( `The run started but could not be read back: ${e}. Reload the runs list to see it.`, @@ -407,8 +483,6 @@ /** The dataset is gone and every run of it with it: back to the list, on no dataset. */ async function datasetDeleted(path: string) { - // A run dialog waiting behind the drawer has nothing to come back to. - resumeRunDialog = false if (selectedDataset === path) { viewingRun = false selectedCaseId = undefined @@ -442,6 +516,13 @@ } let selectedRow = $derived(displayRows.find((row) => row.case_id === selectedCaseId)) + /** The case the side panel is showing. Held rather than read straight off the selection: the + * pane animates shut over a few hundred milliseconds, and the selection is gone on the first + * of them, which would empty the panel before it had finished closing. */ + let openRow = $state(undefined) + $effect(() => { + if (selectedRow) openRow = selectedRow + }) $effect(() => { if (!ws) return @@ -570,34 +651,52 @@
+ {#if loaded && loadError} +
+ Could not load evals + + The datasets or runs could not be read. Check your access to this agent and reload. + +
+ {:else} + + { + // Right opens the run under the highlight, falling back to whichever was open before; + // left is the way back, the same as the breadcrumb. + if (key === 'run') { + // Both branches go through `openRun`: it is what brings the run's own dataset back, + // and the fallback run may be of a dataset the list has since moved off. + const id = highlightedRunId ?? experimentId + if (id) openRun(id) + } else if (key === 'list') { + viewingRun = false + selectedCaseId = undefined + } + }} + pages={[ + { key: 'list', content: listPage }, + { key: 'run', content: runPage } + ]} + /> + {/if} +
+ +{#snippet listPage()} + +

+ Each run answers a dataset of cases with this agent and scores the answers, so runs can be + compared. +

- {#if viewingRun} - - {/if}
- {#if viewingRun && experiment?.run_job_id} - - Open the job - - - {/if} - {#if !viewingRun && loaded && datasets.length > 0} + {#if loaded && datasets.length > 0} -
- {:else if !viewingRun || !loaded} - openRun(e.id)} - onEditDataset={async (path) => { - if (await useDataset(path)) datasetDrawer?.openDrawer('edit') - }} - onNew={() => (runDialogOpen = true)} - /> - {:else} - - - - - {#each scorers as scorer (scorer.id)} - - {/each} - - - - Case - Answer - {#each scorers as scorer, index (scorer.id)} - {@const mean = means.find((m) => m.scorer_id === scorer.id)} - {@const headline = columnHeadline(scorer, mean)} - - -
- - {#if scorer.kind === 'agent'} - - {:else} - - {/if} - {scorerLabel(scorer)} - - - {#if headline} - - {headline.value} + + + + + {#each scorers as scorer (scorer.id)} + + {/each} + + + + Case + Answer + {#each scorers as scorer, index (scorer.id)} + {@const mean = means.find((m) => m.scorer_id === scorer.id)} + {@const headline = columnHeadline(scorer, mean)} + + +
+ + {#if scorer.kind === 'agent'} + + {:else} + + {/if} + {scorerLabel(scorer)} + + + {#if headline} + + {headline.value} + + {#if headline.delta && headline.direction !== 0} + 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} + > + {headline.delta} - {#if headline.delta && headline.direction !== 0} - 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} - > - {headline.delta} - - {/if} {/if} - -
- - {/each} + {/if} + +
+ + {/each} + + + + {#if datasetLoading} + + + + + - - + {:else} {#each displayRows as row (row.case_id)} {@const status = statusOf(row.status)} {/each} - - - {/if} + {/if} + +
- {#if selectedRow} - {@const openRow = selectedRow} - + + + {#if openRow}
-
- + +
+ {openRow.input?.user_message ?? caseLabel(openRow)} -
- {#if openRow.job_id} - - Open the case job - - - {/if} -
-
+ +
{#if openRow.expected != undefined && openRow.expected !== ''} - +
+
+ Expected +
+
+ + {typeof openRow.expected === 'string' + ? openRow.expected + : JSON.stringify(openRow.expected, null, 2)} + +
+
{/if} {#if scorers.length > 0 && openRow.scores.length > 0} -
-
+{/snippet} { - if (await useDataset(path)) { - resumeRunDialog = true - datasetDrawer?.openDrawer('edit') - } - }} - onNewDataset={() => { - resumeRunDialog = true - datasetDrawer?.openDrawer('new') + if (await useDataset(path)) datasetDrawer?.openDrawer('edit') }} + onNewDataset={() => datasetDrawer?.openDrawer('new')} /> { - if (!resumeRunDialog) return - resumeRunDialog = false - // On the dataset the drawer was just in: the dialog opens on the pane's own, which - // creating or editing one has already moved to it. - runDialogOpen = true - }} /> + + diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css deleted file mode 100644 index 1ac9f8b31c..0000000000 --- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css +++ /dev/null @@ -1,26 +0,0 @@ -/* MultilineCellEditor: a popup positioned over the cell, so it has to paint the cell's own frame - rather than inherit it. */ -.ag-theme-alpine .wm-multiline-cell-editor, -.ag-theme-alpine-dark .wm-multiline-cell-editor { - background-color: var(--ag-background-color); -} -.ag-theme-alpine .wm-multiline-cell-editor textarea, -.ag-theme-alpine-dark .wm-multiline-cell-editor textarea { - display: block; - box-sizing: border-box; - /* Horizontal only: the vertical padding is set by the editor, which knows the height of the row - it is replacing. `line-height` here is what it computes against. */ - padding: 0 calc(var(--ag-cell-horizontal-padding) - 1px); - border: 1px solid var(--ag-input-focus-border-color); - border-radius: 3px; - outline: none; - resize: none; - /* Past this it scrolls rather than growing. */ - max-height: 40vh; - overflow-y: auto; - background-color: var(--ag-background-color); - color: var(--ag-foreground-color); - font: inherit; - line-height: 20px; - white-space: pre-wrap; -} diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts deleted file mode 100644 index 00975955c9..0000000000 --- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts +++ /dev/null @@ -1,108 +0,0 @@ -import type { ColDef, ICellEditorComp, ICellEditorParams } from 'ag-grid-community' -// Beside the editor rather than in the AgGrid theme: that file is the vendored theme, and a rule -// added to it is one the next copy of it drops. -import './multilineCellEditor.css' - -/** Kept in step with the `line-height` the stylesheet gives the textarea. */ -const LINE_HEIGHT = 20 - -/** - * A text cell editor that starts the height of the cell and grows as lines are added, for columns - * holding prose rather than a value. Enter commits, Shift+Enter adds a line, Escape cancels. - * - * Rendered as a popup positioned over the cell: an in-cell editor is clipped to the row height, so - * growing is only visible if the editor is allowed to paint outside it. - */ -export class MultilineCellEditor implements ICellEditorComp { - private eGui!: HTMLDivElement - private textarea!: HTMLTextAreaElement - private params!: ICellEditorParams - private wasEmpty = false - - init(params: ICellEditorParams) { - this.params = params - this.eGui = document.createElement('div') - this.eGui.className = 'wm-multiline-cell-editor' - - this.wasEmpty = params.value == undefined - - this.textarea = document.createElement('textarea') - this.textarea.rows = 1 - // A keystroke that opened the edit replaces the value, as it does in every other cell; F2 - // and double-click keep it to be edited. - this.textarea.value = params.eventKey?.length === 1 ? params.eventKey : (params.value ?? '') - this.textarea.style.width = `${params.column.getActualWidth() - 2}px` - // Padded so one line fills the cell it replaces and a second costs a line rather than a row. - // From the row rather than from `--ag-row-height`, which is the theme's figure and not - // necessarily this grid's. - const rowHeight = params.node.rowHeight ?? 28 - const padding = Math.max(0, (rowHeight - LINE_HEIGHT - 2) / 2) - this.textarea.style.paddingTop = `${padding}px` - this.textarea.style.paddingBottom = `${padding}px` - - this.textarea.addEventListener('input', () => this.resize()) - this.textarea.addEventListener('keydown', (e) => { - if (e.key === 'Escape') { - // Kept from whatever is around the grid: a grid in a drawer or a dialog is under a - // surface that closes on Escape, and leaving an edit is not asking to leave that. - e.preventDefault() - e.stopPropagation() - this.params.api.stopEditing(true) - return - } - if (e.key !== 'Enter' || e.isComposing) return - // Both branches keep the key from the grid, which ends the edit on Enter whether or not - // Shift is held: Shift+Enter falls through to the textarea's own newline, and plain Enter - // ends the edit here instead. - e.stopPropagation() - if (!e.shiftKey) { - e.preventDefault() - this.params.stopEditing() - } - }) - this.eGui.appendChild(this.textarea) - } - - private resize() { - this.textarea.style.height = 'auto' - this.textarea.style.height = `${this.textarea.scrollHeight}px` - } - - getGui() { - return this.eGui - } - - afterGuiAttached() { - this.resize() - this.textarea.focus() - // At the end rather than selected: a selection is a keystroke away from erasing the cell. - const end = this.textarea.value.length - this.textarea.setSelectionRange(end, end) - } - - getValue() { - // Nothing typed into a cell that held nothing is not an edit: returning '' here would write - // an empty string over a null, which the grid would see as a change and commit. - if (this.wasEmpty && this.textarea.value === '') return this.params.value - return this.textarea.value - } - - isPopup() { - return true - } - - getPopupPosition(): 'over' | 'under' { - return 'over' - } -} - -/** - * What a column of prose needs, ready to spread into a colDef. `suppressKeyboardEvent` as well as - * the editor: the grid ends an edit on Enter from a handler a popup editor's DOM does not sit - * under, so the editor cannot keep Shift+Enter for itself on its own. - */ -export const multilineCellColDef: Pick = { - cellEditor: MultilineCellEditor, - suppressKeyboardEvent: (p) => - p.editing && (p.event as KeyboardEvent).key === 'Enter' && (p.event as KeyboardEvent).shiftKey -} diff --git a/frontend/src/lib/components/common/EditableTextarea.svelte b/frontend/src/lib/components/common/EditableTextarea.svelte new file mode 100644 index 0000000000..26a6574027 --- /dev/null +++ b/frontend/src/lib/components/common/EditableTextarea.svelte @@ -0,0 +1,173 @@ + + + +{#if editing} + + +{:else} + + +{/if} diff --git a/frontend/src/lib/components/common/drawer/Drawer.svelte b/frontend/src/lib/components/common/drawer/Drawer.svelte index d375bf506e..6bb5358940 100644 --- a/frontend/src/lib/components/common/drawer/Drawer.svelte +++ b/frontend/src/lib/components/common/drawer/Drawer.svelte @@ -8,6 +8,7 @@ import { onMount, createEventDispatcher, setContext, untrack } from 'svelte' import { BROWSER } from 'esm-env' import Disposable from './Disposable.svelte' + import { setTopmostSurface } from '$lib/components/common/overlayHost.svelte' import ConditionalPortal from './ConditionalPortal.svelte' import { chatState } from '$lib/components/copilot/chat/sharedChatState.svelte' import { useReducedMotion } from '$lib/svelte5Utils.svelte' @@ -51,6 +52,11 @@ let disposable: Disposable | undefined = $state(undefined) + // A drawer stacks like a dialog does, so content inside it gets the same answer about whether + // its keys are meant for it. Without this, a drawer opened over a dialog would inherit the + // dialog's answer — false, because the drawer itself is now on top — and go deaf. + setTopmostSurface(() => disposable?.isTopmost() ?? true) + let reducedMotion = useReducedMotion() let duration = $derived(reducedMotion.val ? 0 : _duration) let durationMs = $derived(duration * 1000) diff --git a/frontend/src/lib/components/common/emptyState/EmptyState.svelte b/frontend/src/lib/components/common/emptyState/EmptyState.svelte index 8715065ad0..49c3cd2b77 100644 --- a/frontend/src/lib/components/common/emptyState/EmptyState.svelte +++ b/frontend/src/lib/components/common/emptyState/EmptyState.svelte @@ -10,6 +10,16 @@ label: string icon?: any onClick: () => void + /** + * `default` unless the surface has no other live call to action. Accent is for the case + * where this button is the only thing to press — a form whose submit is disabled until + * this is done, say — so it is not competing with one. + */ + variant?: 'default' | 'accent' + /** Same write lock the surface's other controls take. An empty state is still a live + * control: without this it stays clickable while a request that has already read the + * empty list is in flight, and whatever it adds is discarded when that request lands. */ + disabled?: boolean aiId?: string aiDescription?: string } @@ -32,11 +42,13 @@ {/if}
{#if action} - +
diff --git a/frontend/src/lib/components/common/modal/PagedContent.svelte b/frontend/src/lib/components/common/modal/PagedContent.svelte new file mode 100644 index 0000000000..430a53e66f --- /dev/null +++ b/frontend/src/lib/components/common/modal/PagedContent.svelte @@ -0,0 +1,202 @@ + + + + + + + +
+ {#each pages as page, i (page.key)} + {#if visited.includes(page.key) || warmed} + + +
+ {@render page.content()} +
+ {/if} + {/each} +
+ + diff --git a/frontend/src/lib/components/common/overlayHost.svelte.ts b/frontend/src/lib/components/common/overlayHost.svelte.ts index 0e76709cbf..7c4374115b 100644 --- a/frontend/src/lib/components/common/overlayHost.svelte.ts +++ b/frontend/src/lib/components/common/overlayHost.svelte.ts @@ -72,3 +72,23 @@ export function overlayHostActive(): () => boolean { const host = getOverlayHost() return () => host?.active() ?? true } + +const TOPMOST_SURFACE_KEY = 'topmostSurface' + +/** + * Declare whether the surface enclosing this subtree is the one on top. Set by whatever owns the + * stacking — a dialog, a drawer — so content inside it can tell a key meant for itself from one + * meant for something opened over it. + */ +export function setTopmostSurface(isTopmost: () => boolean) { + setContext(TOPMOST_SURFACE_KEY, isTopmost) +} + +/** + * Whether the enclosing surface is on top. True when nothing declared otherwise, so content that + * is not inside such a surface is not silently made deaf. + */ +export function topmostSurface(): () => boolean { + const isTopmost = getContext<(() => boolean) | undefined>(TOPMOST_SURFACE_KEY) + return () => isTopmost?.() ?? true +} diff --git a/frontend/src/lib/components/text_input/TextInput.svelte b/frontend/src/lib/components/text_input/TextInput.svelte index c57f57120a..03fcedbd8e 100644 --- a/frontend/src/lib/components/text_input/TextInput.svelte +++ b/frontend/src/lib/components/text_input/TextInput.svelte @@ -81,6 +81,11 @@ size?: ButtonType.UnifiedSize unifiedHeight?: boolean underlyingInputEl?: UnderlyingInputElT + /** + * Passed to the `autosize` action on the `textarea` variant. Chiefly `minHeight: 0`, for a + * field that hugs one line instead of reserving the action's 30px floor. + */ + autosizeParams?: import('$lib/autosize').AutosizeParams } export function focus() { @@ -108,7 +113,8 @@ error, size = 'md', unifiedHeight = true, - underlyingInputEl: _underlyingInputEl + underlyingInputEl: _underlyingInputEl, + autosizeParams }: Props = $props() let underlyingInputEl = $derived(_underlyingInputEl ?? ('input' as const)) @@ -152,7 +158,7 @@ onpointerdown={(e) => e.stopImmediatePropagation()} bind:this={inputEl} bind:value - use:autosize + use:autosize={autosizeParams} > {:else if underlyingInputEl === 'input'} Date: Mon, 31 Aug 2026 20:09:53 +0200 Subject: [PATCH 2/3] feat: free AI tokens + home search/filter revamp (#10020) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add free Claude Opus tier with per-user token limit Co-Authored-By: Claude Opus 4.8 (1M context) * nit move alert * Home AI Chat * wire home ai chat * auto send prompt * refactor: remove keyboard arrow-navigation from home list Co-Authored-By: Claude Opus 4.8 (1M context) * feat: replace home search bar with unified FilterSearchbar Co-Authored-By: Claude Opus 4.8 (1M context) * feat: replace home quick tags with FilterSearchbar presets Co-Authored-By: Claude Opus 4.8 (1M context) * feat: add content filter to home FilterSearchbar with EE-gated content view - Clear the kind filter by deleting the key (was showing a 'kind: null' tag on All) - Remove the standalone Content button - Add a 'content' filter; when set, render the Ctrl-K content-search view (ContentSearchInner) which shows text-match snippets and its own EE warning Co-Authored-By: Claude Opus 4.8 (1M context) * feat: disable home AI chat and prompt to configure AI when no model Co-Authored-By: Claude Opus 4.8 (1M context) * track cost instead of tokens * nit * fix: load copilot config on home so AI chat isn't wrongly gated Co-Authored-By: Claude Opus 4.8 (1M context) * Home page update * nits * example prompts * nit * feat: switch free AI tier to DeepSeek with daily cost budgets Co-Authored-By: Claude Opus 4.8 (1M context) * nit * Move bottom buttons to HomeAIChat * [ee] feat: surface free AI tier state and make its metering abort-proof Makes the free Windmill AI tier legible to the user and closes an abuse hole. Backend: - AIConfig gains a response-only free_tier marker (skip_deserializing so a client can't store a forged one via edit_copilot_config). get_copilot_info keeps returning it once the grant is spent, so the client knows AI is off because the grant ran out, not because nothing was configured. - Per-user grant becomes one-time (migration drops the day key from ai_free_token_usage); the daily table stays as the instance kill-switch. - Reserve-then-reconcile metering (see EE commit) so a mid-stream disconnect can no longer dodge the usage report and get metered zero. Frontend: - copilotInfo carries freeTier; model settings show a "Free" pill and a usage meter that warns past 80%. - The home chat and the session chat show a dedicated "you've used your free Windmill AI, add your own API key" state instead of the generic "no provider configured" one. - A failed send re-fetches copilot_info so the exhausted state (and its banner) appears live, without a page reload. Bumps ee-repo-ref.txt to the matching EE commit. Co-Authored-By: Claude Opus 4.8 (1M context) * feat: free AI usage meter reusing the context-usage gauge Show free-tier spend with the same gauge as context usage instead of a bespoke block: - Extract the meter+tooltip into a shared UsageMeter; ContextUsageIndicator uses it, and a new FreeTierUsageIndicator renders it from copilotInfo.freeTier. Placed in the session-chat toolbar and next to the home-chat model settings; the old meter block in the model-settings dropdown is removed (the "Free" pill stays). - Hide the context-usage bar while on the free tier so the free meter takes that slot. - Refresh copilotInfo after every free-tier turn (AIChatManager finally) so the meter advances live and the turn that exhausts the grant flips to the exhausted state, instead of both only updating on reload. Gated to active free-tier users, so it costs nothing for configured-key users. Co-Authored-By: Claude Opus 4.8 (1M context) * docs: fix stale free-tier comments after DeepSeek/cost rework Co-Authored-By: Claude Opus 4.8 (1M context) * feat: always show context bar, replace free-tier meter with usage banner Co-Authored-By: Claude Opus 4.8 (1M context) * nit * fix: atomic free-tier budget reservation (ee ref + sqlx) Co-Authored-By: Claude Opus 4.8 (1M context) * fix: keep CLI/MCP and Hub buttons unblurred on AI chat hover Co-Authored-By: Claude Opus 4.8 (1M context) * Add back arrow nav * nit * nit * fix: three review P1s in the home AI chat & search - AIChatManager: refreshFreeTierUsage now bails unless the global copilot state still belongs to the completing manager's workspace, so a warm session finishing after a workspace switch can't reload its (background) workspace over the active one's models/client/copilotWorkspace. - HomeAIChat: block submission until the copilot config is loaded AND enabled (new `canSend`), so a prompt submitted during the unknown-config window isn't handed to a session that never sends it and silently lost. The disabled overlay still gates on config-loaded to avoid a flash. - ItemsList: the content-search reload effect now depends on $workspaceStore so content results follow the active workspace instead of showing the previous one's. Co-Authored-By: Claude Opus 4.8 (1M context) * [ee] fix: harden the three home-AI-chat/search P1s after deeper review Follow-up to the previous P1 commit; sharper review found the earlier guards insufficient: - refreshFreeTierUsage now compares against the most-recently-*requested* workspace (new copilotWorkspaceRequested in aiStore, set synchronously in loadCopilot), not the last-*resolved* one — otherwise a warm session finishing while a newer workspace's load is still in flight could win the monotonic token and restore its stale workspace over the one being loaded. - The content-search view is keyed by workspace ({#key $workspaceStore}) so a switch remounts ContentSearchInner; late in-flight responses from the previous workspace can no longer land in the new one's component. Backend (EE, via ee-repo-ref bump to 03ef0eb): the free-tier reservation now also prices the worst-case input cap (at the cache-miss rate), and enforce_free_tier_body rejects oversized prompts and pins n=1 — so an aborted large-prompt request can no longer dodge the input bill that reconciliation would otherwise charge. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: exclude service accounts from the free AI tier Free-tier eligibility was keyed solely on authed.email. Workspace admins can create and impersonate arbitrary service accounts (synthetic *.sa.wm.dev identities), each of which would receive its own one-time grant — letting one tenant mint many grants and drain the instance-wide daily allowance. Skip the free-tier fallback for *.sa.wm.dev identities. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: activate free AI tier when clearing a workspace provider edit_copilot_config returned AIConfig::default() when the saved workspace config had no providers and no instance config existed; the frontend applies that response immediately, disabling AI even though the free-tier key is available. A later get_copilot_info (on reload) returns the synthetic free-tier config, so clearing a provider behaved inconsistently until reload. Give this response path the same free-tier fallback as get_copilot_info. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: gate the home AI composer behind the global-AI dev flag The "Build with AI" composer starts a session and navigates to /sessions, which lives behind the same wm_dev_global_ai dev gate as the global AI chat. With the gate off (the default), /sessions renders only its gate message, SessionWrapper never mounts, and the queued prompt is silently dropped. Hide the home entry point behind isGlobalAiEnabled() so it isn't exposed before the sessions gate opens. Co-Authored-By: Claude Opus 4.8 (1M context) * [ee] chore: bump ee-repo-ref for deepseek-v4-flash price/model fix Points at the EE commit that pins deepseek-v4-flash and its real prices (pico-precision accounting). Co-Authored-By: Claude Opus 4.8 (1M context) * [ee] fix: provable byte bound for the free-tier input cap (ee-repo-ref) Bumps ee-repo-ref to the EE commit that caps the raw request body byte length directly (token_count <= byte_count is provable), replacing the unsafe body.len()/2 token estimate that high-entropy prompts could beat. Co-Authored-By: Claude Opus 4.8 (1M context) * nit isGlobalAiEnabled * empty commit * fix(frontend): address Codex review on free-tier / home filters - P1: home filters now sync from the URL reactively, so browser Back/Forward updates the chips, kind toggle and results (and clears keys dropped from the URL) instead of leaving them stale until the next filter edit. - Free-tier banner buttons drop deprecated Button props (size/color/border variant) for unifiedSize + a supported variant. - Condense refreshFreeTierUsage comments to a single race-condition constraint beside the guard. Co-Authored-By: Claude Opus 4.8 * fix(frontend): hide empty kind badge on draft-only scripts A draft-only script can carry an empty `kind`, which still isn't 'script' so the row rendered a blue badge whose only content was capitalize('') — an empty pill left of the "Draft only" badge. Guard the badge on a non-empty kind. Co-Authored-By: Claude Opus 4.8 * feat(frontend): animate home tree-view group expand/collapse Wrap each owner group's children in ResizeTransitionWrapper so height changes animate. A slide transition only animates the initial mount, but a freshly-opened owner fetches its rows and passes through a transient empty state before they land — the ResizeObserver animates that second growth too. Nested TreeViews inherit the wrapper's context and skip their own, so one observer per top-level owner animates the whole subtree. Co-Authored-By: Claude Opus 4.8 * feat(frontend): FilterSearchbar boolean auto-set and string-filter presets - A default-false boolean filter has only one useful value, so selecting it sets true immediately instead of opening a true/false picker. A default-true boolean (e.g. "Include library scripts") still shows the picker, where false is the meaningful choice — expressed via a new optional `default` on the schema. - A plain string filter now surfaces any presets targeting it (`:`) as suggestions once selected, integrated into menuItems so keyboard nav works — previously selecting e.g. "Owner" showed nothing. Co-Authored-By: Claude Opus 4.8 * feat(frontend): home page toolbar and content-filter revamp - "New" create-menu button (scripts/flows/apps/…) replaces the old Content button; the search bar moves to the right of the toggle group. - Restore the content filter dropped in a merge: a `content` searchbar filter swaps the list for the full-text ContentSearchInner view (EE), aligned flush with -mx-2. - Move the owner/group and label chips off the page into FilterSearchbar presets; ownerFilter/labelFilter now derive from the searchbar keys (data layer unchanged). - Move the list controls (select / tree view / expand-all / sort) inline into the top row between the toggle group and search bar; add margin above the list. - Beta tag on the home AI chat; a bit more bottom margin under it; tighten the gap between the admin/tutorial banners and the list. Co-Authored-By: Claude Opus 4.8 * fix(ai): pass the request body to the free-tier reservation Thread the prompt body into resolve_free_tier_credentials so the free tier can size its upfront reservation from the actual request length instead of a fixed worst case (EE c2e248b), fixing normal chats being rejected as "too large". Updates the OSS stub signature and bumps ee-repo-ref. Co-Authored-By: Claude Opus 4.8 * fix(frontend): gate home Create/Import menu on edit permissions The relocated CreateActionsMenu rendered unconditionally, so operators and users in workspaces protected from direct deployment saw create/import actions they can't use. Restore the original gate (!operator && showEditButtons, the latter from NoDirectDeployAlert). Co-Authored-By: Claude Opus 4.8 * fix(frontend): address Codex review on filter searchbar - P1: the boolean shortcut now goes through the same tag-insertion path as the normal branch, so it removes the typed search segment instead of leaving it as a stray free-text (_default_) term. - Mark the Runs `show_future_jobs` filter default: true so selecting it opens the picker (false is the meaningful choice) rather than being a no-op. - Home owner/label presets now emit the canonical `key:\ value` form so the applied-preset check matches after a reparse and can't re-offer a duplicate; update the suggestion extraction to strip the leading separator. - Replace deprecated Button props (size/spacingSize/color) on the relocated list controls with unifiedSize. - Fix stale comments: UsageMeter no longer claims a free-tier consumer; the home filter schema comment describes presets, not the removed ListFilters/label badges. Co-Authored-By: Claude Opus 4.8 * fix(frontend): boolean filter shortcut sets value canonically The round-1 shortcut baked `true` into the tag text, which merged into a following tag (e.g. `archived:\ truekind:\ flow`). Instead remove the typed segment, set the value, and reparse so the text is rebuilt canonically — no lingering free-text and no merge. Co-Authored-By: Claude Opus 4.8 * docs(ai): restate free-tier caller identity contract in the OSS stub; bump ee-repo-ref Co-Authored-By: Claude Opus 4.8 * fix(frontend): keep flanking tags separate when boolean shortcut drops a segment Joining `before`/`after` directly fused the tags a removed mid-segment sat between (e.g. `kind:\ flowsummary:\ bar`). Join with a space; reparse then canonicalizes. Also trims the comment to the essential constraint. Co-Authored-By: Claude Opus 4.8 * chore(ai): update sqlx cache for free-tier daily-day queries; bump ee-repo-ref The reserve/reconcile daily-usage queries now bind the reservation day (EE change); refresh their offline query cache and point ee-repo-ref at the EE commit. Co-Authored-By: Claude Opus 4.8 * fix(ai): activate free tier when instance ai_config has no provider An instance ai_config row won precedence just by existing, so an empty {} (valid via global settings / declarative config) suppressed the free-tier fallback and left AI disabled — even though build_copilot_settings_state already treats it as unconfigured. Apply the same has_providers() check to the instance config in the proxy and edit_copilot_config paths. Also refresh the sqlx cache for the reservation ceiling change and bump ee-repo-ref. Co-Authored-By: Claude Opus 4.8 * fix(frontend): migrate legacy Home filter URLs to the searchbar keys The old Home UI stored free-text in `search`, owner scope in `filter`, and could write `kind=all`; the generic searchbar sync uses `_default_`, `owner`, and a kind enum without `all`. Rewrite those params once before the sync reads the URL so shared/bookmarked links restore, and drop `kind=all` which would otherwise wedge later filter edits. Co-Authored-By: Claude Opus 4.8 * fix(ai): empty instance config in get_copilot_info; label user-disabled Home AI - get_copilot_info returned any existing instance ai_config row before the free-tier fallback, so an empty {} disabled AI in the copilot-info UI even though the proxy now serves the free tier. Apply the same has_providers() gate here. - The Home chat overlay said "No AI provider is configured" when the user had disabled AI in account settings (providers still present). Distinguish that state ("Windmill AI is disabled in your account settings") as the docked chat does, and drop the misleading workspace-config button in that case. Co-Authored-By: Claude Opus 4.8 * chore(ai): drop redundant proxy service-account check; trim TreeView comment The service-account exclusion now lives in the free-tier helper, so the proxy calls it directly. Also condense the tree-view resize-transition comment to the essential reason. Bumps ee-repo-ref. Co-Authored-By: Claude Opus 4.8 * docs(frontend): the Home content filter is not EE-gated ContentSearchInner loads the workspace's scripts/flows/apps/resources and matches their contents client-side, so it works on any instance. Drop the misleading "(EE)" from the filter label and the "EE indexer / off-EE fallback" comments. Co-Authored-By: Claude Opus 4.8 * chore(ee): bump ee-repo-ref for free-tier pricing + exhaustion fixes Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): show disabled Home AI overlay statically, not on hover The disabled-state overlay (reason + configure/add-key action) was opacity-0 and pointer-events-none until group-hover, so keyboard and touch users saw an inert composer with no visible remedy. Render it and the composer blur statically when disabled instead. Also bumps ee-repo-ref for the trimmed free-tier comments. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): give account-disabled Home AI overlay a recovery action The account-disabled branch showed a reason but hid every action, on the mistaken premise that account settings has no linkable route. It opens from the #user-settings hash (the same one the sidebar Account menu uses), so link there. Bumps ee-repo-ref for the free-tier fixes. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): gate Home AI composer for operators; a11y and filter-sync fixes - Home composer now uses prefersSessionHandoff($userStore?.operator) instead of isGlobalAiEnabled(): operators reached this route and could submit a prompt into a /sessions page that refuses them, silently dropping it. Also drops the leftover empty header spacer div above the chat. - HomeAIChat: mark the blurred/disabled subtrees inert so keyboard users can't tab into the unreadable textarea (pointer-events-none didn't stop Tab). - ItemsList: keep the role-dependent searchbar keys (include_library, only_user_folders) in the schema unconditionally and toggle `hidden` instead, so useUrlSyncedFilterInstance (which snapshots the key set once) still URL-syncs a key that first appears after a workspace switch. - Bumps ee-repo-ref for the indexer non-parquet build fix. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): keep CLI/MCP connect row for operators; trim filter comment The previous commit gated all of HomeAIChat behind the operator/session check, which also removed the AI-independent CLI/MCP "Connect workspace" drawer that operators (and the sessions-beta opt-out) had on main. Render HomeAIChat for the same audience as before (isGlobalAiEnabled) and gate only the composer (title, input, examples, overlay) on operator status inside the component; the connect row always shows. Also trims the role-dependent filter-schema comment to the <=4 line rule. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): reconnect Home keyboard navigation to the unified searchbar The searchbar migration replaced the the ItemsList keyboard handler keys off, so Arrow/Enter no longer drove the results list. Thread an `id` down to the searchbar's contenteditable (via TaggedTextInput/FilterSearchbar `inputId`) so the handler and the workspace-switch focus restoration find it again; read the caret through the Selection API instead of an 's selectionStart/End; and stand the list's arrows down while the searchbar's suggestion dropdown is open (tracked via onDropdownVisibleChange). In free-text mode the searchbar no longer opens its dropdown on a bare arrow key, so an empty box passes Arrow/Enter to the list as before. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): stop searchbar Enter inserting a newline; idle typewriter for operators - TaggedTextInput is a single-line filter input, so Enter now preventDefaults the contenteditable's newline insertion (surrounding suggestion-select / list-open handlers still run on bubble). Previously Enter with no row highlighted dropped a literal \n into the query. - HomeAIChat's placeholder typewriter effect now runs only while the composer is shown, so it no longer loops forever driving an unrendered input for operators. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * chore: update ee-repo-ref to f2a31156ac08ecb02d89dbc66d72be58e9c877ff This commit updates the EE repository reference after PR #652 was merged in windmill-ee-private. Previous ee-repo-ref: e59b96a2eea5d1110b40c842f17b337ab051bdd3 New ee-repo-ref: f2a31156ac08ecb02d89dbc66d72be58e9c877ff Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...64fc5e65995ce81e622c174a89befc1a527e5.json | 22 + ...23f37b8d452ea8e199ccc207da238368f1996.json | 23 + ...e6be4cc3adc6aa859c3dbaa57fa50b63741cf.json | 15 + ...fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json | 23 + ...7e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json | 15 + backend/ee-repo-ref.txt | 2 +- ...0260622072905_ai_free_token_usage.down.sql | 2 + .../20260622072905_ai_free_token_usage.up.sql | 19 + backend/summarized_schema.txt | 2 + backend/windmill-api/openapi.yaml | 18 + backend/windmill-api/src/ai.rs | 279 +++++---- backend/windmill-api/src/ai_free_tier_oss.rs | 65 +++ backend/windmill-api/src/lib.rs | 3 + backend/windmill-api/src/workspaces.rs | 58 +- frontend/src/lib/aiStore.ts | 13 +- .../src/lib/components/FilterSearchbar.svelte | 145 ++++- .../src/lib/components/TaggedTextInput.svelte | 11 +- .../lib/components/common/table/AppRow.svelte | 2 +- .../components/common/table/FlowRow.svelte | 2 +- .../components/common/table/RawAppRow.svelte | 2 +- .../lib/components/common/table/Row.svelte | 2 + .../components/common/table/ScriptRow.svelte | 6 +- .../lib/components/copilot/chat/AIChat.svelte | 28 +- .../copilot/chat/AIChatDisplay.svelte | 63 +++ .../copilot/chat/AIChatManager.svelte.ts | 30 +- .../copilot/chat/AIChatModelSettings.svelte | 14 + .../copilot/chat/ContextUsageIndicator.svelte | 28 +- .../components/copilot/chat/UsageMeter.svelte | 39 ++ .../src/lib/components/copilot/loadCopilot.ts | 9 + .../src/lib/components/home/HomeAIChat.svelte | 255 +++++++++ .../src/lib/components/home/ItemsList.svelte | 529 ++++++++++-------- .../lib/components/home/NoItemFound.svelte | 1 - .../src/lib/components/home/TreeView.svelte | 205 +++---- .../src/lib/components/runs/runsFilter.ts | 4 +- .../components/select/GenericDropdown.svelte | 15 +- .../sessions/sessionSwitch.svelte.ts | 6 +- .../src/routes/(root)/(logged)/+page.svelte | 67 +-- 37 files changed, 1457 insertions(+), 565 deletions(-) create mode 100644 backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json create mode 100644 backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json create mode 100644 backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json create mode 100644 backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json create mode 100644 backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json create mode 100644 backend/migrations/20260622072905_ai_free_token_usage.down.sql create mode 100644 backend/migrations/20260622072905_ai_free_token_usage.up.sql create mode 100644 backend/windmill-api/src/ai_free_tier_oss.rs create mode 100644 frontend/src/lib/components/copilot/chat/UsageMeter.svelte create mode 100644 frontend/src/lib/components/home/HomeAIChat.svelte diff --git a/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json new file mode 100644 index 0000000000..4ae74ff53c --- /dev/null +++ b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT cost_nanos FROM ai_free_token_usage WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5" +} diff --git a/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json new file mode 100644 index 0000000000..2253944269 --- /dev/null +++ b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, $1::bigint, now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = ai_free_token_daily_usage.cost_nanos + $1::bigint,\n updated_at = now()\n RETURNING cost_nanos", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Date" + ] + }, + "nullable": [ + false + ] + }, + "hash": "44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996" +} diff --git a/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json new file mode 100644 index 0000000000..a741ee1c0d --- /dev/null +++ b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, GREATEST(0, $1::bigint), now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_daily_usage.cost_nanos + $1::bigint),\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Date" + ] + }, + "nullable": [] + }, + "hash": "acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf" +} diff --git a/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json new file mode 100644 index 0000000000..7078c32f1a --- /dev/null +++ b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, $2::bigint, now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = ai_free_token_usage.cost_nanos + $2::bigint,\n updated_at = now()\n RETURNING cost_nanos", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956" +} diff --git a/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json new file mode 100644 index 0000000000..32f7e8cf1f --- /dev/null +++ b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, GREATEST(0, $2::bigint), now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_usage.cost_nanos + $2::bigint),\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 51788b0e1d..2bb93f6bd9 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -df60763d1f243b0048dfc3fe700bc026b257bea8 +f2a31156ac08ecb02d89dbc66d72be58e9c877ff diff --git a/backend/migrations/20260622072905_ai_free_token_usage.down.sql b/backend/migrations/20260622072905_ai_free_token_usage.down.sql new file mode 100644 index 0000000000..95879034e4 --- /dev/null +++ b/backend/migrations/20260622072905_ai_free_token_usage.down.sql @@ -0,0 +1,2 @@ +DROP TABLE ai_free_token_daily_usage; +DROP TABLE ai_free_token_usage; diff --git a/backend/migrations/20260622072905_ai_free_token_usage.up.sql b/backend/migrations/20260622072905_ai_free_token_usage.up.sql new file mode 100644 index 0000000000..ae0cf47e45 --- /dev/null +++ b/backend/migrations/20260622072905_ai_free_token_usage.up.sql @@ -0,0 +1,19 @@ +-- One-time grant of the Windmill-provided free AI tier, measured as cost in nano-dollars +-- (1e-9 USD) rather than raw tokens — a prompt-cache hit costs a fraction of a fresh input +-- token, so a token count wildly overstates the real bill. The grant never resets: once +-- spent, the user must bring their own API key. Keyed by normalized email so the allowance +-- is shared across a user's workspaces (and is resistant to +tag / gmail-dot aliasing). +CREATE TABLE ai_free_token_usage ( + email VARCHAR(255) PRIMARY KEY, + cost_nanos BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Instance-wide daily cost ceiling (nano-dollars) for the free tier — a kill-switch +-- independent of the per-user grant, bounding the blast radius of a bad day. One row per +-- UTC day. +CREATE TABLE ai_free_token_daily_usage ( + day DATE PRIMARY KEY, + cost_nanos BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 607ef99e0c..dfd0305e57 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -38,6 +38,8 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien FK: (workspace_id) -> workspace(id) agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char) ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) +ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts) +ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts) ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 09a3cf4195..31f0045ce0 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -27256,11 +27256,29 @@ components: type: integer minimum: 1 maximum: 2000000 + free_tier: + $ref: "#/components/schemas/FreeTierInfo" model_pricing: type: object additionalProperties: $ref: "#/components/schemas/ModelPriceOverride" + FreeTierInfo: + type: object + description: >- + Read-only. Present when the workspace has no AI provider of its own and is running + on Windmill's free tier. Ignored on write. + properties: + exhausted: + type: boolean + description: The one-time grant is spent; no provider is served and the user must add their own API key. + used_ratio: + type: number + description: Fraction of the grant consumed, 0 to 1. + required: + - exhausted + - used_ratio + ModelPriceOverride: type: object description: negotiated rates in USD per million tokens, keyed `provider:model` diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 1a5f9dba87..c684830b27 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -409,6 +409,19 @@ impl ExpiringProviderCredentials { } } +/// Set on the copilot config when the workspace has no AI provider of its own and is +/// running on Windmill's free tier, so the client can label the lent model as free, warn +/// before the grant runs out, and tell the user to add their own key once it has — rather +/// than showing the same "no provider configured" state a never-configured workspace gets. +#[derive(Serialize, Deserialize, Debug, Default, Clone)] +pub struct FreeTierInfo { + /// The grant is spent: no provider is served and the user must bring their own key. + pub exhausted: bool, + /// Fraction of the grant consumed, 0.0..=1.0. A ratio, not a dollar amount — the + /// pricing model stays server-side. + pub used_ratio: f64, +} + #[derive(Serialize, Deserialize, Debug, Default)] pub struct AIConfig { #[serde(skip_serializing_if = "Option::is_none")] @@ -423,6 +436,11 @@ pub struct AIConfig { pub custom_prompts: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens_per_model: Option>, + /// Response-only: this same struct is the request body for saving a workspace's AI + /// config, and `skip_deserializing` is what stops a client from storing a forged + /// free-tier marker. Only the server sets it, per-request. + #[serde(skip_serializing_if = "Option::is_none", skip_deserializing)] + pub free_tier: Option, /// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`. /// Only models whose rates differ from the built-in table are stored. #[serde(skip_serializing_if = "Option::is_none")] @@ -1013,83 +1031,119 @@ async fn proxy( check_scopes(&authed, || format!("resources:read:{}", resource_path))?; } - let mut credentials = match workspace_cache { - Some(request_cache) if !request_cache.is_expired() && forced_resource_path.is_none() => { - request_cache.credentials - } - _ => { - let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) = - if let Some(resource_path) = forced_resource_path { - // forced resource path - (resource_path, false, w_id.clone(), None) - } else { - let workspace_ai_config = sqlx::query_scalar!( - "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; + // Set when serving the request through Windmill's free AI tier (the lent key). Holds + // the per-user concurrency lock and drives response metering. + let mut free_lease: Option = None; + let mut credentials = 'cred: { + match workspace_cache { + Some(request_cache) + if !request_cache.is_expired() && forced_resource_path.is_none() => + { + request_cache.credentials + } + _ => { + let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) = + if let Some(resource_path) = forced_resource_path { + // forced resource path + (resource_path, false, w_id.clone(), None) + } else { + let workspace_ai_config = sqlx::query_scalar!( + "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; - let (ai_config_value, resource_workspace, instance_ai_config_revision) = { - let ws_has_config = workspace_ai_config - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .is_some_and(|config| config.has_providers()); + let (ai_config_value, resource_workspace, instance_ai_config_revision) = { + let ws_has_config = workspace_ai_config + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .is_some_and(|config| config.has_providers()); - if ws_has_config { - (workspace_ai_config.unwrap(), w_id.clone(), None) - } else { - let instance_config = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = 'ai_config'" - ) - .fetch_optional(&db) - .await?; + if ws_has_config { + (workspace_ai_config.unwrap(), w_id.clone(), None) + } else { + let instance_config = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = 'ai_config'" + ) + .fetch_optional(&db) + .await?; - match instance_config { - Some(config) => ( - config, - "admins".to_string(), - Some(current_instance_ai_config_revision()), - ), - None => { - return Err(Error::internal_err( - "AI resource not configured".to_string(), - )); + let instance_has_config = + instance_config.as_ref().is_some_and(|v| { + serde_json::from_value::(v.clone()) + .ok() + .is_some_and(|c| c.has_providers()) + }); + match instance_config { + // An instance `ai_config` row with no usable provider (e.g. `{}` + // or `{"providers":{}}`) is treated as unconfigured, exactly as + // build_copilot_settings_state does — otherwise its mere presence + // would suppress the free-tier fallback below. + Some(config) if instance_has_config => ( + config, + "admins".to_string(), + Some(current_instance_ai_config_revision()), + ), + _ => { + // Nothing configured: fall back to Windmill's free AI tier + // (EE-only) if a lent key is set and both the user's + // one-time grant and the instance's daily cap have room. + // Errors once the grant is spent, the day is capped, or the + // user already has a request in flight; None otherwise. + // Ineligible identities (e.g. service accounts) are refused + // inside the helper, so every path treats them alike. + let free = + crate::ai_free_tier_oss::resolve_free_tier_credentials( + &provider, + &db, + &ai_path, + &authed.email, + &body, + ) + .await?; + if let Some((free_credentials, lease)) = free { + free_lease = Some(lease); + break 'cred free_credentials; + } + return Err(Error::internal_err( + "AI resource not configured".to_string(), + )); + } } } + }; + + let mut ai_config = serde_json::from_value::(ai_config_value) + .map_err(|e| Error::BadRequest(e.to_string()))?; + + let provider_config = ai_config + .providers + .as_mut() + .and_then(|providers| providers.remove(&provider)) + .ok_or_else(|| { + Error::BadRequest(format!("Provider {:?} not configured", provider)) + })?; + + if provider_config.resource_path.is_empty() { + return Err(Error::BadRequest("Resource path is empty".to_string())); } + + ( + provider_config.resource_path, + true, + resource_workspace, + instance_ai_config_revision, + ) }; - let mut ai_config = serde_json::from_value::(ai_config_value) - .map_err(|e| Error::BadRequest(e.to_string()))?; - - let provider_config = ai_config - .providers - .as_mut() - .and_then(|providers| providers.remove(&provider)) - .ok_or_else(|| { - Error::BadRequest(format!("Provider {:?} not configured", provider)) - })?; - - if provider_config.resource_path.is_empty() { - return Err(Error::BadRequest("Resource path is empty".to_string())); - } - - ( - provider_config.resource_path, - true, - resource_workspace, - instance_ai_config_revision, - ) - }; - - // For user-specified resources, fetch through an RLS-scoped - // connection so PostgreSQL row-level security enforces the same - // folder/group boundaries as the regular resource API. For the - // workspace/instance ai_config path, the resource_path was already - // validated by an admin/devops user when configuring the workspace, - // so the raw pool is used. - let resource = if is_user_specified_resource { + // For user-specified resources, fetch through an RLS-scoped + // connection so PostgreSQL row-level security enforces the same + // folder/group boundaries as the regular resource API. For the + // workspace/instance ai_config path, the resource_path was already + // validated by an admin/devops user when configuring the workspace, + // so the raw pool is used. + let resource = if is_user_specified_resource { let mut tx = user_db.clone().begin(&authed).await?; let res = sqlx::query_scalar::<_, Option>>>( "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", @@ -1112,38 +1166,45 @@ async fn proxy( .ok_or_else(|| Error::NotFound(format!("Could not find the resource {}, update the resource path in the workspace settings", resource_path)))? .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", resource_path)))?; - let resource = serde_json::from_str::(resource.0.get()) - .map_err(|e| Error::BadRequest(e.to_string()))?; + let resource = serde_json::from_str::(resource.0.get()) + .map_err(|e| Error::BadRequest(e.to_string()))?; - // Enforce RLS on $var: resolution when the resource path was - // user-specified (X-Resource-Path header) so users can only read - // variables they have permission to access. - let enforce_authed = if is_user_specified_resource { - Some(&authed) - } else { - None - }; - let credentials = resolve_provider_credentials( - &provider, - &db, - &resource_workspace, - resource, - enforce_authed, - ) - .await?; - if save_to_cache { - AI_REQUEST_CACHE.insert( - (w_id.clone(), provider.clone()), - ExpiringProviderCredentials::new( - credentials.clone(), - instance_ai_config_revision, - ), - ); + // Enforce RLS on $var: resolution when the resource path was + // user-specified (X-Resource-Path header) so users can only read + // variables they have permission to access. + let enforce_authed = if is_user_specified_resource { + Some(&authed) + } else { + None + }; + let credentials = resolve_provider_credentials( + &provider, + &db, + &resource_workspace, + resource, + enforce_authed, + ) + .await?; + if save_to_cache { + AI_REQUEST_CACHE.insert( + (w_id.clone(), provider.clone()), + ExpiringProviderCredentials::new( + credentials.clone(), + instance_ai_config_revision, + ), + ); + } + credentials } - credentials } }; + // Free tier: pin the model and clamp max_tokens server-side before forwarding, + // since the request body is otherwise client-controlled. + if free_lease.is_some() { + body = crate::ai_free_tier_oss::enforce_free_tier_body(&body)?; + } + if let Some(fim_transform) = maybe_transform_fim_request(&provider, &ai_path, &credentials.base_url, &body)? { @@ -1291,8 +1352,32 @@ async fn proxy( let status_code = response.status(); let headers = response.headers().clone(); + let is_sse = is_sse_response(&headers); + + // Free tier: reconcile the cost reserved up-front against what the response actually + // used, holding the per-user lock (via the lease) until it is recorded. The chat + // streams (SSE), where the usage report only arrives in the final chunk; the + // non-streaming JSON path is handled for completeness. + if let Some(lease) = free_lease { + let body = if is_sse { + axum::body::Body::from_stream(inject_keepalives( + Box::pin(crate::ai_free_tier_oss::meter_usage( + response.bytes_stream(), + db.clone(), + lease, + )), + Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )) + } else { + let bytes = response.bytes().await.map_err(to_anyhow)?; + crate::ai_free_tier_oss::record_json_usage(db.clone(), lease, &bytes); + axum::body::Body::from(bytes) + }; + return Ok((status_code, headers, body)); + } + let stream = response.bytes_stream(); - let body = if is_sse_response(&headers) { + let body = if is_sse { axum::body::Body::from_stream(inject_keepalives( stream, Duration::from_secs(KEEPALIVE_INTERVAL_SECS), diff --git a/backend/windmill-api/src/ai_free_tier_oss.rs b/backend/windmill-api/src/ai_free_tier_oss.rs new file mode 100644 index 0000000000..7a183ff8ed --- /dev/null +++ b/backend/windmill-api/src/ai_free_tier_oss.rs @@ -0,0 +1,65 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ai_free_tier_ee::*; + +// Open-source build: Windmill's free AI tier does not exist. These stubs make the +// callers in `ai.rs` / `workspaces.rs` compile while disabling the feature entirely — +// `resolve_free_tier_credentials` never opts in, so the proxy falls through to its +// normal "AI resource not configured" path and the copilot stays hidden. +// +// Caller contract (enforced by the private impl, restated here for parity): the `email` +// passed to `resolve_free_tier_credentials` / `free_tier_copilot_config` MUST be the +// authenticated caller's own identity (an `ApiAuthed` email), never a client-supplied one — +// it selects whose lent-key grant is spent and whose usage is read. + +#[cfg(not(feature = "private"))] +use crate::ai::AIConfig; +#[cfg(not(feature = "private"))] +use crate::db::DB; +#[cfg(not(feature = "private"))] +use axum::body::Bytes; +#[cfg(not(feature = "private"))] +use windmill_ai::ai_providers::AIProvider; +#[cfg(not(feature = "private"))] +use windmill_ai::credentials::ProviderCredentials; +#[cfg(not(feature = "private"))] +use windmill_common::error::Result; + +#[cfg(not(feature = "private"))] +pub struct FreeTierLease; + +#[cfg(not(feature = "private"))] +pub async fn resolve_free_tier_credentials( + _provider: &AIProvider, + _db: &DB, + _ai_path: &str, + _email: &str, + _body: &Bytes, +) -> Result> { + Ok(None) +} + +#[cfg(not(feature = "private"))] +pub fn enforce_free_tier_body(body: &Bytes) -> Result { + Ok(body.clone()) +} + +#[cfg(not(feature = "private"))] +pub async fn free_tier_copilot_config(_db: &DB, _email: &str) -> Result> { + Ok(None) +} + +#[cfg(not(feature = "private"))] +pub fn record_json_usage(_db: DB, _lease: FreeTierLease, _bytes: &[u8]) {} + +#[cfg(not(feature = "private"))] +pub fn meter_usage( + upstream: S, + _db: DB, + _lease: FreeTierLease, +) -> impl futures::Stream> +where + S: futures::Stream> + Unpin, +{ + upstream +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index de49949043..fc98020d1b 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -66,6 +66,9 @@ use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; mod ai; +#[cfg(feature = "private")] +mod ai_free_tier_ee; +mod ai_free_tier_oss; mod ai_skills; mod apps; mod apps_raw_bundle; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 01eb32b7f6..76378e335a 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -18,7 +18,6 @@ use crate::teams_oss::{ connect_teams, edit_teams_command, run_teams_message_test_job, workspaces_list_available_teams_channels, workspaces_list_available_teams_ids, }; - use axum::{ extract::{Extension, Path}, routing::{get, post}, @@ -153,10 +152,23 @@ async fn edit_copilot_config( .await?; let settings_state = build_copilot_settings_state(workspace_has_config, instance_ai_config.as_ref()); + // A provider-less instance config (e.g. `{}`) is unconfigured, same as build_copilot_settings_state + // treats it — so it must not shadow the free-tier fallback here either. + let instance_config_with_providers = instance_ai_config + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .filter(|c| c.has_providers()); let effective_ai_config = if workspace_has_config { ai_config - } else if let Some(instance_ai_config) = instance_ai_config { - serde_json::from_value::(instance_ai_config).unwrap_or_default() + } else if let Some(instance_config) = instance_config_with_providers { + instance_config + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? + { + // Same fallback as get_copilot_info: with nothing configured, surface Windmill's free + // tier (EE-only) so clearing a workspace provider activates it immediately, instead of + // returning an empty config that disables AI until the next page reload re-fetches it. + free_config } else { AIConfig::default() }; @@ -179,6 +191,7 @@ struct EditCopilotConfigResponse { } async fn get_copilot_info( + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, ) -> JsonResult { @@ -194,16 +207,25 @@ async fn get_copilot_info( )) })?; - if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { - Ok(Json(workspace_ai_config.0)) - } else if let Some(instance_config) = + let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) .await? + .and_then(|v| serde_json::from_value::(v).ok()) + // A provider-less instance config (e.g. `{}`) is unconfigured; don't let it shadow the + // free-tier fallback, matching the proxy and edit_copilot_config paths. + .filter(|c| c.has_providers()); + if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { + Ok(Json(workspace_ai_config.0)) + } else if let Some(instance_config) = instance_config { + Ok(Json(instance_config)) + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? { - Ok(Json( - serde_json::from_value::(instance_config).unwrap_or_default(), - )) + // Nothing configured: fall back to Windmill's free tier (EE-only). The config + // carries a `free_tier` marker even once the user's grant is spent — with no + // providers, but telling the client *why* AI is off. + Ok(Json(free_config)) } else { Ok(Json(AIConfig::default())) } @@ -216,7 +238,14 @@ pub async fn get_critical_alerts( authed: ApiAuthed, Query(params): Query, ) -> JsonResult { - require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, authed.job_id.is_some(), &db).await?; + require_admin_or_devops( + authed.is_admin, + &authed.username, + &authed.email, + authed.job_id.is_some(), + &db, + ) + .await?; crate::utils::get_critical_alerts(db, params, Some(w_id)).await } @@ -232,7 +261,14 @@ pub async fn acknowledge_critical_alert( Path((w_id, id)): Path<(String, i32)>, authed: ApiAuthed, ) -> Result { - require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, authed.job_id.is_some(), &db).await?; + require_admin_or_devops( + authed.is_admin, + &authed.username, + &authed.email, + authed.job_id.is_some(), + &db, + ) + .await?; crate::utils::acknowledge_critical_alert(db, Some(w_id), id).await } diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 0955f68ee5..2d6a98c625 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -7,6 +7,7 @@ import { type AIProviderModel, type AIProvider, type AIConfig, + type FreeTierInfo, type ModelPriceOverride } from './gen' import { @@ -49,6 +50,10 @@ export const copilotInfo = writable<{ /** Negotiated rates per `provider:model`, overriding the built-in price table. */ modelPricing?: Record webSearchEnabledProviders?: Partial> + // Set only when the workspace has no AI provider of its own and is running on + // Windmill's free tier. `exhausted` means the grant is spent: there is no model, but + // that is a different state from "never configured" and the UI must say so. + freeTier?: FreeTierInfo }>({ enabled: false, codeCompletionModel: undefined, @@ -132,8 +137,9 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + webSearchEnabledProviders, modelPricing: aiConfig.model_pricing ?? {}, - webSearchEnabledProviders + freeTier: aiConfig.free_tier }) } else { copilotSessionModel.set(undefined) @@ -146,8 +152,11 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + webSearchEnabledProviders: {}, modelPricing: {}, - webSearchEnabledProviders: {} + // An exhausted free grant lands here — no providers, but the reason AI is off + // is "you used it up", not "you never set it up". + freeTier: aiConfig.free_tier }) } } diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 5078e0a83b..51026f10fd 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -9,6 +9,11 @@ type: 'string' | 'number' | 'boolean' allowMultiple?: boolean format?: 'json' + /** Boolean only: the value the filter holds while unset (defaults to false). + * Selecting a filter whose default is false sets it to true immediately rather + * than opening a true/false picker whose only useful choice is true. A default-true + * boolean still opens the picker, since choosing false is the meaningful action. */ + default?: boolean } | { type: 'date' @@ -121,16 +126,34 @@ // Create the filter instance object const filterInstance: { val: Partial> } = $state({ val: {} }) - // Sync URL params to filter instance on initialization and when URL changes + // Sync URL params to filter instance, reactively. Reading urlFilter[key] tracked + // means browser Back/Forward — which mutates useSearchParams' cells on popstate — + // flows into the instance (chips, kind toggle, results), not just the first render. + // The write happens untracked so it can't self-trigger, and the equality check plus + // the reverse effect's own guard keep the two directions from ping-ponging. for (const key of Object.keys(schemaRec)) { - let urlValue = urlFilter[key] - if (schemaRec[key].type === 'date' && typeof urlValue === 'string') { - const d = new Date(urlValue) - urlValue = isNaN(d.getTime()) ? null : d - } - if (urlValue !== undefined && urlValue !== null) { - ;(filterInstance.val as any)[key] = urlValue - } + $effect(() => { + let urlValue = urlFilter[key] + if (schemaRec[key].type === 'date' && typeof urlValue === 'string') { + const d = new Date(urlValue) + urlValue = isNaN(d.getTime()) ? null : d + } + untrack(() => { + const current = (filterInstance.val as any)[key] + const same = + urlValue instanceof Date && current instanceof Date + ? urlValue.getTime() === current.getTime() + : current === (urlValue ?? undefined) + if (same) return + if (urlValue !== undefined && urlValue !== null) { + ;(filterInstance.val as any)[key] = urlValue + } else if (current !== undefined) { + // Key dropped from the URL (Back to a state without it): clear it so a + // stale chip / filter doesn't linger against the navigated-to URL. + delete (filterInstance.val as any)[key] + } + }) + }) } // Sync filter instance changes back to URL params @@ -275,6 +298,17 @@ class?: string placeholder?: string autofocus?: boolean + // Applied as the id of the underlying editable, so a parent can focus it or recognise its + // key events by id (the searchbar is a contenteditable, not an ). + inputId?: string + // Free-text mode: while the input holds only free text (no specific filter tag is + // being edited and no non-default filter is set), suppress the suggestions dropdown + // so it behaves like a plain search box. This frees the arrow keys for the + // surrounding UI (e.g. a results list). The dropdown returns the moment a specific + // filter is present (e.g. `path: u/me/abc`). + hideDropdownOnFreeText?: boolean + // Notified whenever the dropdown's effective visibility changes + onDropdownVisibleChange?: (visible: boolean) => void } type SchemaT = FilterSchemaRec // TODO: Generic @@ -284,7 +318,10 @@ presets: _presets = [], class: className, placeholder = 'Filter...', - autofocus + autofocus, + hideDropdownOnFreeText = false, + onDropdownVisibleChange, + inputId }: Props = $props() let _value = new DebouncedTempValue( @@ -298,6 +335,24 @@ let currentTag: keyof SchemaT | undefined = $state() let currentTextSegment = $state({ text: '', start: 0, end: 0 }) let open = $state(false) + + // A specific filter is in play when a tag is being edited or any non-free-text filter + // is set. + let hasSpecificFilter = $derived( + !!currentTag || Object.keys(value).some((k) => k !== '_default_') + ) + // A plain search term is being typed (free text, no specific filter). + let hasFreeText = $derived(!!String(value['_default_'] ?? '').trim()) + // Effective dropdown visibility. Free-text mode suppresses the dropdown ONLY while the + // user is typing a plain search term: it still opens when the input is empty (so the + // available filters stay discoverable) and whenever a specific filter is set or being + // edited. That leaves the arrow keys for the surrounding list only during free-text search. + let dropdownVisible = $derived( + open && (!hideDropdownOnFreeText || hasSpecificFilter || !hasFreeText) + ) + $effect(() => { + onDropdownVisibleChange?.(dropdownVisible) + }) let inputElement: HTMLDivElement | undefined = $state() let highlightedIndex = $state(0) let taggedTextInput: TaggedTextInput | undefined = $state() @@ -347,9 +402,17 @@ key, filterSchema, onClick: () => { - // Replace the text segment with the new filter tag const before = asText.val.slice(0, currentTextSegment.start) const after = asText.val.slice(currentTextSegment.end) + if (schema[key].type === 'boolean' && schema[key].default !== true) { + // Set the only useful value and reparse to canonical text. The space is + // required: dropping the segment must not fuse the tags that flanked it. + asText.val = `${before} ${after}` + value[key] = true as any + asText.reparse() + return + } + // Replace the text segment with the new (empty) filter tag; the value picker opens. asText.val = `${before}${before && !before.endsWith(' ') ? ' ' : ''}${key}:\\\u00A0${after}`.trim() + '\u00A0' @@ -406,6 +469,28 @@ onClick: () => setValueForCurrentTag(false) } ] + } else if (filter.type === 'string' && filter.format !== 'json') { + // A plain string filter has no fixed options, but any presets targeting this tag + // (`:`) are exactly its useful values — surface them as suggestions so + // picking one is a click, matching the top-level preset row. Unescape the tagged + // syntax's `\ ` back to a real space for the stored value. + const prefix = `${String(currentTag)}:` + const suffix = String(value[currentTag!] ?? '') + .trim() + .toLowerCase() + return _presets + .filter((p) => p.value.startsWith(prefix) && !asText.val.includes(p.value)) + .map((p) => { + const raw = p.value.slice(prefix.length).replace(/^\\ /, '').replace(/\\ /g, ' ') + return { name: p.name, raw } + }) + .filter((p) => !suffix || p.raw.toLowerCase().includes(suffix)) + .map((p) => ({ + type: 'option' as const, + option: { value: p.raw, label: p.name }, + onClick: () => appendOrSetValueForCurrentTag(p.raw), + onNegativeClick: undefined + })) } } return [] @@ -514,7 +599,9 @@ } function handleKeyDown(e: KeyboardEvent) { - if (!open) return + // In free-text mode the dropdown is hidden; let arrow/enter keys pass through to + // the surrounding UI (e.g. list navigation) rather than steering a hidden menu. + if (!dropdownVisible) return if (e.key === 'Escape') { open = false return @@ -601,6 +688,7 @@ > (open = true)} + onKeyDown={(e) => { + // In free-text mode the searchbar coexists with a list that owns Arrow/Enter, so opening + // the dropdown on a bare navigation key would steal them from an empty box. Typing, click, + // or an already-open dropdown still open/keep it. Other searchbars keep opening on any key. + if ( + !hideDropdownOnFreeText || + !['ArrowDown', 'ArrowUp', 'ArrowLeft', 'ArrowRight', 'Enter', 'Escape', 'Tab'].includes( + e.key + ) + ) { + open = true + } + }} {autofocus} /> {#if asText.val} @@ -630,9 +730,10 @@
inputElement?.getBoundingClientRect() ?? new DOMRect()} - innerClass="!max-h-[30rem]" + innerClass="!max-h-[25rem]" strictWidth > @@ -747,6 +848,20 @@ class="border border-border-light rounded min-h-[4rem]" />
+ {:else if filter.type === 'string'} + {#if menuItems.length} +
+ {#each menuItems as item, index} + {#if item.type === 'option' && item.option} + {@render menuItem({ + onClick: item.onClick, + label: item.option.label || item.option.value, + highlighted: index === highlightedIndex + })} + {/if} + {/each} +
+ {/if} {/if} {/snippet} diff --git a/frontend/src/lib/components/TaggedTextInput.svelte b/frontend/src/lib/components/TaggedTextInput.svelte index 23bbc65e6b..9f19195b85 100644 --- a/frontend/src/lib/components/TaggedTextInput.svelte +++ b/frontend/src/lib/components/TaggedTextInput.svelte @@ -8,6 +8,7 @@ onTextSegmentAtCursorChange, onKeyDown, autofocus, + id, class: className = '' }: { tags: { regex: RegExp; id: string; onClear?: () => void }[] @@ -18,6 +19,7 @@ onTextSegmentAtCursorChange?: (segment: { text: string; start: number; end: number }) => void onKeyDown?: (e: KeyboardEvent) => void autofocus?: boolean + id?: string class?: string } = $props() @@ -337,7 +339,13 @@ function handleKeyDown(e: KeyboardEvent) { onKeyDown?.(e) - if (e.key === 'ArrowDown' || e.key === 'ArrowUp' || e.key === 'Enter') return + // Single-line filter input: block Enter's default newline insertion. Surrounding handlers + // (suggestion select, list open) still run on bubble; only the contenteditable break is gone. + if (e.key === 'Enter') { + e.preventDefault() + return + } + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') return const cursorPos = getCursorPosition() const text = getTextContent() @@ -509,6 +517,7 @@
{#snippet badges()} diff --git a/frontend/src/lib/components/common/table/FlowRow.svelte b/frontend/src/lib/components/common/table/FlowRow.svelte index a155ca3a97..8d88fe8f01 100644 --- a/frontend/src/lib/components/common/table/FlowRow.svelte +++ b/frontend/src/lib/components/common/table/FlowRow.svelte @@ -132,13 +132,13 @@ : `${base}/flows/get/${flow.path}?workspace=${$workspaceStore}`} kind="flow" workspaceId={flow.workspace_id ?? $workspaceStore ?? ''} + {keyboardSelected} {marked} path={flow.draft_path ?? flow.path} summary={flow.is_draft ? `${flow.summary || flow.draft_path || flow.path}*` : flow.summary} {errorHandlerMuted} canFavorite={!flow.draft_only} {depth} - {keyboardSelected} {rowSelection} > {#snippet badges()} diff --git a/frontend/src/lib/components/common/table/RawAppRow.svelte b/frontend/src/lib/components/common/table/RawAppRow.svelte index 3f74356506..ff31eb1aea 100644 --- a/frontend/src/lib/components/common/table/RawAppRow.svelte +++ b/frontend/src/lib/components/common/table/RawAppRow.svelte @@ -35,13 +35,13 @@ {#snippet badges()} diff --git a/frontend/src/lib/components/common/table/Row.svelte b/frontend/src/lib/components/common/table/Row.svelte index 849d77e2d9..77434d3c59 100644 --- a/frontend/src/lib/components/common/table/Row.svelte +++ b/frontend/src/lib/components/common/table/Row.svelte @@ -13,6 +13,8 @@ interface Props { marked: string | undefined selected?: boolean + /** Highlighted by the list's keyboard arrow-navigation (distinct from `selected`, + * which is the checkbox multi-select state). Scrolls itself into view. */ keyboardSelected?: boolean disabled?: boolean canFavorite?: boolean diff --git a/frontend/src/lib/components/common/table/ScriptRow.svelte b/frontend/src/lib/components/common/table/ScriptRow.svelte index 25b557690e..de0a327b1b 100644 --- a/frontend/src/lib/components/common/table/ScriptRow.svelte +++ b/frontend/src/lib/components/common/table/ScriptRow.svelte @@ -147,6 +147,7 @@ ? `${base}/scripts/edit/${script.path}` : `${base}/scripts/get/${script.hash}?workspace=${$workspaceStore}`} kind="script" + {keyboardSelected} {marked} path={script.draft_path ?? script.path} summary={script.is_draft @@ -156,7 +157,6 @@ workspaceId={$workspaceStore ?? ''} canFavorite={!script.draft_only} {depth} - {keyboardSelected} {rowSelection} > {#snippet badges()} @@ -187,7 +187,9 @@ CI test {/if} - {#if script.kind !== 'script'} + + {#if script.kind && script.kind !== 'script'} {script.kind === 'failure' ? 'Error handler' : capitalize(script.kind)} diff --git a/frontend/src/lib/components/copilot/chat/AIChat.svelte b/frontend/src/lib/components/copilot/chat/AIChat.svelte index 2a51b1b2e5..e4dba770eb 100644 --- a/frontend/src/lib/components/copilot/chat/AIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChat.svelte @@ -51,20 +51,26 @@ aiChatManager.scriptEditorOptions?.lang && !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang)) ) + // A spent free grant is not an unconfigured workspace: AIChatDisplay already shows an + // in-thread banner naming the real cause and linking to the key settings, so the generic + // "enable Windmill AI" line would both duplicate it and misstate why the chat is off. + const freeTierExhausted = $derived($copilotInfo.freeTier?.exhausted === true) const disabledMessage = $derived( forceDisabled ? forceDisabledMessage - : !hasCopilot - ? $aiUserDisabled - ? 'Windmill AI is disabled in your account settings' - : isAdmin - ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` - : 'Ask an admin to enable Windmill AI in this workspace to use this chat' - : aiChatManager.mode === AIMode.SCRIPT && - aiChatManager.scriptEditorOptions?.lang && - !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) - ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` - : '' + : freeTierExhausted + ? '' + : !hasCopilot + ? $aiUserDisabled + ? 'Windmill AI is disabled in your account settings' + : isAdmin + ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` + : 'Ask an admin to enable Windmill AI in this workspace to use this chat' + : aiChatManager.mode === AIMode.SCRIPT && + aiChatManager.scriptEditorOptions?.lang && + !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) + ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` + : '' ) const suggestions = [ diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index c68945695b..2ab29499ce 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -15,6 +15,7 @@ Folder, Hand, HistoryIcon, + KeyRound, MousePointer2, Plug, Plus, @@ -59,9 +60,22 @@ readDroppedEntries } from './files/fsAccess' import { sendUserToast } from '$lib/toast' + import Alert from '$lib/components/common/alert/Alert.svelte' + import { copilotInfo } from '$lib/aiStore' + import { base } from '$lib/base' const MAX_YOLO_TOOLTIP_TOOLS = 8 const aiChatManager = getAiChatManager() + + // The user spent their one-time free Windmill AI grant: there is no model left to send + // to, so say so in the thread itself rather than only failing on send. + let freeTierExhausted = $derived($copilotInfo.freeTier?.exhausted === true) + // Still on the free grant: keep how much is left in view right above the composer, so + // running out isn't a surprise. Once spent, the exhausted banner replaces it. + let freeTier = $derived($copilotInfo.freeTier) + let freeTierUsedPct = $derived(Math.min(100, Math.round((freeTier?.used_ratio ?? 0) * 100))) + let showFreeTierUsage = $derived(!!freeTier && !freeTier.exhausted) + // One row per autonomy posture, in picker order, so adding one touches only this // table. `isAvailable` hides the postures that would do nothing in the current AI // mode, which is why the picker can be shorter than this list. @@ -562,6 +576,44 @@ ) +{#snippet freeTierExhaustedBanner()} +
+ +
+ + You have used all of your free Windmill AI tokens. Add your own API key to keep using AI. + + +
+ +
+{/snippet} + +{#snippet freeTierUsageBanner()} +
+ + {freeTierUsedPct}% of your free Windmill AI used + + +
+{/snippet} +
script editor to modify selected lines. {/if} + {#if freeTierExhausted} +
+ {@render freeTierExhaustedBanner()} +
+ {/if} {/if} {#if messages.length > 0} @@ -699,6 +756,9 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> isLast={messageIndex === messages.length - 1} /> {/each} + {#if freeTierExhausted} + {@render freeTierExhaustedBanner()} + {/if} {#if showTypingIndicator}
{#if inputPreface} {@render inputPreface()} {/if} + {#if showFreeTierUsage} + {@render freeTierUsageBanner()} + {/if} = 80) + let capability = $derived( getReasoningCapability(providerModel.provider as AIProvider, providerModel.model) ) @@ -312,6 +319,13 @@ {#if effortLabel} · {effortLabel} {/if} + {#if freeTier && !freeTier.exhausted} + Free + {/if}
diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index ffa06ca5df..96b452120b 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -3,7 +3,7 @@ import { getKnownModelContextWindow, getModelContextWindow } from '../modelConfig' import { getAiChatManager } from './aiChatManagerContext' import { AIMode } from './AIChatManager.svelte' - import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' + import UsageMeter from './UsageMeter.svelte' import { formatTokenCount } from './tokenUsage' const aiChatManager = getAiChatManager() @@ -49,25 +49,11 @@ {#if visible} - - -
-
-
-
-
- {#snippet text()} + + + {#snippet tooltip()}

Context usage

@@ -85,5 +71,5 @@ {/if}

{/snippet} - + {/if} diff --git a/frontend/src/lib/components/copilot/chat/UsageMeter.svelte b/frontend/src/lib/components/copilot/chat/UsageMeter.svelte new file mode 100644 index 0000000000..26e4758953 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/UsageMeter.svelte @@ -0,0 +1,39 @@ + + + +
+
+
+
+
+ {#snippet text()} + {@render tooltip()} + {/snippet} + diff --git a/frontend/src/lib/components/copilot/loadCopilot.ts b/frontend/src/lib/components/copilot/loadCopilot.ts index 4634833b0b..496fb89fdc 100644 --- a/frontend/src/lib/components/copilot/loadCopilot.ts +++ b/frontend/src/lib/components/copilot/loadCopilot.ts @@ -1,6 +1,14 @@ import { WorkspaceService } from '$lib/gen' import { copilotWorkspace, setCopilotInfo } from '$lib/aiStore' import { workspaceAIClients } from './lib' +import { writable } from 'svelte/store' + +// The workspace of the most recent loadCopilot *request*, set synchronously before the +// await — as opposed to `copilotWorkspace`, which only updates once a load resolves. A +// background refresh (e.g. free-tier usage) compares against this so it can't supersede an +// in-flight load for a newer workspace (which would otherwise win the token and restore +// stale state). +export const copilotWorkspaceRequested = writable(undefined) // Lives here, not in $lib/aiStore, purely so that module needs no import of the AI // client — it is the one thing that wanted both. Moving it back recreates the @@ -20,6 +28,7 @@ let loadCopilotToken = 0 let inFlight: { workspace: string; promise: Promise } | undefined export function loadCopilot(workspace: string): Promise { + copilotWorkspaceRequested.set(workspace) if (inFlight?.workspace === workspace) { return inFlight.promise } diff --git a/frontend/src/lib/components/home/HomeAIChat.svelte b/frontend/src/lib/components/home/HomeAIChat.svelte new file mode 100644 index 0000000000..d0e75538ca --- /dev/null +++ b/frontend/src/lib/components/home/HomeAIChat.svelte @@ -0,0 +1,255 @@ + + + + +
+
+ {#if showComposer} +
+
+

Build with AI

+ Beta +
+ +
+ + +
+ +
+
+
+ {/if} + +
+ {#if showComposer} +
+ {#each homeAIExamples as example (example.label)} + + {/each} +
+ {:else} +
+ {/if} + + +
+ + {#if !$userStore?.operator && HOME_SHOW_HUB} + + {/if} +
+
+ {#if showComposer && disabled} +
+

+ {#if $aiUserDisabled} + Windmill AI is disabled in your account settings + {:else if freeTierExhausted} + You have used all of your free Windmill AI tokens + {:else} + No AI provider is configured + {/if} +

+ {#if $aiUserDisabled} + + + {:else} + + {/if} +
+ {/if} +
+
+ + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index d7717d1e04..0f0cb6a6c0 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -1,7 +1,7 @@