diff --git a/ai_evals/README.md b/ai_evals/README.md index 6982d70da9..4804ad2543 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -142,6 +142,15 @@ For `global` mode, `validate` can express draft-level requirements such as: - required or forbidden draft counts - forbidden draft paths +Global initial fixtures can also seed `liveEditorDrafts` with `type`, +`storagePath`, `effectivePath`, and `value` fields. These drafts emulate the +currently open script, flow, or raw app editor so cases can test prompts that +refer to "this" or the "current" item. + +Set `WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT=1` to run those cases with +the old behavior where the live editor is only discoverable through +`list_workspace_items`. + App fixtures can also include an optional `datatables.json` file at the fixture root. For `flow` mode, an `initial` fixture can also include a benchmark workspace catalog of @@ -189,11 +198,15 @@ If `--record` is used, the CLI also appends one compact JSON line to: Each recorded line contains: - run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `judgeModel`) -- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averageJudgeScore`) -- average token usage (`averageTokenUsagePerAttempt`) -- per-case metrics under `cases[]` (`averageDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, pass rate) +- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averagePassedDurationMs`, `averageJudgeScore`) +- average token usage (`averageTokenUsagePerAttempt`, `averageTokenUsagePerPassedAttempt`) +- per-case metrics under `cases[]` (`averageDurationMs`, `averagePassedDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, `averageTokenUsagePerPassedAttempt`, pass rate) - `failedCaseIds` +The CLI headline duration and token averages use passed attempts only. +All-attempt averages are still recorded to make failures auditable without +letting failed attempts skew success cost comparisons. + Example: - summary: `ai_evals/results/2026-04-09T09-40-33.051Z__flow.json` diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index 5e00dd6f34..058adc3644 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -7,8 +7,12 @@ import { prepareGlobalSystemMessage, prepareGlobalUserMessage, } from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; -import { globalDraftStore } from "../../../../../frontend/src/lib/components/copilot/chat/global/draftStore.svelte"; +import { + clearGlobalDrafts, + listGlobalDrafts, +} from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import { UserDraft } from "../../../../../frontend/src/lib/userDraft.svelte"; import type { ModeRunContext } from "../../../../core/types"; import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; @@ -24,6 +28,21 @@ const MUTATING_GLOBAL_TOOLS = new Set([ "deploy_workspace_item", "delete_workspace_item", ]); +const DISABLE_ACTIVE_EDITOR_CONTEXT_ENV = + "WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT"; + +const LIVE_EDITOR_ITEM_KINDS = { + script: "script", + flow: "flow", + app: "raw_app", +} as const; + +export interface GlobalLiveEditorDraftFixture { + type: keyof typeof LIVE_EDITOR_ITEM_KINDS; + storagePath?: string; + effectivePath?: string; + value?: unknown; +} export interface GlobalEvalResult { success: boolean; @@ -38,6 +57,7 @@ export interface GlobalEvalResult { export interface GlobalEvalOptions { workspaceFixtures?: BenchmarkWorkspaceRunnables; + liveEditorDrafts?: GlobalLiveEditorDraftFixture[]; model?: string; maxIterations?: number; provider?: AIProvider; @@ -55,19 +75,26 @@ export async function runGlobalEval( options.workspaceRoot ?? (await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-"))); - globalDraftStore.clearDrafts(workspaceRoot); + clearGlobalDrafts(workspaceRoot); registerBenchmarkWorkspaceRunnables(workspaceRoot, options.workspaceFixtures ?? {}); + seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); try { const model = options.model ?? "claude-haiku-4-5-20251001"; + const injectActiveEditorContext = + process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1"; const rawResult = await runEval({ userPrompt, systemMessage: prepareGlobalSystemMessage(), - userMessage: prepareGlobalUserMessage(userPrompt), + userMessage: prepareGlobalUserMessage( + userPrompt, + [], + injectActiveEditorContext ? { workspace: workspaceRoot } : {}, + ), tools: getGlobalEvalTools(), helpers: {}, apiKey, - getOutput: () => ({ drafts: globalDraftStore.listDrafts(workspaceRoot) }), + getOutput: () => ({ drafts: listGlobalDrafts(workspaceRoot) }), onAssistantMessageStart: options.runContext?.onAssistantMessageStart, onAssistantToken: options.runContext?.onAssistantChunk, onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd, @@ -94,7 +121,8 @@ export async function runGlobalEval( tokenUsage: rawResult.tokenUsage, }; } finally { - globalDraftStore.clearDrafts(workspaceRoot); + clearGlobalDrafts(workspaceRoot); + clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); unregisterBenchmarkWorkspaceRunnables(workspaceRoot); if (!options.workspaceRoot) { await rm(workspaceRoot, { recursive: true, force: true }); @@ -102,6 +130,36 @@ export async function runGlobalEval( } } +function seedLiveEditorDrafts( + workspace: string, + fixtures: GlobalLiveEditorDraftFixture[], +): void { + for (const fixture of fixtures) { + const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type]; + const storagePath = fixture.storagePath ?? fixture.effectivePath ?? ""; + if (fixture.value !== undefined) { + UserDraft.save(itemKind, storagePath, fixture.value, { workspace }); + } + UserDraft.setLiveEditorDraft({ + workspace, + itemKind, + storagePath, + effectivePath: fixture.effectivePath ?? fixture.storagePath, + }); + } +} + +function clearLiveEditorDrafts( + workspace: string, + fixtures: GlobalLiveEditorDraftFixture[], +): void { + for (const fixture of fixtures) { + const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type]; + const storagePath = fixture.storagePath ?? fixture.effectivePath ?? ""; + UserDraft.clearLiveEditorDraft(itemKind, { workspace, storagePath }); + } +} + function getGlobalEvalTools(): ProductionTool<{}>[] { return (globalTools as ProductionTool<{}>[]).map((tool) => { if (!MUTATING_GLOBAL_TOOLS.has(tool.def.function.name)) { diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 1275acf0b4..92c43aab06 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -79,6 +79,16 @@ vi.mock('$lib/gen', async () => { } return actual.ScriptService.getScriptByPath(data) }, + getScriptByPathWithDraft: async (data: { workspace: string; path: string }) => { + if (hasBenchmarkWorkspace(data.workspace)) { + const script = getBenchmarkScriptByPath(data.workspace, data.path) + if (!script) { + throw new Error(`Script "${data.path}" not found in benchmark workspace`) + } + return script + } + return actual.ScriptService.getScriptByPathWithDraft(data) + }, getScriptByHash: async (data: { workspace: string; hash: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { const script = getBenchmarkScriptByHash(data.workspace, data.hash) @@ -108,6 +118,26 @@ vi.mock('$lib/gen', async () => { return flow } return actual.FlowService.getFlowByPath(data) + }, + getFlowByPathWithDraft: async (data: { workspace: string; path: string }) => { + if (hasBenchmarkWorkspace(data.workspace)) { + const flow = getBenchmarkFlowByPath(data.workspace, data.path) + if (!flow) { + throw new Error(`Flow "${data.path}" not found in benchmark workspace`) + } + return flow + } + return actual.FlowService.getFlowByPathWithDraft(data) + }, + getFlowLatestVersion: async (data: { workspace: string; path: string }) => { + if (hasBenchmarkWorkspace(data.workspace)) { + const flow = getBenchmarkFlowByPath(data.workspace, data.path) + if (!flow) { + throw new Error(`Flow "${data.path}" not found in benchmark workspace`) + } + return { id: 1 } + } + return actual.FlowService.getFlowLatestVersion(data) } }), JobService: wrapService(actual.JobService, { diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index b4526f8a85..36238e8ccd 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -87,3 +87,346 @@ - the flow accepts numeric inputs a and b - the flow returns the sum of a and b - the result stays as an AI draft and is not deployed or saved to the workspace + +- id: global-test4-multi-artifact-notification-job + prompt: |- + Set up a draft stale-trial notification job. + Create a Bun script at `f/evals/global/check_stale_trials` that accepts `max_age_days`, uses mocked inline trial account data, and returns the stale trial account IDs. + Also create a weekday 09:00 UTC schedule at `f/evals/global/check_stale_trials_weekday` for that script with `max_age_days` set to 14. + Add an HTTP POST trigger at `f/evals/global/check_stale_trials_manual` with route path `evals/check-stale-trials` that runs the same script manually. + Leave everything as AI drafts only; do not deploy or save anything to the workspace. + runtime: + maxTurns: 12 + validate: + draftCountExactly: 3 + requiredDrafts: + - type: script + path: f/evals/global/check_stale_trials + language: bun + valueIncludes: + - max_age_days + - trial + - type: schedule + path: f/evals/global/check_stale_trials_weekday + valueIncludes: + - f/evals/global/check_stale_trials + - UTC + - "14" + - type: trigger + triggerKind: http + path: f/evals/global/check_stale_trials_manual + valueIncludes: + - evals/check-stale-trials + - f/evals/global/check_stale_trials + toolExpect: + requiredToolsUsed: + - write_script + - write_schedule + - write_trigger + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - creates a Bun script draft for stale trial accounts + - creates a weekday 09:00 UTC schedule draft for the script with max_age_days set to 14 + - creates an HTTP POST trigger draft with route path evals/check-stale-trials for the same script + - leaves all artifacts as drafts only and does not deploy + +- id: global-test5-existing-flow-inline-code-edit + prompt: |- + Update the existing flow at `f/evals/global/process_invoice`. + Only change the `calculate_total` inline code so it applies 8% tax and returns an object containing `subtotal`, `tax`, and `total`. + Leave the updated flow as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + path: f/evals/global/process_invoice + valueIncludes: + - calculate_total + - tax + - total + toolExpect: + requiredToolsUsed: + - read_workspace_item + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - reads the existing process_invoice flow before editing it + - updates the calculate_total inline code to apply 8% tax + - returns subtotal, tax, and total from the updated flow logic + - leaves the result as an AI draft only + +- id: global-test6-secret-variable-draft + prompt: |- + Create a secret variable draft at `f/evals/global/slack_bot_token`. + Use the placeholder value `xoxb-redacted-test-token` and description `Slack bot token for eval notifications`. + Do not create any resource or deploy anything. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: variable + path: f/evals/global/slack_bot_token + valueIncludes: + - Slack bot token + - "true" + forbiddenDrafts: + - type: resource + path: f/evals/global/slack_bot_token + toolExpect: + requiredToolsUsed: + - write_variable + forbiddenToolsUsed: + - write_resource + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: write_variable + field: value + stringStartsWithAnyOf: + - xoxb-redacted-test-token + skipJudge: true + judgeChecklist: + - creates exactly one secret variable draft at f/evals/global/slack_bot_token + - uses the requested placeholder value and description + - does not create a resource or deploy anything + +- id: global-test7-ambiguous-app-asks-question + prompt: |- + Create a new raw app for triaging support tickets. + runtime: + maxTurns: 4 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - askUserQuestion + forbiddenToolsUsed: + - init_app + - write_app_file + - write_app_runnable + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + +- id: global-test8-human-script-infer-path-language + prompt: |- + I need a small helper that formats a customer-facing welcome line. + It should take a person's name and return "Welcome aboard, !". + Please just stage it as a draft for now. + runtime: + maxTurns: 8 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + valueIncludes: + - Welcome aboard + - name + toolExpect: + requiredToolsUsed: + - write_script + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - creates a single script draft for a welcome-line helper + - accepts a person's name as input + - returns a message containing Welcome aboard, the provided name, and an exclamation mark + - chooses a reasonable workspace path and script language without needing the user to specify them + - leaves the result as an AI draft only + +- id: global-test9-human-weekday-trial-job + prompt: |- + Can you set up a draft daily job that checks a few hard-coded trial accounts and returns the ones whose trial has ended? + It should run every weekday morning around 9 in UTC with a 30 day cutoff. + Keep it as draft work only. + runtime: + maxTurns: 10 + validate: + draftCountExactly: 2 + requiredDrafts: + - type: script + pathIncludes: + - trial + valueIncludes: + - trial + - "30" + - type: schedule + pathIncludes: + - trial + valueIncludes: + - UTC + toolExpect: + requiredToolsUsed: + - write_script + - write_schedule + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - creates a script draft that checks hard-coded trial accounts + - returns the accounts whose trial has ended based on a 30 day cutoff + - creates a schedule draft for weekday mornings around 09:00 UTC + - links the schedule to the generated script + - leaves both artifacts as drafts only + +- id: global-test10-human-secret-variable + prompt: |- + I need a placeholder Slack bot token stored securely for future notification work. + Use xoxb-redacted-test-token and note that it is for eval notifications. + Only prepare a draft. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: variable + pathIncludes: + - slack + valueIncludes: + - eval notifications + - "true" + toolExpect: + requiredToolsUsed: + - write_variable + forbiddenToolsUsed: + - write_resource + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: write_variable + field: value + stringStartsWithAnyOf: + - xoxb-redacted-test-token + skipJudge: true + judgeChecklist: + - creates a single secret variable draft for the Slack bot token placeholder + - uses the requested placeholder value + - includes a note or description that it is for eval notifications + - does not create a resource or deploy anything + +- id: global-test11-human-existing-flow-informal-edit + prompt: |- + There is an invoice processing flow in this workspace. + Can you adjust its total calculation so it adds 8% tax and returns subtotal, tax, and total? + Keep the change as a draft. + initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + pathIncludes: + - invoice + valueIncludes: + - calculate_total + - tax + - total + toolExpect: + requiredToolsUsed: + - read_workspace_item + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - finds and edits the existing invoice processing flow without the user providing its exact path + - updates the total calculation to apply 8% tax + - returns subtotal, tax, and total from the updated flow logic + - leaves the result as an AI draft only + +- id: global-test12-current-live-script-edit + prompt: |- + The script I have open formats greetings. + Can you update this script so it uppercases the name before greeting them and ends with an exclamation mark? + Keep it as draft work. + initial: ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json + runtime: + maxTurns: 8 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + path: f/evals/global/current_greeting + language: bun + valueIncludes: + - toUpperCase + - "!" + forbiddenDrafts: + - type: script + path: f/evals/global/format_greeting + - type: script + path: f/evals/global/format_greeting_archive + toolExpect: + requiredToolsUsed: + - read_workspace_item + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - resolves "this script" to the active live editor script instead of another similarly named workspace script + - updates the greeting logic to uppercase the provided name + - returns a greeting ending with an exclamation mark + - leaves the result as a draft only + +- id: global-test13-current-live-flow-edit + prompt: |- + I have the invoice flow open. + In the current flow, update the total calculation to add 8% tax and return subtotal, tax, and total. + Keep the change as a draft. + initial: ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + path: f/evals/global/current_invoice_flow + valueIncludes: + - calculate_total + - tax + - total + forbiddenDrafts: + - type: flow + path: f/evals/global/process_invoice + - type: flow + path: f/evals/global/process_refund + toolExpect: + requiredToolsUsed: + - read_workspace_item + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - resolves "current flow" to the active live editor flow + - does not edit the similarly named deployed invoice or refund flows + - updates the calculate_total logic to apply 8% tax + - returns subtotal, tax, and total from the updated flow logic + - leaves the result as a draft only + +- id: global-test14-current-without-live-editor-asks-question + prompt: |- + Please update this script so it returns `ok`. + Keep it as a draft. + runtime: + maxTurns: 4 + validate: + draftCountExactly: 0 + toolExpect: + forbiddenToolsUsed: + - write_script + - edit_script + - write_flow + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - asks which script to update when the user refers to "this script" without selected or active editor context + - does not guess a path or create a new script draft diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index 8ed61740c8..f92d6d7027 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -211,7 +211,7 @@ async function handleRun(input: { const summaries: Array<{ label: string; passRate: number; - averageDurationMs: number; + averagePassedDurationMs: number | null; }> = []; for (const [index, model] of models.entries()) { @@ -259,7 +259,7 @@ async function handleRun(input: { summaries.push({ label: `${model.id} (${runModel})`, passRate: result.passRate, - averageDurationMs: result.averageDurationMs, + averagePassedDurationMs: result.averagePassedDurationMs ?? null, }); } @@ -267,7 +267,7 @@ async function handleRun(input: { process.stdout.write("\nModel summary\n"); for (const summary of summaries) { process.stdout.write( - `- ${summary.label}: ${formatPercent(summary.passRate)} | ${Math.round(summary.averageDurationMs)}ms\n`, + `- ${summary.label}: ${formatPercent(summary.passRate)} | passed avg ${formatNullableDuration(summary.averagePassedDurationMs)}\n`, ); } } @@ -351,6 +351,10 @@ function formatPercent(value: number): string { return `${(value * 100).toFixed(1)}%`; } +function formatNullableDuration(value: number | null): string { + return value === null ? "n/a" : `${Math.round(value)}ms`; +} + void main().catch((error) => { const message = error instanceof Error ? error.message : String(error); process.stderr.write(`${message}\n`); diff --git a/ai_evals/core/cases.test.ts b/ai_evals/core/cases.test.ts index 733d34ddd2..526ea7c70f 100644 --- a/ai_evals/core/cases.test.ts +++ b/ai_evals/core/cases.test.ts @@ -203,6 +203,34 @@ describe("loadCases", () => { }); }); + it("loads global active-editor eval cases", async () => { + const globalCases = await loadCases("global"); + const scriptCase = globalCases.find( + (entry) => entry.id === "global-test12-current-live-script-edit" + ); + const flowCase = globalCases.find( + (entry) => entry.id === "global-test13-current-live-flow-edit" + ); + + expect(scriptCase?.initialPath).toContain( + "ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json" + ); + expect(scriptCase?.toolExpect).toMatchObject({ + requiredToolsUsed: ["read_workspace_item"], + }); + expect(flowCase?.initialPath).toContain( + "ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json" + ); + expect(flowCase?.validate).toMatchObject({ + requiredDrafts: [ + { + type: "flow", + path: "f/evals/global/current_invoice_flow", + }, + ], + }); + }); + it("loads tool expectations for workspace mutation cases", async () => { const scriptCases = await loadCases("script"); const caseEntry = scriptCases.find( diff --git a/ai_evals/core/results.test.ts b/ai_evals/core/results.test.ts new file mode 100644 index 0000000000..2d6077c5bd --- /dev/null +++ b/ai_evals/core/results.test.ts @@ -0,0 +1,242 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { describe, expect, it } from "bun:test"; +import { + appendHistoryRecord, + buildRunResult, + formatRunSummary, +} from "./results"; +import type { BenchmarkCaseResult } from "./types"; + +function caseResult( + attempts: BenchmarkCaseResult["attempts"], +): BenchmarkCaseResult { + return { + id: "case-1", + prompt: "Do the thing", + attempts, + }; +} + +describe("benchmark results", () => { + it("keeps success cost metrics separate from failed attempts", () => { + const result = buildRunResult({ + mode: "global", + runs: 1, + runModel: "model-under-test", + judgeModel: "judge-model", + caseResults: [ + caseResult([ + { + attempt: 1, + passed: true, + durationMs: 1000, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["edit_script"], + skillsInvoked: [], + checks: [{ name: "edited", passed: true }], + judgeScore: 100, + judgeSummary: "ok", + error: null, + tokenUsage: { prompt: 100, completion: 20, total: 120 }, + }, + { + attempt: 2, + passed: false, + durationMs: 100, + assistantMessageCount: 1, + toolCallCount: 0, + toolsUsed: [], + skillsInvoked: [], + checks: [{ name: "edited", passed: false }], + judgeScore: 10, + judgeSummary: "missed", + error: "failed", + tokenUsage: { prompt: 10, completion: 5, total: 15 }, + }, + ]), + ], + }); + + expect(result.attemptCount).toBe(2); + expect(result.passedAttempts).toBe(1); + expect(result.passRate).toBe(0.5); + expect(result.averageDurationMs).toBe(550); + expect(result.averagePassedDurationMs).toBe(1000); + expect(result.totalTokenUsage).toEqual({ + prompt: 110, + completion: 25, + total: 135, + }); + expect(result.totalPassedTokenUsage).toEqual({ + prompt: 100, + completion: 20, + total: 120, + }); + expect(result.averageTokenUsagePerAttempt).toEqual({ + prompt: 55, + completion: 12.5, + total: 67.5, + }); + expect(result.averageTokenUsagePerPassedAttempt).toEqual({ + prompt: 100, + completion: 20, + total: 120, + }); + + const summary = formatRunSummary(result); + expect(summary).toContain("Average duration (passed): 1000ms"); + expect(summary).toContain("Average tokens (passed): 120 total"); + expect(summary).toContain("Average duration (all attempts): 550ms"); + }); + + it("reports passed averages as unavailable when no attempt passes", () => { + const result = buildRunResult({ + mode: "global", + runs: 1, + runModel: "model-under-test", + judgeModel: "judge-model", + caseResults: [ + caseResult([ + { + attempt: 1, + passed: false, + durationMs: 100, + assistantMessageCount: 1, + toolCallCount: 0, + toolsUsed: [], + skillsInvoked: [], + checks: [{ name: "edited", passed: false }], + judgeScore: 10, + judgeSummary: "missed", + error: "failed", + tokenUsage: { prompt: 10, completion: 5, total: 15 }, + }, + ]), + ], + }); + + expect(result.averagePassedDurationMs).toBeNull(); + expect(result.totalPassedTokenUsage).toBeNull(); + expect(result.averageTokenUsagePerPassedAttempt).toBeNull(); + expect(formatRunSummary(result)).toContain( + "Average duration (passed): n/a", + ); + }); + + it("normalizes passed token averages by passed attempts", () => { + const result = buildRunResult({ + mode: "global", + runs: 1, + runModel: "model-under-test", + judgeModel: "judge-model", + caseResults: [ + caseResult([ + { + attempt: 1, + passed: true, + durationMs: 1000, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["edit_script"], + skillsInvoked: [], + checks: [{ name: "edited", passed: true }], + judgeScore: 100, + judgeSummary: "ok", + error: null, + tokenUsage: { prompt: 100, completion: 20, total: 120 }, + }, + { + attempt: 2, + passed: true, + durationMs: 1200, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["edit_script"], + skillsInvoked: [], + checks: [{ name: "edited", passed: true }], + judgeScore: 100, + judgeSummary: "ok", + error: null, + tokenUsage: null, + }, + ]), + ], + }); + + expect(result.passedAttempts).toBe(2); + expect(result.totalPassedTokenUsage).toEqual({ + prompt: 100, + completion: 20, + total: 120, + }); + expect(result.averageTokenUsagePerPassedAttempt).toEqual({ + prompt: 50, + completion: 10, + total: 60, + }); + }); + + it("records passed-attempt metrics in history", async () => { + const tempDir = await mkdtemp(join(tmpdir(), "windmill-ai-evals-")); + try { + const historyPath = join(tempDir, "history.jsonl"); + const result = buildRunResult({ + mode: "global", + runs: 1, + runModel: "model-under-test", + judgeModel: "judge-model", + caseResults: [ + caseResult([ + { + attempt: 1, + passed: true, + durationMs: 1000, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["edit_script"], + skillsInvoked: [], + checks: [{ name: "edited", passed: true }], + judgeScore: 100, + judgeSummary: "ok", + error: null, + tokenUsage: { prompt: 100, completion: 20, total: 120 }, + }, + { + attempt: 2, + passed: false, + durationMs: 100, + assistantMessageCount: 1, + toolCallCount: 0, + toolsUsed: [], + skillsInvoked: [], + checks: [{ name: "edited", passed: false }], + judgeScore: 10, + judgeSummary: "missed", + error: "failed", + tokenUsage: { prompt: 10, completion: 5, total: 15 }, + }, + ]), + ], + }); + + await appendHistoryRecord(result, historyPath); + const record = JSON.parse(await readFile(historyPath, "utf8")); + + expect(record.averageDurationMs).toBe(550); + expect(record.averagePassedDurationMs).toBe(1000); + expect(record.averageTokenUsagePerAttempt.total).toBe(67.5); + expect(record.averageTokenUsagePerPassedAttempt.total).toBe(120); + expect(record.cases[0].averageDurationMs).toBe(550); + expect(record.cases[0].averagePassedDurationMs).toBe(1000); + expect(record.cases[0].averageTokenUsagePerAttempt.total).toBe(67.5); + expect(record.cases[0].averageTokenUsagePerPassedAttempt.total).toBe( + 120, + ); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/ai_evals/core/results.ts b/ai_evals/core/results.ts index e58840f911..0b84497165 100644 --- a/ai_evals/core/results.ts +++ b/ai_evals/core/results.ts @@ -4,12 +4,20 @@ import { execFileSync } from "node:child_process"; import { getAiEvalsRoot, getRepoRoot } from "./cases"; import type { BenchmarkArtifactFile, + BenchmarkAttemptResult, BenchmarkCaseResult, BenchmarkRunResult, BenchmarkTokenUsage, EvalMode, } from "./types"; +type AttemptAggregate = { + attemptCount: number; + durationTotal: number; + tokenUsageAttemptCount: number; + tokenUsageTotal: BenchmarkTokenUsage | null; +}; + export async function writeRunResult( result: BenchmarkRunResult, outputPath?: string, @@ -77,36 +85,12 @@ export function buildRunResult(input: { judgeModel: string | null; caseResults: BenchmarkCaseResult[]; }): BenchmarkRunResult { - const attemptCount = input.caseResults.reduce( - (sum, entry) => sum + entry.attempts.length, - 0, - ); - const passedAttempts = input.caseResults.reduce( - (sum, entry) => - sum + entry.attempts.filter((attempt) => attempt.passed).length, - 0, - ); - const durationTotal = input.caseResults.reduce( - (sum, entry) => - sum + - entry.attempts.reduce((inner, attempt) => inner + attempt.durationMs, 0), - 0, - ); - const tokenUsageTotal = input.caseResults.reduce( - (sum, entry) => { - for (const attempt of entry.attempts) { - if (!attempt.tokenUsage) { - continue; - } - sum ??= { prompt: 0, completion: 0, total: 0 }; - sum.prompt += attempt.tokenUsage.prompt; - sum.completion += attempt.tokenUsage.completion; - sum.total += attempt.tokenUsage.total; - } - return sum; - }, - null, - ); + const attempts = input.caseResults.flatMap((entry) => entry.attempts); + const passedAttemptResults = attempts.filter((attempt) => attempt.passed); + const attemptAggregate = aggregateAttempts(attempts); + const passedAttemptAggregate = aggregateAttempts(passedAttemptResults); + const attemptCount = attemptAggregate.attemptCount; + const passedAttempts = passedAttemptAggregate.attemptCount; return { version: 1, @@ -120,16 +104,19 @@ export function buildRunResult(input: { attemptCount, passedAttempts, passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount, - averageDurationMs: attemptCount === 0 ? 0 : durationTotal / attemptCount, - totalTokenUsage: tokenUsageTotal, + averageDurationMs: + attemptCount === 0 ? 0 : attemptAggregate.durationTotal / attemptCount, + averagePassedDurationMs: averageDuration(passedAttemptAggregate), + totalTokenUsage: attemptAggregate.tokenUsageTotal, + totalPassedTokenUsage: passedAttemptAggregate.tokenUsageTotal, averageTokenUsagePerAttempt: - attemptCount === 0 || !tokenUsageTotal + attemptCount === 0 ? null - : { - prompt: tokenUsageTotal.prompt / attemptCount, - completion: tokenUsageTotal.completion / attemptCount, - total: tokenUsageTotal.total / attemptCount, - }, + : averageTokenUsage(attemptAggregate, attemptCount), + averageTokenUsagePerPassedAttempt: averageTokenUsage( + passedAttemptAggregate, + passedAttempts, + ), cases: input.caseResults, }; } @@ -138,9 +125,25 @@ export function formatRunSummary(result: BenchmarkRunResult): string { const lines = [ `${result.mode} benchmark complete`, `Pass rate: ${formatPercent(result.passRate)} (${result.passedAttempts}/${result.attemptCount})`, - `Average duration: ${Math.round(result.averageDurationMs)}ms`, + `Average duration (passed): ${formatNullableDuration(result.averagePassedDurationMs ?? null)}`, ]; + if (result.averageTokenUsagePerPassedAttempt) { + lines.push( + `Average tokens (passed): ${formatTokenUsage(result.averageTokenUsagePerPassedAttempt)}`, + ); + } + if (result.passedAttempts < result.attemptCount) { + lines.push( + `Average duration (all attempts): ${Math.round(result.averageDurationMs)}ms`, + ); + if (result.averageTokenUsagePerAttempt) { + lines.push( + `Average tokens (all attempts): ${formatTokenUsage(result.averageTokenUsagePerAttempt)}`, + ); + } + } + const failures = collectFailures(result); if (failures.length > 0) { lines.push("Failures:"); @@ -172,6 +175,60 @@ function collectFailures(result: BenchmarkRunResult): string[] { return failures; } +function aggregateAttempts(attempts: BenchmarkAttemptResult[]): AttemptAggregate { + const aggregate: AttemptAggregate = { + attemptCount: attempts.length, + durationTotal: 0, + tokenUsageAttemptCount: 0, + tokenUsageTotal: null, + }; + + for (const attempt of attempts) { + aggregate.durationTotal += attempt.durationMs; + if (!attempt.tokenUsage) { + continue; + } + aggregate.tokenUsageAttemptCount += 1; + aggregate.tokenUsageTotal ??= { prompt: 0, completion: 0, total: 0 }; + aggregate.tokenUsageTotal.prompt += attempt.tokenUsage.prompt; + aggregate.tokenUsageTotal.completion += attempt.tokenUsage.completion; + aggregate.tokenUsageTotal.total += attempt.tokenUsage.total; + } + + return aggregate; +} + +function averageDuration(aggregate: AttemptAggregate): number | null { + return aggregate.attemptCount === 0 + ? null + : aggregate.durationTotal / aggregate.attemptCount; +} + +function averageTokenUsage( + aggregate: AttemptAggregate, + denominator: number, +): BenchmarkTokenUsage | null { + if (denominator === 0 || !aggregate.tokenUsageTotal) { + return null; + } + return { + prompt: aggregate.tokenUsageTotal.prompt / denominator, + completion: aggregate.tokenUsageTotal.completion / denominator, + total: aggregate.tokenUsageTotal.total / denominator, + }; +} + +function formatNullableDuration(value: number | null): string { + return value === null ? "n/a" : `${Math.round(value)}ms`; +} + +function formatTokenUsage(value: BenchmarkTokenUsage): string { + const total = Math.round(value.total); + const prompt = Math.round(value.prompt); + const completion = Math.round(value.completion); + return `${total} total (${prompt} prompt, ${completion} completion)`; +} + function defaultFileName(mode: EvalMode): string { return `${new Date().toISOString().replaceAll(":", "-")}__${mode}.json`; } @@ -252,12 +309,15 @@ function toHistoryRecord(result: BenchmarkRunResult) { passedAttempts: result.passedAttempts, passRate: result.passRate, averageDurationMs: result.averageDurationMs, + averagePassedDurationMs: result.averagePassedDurationMs ?? null, averageJudgeScore: judgeScores.length === 0 ? null : judgeScores.reduce((sum, score) => sum + score, 0) / judgeScores.length, averageTokenUsagePerAttempt: result.averageTokenUsagePerAttempt ?? null, + averageTokenUsagePerPassedAttempt: + result.averageTokenUsagePerPassedAttempt ?? null, failedCaseIds: Array.from( new Set( result.cases @@ -268,31 +328,15 @@ function toHistoryRecord(result: BenchmarkRunResult) { ), ), cases: result.cases.map((caseResult) => { - const attemptCount = caseResult.attempts.length; - const passedAttempts = caseResult.attempts.filter( - (attempt) => attempt.passed, - ).length; - const totalDurationMs = caseResult.attempts.reduce( - (sum, attempt) => sum + attempt.durationMs, - 0, + const attemptAggregate = aggregateAttempts(caseResult.attempts); + const passedAttemptAggregate = aggregateAttempts( + caseResult.attempts.filter((attempt) => attempt.passed), ); + const attemptCount = attemptAggregate.attemptCount; + const passedAttempts = passedAttemptAggregate.attemptCount; const judgeScores = caseResult.attempts.flatMap((attempt) => typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : [], ); - const totalTokenUsage = - caseResult.attempts.reduce( - (sum, attempt) => { - if (!attempt.tokenUsage) { - return sum; - } - sum ??= { prompt: 0, completion: 0, total: 0 }; - sum.prompt += attempt.tokenUsage.prompt; - sum.completion += attempt.tokenUsage.completion; - sum.total += attempt.tokenUsage.total; - return sum; - }, - null, - ); return { id: caseResult.id, @@ -300,20 +344,23 @@ function toHistoryRecord(result: BenchmarkRunResult) { passedAttempts, passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount, averageDurationMs: - attemptCount === 0 ? 0 : totalDurationMs / attemptCount, + attemptCount === 0 + ? 0 + : attemptAggregate.durationTotal / attemptCount, + averagePassedDurationMs: averageDuration(passedAttemptAggregate), averageJudgeScore: judgeScores.length === 0 ? null : judgeScores.reduce((sum, score) => sum + score, 0) / judgeScores.length, averageTokenUsagePerAttempt: - attemptCount === 0 || !totalTokenUsage + attemptCount === 0 ? null - : { - prompt: totalTokenUsage.prompt / attemptCount, - completion: totalTokenUsage.completion / attemptCount, - total: totalTokenUsage.total / attemptCount, - }, + : averageTokenUsage(attemptAggregate, attemptCount), + averageTokenUsagePerPassedAttempt: averageTokenUsage( + passedAttemptAggregate, + passedAttempts, + ), }; }), }; diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 2b42a0dfc5..ecc46591fc 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -110,7 +110,9 @@ export interface AppValidationSpec { export interface GlobalDraftRequirement { type: string; - path: string; + path?: string; + pathIncludes?: string[]; + pathStartsWith?: string; triggerKind?: string; language?: string; summaryIncludes?: string[]; @@ -324,8 +326,11 @@ export interface BenchmarkRunResult { passedAttempts: number; passRate: number; averageDurationMs: number; + averagePassedDurationMs?: number | null; totalTokenUsage?: BenchmarkTokenUsage | null; + totalPassedTokenUsage?: BenchmarkTokenUsage | null; averageTokenUsagePerAttempt?: BenchmarkTokenUsage | null; + averageTokenUsagePerPassedAttempt?: BenchmarkTokenUsage | null; artifactsPath?: string | null; cases: BenchmarkCaseResult[]; } diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index d2a6e954bb..6010f6351a 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -195,6 +195,69 @@ describe("validateGlobalState", () => { }); }); + it("accepts a required script draft without an exact path", () => { + const checks = validateGlobalState({ + actual: { + drafts: [ + { + type: "script", + path: "f/team_tools/friendly_greeting", + language: "bun", + summary: "Friendly greeting helper", + value: + "export async function main(name: string) {\n return `Hello, ${name}!`\n}\n", + isDraft: true, + }, + ], + }, + validate: { + draftCountExactly: 1, + requiredDrafts: [ + { + type: "script", + pathIncludes: ["greeting"], + language: "bun", + summaryIncludes: ["Friendly"], + valueIncludes: ["Hello"], + }, + ], + }, + }); + + expect(checks.every((check) => check.passed)).toBe(true); + }); + + it("reports flexible global draft path filters when no draft matches", () => { + const checks = validateGlobalState({ + actual: { + drafts: [ + { + type: "script", + path: "f/team_tools/friendly_greeting", + language: "bun", + value: + "export async function main(name: string) {\n return `Hello, ${name}!`\n}\n", + isDraft: true, + }, + ], + }, + validate: { + requiredDrafts: [ + { + type: "script", + pathIncludes: ["invoice"], + }, + ], + }, + }); + + expect(checks).toContainEqual({ + name: "global includes script draft (path includes invoice)", + passed: false, + details: "drafts: script:f/team_tools/friendly_greeting", + }); + }); + it("does not require a TypeScript entrypoint for non-TypeScript script drafts", () => { const checks = validateGlobalState({ actual: { diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 4f59368113..693d34a013 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -315,10 +315,11 @@ export function validateGlobalState(input: { } for (const required of validate.requiredDrafts ?? []) { - const draft = findGlobalDraft(drafts, required.type, required.path, required.triggerKind); + const requirementLabel = formatGlobalDraftRequirement(required); + const draft = findGlobalDraft(drafts, required); checks.push( check( - `global includes ${required.type} draft ${required.path}`, + `global includes ${requirementLabel}`, Boolean(draft), summarizeGlobalDrafts(drafts) ) @@ -330,7 +331,7 @@ export function validateGlobalState(input: { if (required.language !== undefined) { checks.push( check( - `${required.type} draft ${required.path} uses ${required.language}`, + `${requirementLabel} uses ${required.language}`, draft.language === required.language, `language=${draft.language ?? "(none)"}` ) @@ -340,7 +341,7 @@ export function validateGlobalState(input: { for (const snippet of required.summaryIncludes ?? []) { checks.push( check( - `${required.type} draft ${required.path} summary includes '${snippet}'`, + `${requirementLabel} summary includes '${snippet}'`, normalizeText(draft.summary ?? "").includes(normalizeText(snippet)), `summary=${draft.summary ?? ""}` ) @@ -351,7 +352,7 @@ export function validateGlobalState(input: { for (const snippet of required.valueIncludes ?? []) { checks.push( check( - `${required.type} draft ${required.path} value includes '${snippet}'`, + `${requirementLabel} value includes '${snippet}'`, normalizeText(valueText).includes(normalizeText(snippet)), truncateForDetails(valueText) ) @@ -361,7 +362,7 @@ export function validateGlobalState(input: { for (const snippet of required.valueExcludes ?? []) { checks.push( check( - `${required.type} draft ${required.path} value excludes '${snippet}'`, + `${requirementLabel} value excludes '${snippet}'`, !normalizeText(valueText).includes(normalizeText(snippet)), truncateForDetails(valueText) ) @@ -373,7 +374,7 @@ export function validateGlobalState(input: { checks.push( check( `global does not include ${forbidden.type} draft ${forbidden.path}`, - !findGlobalDraft(drafts, forbidden.type, forbidden.path, forbidden.triggerKind), + !findGlobalDraft(drafts, forbidden), summarizeGlobalDrafts(drafts) ) ); @@ -615,16 +616,100 @@ function summarizeProblems(problems: string[], limit = 5): string | undefined { function findGlobalDraft( drafts: GlobalDraft[], - type: string, - path: string, - triggerKind?: string + requirement: { + type: string; + path?: string; + pathIncludes?: string[]; + pathStartsWith?: string; + triggerKind?: string; + summaryIncludes?: string[]; + valueIncludes?: string[]; + valueExcludes?: string[]; + } ): GlobalDraft | undefined { - return drafts.find( - (draft) => - draft.type === type && - draft.path === path && - (triggerKind === undefined || draft.triggerKind === triggerKind) + const candidates = drafts.filter((draft) => + globalDraftMatchesLocator(draft, requirement) ); + return ( + candidates.find((draft) => globalDraftMatchesContent(draft, requirement)) ?? + candidates[0] + ); +} + +function globalDraftMatchesLocator( + draft: GlobalDraft, + requirement: { + type: string; + path?: string; + pathIncludes?: string[]; + pathStartsWith?: string; + triggerKind?: string; + } +): boolean { + return ( + draft.type === requirement.type && + (requirement.path === undefined || draft.path === requirement.path) && + (requirement.pathStartsWith === undefined || + draft.path.startsWith(requirement.pathStartsWith)) && + (requirement.pathIncludes ?? []).every((snippet) => + normalizeText(draft.path).includes(normalizeText(snippet)) + ) && + (requirement.triggerKind === undefined || + draft.triggerKind === requirement.triggerKind) + ); +} + +function globalDraftMatchesContent( + draft: GlobalDraft, + requirement: { + summaryIncludes?: string[]; + valueIncludes?: string[]; + valueExcludes?: string[]; + } +): boolean { + const summary = normalizeText(draft.summary ?? ""); + const value = normalizeText(stringifyGlobalDraftValue(draft.value)); + return ( + (requirement.summaryIncludes ?? []).every((snippet) => + summary.includes(normalizeText(snippet)) + ) && + (requirement.valueIncludes ?? []).every((snippet) => + value.includes(normalizeText(snippet)) + ) && + (requirement.valueExcludes ?? []).every( + (snippet) => !value.includes(normalizeText(snippet)) + ) + ); +} + +function formatGlobalDraftRequirement( + requirement: { + type: string; + path?: string; + pathIncludes?: string[]; + pathStartsWith?: string; + triggerKind?: string; + } +): string { + const typeLabel = + requirement.triggerKind === undefined + ? requirement.type + : `${requirement.triggerKind} ${requirement.type}`; + if (requirement.path !== undefined) { + return `${typeLabel} draft ${requirement.path}`; + } + + const filters = [ + ...(requirement.pathStartsWith === undefined + ? [] + : [`path starts with ${requirement.pathStartsWith}`]), + ...(requirement.pathIncludes ?? []).map( + (snippet) => `path includes ${snippet}` + ), + ]; + return filters.length === 0 + ? `${typeLabel} draft` + : `${typeLabel} draft (${filters.join(", ")})`; } function summarizeGlobalDrafts(drafts: GlobalDraft[]): string { diff --git a/ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json b/ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json new file mode 100644 index 0000000000..98538a1ccb --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json @@ -0,0 +1,66 @@ +{ + "workspace": { + "scripts": [ + { + "path": "f/evals/global/format_greeting", + "summary": "Format a deployed greeting", + "description": "Returns a plain greeting for a provided name.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": ["name"] + }, + "content": "export async function main(name: string) {\n return `Hello, ${name}`\n}\n" + }, + { + "path": "f/evals/global/format_greeting_archive", + "summary": "Archived greeting formatter", + "description": "Older greeting formatter kept for reference.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": ["name"] + }, + "content": "export async function main(name: string) {\n return `Hi, ${name}`\n}\n" + } + ] + }, + "liveEditorDrafts": [ + { + "type": "script", + "storagePath": "f/evals/global/current_greeting", + "effectivePath": "f/evals/global/current_greeting", + "value": { + "path": "f/evals/global/current_greeting", + "summary": "Open greeting formatter", + "description": "Formats a greeting in the live editor.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": ["name"] + }, + "content": "export async function main(name: string) {\n return `Hello, ${name}`\n}\n", + "is_template": false, + "kind": "script" + } + } + ] +} diff --git a/ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json b/ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json new file mode 100644 index 0000000000..5c2ffcb0f0 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json @@ -0,0 +1,118 @@ +{ + "workspace": { + "flows": [ + { + "path": "f/evals/global/process_invoice", + "summary": "Deployed invoice processor", + "description": "Calculates invoice totals from a subtotal.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "subtotal": { + "type": "number" + } + }, + "required": ["subtotal"] + }, + "value": { + "modules": [ + { + "id": "calculate_total", + "summary": "Calculate total from subtotal", + "value": { + "type": "rawscript", + "language": "bun", + "content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n", + "input_transforms": { + "subtotal": { + "type": "javascript", + "expr": "flow_input.subtotal" + } + } + } + } + ] + } + }, + { + "path": "f/evals/global/process_refund", + "summary": "Refund processor", + "description": "Calculates refund totals.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "subtotal": { + "type": "number" + } + }, + "required": ["subtotal"] + }, + "value": { + "modules": [ + { + "id": "calculate_total", + "summary": "Calculate refund total", + "value": { + "type": "rawscript", + "language": "bun", + "content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n", + "input_transforms": { + "subtotal": { + "type": "javascript", + "expr": "flow_input.subtotal" + } + } + } + } + ] + } + } + ] + }, + "liveEditorDrafts": [ + { + "type": "flow", + "storagePath": "f/evals/global/current_invoice_flow", + "effectivePath": "f/evals/global/current_invoice_flow", + "value": { + "path": "f/evals/global/current_invoice_flow", + "summary": "Open invoice processor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "subtotal": { + "type": "number" + } + }, + "required": ["subtotal"] + }, + "value": { + "modules": [ + { + "id": "calculate_total", + "summary": "Calculate total from subtotal", + "value": { + "type": "rawscript", + "language": "bun", + "content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n", + "input_transforms": { + "subtotal": { + "type": "javascript", + "expr": "flow_input.subtotal" + } + } + } + } + ] + }, + "edited_by": "", + "edited_at": "", + "archived": false, + "extra_perms": {} + } + } + ] +} diff --git a/ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json b/ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json new file mode 100644 index 0000000000..b9b4c675c5 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json @@ -0,0 +1,40 @@ +{ + "workspace": { + "flows": [ + { + "path": "f/evals/global/process_invoice", + "summary": "Process an invoice subtotal", + "description": "Calculates invoice totals from a subtotal.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "subtotal": { + "type": "number" + } + }, + "required": ["subtotal"] + }, + "value": { + "modules": [ + { + "id": "calculate_total", + "summary": "Calculate total from subtotal", + "value": { + "type": "rawscript", + "language": "bun", + "content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n", + "input_transforms": { + "subtotal": { + "type": "javascript", + "expr": "flow_input.subtotal" + } + } + } + } + ] + } + } + ] + } +} diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index d68df9f5f8..f3cbf6fd86 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -1,5 +1,8 @@ import { readFile } from "node:fs/promises"; -import { runGlobalEval } from "../adapters/frontend/core/global/globalEvalRunner"; +import { + runGlobalEval, + type GlobalLiveEditorDraftFixture, +} from "../adapters/frontend/core/global/globalEvalRunner"; import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend"; import type { FrontendEvalModelConfig } from "../core/models"; import type { BenchmarkArtifactFile, GlobalValidationSpec, ModeRunner } from "../core/types"; @@ -9,6 +12,7 @@ import { getFrontendApiKey } from "./frontendCommon"; export interface GlobalInitialFixture { workspace?: BenchmarkWorkspaceRunnables; + liveEditorDrafts?: GlobalLiveEditorDraftFixture[]; } export function createGlobalModeRunner( @@ -31,6 +35,7 @@ export function createGlobalModeRunner( getFrontendApiKey(modelConfig.provider), { workspaceFixtures: initial?.workspace, + liveEditorDrafts: initial?.liveEditorDrafts, maxIterations: context.evalCase?.runtime?.maxTurns, provider: modelConfig.provider, model: modelConfig.model, @@ -73,6 +78,7 @@ async function loadGlobalInitialFixture(path: string): Promise, tag: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + used_by_workspaces: Vec, } #[derive(Deserialize, Debug, Serialize, Default)] @@ -1154,7 +1160,7 @@ struct CustomInstanceDbLogs { } async fn list_custom_instance_pg_databases( - _authed: ApiAuthed, + authed: ApiAuthed, Extension(db): Extension, ) -> JsonResult> { let result = sqlx::query_scalar!( @@ -1163,12 +1169,57 @@ async fn list_custom_instance_pg_databases( .fetch_one(&db) .await? .ok_or_else(|| error::Error::ExecutionErr("Couldn't find custom_instance_pg_databases".to_string()))?; - let result = serde_json::from_value(result).map_err(|e| { - error::Error::ExecutionErr(format!( - "couldn't parse custom_instance_pg_databases.databases : {}", - e.to_string() - )) - })?; + let mut result: HashMap = + serde_json::from_value(result).map_err(|e| { + error::Error::ExecutionErr(format!( + "couldn't parse custom_instance_pg_databases.databases : {}", + e.to_string() + )) + })?; + + if is_super_admin_email(&db, &authed.email).await? { + // Enrich each database with the list of workspaces referencing it through + // either a ducklake catalog or a datatable database whose resource_type is + // 'instance'. Not stored in DB to avoid drift. + let usages = sqlx::query!( + r#" + SELECT ws.workspace_id AS "workspace_id!", entry->'catalog'->>'resource_path' AS dbname + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each( + CASE WHEN jsonb_typeof(ws.ducklake->'ducklakes') = 'object' + THEN ws.ducklake->'ducklakes' + ELSE '{}'::jsonb END + ) AS dl(k, entry) + WHERE entry->'catalog'->>'resource_type' = 'instance' + AND entry->'catalog'->>'resource_path' IS NOT NULL + UNION ALL + SELECT ws.workspace_id AS "workspace_id!", entry->'database'->>'resource_path' AS dbname + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each( + CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object' + THEN ws.datatable->'datatables' + ELSE '{}'::jsonb END + ) AS dt(k, entry) + WHERE entry->'database'->>'resource_type' = 'instance' + AND entry->'database'->>'resource_path' IS NOT NULL + "#, + ) + .fetch_all(&db) + .await?; + + let mut by_db: HashMap> = HashMap::new(); + for row in usages { + if let Some(dbname) = row.dbname { + by_db.entry(dbname).or_default().insert(row.workspace_id); + } + } + for (dbname, entry) in result.iter_mut() { + if let Some(workspaces) = by_db.remove(dbname) { + entry.used_by_workspaces = workspaces.into_iter().collect(); + } + } + } + return Ok(Json(result)); } @@ -1196,7 +1247,8 @@ async fn setup_custom_instance_pg_database( let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await; let success = result.is_ok(); let error = result.err().map(|e| e.to_string()); - let status = CustomInstanceDb { logs, success, error, tag: body.tag }; + let status = + CustomInstanceDb { logs, success, error, tag: body.tag, used_by_workspaces: vec![] }; let status_json = serde_json::to_value(&status).map_err(to_anyhow)?; // Save that the database was setup successfully sqlx::query!( diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c101627079..6fea5b17fd 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -25476,6 +25476,11 @@ components: example: "Connection timeout" tag: $ref: "#/components/schemas/CustomInstanceDbTag" + used_by_workspaces: + type: array + items: + type: string + description: Workspaces that reference this database via a ducklake catalog or datatable database with resource_type 'instance'. Computed at request time, not persisted. NewSqsTrigger: type: object diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 5a7c6d2bfa..92ebf08477 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -252,6 +252,17 @@ lazy_static::lazy_static! { pub static ref WORKSPACE_FAIRNESS_OVERLOADED: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(vec![]); pub static ref WORKSPACE_FAIRNESS_LAST_REFRESH_MICROS: AtomicI64 = AtomicI64::new(0); + /// Stochastic admission probability for capped workspaces, expressed in + /// parts per 10_000 (so `420` = 4.2%). The refresh computes this from the + /// observed worker-second distribution and the configured cap so that + /// admission converges to the target *worker-second* share — independent + /// of how the capped vs uncapped workspaces compare on per-job durations. + /// See `workspace_fairness_ee::refresh_overloaded` for the derivation. + /// `10_000` (= admit all) is the default until the first refresh + /// classifies an overloaded set — before that, no workspace is capped so + /// `should_admit_capped` is moot and "admit all" is the correct no-op. + pub static ref WORKSPACE_FAIRNESS_ADMISSION_PPM: AtomicU32 = AtomicU32::new(10_000); + pub static ref SMTP_CONFIG: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(None); pub static ref INDEXER_CONFIG: arc_swap::ArcSwap = arc_swap::ArcSwap::from_pointee(TantivyIndexerSettings::default()); diff --git a/backend/windmill-queue/src/workspace_fairness.rs b/backend/windmill-queue/src/workspace_fairness.rs index 9f4ba00211..8eb3a69394 100644 --- a/backend/windmill-queue/src/workspace_fairness.rs +++ b/backend/windmill-queue/src/workspace_fairness.rs @@ -1,15 +1,253 @@ -//! Per-workspace fairness for the shared worker pool (Enterprise feature). +//! # Per-workspace fairness for the shared worker pool (Enterprise feature) //! -//! The real algorithm — overloaded-set aggregation, coordinated refresh on -//! `background_task_state`, audit emission, stochastic admission decision — -//! lives in [`crate::workspace_fairness_ee`] and only compiles when the -//! `private` feature is on. This module is the public surface used by the -//! pull dispatch in `jobs.rs` and the integration tests; when EE is on it -//! transparently re-exports the EE implementation, when EE is off it -//! provides no-op stubs so the OSS build stays bit-identical to the -//! pre-fairness pull path. +//! On multi-tenant deployments (notably `app.windmill.dev`, and any EE +//! cluster with a single shared worker group) a single workspace flooding +//! the queue with jobs can degrade quality of service for everyone else. +//! This module computes the set of "overloaded" workspaces whose share of +//! the worker pool must be capped, and the dispatch in `jobs.rs` uses a +//! **duration-weighted stochastic admission rule** at pull time to enforce +//! the cap as a *worker-second* share, not a pull-count share. //! -//! See [`crate::workspace_fairness_ee`] for design notes and SQL details. +//! The full algorithm lives in [`crate::workspace_fairness_ee`] behind the +//! `private` feature — this OSS-facing module is the public surface that +//! the pull dispatch and integration tests call. When EE is on, the symbols +//! here transparently re-export the EE implementation. When EE is off, they +//! are no-ops: `maybe_refresh_overloaded` does nothing, `should_admit_capped` +//! always returns `true`, and the pull path is bit-identical to its +//! pre-fairness shape. **Both runtime correctness and the entire reasoning +//! below assume the EE module is compiled in**; the OSS build is a stub. +//! +//! Every numerical default mentioned below (`MAX_PERCENT = 50`, +//! `DURATION_SECS = 10`, `MIN_TOTAL = 4`, `WORKER_PING_LIVE_SECS = 60`, +//! `ADMISSION_EPSILON_PERCENT = 5`) is tunable via global settings or +//! constants; the values here are the as-shipped defaults at the time of +//! writing and what the design discussion below was calibrated against. +//! +//! ## 1. What "overloaded" means — worker-seconds, not jobs +//! +//! A workspace is overloaded when, over a rolling +//! `WORKSPACE_FAIRNESS_DURATION_SECS = 10s` window, it has consumed at least +//! `WORKSPACE_FAIRNESS_MAX_PERCENT = 50%` of cluster worker-time. Activity +//! is measured in **worker-seconds**: each job contributes the wall-clock +//! time it actually held a worker, intersected with the window. A +//! count-based signal — "what fraction of jobs in the window are from this +//! workspace" — gets badly fooled by job-duration heterogeneity: 600 +//! short (100ms) jobs and one long (60s) job consume the same worker-time +//! but the count-based form attributes 600× more weight to the spammy +//! workspace. Worker-seconds put both patterns on the same scale. +//! +//! Two sources contribute to a workspace's worker-second total: +//! +//! - **Running** (live, currently-on-a-worker): driven from `v2_job_runtime` +//! filtered on `ping > now() - WORKER_PING_LIVE_SECS` (60s, ≈ 2× worker +//! heartbeat interval), then PK-joined to `v2_job` for the `kind` filter +//! and `v2_job_queue` for `started_at` / `suspend_until`. Contribution is +//! `clamp(min(now, ping) − max(started_at, window_start), 0, window)`. +//! End-of-interval is the per-job `ping`, which both (a) implements the +//! zombie defense — a worker that stopped pinging stops accruing +//! worker-seconds at its last heartbeat, so a backlog of stuck +//! `running = true` rows can't dominate the denominator — and (b) matches +//! the semantic of "worker-seconds the worker has confirmed". `v2_job_runtime` +//! is small (rows deleted on completion), so driving the scan from there +//! keeps the per-refresh cost bounded by the *in-flight* count rather +//! than by the queue size, even when one workspace has thousands of +//! `running = true` rows. +//! +//! - **Completed** (recently finished): pulled by an index scan over +//! `v2_job_completed (completed_at)`, then PK-joined to `v2_job`. The +//! index hit is critical — see "Why no `WITH params AS (...)` CTE" below. +//! Contribution is `clamp(min(completed_at, now) − max(started_at, +//! completed_at − duration_ms, window_start), 0, window)`. Clamping +//! start-of-interval by `completed_at − duration_ms` defends against +//! zombie rows that `zombie_monitor` force-failed: `started_at` may be +//! far in the past, but `duration_ms` reflects the actual measured worker +//! time, so the row only contributes its real runtime, not the idle wait +//! before force-fail. +//! +//! Both halves exclude **flow-orchestration kinds** +//! (`flow, flowpreview, flownode, singlestepflow`) and **concurrency- +//! suspended rows** (`suspend_until IS NOT NULL`) — these hold +//! `running = true` but consume no worker slot. Same predicate as +//! `handle_zombie_jobs` in `monitor.rs`. +//! +//! `WORKSPACE_FAIRNESS_MIN_TOTAL = 4` is also in worker-seconds (≈ 40 % +//! utilization of one worker over a 10s window) — below the floor, the +//! cluster is too quiet to bother capping anyone. +//! +//! ## 2. The cap is enforced stochastically, weighted by duration +//! +//! The pull dispatch in `jobs.rs` flips a coin on every pull: with +//! probability `p_c` it uses the standard pull query (capped workspaces +//! are admissible — FIFO will pick them if they're at the head), and with +//! probability `1 − p_c` it uses the *fairness pull query* which excludes +//! the overloaded workspaces. Doing it as a probabilistic split rather +//! than a binary cap/uncap gate keeps victim latency flat instead of +//! breathing in/out with each refresh cycle. +//! +//! The key design choice is how `p_c` is set. The natural first try is +//! `p_c = (MAX_PERCENT + ε) / 100` — a constant. That converges the +//! *pull-count* ratio to `MAX_PERCENT`, but only matches the worker-second +//! ratio when capped and uncapped workspaces share the same mean job +//! duration. The steady-state share equation is: +//! +//! `share = p_c · D_c / (p_c · D_c + (1 − p_c) · D_u)` +//! +//! where `D_c` and `D_u` are the per-job mean durations of capped and +//! uncapped workspaces respectively. With `D_c = 34s` and `D_u = 1s` (the +//! exact numbers observed during the lancom01-prod / jps-internal cloud +//! incident), a constant `p_c = 0.65` (60 % + 5 % ε) yields +//! +//! `share = 0.65 · 34 / (0.65 · 34 + 0.35 · 1) = 22.1 / 22.45 ≈ 98%` +//! +//! — i.e., the "60 % cap" was in practice giving capped workspaces 98 % +//! of worker-seconds. Victims were observed waiting 15s+ for pickup +//! despite the cap firing on every pull. +//! +//! Inverting the equation for the desired share `t = (MAX_PERCENT + ε) / 100`: +//! +//! `p_c = t · D_u / ((1 − t) · D_c + t · D_u)` +//! +//! Same numbers, target 0.65: `p_c ≈ 0.054` — about 12× tighter than the +//! count-based form. The refresh computes `p_c` and stores it in +//! [`WORKSPACE_FAIRNESS_ADMISSION_PPM`] (parts-per-10_000, fits in an +//! `AtomicU32`). The pull-time check is one atomic load plus one +//! `rand::rng().random_range(0..10_000)` draw — same hot-path cost as the +//! count-based form. +//! +//! ### `D_c`/`D_u` come from a separate, longer service-time window +//! +//! Crucially, `D_c` and `D_u` must be **true mean service times**, because +//! the share equation above is Little's-law-based +//! (`occupancy = arrival_rate × mean_service_time`). They are **not** taken +//! from the occupancy aggregation: that aggregation clamps each job's +//! contribution to the short occupancy window (`DURATION_SECS`, 10s), so a +//! job longer than the window contributes at most 10s — fine for measuring +//! *share*, but it would truncate `D_c` to ≤ 10s and systematically +//! under-admit the skew exactly when capped jobs are long (the case the cap +//! exists for: e.g. true `D_c = 34s` clamped to 10s gives `p_c ≈ 0.157`, an +//! 86 % effective share instead of 65 %). Instead, the refresh samples true +//! unclamped `duration_ms` of completed jobs over a longer, decoupled +//! service-time window (`DURATION_SAMPLE_SECS`, 60s) — long enough to avoid +//! truncation and to keep the mean stable when few jobs complete within the +//! 10s occupancy window. So the refresh emits two per-workspace signals: +//! windowed occupancy worker-seconds (for classification) and a 60s +//! service-time `(Σ duration_ms, count)` (for admission), merged per +//! workspace. +//! +//! ### Why we kept the fallback when the fairness pull returns empty +//! +//! The 100 − `p_c` % of pulls that try the fairness query (excluding +//! capped workspaces) fall back to the standard query if the fairness +//! query returns no row. The alternative — idle the worker, holding the +//! slot open in case a victim shows up — was considered but rejected for +//! the first iteration: with `p_c` correctly tightened, victims do get the +//! slot they need *when they exist*, and absent victims, falling back to +//! the capped pool is the right behaviour (otherwise the cluster +//! under-utilises itself for no benefit). Adding a reserve-capacity skip +//! is a fine-tuning lever for bursty victim arrival patterns and is left +//! as a follow-up. +//! +//! ### Degenerate cases +//! +//! If either bucket is empty — no capped jobs, no uncapped jobs, or a +//! capped workspace with zero completions in the 60s service-time window +//! (all its jobs still running) — the formula is undefined. The refresh +//! falls back to the count-based `p_c = t` in those cases — it matches +//! the pre-refactor behaviour and is the safest thing to do when there's +//! no service-time signal yet to weight on. +//! +//! ## 3. Coordinated refresh — exactly once per cycle, cluster-wide +//! +//! The aggregation is too expensive to run on every worker process every +//! pull (and would produce no new information on the sub-second +//! timescale). It runs **at most once every `refresh_interval` seconds +//! across the entire fleet**, gated by both a per-process CAS and a +//! DB-side row lock: +//! +//! 1. **Per-process gate** — `maybe_refresh_overloaded` (called from the +//! pull path) does `LAST_REFRESH_MICROS.compare_exchange` to ensure at +//! most one in-flight refresh per process per interval. If the CAS +//! fails or the interval hasn't elapsed yet, the call is a no-op. +//! Cost on the hot path: one atomic load, optionally one CAS. +//! +//! 2. **DB-side claim** — `refresh_overloaded` first does a cheap upsert +//! (`INSERT ... ON CONFLICT ON background_task_state ... WHERE +//! updated_at < NOW() − refresh_interval RETURNING true`). The `VALUES` +//! clause is all constants, so Postgres has no expensive work to do +//! even for losers. Only the unique winner per cycle gets `Some(true)`; +//! losers get `None` and skip the aggregation entirely. +//! +//! 3. **Winner-only aggregation** — the winner runs the +//! `v2_job_runtime ∪ v2_job_completed` worker-second aggregation +//! returning per-workspace `(workspace_id, worker_seconds, jobs)`, +//! classifies into overloaded/uncapped, computes `p_c`, and writes the +//! new payload `{"overloaded": [...], "admission_ppm": N}` back to +//! `background_task_state.workspace_fairness`. +//! +//! 4. **Everyone reads** — winner and losers alike then `SELECT` the +//! current value, parse it, and update their in-process +//! `WORKSPACE_FAIRNESS_OVERLOADED` and `WORKSPACE_FAIRNESS_ADMISSION_PPM` +//! atomics. This is what makes losers eventually see the winner's +//! decision; they just don't pay the aggregation cost. +//! +//! The refresh interval is `ACTIVE_REFRESH_SECS = 2s` when the cluster +//! currently has a capped workspace (faster — we want the cap to lift +//! promptly once load drops) and `IDLE_REFRESH_SECS = 5s` otherwise +//! (slower — minimise DB load during normal operation). The DB-side guard +//! always uses the tighter `ACTIVE_REFRESH_SECS` to bound the race +//! window; the per-process gate enforces the idle cadence. +//! +//! If a refresh fails (DB error, timeout > 5s), `LAST_REFRESH_MICROS` is +//! left set to the attempt's timestamp so the next attempt has to wait a +//! full interval — exactly the same cooldown as a successful refresh. +//! Resetting to `0` on failure would remove the rate limit entirely +//! precisely when DB load is highest, which is the wrong direction. +//! +//! ## 4. Audit logging +//! +//! Workspaces entering or leaving the capped set produce +//! `workspace_fairness.capped` / `workspace_fairness.uncapped` audit +//! entries scoped to the `admins` workspace, with the affected workspace +//! as the `resource` field. Emitted by the refresh winner only, so a +//! transition produces exactly one audit row regardless of fleet size. +//! The "previous list" diffed against is the DB value (not the per-process +//! cache) so a freshly-restarted worker that happens to win the first +//! claim doesn't emit spurious "newly capped" entries for workspaces that +//! were already capped before it started. +//! +//! ## 5. Notable SQL performance constraints +//! +//! - **No `WITH params AS (...)` CTE for `window_start`.** A natural +//! refactor would be to compute `NOW() - make_interval(secs => N)` once +//! in a CTE and reference it in both halves of the UNION. But Postgres +//! *materialises* the CTE and the optimiser can no longer push the +//! `completed_at > window_start` predicate down to the +//! `ix_job_completed_completed_at` index. On the production cloud DB +//! (~12M `v2_job_completed` rows), that turns a 10 ms index scan into a +//! ~47s full table scan. The query intentionally inlines `NOW()` and +//! `NOW() - make_interval(...)` at every callsite. +//! +//! - **Drive running side from `v2_job_runtime`, not `v2_job_queue`.** +//! Naive ordering ("scan v2_job_queue for `running = true`, join v2_job +//! for the kind filter") does a Seq Scan over ~thousands of running-or- +//! bookkeeping rows and does a PK lookup into `v2_job` for every one of +//! them — ~10 ms in prod, but worse: bounded by *queue size*. Pivoting +//! to drive the scan from `v2_job_runtime` filtered on +//! `ping > NOW() - 60s` narrows to the in-flight set (small, deletes- +//! on-completion) *before* any PK lookups: 1.3 ms, 9× less I/O, +//! bounded by *live worker count*. +//! +//! ## 6. Enterprise gating +//! +//! The cap is an Enterprise feature. `windmill-api-settings` rejects +//! `workspace_fairness_enabled = true` writes from non-EE builds, and on a +//! single-tenant self-hosted deployment the default +//! `workspace_fairness_enabled = false` keeps the pull path identical to +//! the pre-fairness baseline. At runtime the dispatch checks the atomic +//! only — when fairness is off, `maybe_refresh_overloaded` drains the +//! cached state in one pull cycle (resetting `WORKSPACE_FAIRNESS_OVERLOADED` +//! to empty and `WORKSPACE_FAIRNESS_ADMISSION_PPM` to 10_000 = "admit all"), +//! so toggling the feature off without restarting workers is safe. #[cfg(feature = "private")] #[allow(unused)] diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 625fe69780..22cdd5900b 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -22,6 +22,7 @@ import { showConflict, showDiff, extractNativeTriggerInfo, + redactEncryptionKey, } from "../../types.ts"; import { downloadZip } from "./pull.ts"; import { runLint, printReport, checkMissingLocks } from "../lint/lint.ts"; @@ -3095,16 +3096,22 @@ function prettyChanges( ), ); } else if (change.name === "edited") { + const changeType = getTypeStrFromPath(change.path); log.info( colors.yellow( - `~ ${getTypeStrFromPath(change.path)} ` + + `~ ${changeType} ` + displayPath + colors.gray(wsNote) + (change.codebase ? ` (codebase changed)` : ""), ), ); if (change.before != change.after) { - if (change.path.endsWith(".yaml")) { + if (changeType === "encryption_key") { + showDiff( + redactEncryptionKey(change.before), + redactEncryptionKey(change.after), + ); + } else if (change.path.endsWith(".yaml")) { try { showDiff( yamlStringify( diff --git a/cli/src/guidance/writer.ts b/cli/src/guidance/writer.ts index fcaae0671a..033519bf8e 100644 --- a/cli/src/guidance/writer.ts +++ b/cli/src/guidance/writer.ts @@ -207,13 +207,19 @@ async function reconcileIncludingFile(options: { } function referencesIncludeLine(content: string, includeLine: string): boolean { - // Match only when the include sits on a line by itself (allowing leading - // and trailing whitespace). Earlier we split on `\s+`, but that - // false-positives on commented-out includes like `` - // where the middle token equals the include. CRLF is handled by the - // `\r?\n` split. + // Match when the include appears as a whitespace-separated token on any + // line that isn't an HTML comment. We can't require the include to be on a + // line by itself: our own CLAUDE.md default is `Instructions are in + // @AGENTS.md` (one sentence), and a strict equality check made `wmill + // refresh prompts` re-prompt every run on files wmill itself wrote. + // Skipping comment-bearing lines keeps `` from + // false-positiving. for (const line of content.split(/\r?\n/)) { - if (line.trim() === includeLine) { + const trimmed = line.trim(); + if (trimmed.startsWith("")) { + continue; + } + if (trimmed.split(/\s+/).includes(includeLine)) { return true; } } diff --git a/cli/src/types.ts b/cli/src/types.ts index e0d45adb66..75bad535c8 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -129,11 +129,46 @@ export function showDiff(local: string, remote: string) { export function showConflict(path: string, local: string, remote: string) { log.info(colors.yellow(`- ${path}`)); - showDiff(local, remote); + let isEncryptionKey = false; + try { + isEncryptionKey = getTypeStrFromPath(path) === "encryption_key"; + } catch { + // ignore + } + if (isEncryptionKey) { + showDiff(redactEncryptionKey(local), redactEncryptionKey(remote)); + } else { + showDiff(local, remote); + } log.info("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m"); log.info("\n"); } +// Reveal only the first 5 chars of the key so a rotation is still visible in +// the diff (different prefixes), without leaking the whole secret to stdout. +// The remaining chars are replaced with `*`, preserving length so the diff +// keeps showing whether the key length changed. +export function redactEncryptionKey(content: string): string { + if (!content) return content; + // The encryption_key payload is JSON-encoded (a quoted string). Parse it so + // we redact the key value itself, then re-serialize to JSON to preserve the + // file's shape; fall back to raw redaction if parsing fails. + try { + const parsed = JSON.parse(content); + if (typeof parsed === "string") { + return JSON.stringify(redactString(parsed)); + } + } catch { + // not JSON — treat content as the raw key + } + return redactString(content); +} + +function redactString(s: string): string { + if (s.length <= 5) return s; + return s.slice(0, 5) + "*".repeat(s.length - 5); +} + /** * Pushes an object to the workspace server based on its type * @param workspace - The workspace ID to push to diff --git a/cli/test/guidance_writer_unit.test.ts b/cli/test/guidance_writer_unit.test.ts index fa3a91d641..cbeb494d1c 100644 --- a/cli/test/guidance_writer_unit.test.ts +++ b/cli/test/guidance_writer_unit.test.ts @@ -391,6 +391,13 @@ describe("writeAiGuidanceFiles — referencesAgentsCli (via reconciliation)", () ["between blank lines", "before\n\n@AGENTS.cli.md\n\nafter"], ["leading whitespace then include", " @AGENTS.cli.md\n"], ["CRLF line endings", "line one\r\n@AGENTS.cli.md\r\nline three"], + // Mid-sentence include: this is how our own CLAUDE.md default looks + // ("Instructions are in @AGENTS.md"). A strict line-equality check made + // `wmill refresh prompts` re-prompt every run on files wmill wrote. + ["mid-sentence include", "Instructions are in @AGENTS.cli.md\n"], + // `>` blockquote prefix doesn't disable Claude's `@`-import expansion, + // so we treat it as a reference too. + ["blockquoted include", "> @AGENTS.cli.md"], ])("treats %s as a reference (no append)", async (_label, content) => { await withTempDir(async (tempDir) => { await writeFile(join(tempDir, "AGENTS.md"), content, "utf8"); @@ -406,7 +413,6 @@ describe("writeAiGuidanceFiles — referencesAgentsCli (via reconciliation)", () ["@AGENTS-cli-md (lookalike)", "@AGENTS-cli-md"], ["@AGENTS.cli.md without surrounding whitespace", "foo@AGENTS.cli.md"], ["commented-out include", ""], - ["blockquoted include", "> @AGENTS.cli.md"], ])("does not treat %s as a reference (append happens)", async (_label, content) => { await withTempDir(async (tempDir) => { await writeFile(join(tempDir, "AGENTS.md"), content, "utf8"); diff --git a/frontend/scripts/ui_builder_artifact.json b/frontend/scripts/ui_builder_artifact.json index 05a9068963..2863ec67ab 100644 --- a/frontend/scripts/ui_builder_artifact.json +++ b/frontend/scripts/ui_builder_artifact.json @@ -1,5 +1,5 @@ { "baseUrl": "https://pub-06154ed168a24e73a86ab84db6bf15d8.r2.dev", - "version": "00c9834", - "sha256": "5757e5b9cbf79c20d507dc4c588640368e84b873cb48f3806ca8c67fd1aa625f" + "version": "fe13d03", + "sha256": "c588569103ce065a26f334d9b625f616cd23d4204deaacd6fadcfbf16f23462c" } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 79bf64ffb7..6e2f21ce96 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -681,7 +681,11 @@ export class AIChatManager { } else if (this.mode === AIMode.NAVIGATOR) { return prepareNavigatorUserMessage(pendingPrompt) } else if (this.mode === AIMode.GLOBAL) { - return prepareGlobalUserMessage(pendingPrompt, this.contextManager.getSelectedContext()) + return prepareGlobalUserMessage( + pendingPrompt, + this.contextManager.getSelectedContext(), + { workspace: get(workspaceStore) } + ) } return undefined }, @@ -898,7 +902,9 @@ export class AIChatManager { userMessage = prepareApiUserMessage(oldInstructions) break case AIMode.GLOBAL: - userMessage = prepareGlobalUserMessage(oldInstructions, oldSelectedContext) + userMessage = prepareGlobalUserMessage(oldInstructions, oldSelectedContext, { + workspace: get(workspaceStore) + }) break case AIMode.APP: userMessage = prepareAppUserMessage( diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index ddc7e88a29..0569982fcc 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -74,6 +74,8 @@ vi.mock('$lib/gen', async () => { }), AppService: wrapService(actual.AppService, { existsApp: vi.fn(async () => false), + createAppRaw: vi.fn(async () => 'created'), + updateAppRaw: vi.fn(async () => 'updated'), getAppByPath: vi.fn(async () => { throw new Error('getAppByPath mock not configured') }), @@ -96,9 +98,17 @@ vi.mock('$lib/gen', async () => { } }) +vi.mock('./rawAppBundlerBridge', () => ({ + bundleRawAppDraft: vi.fn(async () => ({ + js: 'bundled js', + css: 'bundled css' + })) +})) + import { globalTools, prepareGlobalSystemMessage, prepareGlobalUserMessage } from './core' import { UserDraft, __resetUserDraftForTesting } from '$lib/userDraft.svelte' import { clearGlobalDrafts } from './userDraftAdapter' +import { bundleRawAppDraft } from './rawAppBundlerBridge' import { AppService, FlowService, @@ -960,6 +970,111 @@ describe('global AI tools', () => { expect(UserDraft.get('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() }) + it('deploys a new raw app draft by bundling files and creating a raw app', async () => { + UserDraft.save( + 'raw_app', + 'f/apps/report', + { + summary: 'AI report', + files: { + '/index.tsx': 'console.log("app")', + '/package.json': '{"dependencies":{"react":"19.0.0"}}' + }, + runnables: {}, + data: { tables: [] } + }, + { workspace: WORKSPACE } + ) + + const raw = await callGlobalTool('deploy_workspace_item', { + type: 'app', + path: 'f/apps/report', + deployment_message: 'ship report' + }) + + expect(bundleRawAppDraft).toHaveBeenCalledWith( + expect.objectContaining({ + workspace: WORKSPACE, + files: expect.objectContaining({ + '/index.tsx': 'console.log("app")' + }) + }) + ) + expect(AppService.createAppRaw).toHaveBeenCalledWith({ + workspace: WORKSPACE, + formData: { + app: { + path: 'f/apps/report', + value: { + files: { + '/index.tsx': 'console.log("app")', + '/package.json': '{"dependencies":{"react":"19.0.0"}}' + }, + runnables: {}, + data: { tables: [] } + }, + summary: 'AI report', + policy: expect.objectContaining({ execution_mode: 'publisher' }), + deployment_message: 'ship report', + custom_path: undefined + }, + js: 'bundled js', + css: 'bundled css' + } + }) + expect(AppService.updateAppRaw).not.toHaveBeenCalled() + expect(UserDraft.get('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() + expect(JSON.parse(raw)).toMatchObject({ + success: true, + type: 'app', + path: 'f/apps/report' + }) + }) + + it('deploys an existing raw app draft by bundling files and updating the raw app', async () => { + vi.mocked(AppService.existsApp).mockResolvedValueOnce(true) + UserDraft.save( + 'raw_app', + 'f/apps/report', + { + summary: 'Updated report', + files: { '/index.tsx': 'console.log("updated")' }, + runnables: {}, + data: { tables: ['orders'] }, + policy: { execution_mode: 'anonymous' }, + custom_path: 'kept-by-backend' + }, + { workspace: WORKSPACE } + ) + + await callGlobalTool('deploy_workspace_item', { + type: 'app', + path: 'f/apps/report' + }) + + expect(AppService.updateAppRaw).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/apps/report', + formData: { + app: { + path: 'f/apps/report', + value: { + files: { '/index.tsx': 'console.log("updated")' }, + runnables: {}, + data: { tables: ['orders'] } + }, + summary: 'Updated report', + policy: expect.objectContaining({ execution_mode: 'anonymous' }), + deployment_message: undefined + }, + js: 'bundled js', + css: 'bundled css' + } + }) + expect(AppService.createAppRaw).not.toHaveBeenCalled() + expect(UserDraft.get('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() + }) + it('fills an empty rawscript module through set_flow_module_code', async () => { await callGlobalTool('write_flow', { path: 'f/flows/empty-module', @@ -1168,6 +1283,7 @@ describe('prepareGlobalSystemMessage', () => { expect(content).toContain( 'Use discard_local_draft to remove an unsaved local draft, including the matching open editor draft' ) + expect(content).toContain('If the user message includes an ACTIVE EDITOR section') expect(content).not.toContain('AI draft') expect(content).not.toContain('UserDraft') expect(content).not.toContain('localStorage') @@ -1190,6 +1306,27 @@ describe('prepareGlobalSystemMessage', () => { }) describe('prepareGlobalUserMessage', () => { + it('injects the active editor reference without contents', () => { + __resetUserDraftForTesting() + localStorage.clear() + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'script', + storagePath: '', + effectivePath: 'f/scripts/live_greeting' + }) + + const message = prepareGlobalUserMessage('Update this script', [], { workspace: WORKSPACE }) + + expect(message.content).toContain('## ACTIVE EDITOR') + expect(message.content).toContain('type: script') + expect(message.content).toContain('path: f/scripts/live_greeting') + expect(message.content).toContain('isLiveDraft: true') + expect(message.content).toContain('## INSTRUCTIONS:\nUpdate this script') + expect(message.content).not.toContain('When the user says') + expect(message.content).not.toContain('content') + }) + it('includes selected workspace item references without contents', () => { const message = prepareGlobalUserMessage('Update these items', [ { diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index f093a8d91d..6f2cd10e3b 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -64,6 +64,7 @@ import { import type { ContextElement } from '../context' import { UserDraft, type UserDraftMeta } from '$lib/userDraft.svelte' import { emptySchema } from '$lib/utils' +import { inferArgs } from '$lib/infer' import { resourceRequestSchema, scheduleRequestSchema, @@ -83,6 +84,7 @@ import { type WorkspaceItemType } from './workspaceItems' import { buildFlowDeployRequestBody, buildScriptDeployRequestBody } from './deployRequests' +import { bundleRawAppDraft } from './rawAppBundlerBridge' import { clearEphemeralSecretVariableDraftValue, deleteGlobalDraft, @@ -111,6 +113,28 @@ const INSTRUCTION_SUBJECTS = [ 'app' ] as const satisfies readonly WorkspaceItemType[] const MAX_LIST_LIMIT = 100 +type ActiveGlobalEditorType = Extract +type LiveEditorDraftKind = Parameters[0] + +const ACTIVE_GLOBAL_EDITOR_DRAFTS: readonly { + itemKind: LiveEditorDraftKind + type: ActiveGlobalEditorType +}[] = [ + { itemKind: 'script', type: 'script' }, + { itemKind: 'flow', type: 'flow' }, + { itemKind: 'raw_app', type: 'app' } +] + +export type GlobalActiveEditorContext = { + type: ActiveGlobalEditorType + path: string + isLiveDraft: true +} + +export type GlobalUserMessageOptions = { + workspace?: string + activeEditor?: GlobalActiveEditorContext +} const itemTypeSchema = z.enum(ITEM_TYPES) const instructionSubjectSchema = z.enum(INSTRUCTION_SUBJECTS) @@ -497,7 +521,7 @@ Use tools to inspect workspace items and create local drafts for scripts, flows, Rules: - Draft tools create or update local drafts only; they do not deploy or mutate deployed workspace items. - Use list_workspace_items to find items and read_workspace_item before changing an existing item. For triggers, pass trigger_kind. -- If the user refers to the open editor, use the item marked isLiveDraft=true. +- If the user message includes an ACTIVE EDITOR section, treat it as the currently open item and use it for references like "this", "current", or "open editor". - Use deploy_workspace_item only after the user explicitly asks to deploy. It persists a local draft to the workspace. - Use discard_local_draft to remove an unsaved local draft, including the matching open editor draft. Use delete_workspace_item only to delete a deployed workspace item. - Variable values are never readable. For secrets, create a secret variable and reference it from resources as "$var:path/to/variable". @@ -516,7 +540,7 @@ Raw apps: - Use write_app_file, patch_app_file, and delete_app_file for frontend files. - Use write_app_runnable and delete_app_runnable for backend runnables. - Use init_app only after confirming framework, path, and summary with the user. -- Apps cannot be deployed from chat; tell the user to open the app editor.` +- Use deploy_workspace_item after explicit user deploy intent; raw app deploy bundles JS/CSS before saving.` const DEFAULT_LIST_TYPES = ['script', 'flow'] as const satisfies readonly WorkspaceItemType[] @@ -1211,7 +1235,7 @@ type InstructionSubject = (typeof INSTRUCTION_SUBJECTS)[number] function getAppInstructions(): string { return `# Global draft app instructions -- Global mode edits raw app drafts only; it does not save, deploy, or bundle. +- Global mode edits raw app drafts only; it does not save or deploy unless the user explicitly asks to deploy. - App drafts are addressed by workspace path (e.g. \`f/folder/my_app\`). The first write tool snapshots the workspace app onto the draft, and subsequent writes accumulate. - To create a new app, use \`init_app\` with a path, optional summary, and a framework (\`react19\` / \`react18\` / \`svelte5\` / \`vue\`). Confirm framework + path + summary with the user before calling — do not silently default to \`react19\` even though it is the recommended choice. \`init_app\` errors if an app already exists at the path or a draft is already in flight; in that case, edit the existing one rather than re-initializing. - \`init_app\` seeds a starter inline runnable named \`a\` (bun, \`main(x: string) => string\`) so the React/Svelte demo button works on first render. Replace or remove it once you start building real backend runnables. @@ -1219,7 +1243,7 @@ function getAppInstructions(): string { - Backend inline runnables are addressed as \`backend//main.{ts|py}\` from the file tools, but you create or update them via \`write_app_runnable\` / \`delete_app_runnable\` (which take the runnable shape directly: \`{ name, type, inlineScript?, path?, staticInputs? }\`). - \`/wmill.d.ts\` (or \`wmill.ts\`) is generated automatically from the backend runnables — never write it directly. - Inline runnables only support \`bun\` or \`python3\` in chat. Path runnables (\`script\`/\`flow\`/\`hubscript\`) reference an existing item. -- Apps cannot be deployed from chat. The app editor bundles JS/CSS before save; tell the user to open the app editor to deploy app drafts. +- Use \`deploy_workspace_item\` after explicit user deploy intent. The deploy tool bundles JS/CSS before saving the raw app. - Use \`read_workspace_item\` with \`type: 'app'\` for a metadata summary (file paths and runnable list, no contents). Use \`read_app_file\` to read an individual file. - Note: the authoring reference below mentions the CLI on-disk layout (\`backend/.\`, \`raw_app.yaml\`, \`sql_to_apply/\`). That layout is only relevant for the terminal workflow — in chat, apps are addressed via the tool surface above. @@ -2544,7 +2568,13 @@ async function patchAppFile( } async function recomputeAppPolicy(value: AppDraftValue): Promise { - value.policy = (await updateRawAppPolicy(value.runnables as any, value.policy as any)) as any + const policy = (await updateRawAppPolicy(value.runnables as any, value.policy as any)) as NonNullable< + AppDraftValue['policy'] + > + if (!policy.execution_mode) { + policy.execution_mode = 'publisher' + } + value.policy = policy } async function writeAppRunnable( @@ -2719,12 +2749,6 @@ async function deployDraft( const { workspace, toolId, toolCallbacks } = ctx const { type, path, trigger_kind: triggerKind, deployment_message: deploymentMessage } = args - if (type === 'app') { - throw new Error( - 'Apps cannot be deployed from chat. Open the app editor to deploy (the editor bundles JS/CSS before save).' - ) - } - if (type === 'trigger' && !triggerKind) { throw new Error('trigger_kind is required when deploying a trigger.') } @@ -2748,10 +2772,16 @@ async function deployDraft( const existing = (await ScriptService.existsScriptByPath({ workspace, path })) ? await ScriptService.getScriptByPath({ workspace, path }) : undefined - await ScriptService.createScript({ - workspace, - requestBody: buildScriptDeployRequestBody(path, draft, existing, deploymentMessage) - }) + const requestBody = buildScriptDeployRequestBody(path, draft, existing, deploymentMessage) + // Infer the arg schema from the content so it matches the code, like the editor does. + try { + const schema = emptySchema() + await inferArgs(requestBody.language, requestBody.content, schema) + requestBody.schema = schema + } catch (e) { + console.error('Failed to infer script schema before deploy', e) + } + await ScriptService.createScript({ workspace, requestBody }) break } case 'flow': { @@ -2820,6 +2850,87 @@ async function deployDraft( actions = [createOpenVariableAction(path)] break } + case 'app': { + const appDraft = draft.value as AppDraftValue + const appValue: AppDraftValue = { + ...appDraft, + files: { ...(appDraft.files ?? {}) }, + runnables: { ...(appDraft.runnables ?? {}) }, + data: appDraft.data ?? { ...DEFAULT_RAW_APP_DATA } + } + await recomputeAppPolicy(appValue) + const policy = appValue.policy + if (!policy) { + throw new Error(`Draft app "${path}" has no policy to deploy.`) + } + + toolCallbacks.setToolStatus(toolId, { + content: `Bundling app "${path}"...` + }) + const bundle = await bundleRawAppDraft({ + workspace, + files: appValue.files, + onLog: (delta) => { + const lines = delta + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + const latest = lines[lines.length - 1] + if (latest) { + toolCallbacks.setToolStatus(toolId, { + content: `Bundling app "${path}"... ${latest}` + }) + } + } + }) + + toolCallbacks.setToolStatus(toolId, { + content: `Deploying app "${path}"...` + }) + const rawAppValue = { + files: appValue.files, + runnables: appValue.runnables, + data: appValue.data ?? { ...DEFAULT_RAW_APP_DATA } + } + const summary = appValue.summary ?? draft.summary ?? '' + if (await AppService.existsApp({ workspace, path })) { + // Omit custom_path on update for now. The backend preserves it when absent, while + // sending it requires admin privileges; this chat deploy path does not yet mirror + // the raw app editor's user/admin-specific custom_path handling. + await AppService.updateAppRaw({ + workspace, + path, + formData: { + app: { + path, + value: rawAppValue, + summary, + policy, + deployment_message: deploymentMessage + }, + js: bundle.js, + css: bundle.css + } + }) + } else { + await AppService.createAppRaw({ + workspace, + formData: { + app: { + path, + value: rawAppValue, + summary, + policy, + deployment_message: deploymentMessage, + custom_path: appValue.custom_path + }, + js: bundle.js, + css: bundle.css + } + }) + } + break + } } deleteGlobalDraft(workspace, type, path, triggerKind, { preserveLiveDraft: true }) @@ -2914,15 +3025,36 @@ export function prepareGlobalSystemMessage( } } +export function getActiveGlobalEditorContext( + workspace: string +): GlobalActiveEditorContext | undefined { + for (const { itemKind, type } of ACTIVE_GLOBAL_EDITOR_DRAFTS) { + const liveDraft = UserDraft.getLiveEditorDraft(itemKind, { workspace }) + const path = liveDraft?.effectivePath || liveDraft?.storagePath + if (!path) continue + return { type, path, isLiveDraft: true } + } +} + export function prepareGlobalUserMessage( instructions: string, - selectedContext: ContextElement[] = [] + selectedContext: ContextElement[] = [], + options: GlobalUserMessageOptions = {} ): ChatCompletionUserMessageParam { const selectedWorkspaceItems = selectedContext.filter( (context) => context.type === 'workspace_script' || context.type === 'workspace_flow' ) + const activeEditor = + options.activeEditor ?? (options.workspace ? getActiveGlobalEditorContext(options.workspace) : undefined) let content = '' + if (activeEditor) { + content += '## ACTIVE EDITOR\n' + content += `type: ${activeEditor.type}\n` + content += `path: ${activeEditor.path}\n` + content += `isLiveDraft: true\n\n` + } + if (selectedWorkspaceItems.length > 0) { content += '## SELECTED CONTEXT\n' for (const context of selectedWorkspaceItems) { diff --git a/frontend/src/lib/components/copilot/chat/global/rawAppBundlerBridge.ts b/frontend/src/lib/components/copilot/chat/global/rawAppBundlerBridge.ts new file mode 100644 index 0000000000..27f9f5aaee --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/rawAppBundlerBridge.ts @@ -0,0 +1,144 @@ +import { WorkspaceService } from '$lib/gen' + +export type RawAppBundle = { + js: string + css: string +} + +type BundleRawAppFilesParams = { + files: Record + sharedUiFiles?: Record + bundlerType?: 'esbuild' | 'rolldown' + timeoutMs?: number + onLog?: (delta: string) => void +} + +type BundleRawAppDraftParams = BundleRawAppFilesParams & { + workspace: string +} + +const DEFAULT_TIMEOUT_MS = 120_000 + +function makeRequestId(): string { + return globalThis.crypto?.randomUUID?.() ?? Math.random().toString(36).slice(2) +} + +async function loadSharedUiFiles(workspace: string): Promise> { + try { + const res = (await WorkspaceService.getSharedUi({ workspace })) as { + files?: Record + } + return res.files ?? {} + } catch (e) { + console.warn('Failed to load shared UI for raw app bundling:', e) + return {} + } +} + +export async function bundleRawAppDraft(params: BundleRawAppDraftParams): Promise { + const sharedUiFiles = params.sharedUiFiles ?? (await loadSharedUiFiles(params.workspace)) + return bundleRawAppFiles({ + ...params, + sharedUiFiles + }) +} + +export function bundleRawAppFiles({ + files, + sharedUiFiles = {}, + bundlerType = 'esbuild', + timeoutMs = DEFAULT_TIMEOUT_MS, + onLog +}: BundleRawAppFilesParams): Promise { + if (typeof window === 'undefined' || typeof document === 'undefined') { + return Promise.reject(new Error('Raw app bundling requires a browser environment.')) + } + + return new Promise((resolve, reject) => { + const requestId = makeRequestId() + const iframe = document.createElement('iframe') + let settled = false + let bundleRequestSent = false + + const cleanup = () => { + window.removeEventListener('message', onMessage) + clearTimeout(timeout) + iframe.remove() + } + + const settle = (fn: (value: T) => void, value: T) => { + if (settled) return + settled = true + cleanup() + fn(value) + } + + const postToBundler = (message: Record) => { + if (!iframe.contentWindow) { + settle(reject, new Error('Raw app bundler iframe did not initialize.')) + return + } + iframe.contentWindow.postMessage(message, window.location.origin) + } + + const sendBundleProtocolRequest = () => { + if (settled || bundleRequestSent) return + bundleRequestSent = true + postToBundler({ + type: 'bundleRawApp', + requestId, + files, + sharedUiFiles, + bundlerType + }) + } + + const timeout = window.setTimeout(() => { + settle(reject, new Error('Timed out while bundling raw app.')) + }, timeoutMs) + + function onMessage(event: MessageEvent) { + if (event.source !== iframe.contentWindow) return + const data = event.data + if (!data) return + + if (data.type === 'bundleRawAppReady') { + sendBundleProtocolRequest() + return + } + + if (data.requestId !== requestId) return + + if (data.type === 'appendLogs') { + onLog?.(String(data.delta ?? '')) + } else if (data.type === 'bundleRawAppResult') { + const bundle = data.bundle + if (!bundle?.js) { + settle(reject, new Error('Raw app bundler returned an empty JavaScript bundle.')) + return + } + settle(resolve, { + js: String(bundle.js), + css: String(bundle.css ?? '') + }) + } else if (data.type === 'bundleRawAppError') { + settle(reject, new Error(String(data.error ?? 'Raw app bundle failed.'))) + } + } + + iframe.title = 'Raw app bundler' + iframe.tabIndex = -1 + // Windmill pages use COEP=require-corp; the static UI builder iframe must be credentialless. + iframe.setAttribute('credentialless', '') + iframe.style.position = 'fixed' + iframe.style.width = '0' + iframe.style.height = '0' + iframe.style.border = '0' + iframe.style.opacity = '0' + iframe.style.pointerEvents = 'none' + iframe.src = '/ui_builder/index.html?mode=bundle' + + window.addEventListener('message', onMessage) + document.body.appendChild(iframe) + }) +} diff --git a/frontend/src/lib/components/select/MultiSelect.svelte b/frontend/src/lib/components/select/MultiSelect.svelte index 750ab62b8c..419984b20d 100644 --- a/frontend/src/lib/components/select/MultiSelect.svelte +++ b/frontend/src/lib/components/select/MultiSelect.svelte @@ -72,6 +72,8 @@ let wrapperEl: HTMLDivElement | undefined = $state() let searchInputEl: TextInput | undefined = $state() + let currentValue = $derived(value ?? []) + $effect(() => searchInputEl?.focus()) let processedItems: ProcessedItem[] = $derived.by(() => { @@ -87,18 +89,20 @@ }) let valueEntry = $derived( - value.map((v) => processedItems.find((item) => item.value === v) ?? { value: v, label: v }) + currentValue.map( + (v) => processedItems.find((item) => item.value === v) ?? { value: v, label: v } + ) ) function onAddValue(item: ProcessedItem) { if (item.__is_create && onCreateItem) { onCreateItem(item.value) } else { - value = [...value, item.value] + value = [...currentValue, item.value] } } function onRemoveValue(item: ProcessedItem) { - value = value.filter((v) => v !== item.value) + value = currentValue.filter((v) => v !== item.value) } function clearValue() { @@ -132,7 +136,7 @@ - {#if value.length === 0} + {#if currentValue.length === 0} {placeholder} @@ -149,7 +153,7 @@ {allowClear} onRemove={onRemoveValue} onReorder={reorderable - ? (oldIdx, newIdx) => (value = reorder(value, oldIdx, newIdx)) + ? (oldIdx, newIdx) => (value = reorder(currentValue, oldIdx, newIdx)) : undefined} /> @@ -166,7 +170,7 @@ {disablePortal} onSelectValue={onAddValue} {open} - processedItems={processedItems.filter((item) => !value.includes(item.value))} + processedItems={processedItems.filter((item) => !currentValue.includes(item.value))} value={undefined} {disabled} {filterText} @@ -181,7 +185,7 @@ ulClass="options" > {#snippet header()} - {#if processedItems.length - value.length > 0 || onCreateItem} + {#if processedItems.length - currentValue.length > 0 || onCreateItem}
- + {#snippet children({ createMenu })} - +
+ - {#each secondaryMenuLinks as menuLink (menuLink.href ?? menuLink.label)} - {#if menuLink.subItems} - {@const notificationsCount = computeAllNotificationsCount(menuLink.subItems)} - - {#snippet triggr({ trigger })} - - {/snippet} + {#each secondaryMenuLinks as menuLink (menuLink.href ?? menuLink.label)} + {#if menuLink.subItems} + {@const notificationsCount = computeAllNotificationsCount(menuLink.subItems)} + + {#snippet triggr({ trigger })} + + {/snippet} - {#snippet children({ item })} - {#each menuLink.subItems as subItem (subItem.href ?? subItem.label)} - { - subItem?.['action']?.() - }} - aiId={subItem.aiId} - aiDescription={subItem.aiDescription} - > -
- {#if subItem.icon} - - {/if} - {subItem.label} - {#if subItem?.['notificationCount']} -
- -
- {/if} -
- - {/each} - {/snippet} - - {:else} - - {#snippet children({})} - - {/snippet} - - {/if} - {/each} - {/snippet} - - - - {#snippet children({ createMenu })} - {#each thirdMenuLinks as menuLink (menuLink)} - {#if menuLink.subItems} - - {#snippet triggr({ trigger })} - - {/snippet} - {#snippet children({ item })} - {#each menuLink.subItems as subItem (subItem.href ?? subItem.label)} - -
- {#if subItem.icon} - - {/if} - - {subItem.label} -
- - {/each} - {#if recentChangelogs.length > 0} -
- Latest changelogs - {#each recentChangelogs as changelog} -
- {changelog.label} + {#if subItem.icon} + + {/if} + {subItem.label} + {#if subItem?.['notificationCount']} +
+ +
+ {/if}
{/each} - {/if} - {/snippet} - - {/if} - {/each} + {/snippet} + + {:else} + + {#snippet children({})} + + {/snippet} + + {/if} + {/each} +
+ +
+ {#each thirdMenuLinks as menuLink (menuLink)} + {#if menuLink.subItems} + + {#snippet triggr({ trigger })} + + {/snippet} + {#snippet children({ item })} + {#each menuLink.subItems as subItem (subItem.href ?? subItem.label)} + +
+ {#if subItem.icon} + + {/if} + + {subItem.label} +
+ + {/each} + {#if recentChangelogs.length > 0} +
+ Latest changelogs + {#each recentChangelogs as changelog} + +
+ {changelog.label} +
+ + {/each} + {/if} + {/snippet} + + {/if} + {/each} +
{/snippet}
diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index b7652da98f..fb5f998aad 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -696,7 +696,7 @@ variant="default" startIcon={{ icon: List }} disabled={!allowSchedule || pathError != '' || emptyString(script_path)} - href={`${base}/runs/${script_path}?show_schedules=true&show_future_jobs=true`} + href={`${base}/runs/?schedule_path=${path}&job_trigger_kind=schedule&show_future_jobs=true`} > View runs diff --git a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte index 9fccaa8b35..222e5606c8 100644 --- a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte +++ b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte @@ -9,6 +9,8 @@ import { ArrowRight, TriangleAlert } from 'lucide-svelte' import type { ConfirmationModalHandle } from '../common/confirmationModal/asyncConfirmationModal.svelte' import type { Snippet } from 'svelte' + import Tooltip from '../meltComponents/Tooltip.svelte' + import { workspaceStore } from '$lib/stores' type Props = { value: string | undefined @@ -37,6 +39,11 @@ ) ) let open = $state(false) + + function otherWorkspaces(dbname: string): string[] { + const all = customInstanceDbs.current?.[dbname]?.used_by_workspaces ?? [] + return all.filter((w) => w !== $workspaceStore) + }
@@ -52,11 +59,17 @@ disabled={!$isCustomInstanceDbEnabled} > {#snippet endSnippet({ item })} - {@render customInstanceDbWizardButton(item.value)} +
+ {@render sharedWorkspacesWarning(item.value)} + {@render customInstanceDbWizardButton(item.value)} +
{/snippet} {#if value} - {@render customInstanceDbWizardButton(value, 'absolute right-1.5')} +
+ {@render sharedWorkspacesWarning(value)} + {@render customInstanceDbWizardButton(value)} +
{/if}
@@ -74,12 +87,12 @@ } /> -{#snippet customInstanceDbWizardButton(dbname: string, clazz: string = '')} +{#snippet customInstanceDbWizardButton(dbname: string)} {@const status = customInstanceDbs.current?.[dbname]} {/snippet} + +{#snippet sharedWorkspacesWarning(dbname: string)} + {@const others = otherWorkspaces(dbname)} + {#if others.length > 0} + + + {#snippet text()} + This database is also used by workspace{others.length > 1 ? 's' : ''} + {others.join(', ')}. Any data written here will be shared + with {others.length > 1 ? 'them' : 'it'}. + {/snippet} + + {/if} +{/snippet} diff --git a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte index b9731394a0..2a88266c4e 100644 --- a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte @@ -462,7 +462,7 @@ {/key}