starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-09 16:05:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into bump-cargo-sweep

Browse Source
This commit is contained in:
Pyra
2026-05-27 14:27:17 +02:00
committed by GitHub
parent e6ac30e625 e29dfbaa87
commit 18d72d342f
7 changed files with 497 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai_evals/adapters/frontend/core/global/globalEvalRunner.ts
+7 -4
View File
@@ -7,7 +7,10 @@ import {
prepareGlobalSystemMessage,
prepareGlobalUserMessage,
} from "../../../../../frontend/src/lib/components/copilot/chat/global/core";
import { globalDraftStore } from "../../../../../frontend/src/lib/components/copilot/chat/global/draftStore.svelte";
import {
clearGlobalDrafts,
listGlobalDrafts,
} from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter";
import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
import type { ModeRunContext } from "../../../../core/types";
import type { GlobalDraftState } from "../../../../core/validators";
@@ -55,7 +58,7 @@ export async function runGlobalEval(
options.workspaceRoot ??
(await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-")));
globalDraftStore.clearDrafts(workspaceRoot);
clearGlobalDrafts(workspaceRoot);
registerBenchmarkWorkspaceRunnables(workspaceRoot, options.workspaceFixtures ?? {});
try {
@@ -67,7 +70,7 @@ export async function runGlobalEval(
tools: getGlobalEvalTools(),
helpers: {},
apiKey,
getOutput: () => ({ drafts: globalDraftStore.listDrafts(workspaceRoot) }),
getOutput: () => ({ drafts: listGlobalDrafts(workspaceRoot) }),
onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
onAssistantToken: options.runContext?.onAssistantChunk,
onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
@@ -94,7 +97,7 @@ export async function runGlobalEval(
tokenUsage: rawResult.tokenUsage,
};
} finally {
globalDraftStore.clearDrafts(workspaceRoot);
clearGlobalDrafts(workspaceRoot);
unregisterBenchmarkWorkspaceRunnables(workspaceRoot);
if (!options.workspaceRoot) {
await rm(workspaceRoot, { recursive: true, force: true });
ai_evals/adapters/frontend/vitestAdapter.test.ts
+30
View File
@@ -79,6 +79,16 @@ vi.mock('$lib/gen', async () => {
}
return actual.ScriptService.getScriptByPath(data)
},
getScriptByPathWithDraft: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const script = getBenchmarkScriptByPath(data.workspace, data.path)
if (!script) {
throw new Error(`Script "${data.path}" not found in benchmark workspace`)
}
return script
}
return actual.ScriptService.getScriptByPathWithDraft(data)
},
getScriptByHash: async (data: { workspace: string; hash: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const script = getBenchmarkScriptByHash(data.workspace, data.hash)
@@ -108,6 +118,26 @@ vi.mock('$lib/gen', async () => {
return flow
}
return actual.FlowService.getFlowByPath(data)
},
getFlowByPathWithDraft: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const flow = getBenchmarkFlowByPath(data.workspace, data.path)
if (!flow) {
throw new Error(`Flow "${data.path}" not found in benchmark workspace`)
}
return flow
}
return actual.FlowService.getFlowByPathWithDraft(data)
},
getFlowLatestVersion: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const flow = getBenchmarkFlowByPath(data.workspace, data.path)
if (!flow) {
throw new Error(`Flow "${data.path}" not found in benchmark workspace`)
}
return { id: 1 }
}
return actual.FlowService.getFlowLatestVersion(data)
}
}),
JobService: wrapService(actual.JobService, {
ai_evals/cases/global.yaml
+254
View File
@@ -87,3 +87,257 @@
- the flow accepts numeric inputs a and b
- the flow returns the sum of a and b
- the result stays as an AI draft and is not deployed or saved to the workspace
- id: global-test4-multi-artifact-notification-job
prompt: |-
Set up a draft stale-trial notification job.
Create a Bun script at `f/evals/global/check_stale_trials` that accepts `max_age_days`, uses mocked inline trial account data, and returns the stale trial account IDs.
Also create a weekday 09:00 UTC schedule at `f/evals/global/check_stale_trials_weekday` for that script with `max_age_days` set to 14.
Add an HTTP POST trigger at `f/evals/global/check_stale_trials_manual` with route path `evals/check-stale-trials` that runs the same script manually.
Leave everything as AI drafts only; do not deploy or save anything to the workspace.
runtime:
maxTurns: 12
validate:
draftCountExactly: 3
requiredDrafts:
- type: script
path: f/evals/global/check_stale_trials
language: bun
valueIncludes:
- max_age_days
- trial
- type: schedule
path: f/evals/global/check_stale_trials_weekday
valueIncludes:
- f/evals/global/check_stale_trials
- UTC
- "14"
- type: trigger
triggerKind: http
path: f/evals/global/check_stale_trials_manual
valueIncludes:
- evals/check-stale-trials
- f/evals/global/check_stale_trials
toolExpect:
requiredToolsUsed:
- write_script
- write_schedule
- write_trigger
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- creates a Bun script draft for stale trial accounts
- creates a weekday 09:00 UTC schedule draft for the script with max_age_days set to 14
- creates an HTTP POST trigger draft with route path evals/check-stale-trials for the same script
- leaves all artifacts as drafts only and does not deploy
- id: global-test5-existing-flow-inline-code-edit
prompt: |-
Update the existing flow at `f/evals/global/process_invoice`.
Only change the `calculate_total` inline code so it applies 8% tax and returns an object containing `subtotal`, `tax`, and `total`.
Leave the updated flow as an AI draft only; do not deploy or save it.
initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
runtime:
maxTurns: 10
validate:
draftCountExactly: 1
requiredDrafts:
- type: flow
path: f/evals/global/process_invoice
valueIncludes:
- calculate_total
- tax
- total
toolExpect:
requiredToolsUsed:
- read_workspace_item
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- reads the existing process_invoice flow before editing it
- updates the calculate_total inline code to apply 8% tax
- returns subtotal, tax, and total from the updated flow logic
- leaves the result as an AI draft only
- id: global-test6-secret-variable-draft
prompt: |-
Create a secret variable draft at `f/evals/global/slack_bot_token`.
Use the placeholder value `xoxb-redacted-test-token` and description `Slack bot token for eval notifications`.
Do not create any resource or deploy anything.
runtime:
maxTurns: 6
validate:
draftCountExactly: 1
requiredDrafts:
- type: variable
path: f/evals/global/slack_bot_token
valueIncludes:
- Slack bot token
- "true"
forbiddenDrafts:
- type: resource
path: f/evals/global/slack_bot_token
toolExpect:
requiredToolsUsed:
- write_variable
forbiddenToolsUsed:
- write_resource
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: write_variable
field: value
stringStartsWithAnyOf:
- xoxb-redacted-test-token
skipJudge: true
judgeChecklist:
- creates exactly one secret variable draft at f/evals/global/slack_bot_token
- uses the requested placeholder value and description
- does not create a resource or deploy anything
- id: global-test7-ambiguous-app-asks-question
prompt: |-
Create a new raw app for triaging support tickets.
runtime:
maxTurns: 4
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- askUserQuestion
forbiddenToolsUsed:
- init_app
- write_app_file
- write_app_runnable
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
- id: global-test8-human-script-infer-path-language
prompt: |-
I need a small helper that formats a customer-facing welcome line.
It should take a person's name and return "Welcome aboard, <name>!".
Please just stage it as a draft for now.
runtime:
maxTurns: 8
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
valueIncludes:
- Welcome aboard
- name
toolExpect:
requiredToolsUsed:
- write_script
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- creates a single script draft for a welcome-line helper
- accepts a person's name as input
- returns a message containing Welcome aboard, the provided name, and an exclamation mark
- chooses a reasonable workspace path and script language without needing the user to specify them
- leaves the result as an AI draft only
- id: global-test9-human-weekday-trial-job
prompt: |-
Can you set up a draft daily job that checks a few hard-coded trial accounts and returns the ones whose trial has ended?
It should run every weekday morning around 9 in UTC with a 30 day cutoff.
Keep it as draft work only.
runtime:
maxTurns: 10
validate:
draftCountExactly: 2
requiredDrafts:
- type: script
pathIncludes:
- trial
valueIncludes:
- trial
- "30"
- type: schedule
pathIncludes:
- trial
valueIncludes:
- UTC
toolExpect:
requiredToolsUsed:
- write_script
- write_schedule
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- creates a script draft that checks hard-coded trial accounts
- returns the accounts whose trial has ended based on a 30 day cutoff
- creates a schedule draft for weekday mornings around 09:00 UTC
- links the schedule to the generated script
- leaves both artifacts as drafts only
- id: global-test10-human-secret-variable
prompt: |-
I need a placeholder Slack bot token stored securely for future notification work.
Use xoxb-redacted-test-token and note that it is for eval notifications.
Only prepare a draft.
runtime:
maxTurns: 6
validate:
draftCountExactly: 1
requiredDrafts:
- type: variable
pathIncludes:
- slack
valueIncludes:
- eval notifications
- "true"
toolExpect:
requiredToolsUsed:
- write_variable
forbiddenToolsUsed:
- write_resource
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: write_variable
field: value
stringStartsWithAnyOf:
- xoxb-redacted-test-token
skipJudge: true
judgeChecklist:
- creates a single secret variable draft for the Slack bot token placeholder
- uses the requested placeholder value
- includes a note or description that it is for eval notifications
- does not create a resource or deploy anything
- id: global-test11-human-existing-flow-informal-edit
prompt: |-
There is an invoice processing flow in this workspace.
Can you adjust its total calculation so it adds 8% tax and returns subtotal, tax, and total?
Keep the change as a draft.
initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
runtime:
maxTurns: 10
validate:
draftCountExactly: 1
requiredDrafts:
- type: flow
pathIncludes:
- invoice
valueIncludes:
- calculate_total
- tax
- total
toolExpect:
requiredToolsUsed:
- read_workspace_item
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- finds and edits the existing invoice processing flow without the user providing its exact path
- updates the total calculation to apply 8% tax
- returns subtotal, tax, and total from the updated flow logic
- leaves the result as an AI draft only
ai_evals/core/types.ts
+3 -1
View File
@@ -110,7 +110,9 @@ export interface AppValidationSpec {
export interface GlobalDraftRequirement {
type: string;
path: string;
path?: string;
pathIncludes?: string[];
pathStartsWith?: string;
triggerKind?: string;
language?: string;
summaryIncludes?: string[];
ai_evals/core/validators.test.ts
+63
View File
@@ -195,6 +195,69 @@ describe("validateGlobalState", () => {
});
});
it("accepts a required script draft without an exact path", () => {
const checks = validateGlobalState({
actual: {
drafts: [
{
type: "script",
path: "f/team_tools/friendly_greeting",
language: "bun",
summary: "Friendly greeting helper",
value:
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
isDraft: true,
},
],
},
validate: {
draftCountExactly: 1,
requiredDrafts: [
{
type: "script",
pathIncludes: ["greeting"],
language: "bun",
summaryIncludes: ["Friendly"],
valueIncludes: ["Hello"],
},
],
},
});
expect(checks.every((check) => check.passed)).toBe(true);
});
it("reports flexible global draft path filters when no draft matches", () => {
const checks = validateGlobalState({
actual: {
drafts: [
{
type: "script",
path: "f/team_tools/friendly_greeting",
language: "bun",
value:
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
isDraft: true,
},
],
},
validate: {
requiredDrafts: [
{
type: "script",
pathIncludes: ["invoice"],
},
],
},
});
expect(checks).toContainEqual({
name: "global includes script draft (path includes invoice)",
passed: false,
details: "drafts: script:f/team_tools/friendly_greeting",
});
});
it("does not require a TypeScript entrypoint for non-TypeScript script drafts", () => {
const checks = validateGlobalState({
actual: {
ai_evals/core/validators.ts
+100 -15
View File
@@ -315,10 +315,11 @@ export function validateGlobalState(input: {
}
for (const required of validate.requiredDrafts ?? []) {
const draft = findGlobalDraft(drafts, required.type, required.path, required.triggerKind);
const requirementLabel = formatGlobalDraftRequirement(required);
const draft = findGlobalDraft(drafts, required);
checks.push(
check(
`global includes ${required.type} draft ${required.path}`,
`global includes ${requirementLabel}`,
Boolean(draft),
summarizeGlobalDrafts(drafts)
)
@@ -330,7 +331,7 @@ export function validateGlobalState(input: {
if (required.language !== undefined) {
checks.push(
check(
`${required.type} draft ${required.path} uses ${required.language}`,
`${requirementLabel} uses ${required.language}`,
draft.language === required.language,
`language=${draft.language ?? "(none)"}`
)
@@ -340,7 +341,7 @@ export function validateGlobalState(input: {
for (const snippet of required.summaryIncludes ?? []) {
checks.push(
check(
`${required.type} draft ${required.path} summary includes '${snippet}'`,
`${requirementLabel} summary includes '${snippet}'`,
normalizeText(draft.summary ?? "").includes(normalizeText(snippet)),
`summary=${draft.summary ?? ""}`
)
@@ -351,7 +352,7 @@ export function validateGlobalState(input: {
for (const snippet of required.valueIncludes ?? []) {
checks.push(
check(
`${required.type} draft ${required.path} value includes '${snippet}'`,
`${requirementLabel} value includes '${snippet}'`,
normalizeText(valueText).includes(normalizeText(snippet)),
truncateForDetails(valueText)
)
@@ -361,7 +362,7 @@ export function validateGlobalState(input: {
for (const snippet of required.valueExcludes ?? []) {
checks.push(
check(
`${required.type} draft ${required.path} value excludes '${snippet}'`,
`${requirementLabel} value excludes '${snippet}'`,
!normalizeText(valueText).includes(normalizeText(snippet)),
truncateForDetails(valueText)
)
@@ -373,7 +374,7 @@ export function validateGlobalState(input: {
checks.push(
check(
`global does not include ${forbidden.type} draft ${forbidden.path}`,
!findGlobalDraft(drafts, forbidden.type, forbidden.path, forbidden.triggerKind),
!findGlobalDraft(drafts, forbidden),
summarizeGlobalDrafts(drafts)
)
);
@@ -615,16 +616,100 @@ function summarizeProblems(problems: string[], limit = 5): string | undefined {
function findGlobalDraft(
drafts: GlobalDraft[],
type: string,
path: string,
triggerKind?: string
requirement: {
type: string;
path?: string;
pathIncludes?: string[];
pathStartsWith?: string;
triggerKind?: string;
summaryIncludes?: string[];
valueIncludes?: string[];
valueExcludes?: string[];
}
): GlobalDraft | undefined {
return drafts.find(
(draft) =>
draft.type === type &&
draft.path === path &&
(triggerKind === undefined || draft.triggerKind === triggerKind)
const candidates = drafts.filter((draft) =>
globalDraftMatchesLocator(draft, requirement)
);
return (
candidates.find((draft) => globalDraftMatchesContent(draft, requirement)) ??
candidates[0]
);
}
function globalDraftMatchesLocator(
draft: GlobalDraft,
requirement: {
type: string;
path?: string;
pathIncludes?: string[];
pathStartsWith?: string;
triggerKind?: string;
}
): boolean {
return (
draft.type === requirement.type &&
(requirement.path === undefined || draft.path === requirement.path) &&
(requirement.pathStartsWith === undefined ||
draft.path.startsWith(requirement.pathStartsWith)) &&
(requirement.pathIncludes ?? []).every((snippet) =>
normalizeText(draft.path).includes(normalizeText(snippet))
) &&
(requirement.triggerKind === undefined ||
draft.triggerKind === requirement.triggerKind)
);
}
function globalDraftMatchesContent(
draft: GlobalDraft,
requirement: {
summaryIncludes?: string[];
valueIncludes?: string[];
valueExcludes?: string[];
}
): boolean {
const summary = normalizeText(draft.summary ?? "");
const value = normalizeText(stringifyGlobalDraftValue(draft.value));
return (
(requirement.summaryIncludes ?? []).every((snippet) =>
summary.includes(normalizeText(snippet))
) &&
(requirement.valueIncludes ?? []).every((snippet) =>
value.includes(normalizeText(snippet))
) &&
(requirement.valueExcludes ?? []).every(
(snippet) => !value.includes(normalizeText(snippet))
)
);
}
function formatGlobalDraftRequirement(
requirement: {
type: string;
path?: string;
pathIncludes?: string[];
pathStartsWith?: string;
triggerKind?: string;
}
): string {
const typeLabel =
requirement.triggerKind === undefined
? requirement.type
: `${requirement.triggerKind} ${requirement.type}`;
if (requirement.path !== undefined) {
return `${typeLabel} draft ${requirement.path}`;
}
const filters = [
...(requirement.pathStartsWith === undefined
? []
: [`path starts with ${requirement.pathStartsWith}`]),
...(requirement.pathIncludes ?? []).map(
(snippet) => `path includes ${snippet}`
),
];
return filters.length === 0
? `${typeLabel} draft`
: `${typeLabel} draft (${filters.join(", ")})`;
}
function summarizeGlobalDrafts(drafts: GlobalDraft[]): string {
ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
+40
View File
@@ -0,0 +1,40 @@
{
"workspace": {
"flows": [
{
"path": "f/evals/global/process_invoice",
"summary": "Process an invoice subtotal",
"description": "Calculates invoice totals from a subtotal.",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"subtotal": {
"type": "number"
}
},
"required": ["subtotal"]
},
"value": {
"modules": [
{
"id": "calculate_total",
"summary": "Calculate total from subtotal",
"value": {
"type": "rawscript",
"language": "bun",
"content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n",
"input_transforms": {
"subtotal": {
"type": "javascript",
"expr": "flow_input.subtotal"
}
}
}
}
]
}
}
]
}
}