mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-08 16:03:27 +00:00
Merge remote-tracking branch 'origin/main' into remove-workspace-drafts
# Conflicts: # frontend/src/lib/components/copilot/chat/global/core.test.ts
This commit is contained in:
Diego Imbert
+16
-3
@@ -142,6 +142,15 @@ For `global` mode, `validate` can express draft-level requirements such as:
|
||||
- required or forbidden draft counts
|
||||
- forbidden draft paths
|
||||
|
||||
Global initial fixtures can also seed `liveEditorDrafts` with `type`,
|
||||
`storagePath`, `effectivePath`, and `value` fields. These drafts emulate the
|
||||
currently open script, flow, or raw app editor so cases can test prompts that
|
||||
refer to "this" or the "current" item.
|
||||
|
||||
Set `WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT=1` to run those cases with
|
||||
the old behavior where the live editor is only discoverable through
|
||||
`list_workspace_items`.
|
||||
|
||||
App fixtures can also include an optional `datatables.json` file at the fixture root.
|
||||
|
||||
For `flow` mode, an `initial` fixture can also include a benchmark workspace catalog of
|
||||
@@ -189,11 +198,15 @@ If `--record` is used, the CLI also appends one compact JSON line to:
|
||||
Each recorded line contains:
|
||||
|
||||
- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `judgeModel`)
|
||||
- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averageJudgeScore`)
|
||||
- average token usage (`averageTokenUsagePerAttempt`)
|
||||
- per-case metrics under `cases[]` (`averageDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, pass rate)
|
||||
- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averagePassedDurationMs`, `averageJudgeScore`)
|
||||
- average token usage (`averageTokenUsagePerAttempt`, `averageTokenUsagePerPassedAttempt`)
|
||||
- per-case metrics under `cases[]` (`averageDurationMs`, `averagePassedDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, `averageTokenUsagePerPassedAttempt`, pass rate)
|
||||
- `failedCaseIds`
|
||||
|
||||
The CLI headline duration and token averages use passed attempts only.
|
||||
All-attempt averages are still recorded to make failures auditable without
|
||||
letting failed attempts skew success cost comparisons.
|
||||
|
||||
Example:
|
||||
|
||||
- summary: `ai_evals/results/2026-04-09T09-40-33.051Z__flow.json`
|
||||
|
||||
@@ -7,8 +7,12 @@ import {
|
||||
prepareGlobalSystemMessage,
|
||||
prepareGlobalUserMessage,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/global/core";
|
||||
import { globalDraftStore } from "../../../../../frontend/src/lib/components/copilot/chat/global/draftStore.svelte";
|
||||
import {
|
||||
clearGlobalDrafts,
|
||||
listGlobalDrafts,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter";
|
||||
import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
|
||||
import { UserDraft } from "../../../../../frontend/src/lib/userDraft.svelte";
|
||||
import type { ModeRunContext } from "../../../../core/types";
|
||||
import type { GlobalDraftState } from "../../../../core/validators";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
@@ -24,6 +28,21 @@ const MUTATING_GLOBAL_TOOLS = new Set([
|
||||
"deploy_workspace_item",
|
||||
"delete_workspace_item",
|
||||
]);
|
||||
const DISABLE_ACTIVE_EDITOR_CONTEXT_ENV =
|
||||
"WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT";
|
||||
|
||||
const LIVE_EDITOR_ITEM_KINDS = {
|
||||
script: "script",
|
||||
flow: "flow",
|
||||
app: "raw_app",
|
||||
} as const;
|
||||
|
||||
export interface GlobalLiveEditorDraftFixture {
|
||||
type: keyof typeof LIVE_EDITOR_ITEM_KINDS;
|
||||
storagePath?: string;
|
||||
effectivePath?: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
export interface GlobalEvalResult {
|
||||
success: boolean;
|
||||
@@ -38,6 +57,7 @@ export interface GlobalEvalResult {
|
||||
|
||||
export interface GlobalEvalOptions {
|
||||
workspaceFixtures?: BenchmarkWorkspaceRunnables;
|
||||
liveEditorDrafts?: GlobalLiveEditorDraftFixture[];
|
||||
model?: string;
|
||||
maxIterations?: number;
|
||||
provider?: AIProvider;
|
||||
@@ -55,19 +75,26 @@ export async function runGlobalEval(
|
||||
options.workspaceRoot ??
|
||||
(await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-")));
|
||||
|
||||
globalDraftStore.clearDrafts(workspaceRoot);
|
||||
clearGlobalDrafts(workspaceRoot);
|
||||
registerBenchmarkWorkspaceRunnables(workspaceRoot, options.workspaceFixtures ?? {});
|
||||
seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []);
|
||||
|
||||
try {
|
||||
const model = options.model ?? "claude-haiku-4-5-20251001";
|
||||
const injectActiveEditorContext =
|
||||
process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1";
|
||||
const rawResult = await runEval({
|
||||
userPrompt,
|
||||
systemMessage: prepareGlobalSystemMessage(),
|
||||
userMessage: prepareGlobalUserMessage(userPrompt),
|
||||
userMessage: prepareGlobalUserMessage(
|
||||
userPrompt,
|
||||
[],
|
||||
injectActiveEditorContext ? { workspace: workspaceRoot } : {},
|
||||
),
|
||||
tools: getGlobalEvalTools(),
|
||||
helpers: {},
|
||||
apiKey,
|
||||
getOutput: () => ({ drafts: globalDraftStore.listDrafts(workspaceRoot) }),
|
||||
getOutput: () => ({ drafts: listGlobalDrafts(workspaceRoot) }),
|
||||
onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
|
||||
onAssistantToken: options.runContext?.onAssistantChunk,
|
||||
onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
|
||||
@@ -94,7 +121,8 @@ export async function runGlobalEval(
|
||||
tokenUsage: rawResult.tokenUsage,
|
||||
};
|
||||
} finally {
|
||||
globalDraftStore.clearDrafts(workspaceRoot);
|
||||
clearGlobalDrafts(workspaceRoot);
|
||||
clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []);
|
||||
unregisterBenchmarkWorkspaceRunnables(workspaceRoot);
|
||||
if (!options.workspaceRoot) {
|
||||
await rm(workspaceRoot, { recursive: true, force: true });
|
||||
@@ -102,6 +130,36 @@ export async function runGlobalEval(
|
||||
}
|
||||
}
|
||||
|
||||
function seedLiveEditorDrafts(
|
||||
workspace: string,
|
||||
fixtures: GlobalLiveEditorDraftFixture[],
|
||||
): void {
|
||||
for (const fixture of fixtures) {
|
||||
const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type];
|
||||
const storagePath = fixture.storagePath ?? fixture.effectivePath ?? "";
|
||||
if (fixture.value !== undefined) {
|
||||
UserDraft.save(itemKind, storagePath, fixture.value, { workspace });
|
||||
}
|
||||
UserDraft.setLiveEditorDraft({
|
||||
workspace,
|
||||
itemKind,
|
||||
storagePath,
|
||||
effectivePath: fixture.effectivePath ?? fixture.storagePath,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function clearLiveEditorDrafts(
|
||||
workspace: string,
|
||||
fixtures: GlobalLiveEditorDraftFixture[],
|
||||
): void {
|
||||
for (const fixture of fixtures) {
|
||||
const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type];
|
||||
const storagePath = fixture.storagePath ?? fixture.effectivePath ?? "";
|
||||
UserDraft.clearLiveEditorDraft(itemKind, { workspace, storagePath });
|
||||
}
|
||||
}
|
||||
|
||||
function getGlobalEvalTools(): ProductionTool<{}>[] {
|
||||
return (globalTools as ProductionTool<{}>[]).map((tool) => {
|
||||
if (!MUTATING_GLOBAL_TOOLS.has(tool.def.function.name)) {
|
||||
|
||||
@@ -79,6 +79,16 @@ vi.mock('$lib/gen', async () => {
|
||||
}
|
||||
return actual.ScriptService.getScriptByPath(data)
|
||||
},
|
||||
getScriptByPathWithDraft: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
const script = getBenchmarkScriptByPath(data.workspace, data.path)
|
||||
if (!script) {
|
||||
throw new Error(`Script "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return script
|
||||
}
|
||||
return actual.ScriptService.getScriptByPathWithDraft(data)
|
||||
},
|
||||
getScriptByHash: async (data: { workspace: string; hash: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
const script = getBenchmarkScriptByHash(data.workspace, data.hash)
|
||||
@@ -108,6 +118,26 @@ vi.mock('$lib/gen', async () => {
|
||||
return flow
|
||||
}
|
||||
return actual.FlowService.getFlowByPath(data)
|
||||
},
|
||||
getFlowByPathWithDraft: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
const flow = getBenchmarkFlowByPath(data.workspace, data.path)
|
||||
if (!flow) {
|
||||
throw new Error(`Flow "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return flow
|
||||
}
|
||||
return actual.FlowService.getFlowByPathWithDraft(data)
|
||||
},
|
||||
getFlowLatestVersion: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
const flow = getBenchmarkFlowByPath(data.workspace, data.path)
|
||||
if (!flow) {
|
||||
throw new Error(`Flow "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return { id: 1 }
|
||||
}
|
||||
return actual.FlowService.getFlowLatestVersion(data)
|
||||
}
|
||||
}),
|
||||
JobService: wrapService(actual.JobService, {
|
||||
|
||||
@@ -87,3 +87,346 @@
|
||||
- the flow accepts numeric inputs a and b
|
||||
- the flow returns the sum of a and b
|
||||
- the result stays as an AI draft and is not deployed or saved to the workspace
|
||||
|
||||
- id: global-test4-multi-artifact-notification-job
|
||||
prompt: |-
|
||||
Set up a draft stale-trial notification job.
|
||||
Create a Bun script at `f/evals/global/check_stale_trials` that accepts `max_age_days`, uses mocked inline trial account data, and returns the stale trial account IDs.
|
||||
Also create a weekday 09:00 UTC schedule at `f/evals/global/check_stale_trials_weekday` for that script with `max_age_days` set to 14.
|
||||
Add an HTTP POST trigger at `f/evals/global/check_stale_trials_manual` with route path `evals/check-stale-trials` that runs the same script manually.
|
||||
Leave everything as AI drafts only; do not deploy or save anything to the workspace.
|
||||
runtime:
|
||||
maxTurns: 12
|
||||
validate:
|
||||
draftCountExactly: 3
|
||||
requiredDrafts:
|
||||
- type: script
|
||||
path: f/evals/global/check_stale_trials
|
||||
language: bun
|
||||
valueIncludes:
|
||||
- max_age_days
|
||||
- trial
|
||||
- type: schedule
|
||||
path: f/evals/global/check_stale_trials_weekday
|
||||
valueIncludes:
|
||||
- f/evals/global/check_stale_trials
|
||||
- UTC
|
||||
- "14"
|
||||
- type: trigger
|
||||
triggerKind: http
|
||||
path: f/evals/global/check_stale_trials_manual
|
||||
valueIncludes:
|
||||
- evals/check-stale-trials
|
||||
- f/evals/global/check_stale_trials
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- write_script
|
||||
- write_schedule
|
||||
- write_trigger
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- creates a Bun script draft for stale trial accounts
|
||||
- creates a weekday 09:00 UTC schedule draft for the script with max_age_days set to 14
|
||||
- creates an HTTP POST trigger draft with route path evals/check-stale-trials for the same script
|
||||
- leaves all artifacts as drafts only and does not deploy
|
||||
|
||||
- id: global-test5-existing-flow-inline-code-edit
|
||||
prompt: |-
|
||||
Update the existing flow at `f/evals/global/process_invoice`.
|
||||
Only change the `calculate_total` inline code so it applies 8% tax and returns an object containing `subtotal`, `tax`, and `total`.
|
||||
Leave the updated flow as an AI draft only; do not deploy or save it.
|
||||
initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
|
||||
runtime:
|
||||
maxTurns: 10
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: flow
|
||||
path: f/evals/global/process_invoice
|
||||
valueIncludes:
|
||||
- calculate_total
|
||||
- tax
|
||||
- total
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- read_workspace_item
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- reads the existing process_invoice flow before editing it
|
||||
- updates the calculate_total inline code to apply 8% tax
|
||||
- returns subtotal, tax, and total from the updated flow logic
|
||||
- leaves the result as an AI draft only
|
||||
|
||||
- id: global-test6-secret-variable-draft
|
||||
prompt: |-
|
||||
Create a secret variable draft at `f/evals/global/slack_bot_token`.
|
||||
Use the placeholder value `xoxb-redacted-test-token` and description `Slack bot token for eval notifications`.
|
||||
Do not create any resource or deploy anything.
|
||||
runtime:
|
||||
maxTurns: 6
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: variable
|
||||
path: f/evals/global/slack_bot_token
|
||||
valueIncludes:
|
||||
- Slack bot token
|
||||
- "true"
|
||||
forbiddenDrafts:
|
||||
- type: resource
|
||||
path: f/evals/global/slack_bot_token
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- write_variable
|
||||
forbiddenToolsUsed:
|
||||
- write_resource
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
toolCallArgs:
|
||||
- tool: write_variable
|
||||
field: value
|
||||
stringStartsWithAnyOf:
|
||||
- xoxb-redacted-test-token
|
||||
skipJudge: true
|
||||
judgeChecklist:
|
||||
- creates exactly one secret variable draft at f/evals/global/slack_bot_token
|
||||
- uses the requested placeholder value and description
|
||||
- does not create a resource or deploy anything
|
||||
|
||||
- id: global-test7-ambiguous-app-asks-question
|
||||
prompt: |-
|
||||
Create a new raw app for triaging support tickets.
|
||||
runtime:
|
||||
maxTurns: 4
|
||||
validate:
|
||||
draftCountExactly: 0
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- askUserQuestion
|
||||
forbiddenToolsUsed:
|
||||
- init_app
|
||||
- write_app_file
|
||||
- write_app_runnable
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
skipJudge: true
|
||||
|
||||
- id: global-test8-human-script-infer-path-language
|
||||
prompt: |-
|
||||
I need a small helper that formats a customer-facing welcome line.
|
||||
It should take a person's name and return "Welcome aboard, <name>!".
|
||||
Please just stage it as a draft for now.
|
||||
runtime:
|
||||
maxTurns: 8
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: script
|
||||
valueIncludes:
|
||||
- Welcome aboard
|
||||
- name
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- write_script
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- creates a single script draft for a welcome-line helper
|
||||
- accepts a person's name as input
|
||||
- returns a message containing Welcome aboard, the provided name, and an exclamation mark
|
||||
- chooses a reasonable workspace path and script language without needing the user to specify them
|
||||
- leaves the result as an AI draft only
|
||||
|
||||
- id: global-test9-human-weekday-trial-job
|
||||
prompt: |-
|
||||
Can you set up a draft daily job that checks a few hard-coded trial accounts and returns the ones whose trial has ended?
|
||||
It should run every weekday morning around 9 in UTC with a 30 day cutoff.
|
||||
Keep it as draft work only.
|
||||
runtime:
|
||||
maxTurns: 10
|
||||
validate:
|
||||
draftCountExactly: 2
|
||||
requiredDrafts:
|
||||
- type: script
|
||||
pathIncludes:
|
||||
- trial
|
||||
valueIncludes:
|
||||
- trial
|
||||
- "30"
|
||||
- type: schedule
|
||||
pathIncludes:
|
||||
- trial
|
||||
valueIncludes:
|
||||
- UTC
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- write_script
|
||||
- write_schedule
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- creates a script draft that checks hard-coded trial accounts
|
||||
- returns the accounts whose trial has ended based on a 30 day cutoff
|
||||
- creates a schedule draft for weekday mornings around 09:00 UTC
|
||||
- links the schedule to the generated script
|
||||
- leaves both artifacts as drafts only
|
||||
|
||||
- id: global-test10-human-secret-variable
|
||||
prompt: |-
|
||||
I need a placeholder Slack bot token stored securely for future notification work.
|
||||
Use xoxb-redacted-test-token and note that it is for eval notifications.
|
||||
Only prepare a draft.
|
||||
runtime:
|
||||
maxTurns: 6
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: variable
|
||||
pathIncludes:
|
||||
- slack
|
||||
valueIncludes:
|
||||
- eval notifications
|
||||
- "true"
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- write_variable
|
||||
forbiddenToolsUsed:
|
||||
- write_resource
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
toolCallArgs:
|
||||
- tool: write_variable
|
||||
field: value
|
||||
stringStartsWithAnyOf:
|
||||
- xoxb-redacted-test-token
|
||||
skipJudge: true
|
||||
judgeChecklist:
|
||||
- creates a single secret variable draft for the Slack bot token placeholder
|
||||
- uses the requested placeholder value
|
||||
- includes a note or description that it is for eval notifications
|
||||
- does not create a resource or deploy anything
|
||||
|
||||
- id: global-test11-human-existing-flow-informal-edit
|
||||
prompt: |-
|
||||
There is an invoice processing flow in this workspace.
|
||||
Can you adjust its total calculation so it adds 8% tax and returns subtotal, tax, and total?
|
||||
Keep the change as a draft.
|
||||
initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
|
||||
runtime:
|
||||
maxTurns: 10
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: flow
|
||||
pathIncludes:
|
||||
- invoice
|
||||
valueIncludes:
|
||||
- calculate_total
|
||||
- tax
|
||||
- total
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- read_workspace_item
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- finds and edits the existing invoice processing flow without the user providing its exact path
|
||||
- updates the total calculation to apply 8% tax
|
||||
- returns subtotal, tax, and total from the updated flow logic
|
||||
- leaves the result as an AI draft only
|
||||
|
||||
- id: global-test12-current-live-script-edit
|
||||
prompt: |-
|
||||
The script I have open formats greetings.
|
||||
Can you update this script so it uppercases the name before greeting them and ends with an exclamation mark?
|
||||
Keep it as draft work.
|
||||
initial: ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json
|
||||
runtime:
|
||||
maxTurns: 8
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: script
|
||||
path: f/evals/global/current_greeting
|
||||
language: bun
|
||||
valueIncludes:
|
||||
- toUpperCase
|
||||
- "!"
|
||||
forbiddenDrafts:
|
||||
- type: script
|
||||
path: f/evals/global/format_greeting
|
||||
- type: script
|
||||
path: f/evals/global/format_greeting_archive
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- read_workspace_item
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- resolves "this script" to the active live editor script instead of another similarly named workspace script
|
||||
- updates the greeting logic to uppercase the provided name
|
||||
- returns a greeting ending with an exclamation mark
|
||||
- leaves the result as a draft only
|
||||
|
||||
- id: global-test13-current-live-flow-edit
|
||||
prompt: |-
|
||||
I have the invoice flow open.
|
||||
In the current flow, update the total calculation to add 8% tax and return subtotal, tax, and total.
|
||||
Keep the change as a draft.
|
||||
initial: ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json
|
||||
runtime:
|
||||
maxTurns: 10
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: flow
|
||||
path: f/evals/global/current_invoice_flow
|
||||
valueIncludes:
|
||||
- calculate_total
|
||||
- tax
|
||||
- total
|
||||
forbiddenDrafts:
|
||||
- type: flow
|
||||
path: f/evals/global/process_invoice
|
||||
- type: flow
|
||||
path: f/evals/global/process_refund
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- read_workspace_item
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- resolves "current flow" to the active live editor flow
|
||||
- does not edit the similarly named deployed invoice or refund flows
|
||||
- updates the calculate_total logic to apply 8% tax
|
||||
- returns subtotal, tax, and total from the updated flow logic
|
||||
- leaves the result as a draft only
|
||||
|
||||
- id: global-test14-current-without-live-editor-asks-question
|
||||
prompt: |-
|
||||
Please update this script so it returns `ok`.
|
||||
Keep it as a draft.
|
||||
runtime:
|
||||
maxTurns: 4
|
||||
validate:
|
||||
draftCountExactly: 0
|
||||
toolExpect:
|
||||
forbiddenToolsUsed:
|
||||
- write_script
|
||||
- edit_script
|
||||
- write_flow
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
skipJudge: true
|
||||
judgeChecklist:
|
||||
- asks which script to update when the user refers to "this script" without selected or active editor context
|
||||
- does not guess a path or create a new script draft
|
||||
|
||||
@@ -211,7 +211,7 @@ async function handleRun(input: {
|
||||
const summaries: Array<{
|
||||
label: string;
|
||||
passRate: number;
|
||||
averageDurationMs: number;
|
||||
averagePassedDurationMs: number | null;
|
||||
}> = [];
|
||||
|
||||
for (const [index, model] of models.entries()) {
|
||||
@@ -259,7 +259,7 @@ async function handleRun(input: {
|
||||
summaries.push({
|
||||
label: `${model.id} (${runModel})`,
|
||||
passRate: result.passRate,
|
||||
averageDurationMs: result.averageDurationMs,
|
||||
averagePassedDurationMs: result.averagePassedDurationMs ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -267,7 +267,7 @@ async function handleRun(input: {
|
||||
process.stdout.write("\nModel summary\n");
|
||||
for (const summary of summaries) {
|
||||
process.stdout.write(
|
||||
`- ${summary.label}: ${formatPercent(summary.passRate)} | ${Math.round(summary.averageDurationMs)}ms\n`,
|
||||
`- ${summary.label}: ${formatPercent(summary.passRate)} | passed avg ${formatNullableDuration(summary.averagePassedDurationMs)}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -351,6 +351,10 @@ function formatPercent(value: number): string {
|
||||
return `${(value * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function formatNullableDuration(value: number | null): string {
|
||||
return value === null ? "n/a" : `${Math.round(value)}ms`;
|
||||
}
|
||||
|
||||
void main().catch((error) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
process.stderr.write(`${message}\n`);
|
||||
|
||||
@@ -203,6 +203,34 @@ describe("loadCases", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("loads global active-editor eval cases", async () => {
|
||||
const globalCases = await loadCases("global");
|
||||
const scriptCase = globalCases.find(
|
||||
(entry) => entry.id === "global-test12-current-live-script-edit"
|
||||
);
|
||||
const flowCase = globalCases.find(
|
||||
(entry) => entry.id === "global-test13-current-live-flow-edit"
|
||||
);
|
||||
|
||||
expect(scriptCase?.initialPath).toContain(
|
||||
"ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json"
|
||||
);
|
||||
expect(scriptCase?.toolExpect).toMatchObject({
|
||||
requiredToolsUsed: ["read_workspace_item"],
|
||||
});
|
||||
expect(flowCase?.initialPath).toContain(
|
||||
"ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json"
|
||||
);
|
||||
expect(flowCase?.validate).toMatchObject({
|
||||
requiredDrafts: [
|
||||
{
|
||||
type: "flow",
|
||||
path: "f/evals/global/current_invoice_flow",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("loads tool expectations for workspace mutation cases", async () => {
|
||||
const scriptCases = await loadCases("script");
|
||||
const caseEntry = scriptCases.find(
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
appendHistoryRecord,
|
||||
buildRunResult,
|
||||
formatRunSummary,
|
||||
} from "./results";
|
||||
import type { BenchmarkCaseResult } from "./types";
|
||||
|
||||
function caseResult(
|
||||
attempts: BenchmarkCaseResult["attempts"],
|
||||
): BenchmarkCaseResult {
|
||||
return {
|
||||
id: "case-1",
|
||||
prompt: "Do the thing",
|
||||
attempts,
|
||||
};
|
||||
}
|
||||
|
||||
describe("benchmark results", () => {
|
||||
it("keeps success cost metrics separate from failed attempts", () => {
|
||||
const result = buildRunResult({
|
||||
mode: "global",
|
||||
runs: 1,
|
||||
runModel: "model-under-test",
|
||||
judgeModel: "judge-model",
|
||||
caseResults: [
|
||||
caseResult([
|
||||
{
|
||||
attempt: 1,
|
||||
passed: true,
|
||||
durationMs: 1000,
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 1,
|
||||
toolsUsed: ["edit_script"],
|
||||
skillsInvoked: [],
|
||||
checks: [{ name: "edited", passed: true }],
|
||||
judgeScore: 100,
|
||||
judgeSummary: "ok",
|
||||
error: null,
|
||||
tokenUsage: { prompt: 100, completion: 20, total: 120 },
|
||||
},
|
||||
{
|
||||
attempt: 2,
|
||||
passed: false,
|
||||
durationMs: 100,
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 0,
|
||||
toolsUsed: [],
|
||||
skillsInvoked: [],
|
||||
checks: [{ name: "edited", passed: false }],
|
||||
judgeScore: 10,
|
||||
judgeSummary: "missed",
|
||||
error: "failed",
|
||||
tokenUsage: { prompt: 10, completion: 5, total: 15 },
|
||||
},
|
||||
]),
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.attemptCount).toBe(2);
|
||||
expect(result.passedAttempts).toBe(1);
|
||||
expect(result.passRate).toBe(0.5);
|
||||
expect(result.averageDurationMs).toBe(550);
|
||||
expect(result.averagePassedDurationMs).toBe(1000);
|
||||
expect(result.totalTokenUsage).toEqual({
|
||||
prompt: 110,
|
||||
completion: 25,
|
||||
total: 135,
|
||||
});
|
||||
expect(result.totalPassedTokenUsage).toEqual({
|
||||
prompt: 100,
|
||||
completion: 20,
|
||||
total: 120,
|
||||
});
|
||||
expect(result.averageTokenUsagePerAttempt).toEqual({
|
||||
prompt: 55,
|
||||
completion: 12.5,
|
||||
total: 67.5,
|
||||
});
|
||||
expect(result.averageTokenUsagePerPassedAttempt).toEqual({
|
||||
prompt: 100,
|
||||
completion: 20,
|
||||
total: 120,
|
||||
});
|
||||
|
||||
const summary = formatRunSummary(result);
|
||||
expect(summary).toContain("Average duration (passed): 1000ms");
|
||||
expect(summary).toContain("Average tokens (passed): 120 total");
|
||||
expect(summary).toContain("Average duration (all attempts): 550ms");
|
||||
});
|
||||
|
||||
it("reports passed averages as unavailable when no attempt passes", () => {
|
||||
const result = buildRunResult({
|
||||
mode: "global",
|
||||
runs: 1,
|
||||
runModel: "model-under-test",
|
||||
judgeModel: "judge-model",
|
||||
caseResults: [
|
||||
caseResult([
|
||||
{
|
||||
attempt: 1,
|
||||
passed: false,
|
||||
durationMs: 100,
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 0,
|
||||
toolsUsed: [],
|
||||
skillsInvoked: [],
|
||||
checks: [{ name: "edited", passed: false }],
|
||||
judgeScore: 10,
|
||||
judgeSummary: "missed",
|
||||
error: "failed",
|
||||
tokenUsage: { prompt: 10, completion: 5, total: 15 },
|
||||
},
|
||||
]),
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.averagePassedDurationMs).toBeNull();
|
||||
expect(result.totalPassedTokenUsage).toBeNull();
|
||||
expect(result.averageTokenUsagePerPassedAttempt).toBeNull();
|
||||
expect(formatRunSummary(result)).toContain(
|
||||
"Average duration (passed): n/a",
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes passed token averages by passed attempts", () => {
|
||||
const result = buildRunResult({
|
||||
mode: "global",
|
||||
runs: 1,
|
||||
runModel: "model-under-test",
|
||||
judgeModel: "judge-model",
|
||||
caseResults: [
|
||||
caseResult([
|
||||
{
|
||||
attempt: 1,
|
||||
passed: true,
|
||||
durationMs: 1000,
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 1,
|
||||
toolsUsed: ["edit_script"],
|
||||
skillsInvoked: [],
|
||||
checks: [{ name: "edited", passed: true }],
|
||||
judgeScore: 100,
|
||||
judgeSummary: "ok",
|
||||
error: null,
|
||||
tokenUsage: { prompt: 100, completion: 20, total: 120 },
|
||||
},
|
||||
{
|
||||
attempt: 2,
|
||||
passed: true,
|
||||
durationMs: 1200,
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 1,
|
||||
toolsUsed: ["edit_script"],
|
||||
skillsInvoked: [],
|
||||
checks: [{ name: "edited", passed: true }],
|
||||
judgeScore: 100,
|
||||
judgeSummary: "ok",
|
||||
error: null,
|
||||
tokenUsage: null,
|
||||
},
|
||||
]),
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.passedAttempts).toBe(2);
|
||||
expect(result.totalPassedTokenUsage).toEqual({
|
||||
prompt: 100,
|
||||
completion: 20,
|
||||
total: 120,
|
||||
});
|
||||
expect(result.averageTokenUsagePerPassedAttempt).toEqual({
|
||||
prompt: 50,
|
||||
completion: 10,
|
||||
total: 60,
|
||||
});
|
||||
});
|
||||
|
||||
it("records passed-attempt metrics in history", async () => {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), "windmill-ai-evals-"));
|
||||
try {
|
||||
const historyPath = join(tempDir, "history.jsonl");
|
||||
const result = buildRunResult({
|
||||
mode: "global",
|
||||
runs: 1,
|
||||
runModel: "model-under-test",
|
||||
judgeModel: "judge-model",
|
||||
caseResults: [
|
||||
caseResult([
|
||||
{
|
||||
attempt: 1,
|
||||
passed: true,
|
||||
durationMs: 1000,
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 1,
|
||||
toolsUsed: ["edit_script"],
|
||||
skillsInvoked: [],
|
||||
checks: [{ name: "edited", passed: true }],
|
||||
judgeScore: 100,
|
||||
judgeSummary: "ok",
|
||||
error: null,
|
||||
tokenUsage: { prompt: 100, completion: 20, total: 120 },
|
||||
},
|
||||
{
|
||||
attempt: 2,
|
||||
passed: false,
|
||||
durationMs: 100,
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 0,
|
||||
toolsUsed: [],
|
||||
skillsInvoked: [],
|
||||
checks: [{ name: "edited", passed: false }],
|
||||
judgeScore: 10,
|
||||
judgeSummary: "missed",
|
||||
error: "failed",
|
||||
tokenUsage: { prompt: 10, completion: 5, total: 15 },
|
||||
},
|
||||
]),
|
||||
],
|
||||
});
|
||||
|
||||
await appendHistoryRecord(result, historyPath);
|
||||
const record = JSON.parse(await readFile(historyPath, "utf8"));
|
||||
|
||||
expect(record.averageDurationMs).toBe(550);
|
||||
expect(record.averagePassedDurationMs).toBe(1000);
|
||||
expect(record.averageTokenUsagePerAttempt.total).toBe(67.5);
|
||||
expect(record.averageTokenUsagePerPassedAttempt.total).toBe(120);
|
||||
expect(record.cases[0].averageDurationMs).toBe(550);
|
||||
expect(record.cases[0].averagePassedDurationMs).toBe(1000);
|
||||
expect(record.cases[0].averageTokenUsagePerAttempt.total).toBe(67.5);
|
||||
expect(record.cases[0].averageTokenUsagePerPassedAttempt.total).toBe(
|
||||
120,
|
||||
);
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
+114
-67
@@ -4,12 +4,20 @@ import { execFileSync } from "node:child_process";
|
||||
import { getAiEvalsRoot, getRepoRoot } from "./cases";
|
||||
import type {
|
||||
BenchmarkArtifactFile,
|
||||
BenchmarkAttemptResult,
|
||||
BenchmarkCaseResult,
|
||||
BenchmarkRunResult,
|
||||
BenchmarkTokenUsage,
|
||||
EvalMode,
|
||||
} from "./types";
|
||||
|
||||
type AttemptAggregate = {
|
||||
attemptCount: number;
|
||||
durationTotal: number;
|
||||
tokenUsageAttemptCount: number;
|
||||
tokenUsageTotal: BenchmarkTokenUsage | null;
|
||||
};
|
||||
|
||||
export async function writeRunResult(
|
||||
result: BenchmarkRunResult,
|
||||
outputPath?: string,
|
||||
@@ -77,36 +85,12 @@ export function buildRunResult(input: {
|
||||
judgeModel: string | null;
|
||||
caseResults: BenchmarkCaseResult[];
|
||||
}): BenchmarkRunResult {
|
||||
const attemptCount = input.caseResults.reduce(
|
||||
(sum, entry) => sum + entry.attempts.length,
|
||||
0,
|
||||
);
|
||||
const passedAttempts = input.caseResults.reduce(
|
||||
(sum, entry) =>
|
||||
sum + entry.attempts.filter((attempt) => attempt.passed).length,
|
||||
0,
|
||||
);
|
||||
const durationTotal = input.caseResults.reduce(
|
||||
(sum, entry) =>
|
||||
sum +
|
||||
entry.attempts.reduce((inner, attempt) => inner + attempt.durationMs, 0),
|
||||
0,
|
||||
);
|
||||
const tokenUsageTotal = input.caseResults.reduce<BenchmarkTokenUsage | null>(
|
||||
(sum, entry) => {
|
||||
for (const attempt of entry.attempts) {
|
||||
if (!attempt.tokenUsage) {
|
||||
continue;
|
||||
}
|
||||
sum ??= { prompt: 0, completion: 0, total: 0 };
|
||||
sum.prompt += attempt.tokenUsage.prompt;
|
||||
sum.completion += attempt.tokenUsage.completion;
|
||||
sum.total += attempt.tokenUsage.total;
|
||||
}
|
||||
return sum;
|
||||
},
|
||||
null,
|
||||
);
|
||||
const attempts = input.caseResults.flatMap((entry) => entry.attempts);
|
||||
const passedAttemptResults = attempts.filter((attempt) => attempt.passed);
|
||||
const attemptAggregate = aggregateAttempts(attempts);
|
||||
const passedAttemptAggregate = aggregateAttempts(passedAttemptResults);
|
||||
const attemptCount = attemptAggregate.attemptCount;
|
||||
const passedAttempts = passedAttemptAggregate.attemptCount;
|
||||
|
||||
return {
|
||||
version: 1,
|
||||
@@ -120,16 +104,19 @@ export function buildRunResult(input: {
|
||||
attemptCount,
|
||||
passedAttempts,
|
||||
passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount,
|
||||
averageDurationMs: attemptCount === 0 ? 0 : durationTotal / attemptCount,
|
||||
totalTokenUsage: tokenUsageTotal,
|
||||
averageDurationMs:
|
||||
attemptCount === 0 ? 0 : attemptAggregate.durationTotal / attemptCount,
|
||||
averagePassedDurationMs: averageDuration(passedAttemptAggregate),
|
||||
totalTokenUsage: attemptAggregate.tokenUsageTotal,
|
||||
totalPassedTokenUsage: passedAttemptAggregate.tokenUsageTotal,
|
||||
averageTokenUsagePerAttempt:
|
||||
attemptCount === 0 || !tokenUsageTotal
|
||||
attemptCount === 0
|
||||
? null
|
||||
: {
|
||||
prompt: tokenUsageTotal.prompt / attemptCount,
|
||||
completion: tokenUsageTotal.completion / attemptCount,
|
||||
total: tokenUsageTotal.total / attemptCount,
|
||||
},
|
||||
: averageTokenUsage(attemptAggregate, attemptCount),
|
||||
averageTokenUsagePerPassedAttempt: averageTokenUsage(
|
||||
passedAttemptAggregate,
|
||||
passedAttempts,
|
||||
),
|
||||
cases: input.caseResults,
|
||||
};
|
||||
}
|
||||
@@ -138,9 +125,25 @@ export function formatRunSummary(result: BenchmarkRunResult): string {
|
||||
const lines = [
|
||||
`${result.mode} benchmark complete`,
|
||||
`Pass rate: ${formatPercent(result.passRate)} (${result.passedAttempts}/${result.attemptCount})`,
|
||||
`Average duration: ${Math.round(result.averageDurationMs)}ms`,
|
||||
`Average duration (passed): ${formatNullableDuration(result.averagePassedDurationMs ?? null)}`,
|
||||
];
|
||||
|
||||
if (result.averageTokenUsagePerPassedAttempt) {
|
||||
lines.push(
|
||||
`Average tokens (passed): ${formatTokenUsage(result.averageTokenUsagePerPassedAttempt)}`,
|
||||
);
|
||||
}
|
||||
if (result.passedAttempts < result.attemptCount) {
|
||||
lines.push(
|
||||
`Average duration (all attempts): ${Math.round(result.averageDurationMs)}ms`,
|
||||
);
|
||||
if (result.averageTokenUsagePerAttempt) {
|
||||
lines.push(
|
||||
`Average tokens (all attempts): ${formatTokenUsage(result.averageTokenUsagePerAttempt)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const failures = collectFailures(result);
|
||||
if (failures.length > 0) {
|
||||
lines.push("Failures:");
|
||||
@@ -172,6 +175,60 @@ function collectFailures(result: BenchmarkRunResult): string[] {
|
||||
return failures;
|
||||
}
|
||||
|
||||
function aggregateAttempts(attempts: BenchmarkAttemptResult[]): AttemptAggregate {
|
||||
const aggregate: AttemptAggregate = {
|
||||
attemptCount: attempts.length,
|
||||
durationTotal: 0,
|
||||
tokenUsageAttemptCount: 0,
|
||||
tokenUsageTotal: null,
|
||||
};
|
||||
|
||||
for (const attempt of attempts) {
|
||||
aggregate.durationTotal += attempt.durationMs;
|
||||
if (!attempt.tokenUsage) {
|
||||
continue;
|
||||
}
|
||||
aggregate.tokenUsageAttemptCount += 1;
|
||||
aggregate.tokenUsageTotal ??= { prompt: 0, completion: 0, total: 0 };
|
||||
aggregate.tokenUsageTotal.prompt += attempt.tokenUsage.prompt;
|
||||
aggregate.tokenUsageTotal.completion += attempt.tokenUsage.completion;
|
||||
aggregate.tokenUsageTotal.total += attempt.tokenUsage.total;
|
||||
}
|
||||
|
||||
return aggregate;
|
||||
}
|
||||
|
||||
function averageDuration(aggregate: AttemptAggregate): number | null {
|
||||
return aggregate.attemptCount === 0
|
||||
? null
|
||||
: aggregate.durationTotal / aggregate.attemptCount;
|
||||
}
|
||||
|
||||
function averageTokenUsage(
|
||||
aggregate: AttemptAggregate,
|
||||
denominator: number,
|
||||
): BenchmarkTokenUsage | null {
|
||||
if (denominator === 0 || !aggregate.tokenUsageTotal) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
prompt: aggregate.tokenUsageTotal.prompt / denominator,
|
||||
completion: aggregate.tokenUsageTotal.completion / denominator,
|
||||
total: aggregate.tokenUsageTotal.total / denominator,
|
||||
};
|
||||
}
|
||||
|
||||
function formatNullableDuration(value: number | null): string {
|
||||
return value === null ? "n/a" : `${Math.round(value)}ms`;
|
||||
}
|
||||
|
||||
function formatTokenUsage(value: BenchmarkTokenUsage): string {
|
||||
const total = Math.round(value.total);
|
||||
const prompt = Math.round(value.prompt);
|
||||
const completion = Math.round(value.completion);
|
||||
return `${total} total (${prompt} prompt, ${completion} completion)`;
|
||||
}
|
||||
|
||||
function defaultFileName(mode: EvalMode): string {
|
||||
return `${new Date().toISOString().replaceAll(":", "-")}__${mode}.json`;
|
||||
}
|
||||
@@ -252,12 +309,15 @@ function toHistoryRecord(result: BenchmarkRunResult) {
|
||||
passedAttempts: result.passedAttempts,
|
||||
passRate: result.passRate,
|
||||
averageDurationMs: result.averageDurationMs,
|
||||
averagePassedDurationMs: result.averagePassedDurationMs ?? null,
|
||||
averageJudgeScore:
|
||||
judgeScores.length === 0
|
||||
? null
|
||||
: judgeScores.reduce((sum, score) => sum + score, 0) /
|
||||
judgeScores.length,
|
||||
averageTokenUsagePerAttempt: result.averageTokenUsagePerAttempt ?? null,
|
||||
averageTokenUsagePerPassedAttempt:
|
||||
result.averageTokenUsagePerPassedAttempt ?? null,
|
||||
failedCaseIds: Array.from(
|
||||
new Set(
|
||||
result.cases
|
||||
@@ -268,31 +328,15 @@ function toHistoryRecord(result: BenchmarkRunResult) {
|
||||
),
|
||||
),
|
||||
cases: result.cases.map((caseResult) => {
|
||||
const attemptCount = caseResult.attempts.length;
|
||||
const passedAttempts = caseResult.attempts.filter(
|
||||
(attempt) => attempt.passed,
|
||||
).length;
|
||||
const totalDurationMs = caseResult.attempts.reduce(
|
||||
(sum, attempt) => sum + attempt.durationMs,
|
||||
0,
|
||||
const attemptAggregate = aggregateAttempts(caseResult.attempts);
|
||||
const passedAttemptAggregate = aggregateAttempts(
|
||||
caseResult.attempts.filter((attempt) => attempt.passed),
|
||||
);
|
||||
const attemptCount = attemptAggregate.attemptCount;
|
||||
const passedAttempts = passedAttemptAggregate.attemptCount;
|
||||
const judgeScores = caseResult.attempts.flatMap((attempt) =>
|
||||
typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : [],
|
||||
);
|
||||
const totalTokenUsage =
|
||||
caseResult.attempts.reduce<BenchmarkTokenUsage | null>(
|
||||
(sum, attempt) => {
|
||||
if (!attempt.tokenUsage) {
|
||||
return sum;
|
||||
}
|
||||
sum ??= { prompt: 0, completion: 0, total: 0 };
|
||||
sum.prompt += attempt.tokenUsage.prompt;
|
||||
sum.completion += attempt.tokenUsage.completion;
|
||||
sum.total += attempt.tokenUsage.total;
|
||||
return sum;
|
||||
},
|
||||
null,
|
||||
);
|
||||
|
||||
return {
|
||||
id: caseResult.id,
|
||||
@@ -300,20 +344,23 @@ function toHistoryRecord(result: BenchmarkRunResult) {
|
||||
passedAttempts,
|
||||
passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount,
|
||||
averageDurationMs:
|
||||
attemptCount === 0 ? 0 : totalDurationMs / attemptCount,
|
||||
attemptCount === 0
|
||||
? 0
|
||||
: attemptAggregate.durationTotal / attemptCount,
|
||||
averagePassedDurationMs: averageDuration(passedAttemptAggregate),
|
||||
averageJudgeScore:
|
||||
judgeScores.length === 0
|
||||
? null
|
||||
: judgeScores.reduce((sum, score) => sum + score, 0) /
|
||||
judgeScores.length,
|
||||
averageTokenUsagePerAttempt:
|
||||
attemptCount === 0 || !totalTokenUsage
|
||||
attemptCount === 0
|
||||
? null
|
||||
: {
|
||||
prompt: totalTokenUsage.prompt / attemptCount,
|
||||
completion: totalTokenUsage.completion / attemptCount,
|
||||
total: totalTokenUsage.total / attemptCount,
|
||||
},
|
||||
: averageTokenUsage(attemptAggregate, attemptCount),
|
||||
averageTokenUsagePerPassedAttempt: averageTokenUsage(
|
||||
passedAttemptAggregate,
|
||||
passedAttempts,
|
||||
),
|
||||
};
|
||||
}),
|
||||
};
|
||||
|
||||
@@ -110,7 +110,9 @@ export interface AppValidationSpec {
|
||||
|
||||
export interface GlobalDraftRequirement {
|
||||
type: string;
|
||||
path: string;
|
||||
path?: string;
|
||||
pathIncludes?: string[];
|
||||
pathStartsWith?: string;
|
||||
triggerKind?: string;
|
||||
language?: string;
|
||||
summaryIncludes?: string[];
|
||||
@@ -324,8 +326,11 @@ export interface BenchmarkRunResult {
|
||||
passedAttempts: number;
|
||||
passRate: number;
|
||||
averageDurationMs: number;
|
||||
averagePassedDurationMs?: number | null;
|
||||
totalTokenUsage?: BenchmarkTokenUsage | null;
|
||||
totalPassedTokenUsage?: BenchmarkTokenUsage | null;
|
||||
averageTokenUsagePerAttempt?: BenchmarkTokenUsage | null;
|
||||
averageTokenUsagePerPassedAttempt?: BenchmarkTokenUsage | null;
|
||||
artifactsPath?: string | null;
|
||||
cases: BenchmarkCaseResult[];
|
||||
}
|
||||
|
||||
@@ -195,6 +195,69 @@ describe("validateGlobalState", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts a required script draft without an exact path", () => {
|
||||
const checks = validateGlobalState({
|
||||
actual: {
|
||||
drafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/team_tools/friendly_greeting",
|
||||
language: "bun",
|
||||
summary: "Friendly greeting helper",
|
||||
value:
|
||||
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
|
||||
isDraft: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
validate: {
|
||||
draftCountExactly: 1,
|
||||
requiredDrafts: [
|
||||
{
|
||||
type: "script",
|
||||
pathIncludes: ["greeting"],
|
||||
language: "bun",
|
||||
summaryIncludes: ["Friendly"],
|
||||
valueIncludes: ["Hello"],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(checks.every((check) => check.passed)).toBe(true);
|
||||
});
|
||||
|
||||
it("reports flexible global draft path filters when no draft matches", () => {
|
||||
const checks = validateGlobalState({
|
||||
actual: {
|
||||
drafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/team_tools/friendly_greeting",
|
||||
language: "bun",
|
||||
value:
|
||||
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
|
||||
isDraft: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
validate: {
|
||||
requiredDrafts: [
|
||||
{
|
||||
type: "script",
|
||||
pathIncludes: ["invoice"],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(checks).toContainEqual({
|
||||
name: "global includes script draft (path includes invoice)",
|
||||
passed: false,
|
||||
details: "drafts: script:f/team_tools/friendly_greeting",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not require a TypeScript entrypoint for non-TypeScript script drafts", () => {
|
||||
const checks = validateGlobalState({
|
||||
actual: {
|
||||
|
||||
+100
-15
@@ -315,10 +315,11 @@ export function validateGlobalState(input: {
|
||||
}
|
||||
|
||||
for (const required of validate.requiredDrafts ?? []) {
|
||||
const draft = findGlobalDraft(drafts, required.type, required.path, required.triggerKind);
|
||||
const requirementLabel = formatGlobalDraftRequirement(required);
|
||||
const draft = findGlobalDraft(drafts, required);
|
||||
checks.push(
|
||||
check(
|
||||
`global includes ${required.type} draft ${required.path}`,
|
||||
`global includes ${requirementLabel}`,
|
||||
Boolean(draft),
|
||||
summarizeGlobalDrafts(drafts)
|
||||
)
|
||||
@@ -330,7 +331,7 @@ export function validateGlobalState(input: {
|
||||
if (required.language !== undefined) {
|
||||
checks.push(
|
||||
check(
|
||||
`${required.type} draft ${required.path} uses ${required.language}`,
|
||||
`${requirementLabel} uses ${required.language}`,
|
||||
draft.language === required.language,
|
||||
`language=${draft.language ?? "(none)"}`
|
||||
)
|
||||
@@ -340,7 +341,7 @@ export function validateGlobalState(input: {
|
||||
for (const snippet of required.summaryIncludes ?? []) {
|
||||
checks.push(
|
||||
check(
|
||||
`${required.type} draft ${required.path} summary includes '${snippet}'`,
|
||||
`${requirementLabel} summary includes '${snippet}'`,
|
||||
normalizeText(draft.summary ?? "").includes(normalizeText(snippet)),
|
||||
`summary=${draft.summary ?? ""}`
|
||||
)
|
||||
@@ -351,7 +352,7 @@ export function validateGlobalState(input: {
|
||||
for (const snippet of required.valueIncludes ?? []) {
|
||||
checks.push(
|
||||
check(
|
||||
`${required.type} draft ${required.path} value includes '${snippet}'`,
|
||||
`${requirementLabel} value includes '${snippet}'`,
|
||||
normalizeText(valueText).includes(normalizeText(snippet)),
|
||||
truncateForDetails(valueText)
|
||||
)
|
||||
@@ -361,7 +362,7 @@ export function validateGlobalState(input: {
|
||||
for (const snippet of required.valueExcludes ?? []) {
|
||||
checks.push(
|
||||
check(
|
||||
`${required.type} draft ${required.path} value excludes '${snippet}'`,
|
||||
`${requirementLabel} value excludes '${snippet}'`,
|
||||
!normalizeText(valueText).includes(normalizeText(snippet)),
|
||||
truncateForDetails(valueText)
|
||||
)
|
||||
@@ -373,7 +374,7 @@ export function validateGlobalState(input: {
|
||||
checks.push(
|
||||
check(
|
||||
`global does not include ${forbidden.type} draft ${forbidden.path}`,
|
||||
!findGlobalDraft(drafts, forbidden.type, forbidden.path, forbidden.triggerKind),
|
||||
!findGlobalDraft(drafts, forbidden),
|
||||
summarizeGlobalDrafts(drafts)
|
||||
)
|
||||
);
|
||||
@@ -615,16 +616,100 @@ function summarizeProblems(problems: string[], limit = 5): string | undefined {
|
||||
|
||||
function findGlobalDraft(
|
||||
drafts: GlobalDraft[],
|
||||
type: string,
|
||||
path: string,
|
||||
triggerKind?: string
|
||||
requirement: {
|
||||
type: string;
|
||||
path?: string;
|
||||
pathIncludes?: string[];
|
||||
pathStartsWith?: string;
|
||||
triggerKind?: string;
|
||||
summaryIncludes?: string[];
|
||||
valueIncludes?: string[];
|
||||
valueExcludes?: string[];
|
||||
}
|
||||
): GlobalDraft | undefined {
|
||||
return drafts.find(
|
||||
(draft) =>
|
||||
draft.type === type &&
|
||||
draft.path === path &&
|
||||
(triggerKind === undefined || draft.triggerKind === triggerKind)
|
||||
const candidates = drafts.filter((draft) =>
|
||||
globalDraftMatchesLocator(draft, requirement)
|
||||
);
|
||||
return (
|
||||
candidates.find((draft) => globalDraftMatchesContent(draft, requirement)) ??
|
||||
candidates[0]
|
||||
);
|
||||
}
|
||||
|
||||
function globalDraftMatchesLocator(
|
||||
draft: GlobalDraft,
|
||||
requirement: {
|
||||
type: string;
|
||||
path?: string;
|
||||
pathIncludes?: string[];
|
||||
pathStartsWith?: string;
|
||||
triggerKind?: string;
|
||||
}
|
||||
): boolean {
|
||||
return (
|
||||
draft.type === requirement.type &&
|
||||
(requirement.path === undefined || draft.path === requirement.path) &&
|
||||
(requirement.pathStartsWith === undefined ||
|
||||
draft.path.startsWith(requirement.pathStartsWith)) &&
|
||||
(requirement.pathIncludes ?? []).every((snippet) =>
|
||||
normalizeText(draft.path).includes(normalizeText(snippet))
|
||||
) &&
|
||||
(requirement.triggerKind === undefined ||
|
||||
draft.triggerKind === requirement.triggerKind)
|
||||
);
|
||||
}
|
||||
|
||||
function globalDraftMatchesContent(
|
||||
draft: GlobalDraft,
|
||||
requirement: {
|
||||
summaryIncludes?: string[];
|
||||
valueIncludes?: string[];
|
||||
valueExcludes?: string[];
|
||||
}
|
||||
): boolean {
|
||||
const summary = normalizeText(draft.summary ?? "");
|
||||
const value = normalizeText(stringifyGlobalDraftValue(draft.value));
|
||||
return (
|
||||
(requirement.summaryIncludes ?? []).every((snippet) =>
|
||||
summary.includes(normalizeText(snippet))
|
||||
) &&
|
||||
(requirement.valueIncludes ?? []).every((snippet) =>
|
||||
value.includes(normalizeText(snippet))
|
||||
) &&
|
||||
(requirement.valueExcludes ?? []).every(
|
||||
(snippet) => !value.includes(normalizeText(snippet))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function formatGlobalDraftRequirement(
|
||||
requirement: {
|
||||
type: string;
|
||||
path?: string;
|
||||
pathIncludes?: string[];
|
||||
pathStartsWith?: string;
|
||||
triggerKind?: string;
|
||||
}
|
||||
): string {
|
||||
const typeLabel =
|
||||
requirement.triggerKind === undefined
|
||||
? requirement.type
|
||||
: `${requirement.triggerKind} ${requirement.type}`;
|
||||
if (requirement.path !== undefined) {
|
||||
return `${typeLabel} draft ${requirement.path}`;
|
||||
}
|
||||
|
||||
const filters = [
|
||||
...(requirement.pathStartsWith === undefined
|
||||
? []
|
||||
: [`path starts with ${requirement.pathStartsWith}`]),
|
||||
...(requirement.pathIncludes ?? []).map(
|
||||
(snippet) => `path includes ${snippet}`
|
||||
),
|
||||
];
|
||||
return filters.length === 0
|
||||
? `${typeLabel} draft`
|
||||
: `${typeLabel} draft (${filters.join(", ")})`;
|
||||
}
|
||||
|
||||
function summarizeGlobalDrafts(drafts: GlobalDraft[]): string {
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"workspace": {
|
||||
"scripts": [
|
||||
{
|
||||
"path": "f/evals/global/format_greeting",
|
||||
"summary": "Format a deployed greeting",
|
||||
"description": "Returns a plain greeting for a provided name.",
|
||||
"language": "bun",
|
||||
"schema": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["name"]
|
||||
},
|
||||
"content": "export async function main(name: string) {\n return `Hello, ${name}`\n}\n"
|
||||
},
|
||||
{
|
||||
"path": "f/evals/global/format_greeting_archive",
|
||||
"summary": "Archived greeting formatter",
|
||||
"description": "Older greeting formatter kept for reference.",
|
||||
"language": "bun",
|
||||
"schema": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["name"]
|
||||
},
|
||||
"content": "export async function main(name: string) {\n return `Hi, ${name}`\n}\n"
|
||||
}
|
||||
]
|
||||
},
|
||||
"liveEditorDrafts": [
|
||||
{
|
||||
"type": "script",
|
||||
"storagePath": "f/evals/global/current_greeting",
|
||||
"effectivePath": "f/evals/global/current_greeting",
|
||||
"value": {
|
||||
"path": "f/evals/global/current_greeting",
|
||||
"summary": "Open greeting formatter",
|
||||
"description": "Formats a greeting in the live editor.",
|
||||
"language": "bun",
|
||||
"schema": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["name"]
|
||||
},
|
||||
"content": "export async function main(name: string) {\n return `Hello, ${name}`\n}\n",
|
||||
"is_template": false,
|
||||
"kind": "script"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
{
|
||||
"workspace": {
|
||||
"flows": [
|
||||
{
|
||||
"path": "f/evals/global/process_invoice",
|
||||
"summary": "Deployed invoice processor",
|
||||
"description": "Calculates invoice totals from a subtotal.",
|
||||
"schema": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"subtotal": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": ["subtotal"]
|
||||
},
|
||||
"value": {
|
||||
"modules": [
|
||||
{
|
||||
"id": "calculate_total",
|
||||
"summary": "Calculate total from subtotal",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "bun",
|
||||
"content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n",
|
||||
"input_transforms": {
|
||||
"subtotal": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.subtotal"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"path": "f/evals/global/process_refund",
|
||||
"summary": "Refund processor",
|
||||
"description": "Calculates refund totals.",
|
||||
"schema": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"subtotal": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": ["subtotal"]
|
||||
},
|
||||
"value": {
|
||||
"modules": [
|
||||
{
|
||||
"id": "calculate_total",
|
||||
"summary": "Calculate refund total",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "bun",
|
||||
"content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n",
|
||||
"input_transforms": {
|
||||
"subtotal": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.subtotal"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"liveEditorDrafts": [
|
||||
{
|
||||
"type": "flow",
|
||||
"storagePath": "f/evals/global/current_invoice_flow",
|
||||
"effectivePath": "f/evals/global/current_invoice_flow",
|
||||
"value": {
|
||||
"path": "f/evals/global/current_invoice_flow",
|
||||
"summary": "Open invoice processor",
|
||||
"schema": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"subtotal": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": ["subtotal"]
|
||||
},
|
||||
"value": {
|
||||
"modules": [
|
||||
{
|
||||
"id": "calculate_total",
|
||||
"summary": "Calculate total from subtotal",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "bun",
|
||||
"content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n",
|
||||
"input_transforms": {
|
||||
"subtotal": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.subtotal"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"edited_by": "",
|
||||
"edited_at": "",
|
||||
"archived": false,
|
||||
"extra_perms": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"workspace": {
|
||||
"flows": [
|
||||
{
|
||||
"path": "f/evals/global/process_invoice",
|
||||
"summary": "Process an invoice subtotal",
|
||||
"description": "Calculates invoice totals from a subtotal.",
|
||||
"schema": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"subtotal": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": ["subtotal"]
|
||||
},
|
||||
"value": {
|
||||
"modules": [
|
||||
{
|
||||
"id": "calculate_total",
|
||||
"summary": "Calculate total from subtotal",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "bun",
|
||||
"content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n",
|
||||
"input_transforms": {
|
||||
"subtotal": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.subtotal"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { runGlobalEval } from "../adapters/frontend/core/global/globalEvalRunner";
|
||||
import {
|
||||
runGlobalEval,
|
||||
type GlobalLiveEditorDraftFixture,
|
||||
} from "../adapters/frontend/core/global/globalEvalRunner";
|
||||
import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend";
|
||||
import type { FrontendEvalModelConfig } from "../core/models";
|
||||
import type { BenchmarkArtifactFile, GlobalValidationSpec, ModeRunner } from "../core/types";
|
||||
@@ -9,6 +12,7 @@ import { getFrontendApiKey } from "./frontendCommon";
|
||||
|
||||
export interface GlobalInitialFixture {
|
||||
workspace?: BenchmarkWorkspaceRunnables;
|
||||
liveEditorDrafts?: GlobalLiveEditorDraftFixture[];
|
||||
}
|
||||
|
||||
export function createGlobalModeRunner(
|
||||
@@ -31,6 +35,7 @@ export function createGlobalModeRunner(
|
||||
getFrontendApiKey(modelConfig.provider),
|
||||
{
|
||||
workspaceFixtures: initial?.workspace,
|
||||
liveEditorDrafts: initial?.liveEditorDrafts,
|
||||
maxIterations: context.evalCase?.runtime?.maxTurns,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
@@ -73,6 +78,7 @@ async function loadGlobalInitialFixture(path: string): Promise<GlobalInitialFixt
|
||||
const parsed = JSON.parse(await readFile(path, "utf8")) as GlobalInitialFixture;
|
||||
return {
|
||||
workspace: parsed.workspace ?? {},
|
||||
liveEditorDrafts: parsed.liveEditorDrafts ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
327d23f7438968a21bac9fd42e7f6f027c61477c
|
||||
55c19293232be379a3044eb78f677b545882ffd6
|
||||
@@ -6,7 +6,10 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::{collections::HashMap, time::Duration};
|
||||
use std::{
|
||||
collections::{BTreeSet, HashMap},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
mod audit_logs_s3;
|
||||
@@ -47,6 +50,7 @@ use windmill_common::secret_backend::{
|
||||
AwsSecretsManagerSettings, AzureKeyVaultSettings, SecretMigrationReport, VaultSettings,
|
||||
};
|
||||
use windmill_common::{
|
||||
auth::is_super_admin_email,
|
||||
ee_oss::{get_license_plan, LicensePlan},
|
||||
email_oss::send_email_plain_text,
|
||||
error::{self, JsonResult, Result},
|
||||
@@ -1135,6 +1139,8 @@ struct CustomInstanceDb {
|
||||
success: bool,
|
||||
error: Option<String>,
|
||||
tag: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
used_by_workspaces: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Serialize, Default)]
|
||||
@@ -1154,7 +1160,7 @@ struct CustomInstanceDbLogs {
|
||||
}
|
||||
|
||||
async fn list_custom_instance_pg_databases(
|
||||
_authed: ApiAuthed,
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
) -> JsonResult<HashMap<String, CustomInstanceDb>> {
|
||||
let result = sqlx::query_scalar!(
|
||||
@@ -1163,12 +1169,57 @@ async fn list_custom_instance_pg_databases(
|
||||
.fetch_one(&db)
|
||||
.await?
|
||||
.ok_or_else(|| error::Error::ExecutionErr("Couldn't find custom_instance_pg_databases".to_string()))?;
|
||||
let result = serde_json::from_value(result).map_err(|e| {
|
||||
error::Error::ExecutionErr(format!(
|
||||
"couldn't parse custom_instance_pg_databases.databases : {}",
|
||||
e.to_string()
|
||||
))
|
||||
})?;
|
||||
let mut result: HashMap<String, CustomInstanceDb> =
|
||||
serde_json::from_value(result).map_err(|e| {
|
||||
error::Error::ExecutionErr(format!(
|
||||
"couldn't parse custom_instance_pg_databases.databases : {}",
|
||||
e.to_string()
|
||||
))
|
||||
})?;
|
||||
|
||||
if is_super_admin_email(&db, &authed.email).await? {
|
||||
// Enrich each database with the list of workspaces referencing it through
|
||||
// either a ducklake catalog or a datatable database whose resource_type is
|
||||
// 'instance'. Not stored in DB to avoid drift.
|
||||
let usages = sqlx::query!(
|
||||
r#"
|
||||
SELECT ws.workspace_id AS "workspace_id!", entry->'catalog'->>'resource_path' AS dbname
|
||||
FROM workspace_settings ws
|
||||
CROSS JOIN LATERAL jsonb_each(
|
||||
CASE WHEN jsonb_typeof(ws.ducklake->'ducklakes') = 'object'
|
||||
THEN ws.ducklake->'ducklakes'
|
||||
ELSE '{}'::jsonb END
|
||||
) AS dl(k, entry)
|
||||
WHERE entry->'catalog'->>'resource_type' = 'instance'
|
||||
AND entry->'catalog'->>'resource_path' IS NOT NULL
|
||||
UNION ALL
|
||||
SELECT ws.workspace_id AS "workspace_id!", entry->'database'->>'resource_path' AS dbname
|
||||
FROM workspace_settings ws
|
||||
CROSS JOIN LATERAL jsonb_each(
|
||||
CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object'
|
||||
THEN ws.datatable->'datatables'
|
||||
ELSE '{}'::jsonb END
|
||||
) AS dt(k, entry)
|
||||
WHERE entry->'database'->>'resource_type' = 'instance'
|
||||
AND entry->'database'->>'resource_path' IS NOT NULL
|
||||
"#,
|
||||
)
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
|
||||
let mut by_db: HashMap<String, BTreeSet<String>> = HashMap::new();
|
||||
for row in usages {
|
||||
if let Some(dbname) = row.dbname {
|
||||
by_db.entry(dbname).or_default().insert(row.workspace_id);
|
||||
}
|
||||
}
|
||||
for (dbname, entry) in result.iter_mut() {
|
||||
if let Some(workspaces) = by_db.remove(dbname) {
|
||||
entry.used_by_workspaces = workspaces.into_iter().collect();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(Json(result));
|
||||
}
|
||||
|
||||
@@ -1196,7 +1247,8 @@ async fn setup_custom_instance_pg_database(
|
||||
let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await;
|
||||
let success = result.is_ok();
|
||||
let error = result.err().map(|e| e.to_string());
|
||||
let status = CustomInstanceDb { logs, success, error, tag: body.tag };
|
||||
let status =
|
||||
CustomInstanceDb { logs, success, error, tag: body.tag, used_by_workspaces: vec![] };
|
||||
let status_json = serde_json::to_value(&status).map_err(to_anyhow)?;
|
||||
// Save that the database was setup successfully
|
||||
sqlx::query!(
|
||||
|
||||
@@ -25476,6 +25476,11 @@ components:
|
||||
example: "Connection timeout"
|
||||
tag:
|
||||
$ref: "#/components/schemas/CustomInstanceDbTag"
|
||||
used_by_workspaces:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Workspaces that reference this database via a ducklake catalog or datatable database with resource_type 'instance'. Computed at request time, not persisted.
|
||||
|
||||
NewSqsTrigger:
|
||||
type: object
|
||||
|
||||
@@ -252,6 +252,17 @@ lazy_static::lazy_static! {
|
||||
pub static ref WORKSPACE_FAIRNESS_OVERLOADED: arc_swap::ArcSwap<Vec<String>> = arc_swap::ArcSwap::from_pointee(vec![]);
|
||||
pub static ref WORKSPACE_FAIRNESS_LAST_REFRESH_MICROS: AtomicI64 = AtomicI64::new(0);
|
||||
|
||||
/// Stochastic admission probability for capped workspaces, expressed in
|
||||
/// parts per 10_000 (so `420` = 4.2%). The refresh computes this from the
|
||||
/// observed worker-second distribution and the configured cap so that
|
||||
/// admission converges to the target *worker-second* share — independent
|
||||
/// of how the capped vs uncapped workspaces compare on per-job durations.
|
||||
/// See `workspace_fairness_ee::refresh_overloaded` for the derivation.
|
||||
/// `10_000` (= admit all) is the default until the first refresh
|
||||
/// classifies an overloaded set — before that, no workspace is capped so
|
||||
/// `should_admit_capped` is moot and "admit all" is the correct no-op.
|
||||
pub static ref WORKSPACE_FAIRNESS_ADMISSION_PPM: AtomicU32 = AtomicU32::new(10_000);
|
||||
|
||||
|
||||
pub static ref SMTP_CONFIG: arc_swap::ArcSwap<Option<Smtp>> = arc_swap::ArcSwap::from_pointee(None);
|
||||
pub static ref INDEXER_CONFIG: arc_swap::ArcSwap<TantivyIndexerSettings> = arc_swap::ArcSwap::from_pointee(TantivyIndexerSettings::default());
|
||||
|
||||
@@ -1,15 +1,253 @@
|
||||
//! Per-workspace fairness for the shared worker pool (Enterprise feature).
|
||||
//! # Per-workspace fairness for the shared worker pool (Enterprise feature)
|
||||
//!
|
||||
//! The real algorithm — overloaded-set aggregation, coordinated refresh on
|
||||
//! `background_task_state`, audit emission, stochastic admission decision —
|
||||
//! lives in [`crate::workspace_fairness_ee`] and only compiles when the
|
||||
//! `private` feature is on. This module is the public surface used by the
|
||||
//! pull dispatch in `jobs.rs` and the integration tests; when EE is on it
|
||||
//! transparently re-exports the EE implementation, when EE is off it
|
||||
//! provides no-op stubs so the OSS build stays bit-identical to the
|
||||
//! pre-fairness pull path.
|
||||
//! On multi-tenant deployments (notably `app.windmill.dev`, and any EE
|
||||
//! cluster with a single shared worker group) a single workspace flooding
|
||||
//! the queue with jobs can degrade quality of service for everyone else.
|
||||
//! This module computes the set of "overloaded" workspaces whose share of
|
||||
//! the worker pool must be capped, and the dispatch in `jobs.rs` uses a
|
||||
//! **duration-weighted stochastic admission rule** at pull time to enforce
|
||||
//! the cap as a *worker-second* share, not a pull-count share.
|
||||
//!
|
||||
//! See [`crate::workspace_fairness_ee`] for design notes and SQL details.
|
||||
//! The full algorithm lives in [`crate::workspace_fairness_ee`] behind the
|
||||
//! `private` feature — this OSS-facing module is the public surface that
|
||||
//! the pull dispatch and integration tests call. When EE is on, the symbols
|
||||
//! here transparently re-export the EE implementation. When EE is off, they
|
||||
//! are no-ops: `maybe_refresh_overloaded` does nothing, `should_admit_capped`
|
||||
//! always returns `true`, and the pull path is bit-identical to its
|
||||
//! pre-fairness shape. **Both runtime correctness and the entire reasoning
|
||||
//! below assume the EE module is compiled in**; the OSS build is a stub.
|
||||
//!
|
||||
//! Every numerical default mentioned below (`MAX_PERCENT = 50`,
|
||||
//! `DURATION_SECS = 10`, `MIN_TOTAL = 4`, `WORKER_PING_LIVE_SECS = 60`,
|
||||
//! `ADMISSION_EPSILON_PERCENT = 5`) is tunable via global settings or
|
||||
//! constants; the values here are the as-shipped defaults at the time of
|
||||
//! writing and what the design discussion below was calibrated against.
|
||||
//!
|
||||
//! ## 1. What "overloaded" means — worker-seconds, not jobs
|
||||
//!
|
||||
//! A workspace is overloaded when, over a rolling
|
||||
//! `WORKSPACE_FAIRNESS_DURATION_SECS = 10s` window, it has consumed at least
|
||||
//! `WORKSPACE_FAIRNESS_MAX_PERCENT = 50%` of cluster worker-time. Activity
|
||||
//! is measured in **worker-seconds**: each job contributes the wall-clock
|
||||
//! time it actually held a worker, intersected with the window. A
|
||||
//! count-based signal — "what fraction of jobs in the window are from this
|
||||
//! workspace" — gets badly fooled by job-duration heterogeneity: 600
|
||||
//! short (100ms) jobs and one long (60s) job consume the same worker-time
|
||||
//! but the count-based form attributes 600× more weight to the spammy
|
||||
//! workspace. Worker-seconds put both patterns on the same scale.
|
||||
//!
|
||||
//! Two sources contribute to a workspace's worker-second total:
|
||||
//!
|
||||
//! - **Running** (live, currently-on-a-worker): driven from `v2_job_runtime`
|
||||
//! filtered on `ping > now() - WORKER_PING_LIVE_SECS` (60s, ≈ 2× worker
|
||||
//! heartbeat interval), then PK-joined to `v2_job` for the `kind` filter
|
||||
//! and `v2_job_queue` for `started_at` / `suspend_until`. Contribution is
|
||||
//! `clamp(min(now, ping) − max(started_at, window_start), 0, window)`.
|
||||
//! End-of-interval is the per-job `ping`, which both (a) implements the
|
||||
//! zombie defense — a worker that stopped pinging stops accruing
|
||||
//! worker-seconds at its last heartbeat, so a backlog of stuck
|
||||
//! `running = true` rows can't dominate the denominator — and (b) matches
|
||||
//! the semantic of "worker-seconds the worker has confirmed". `v2_job_runtime`
|
||||
//! is small (rows deleted on completion), so driving the scan from there
|
||||
//! keeps the per-refresh cost bounded by the *in-flight* count rather
|
||||
//! than by the queue size, even when one workspace has thousands of
|
||||
//! `running = true` rows.
|
||||
//!
|
||||
//! - **Completed** (recently finished): pulled by an index scan over
|
||||
//! `v2_job_completed (completed_at)`, then PK-joined to `v2_job`. The
|
||||
//! index hit is critical — see "Why no `WITH params AS (...)` CTE" below.
|
||||
//! Contribution is `clamp(min(completed_at, now) − max(started_at,
|
||||
//! completed_at − duration_ms, window_start), 0, window)`. Clamping
|
||||
//! start-of-interval by `completed_at − duration_ms` defends against
|
||||
//! zombie rows that `zombie_monitor` force-failed: `started_at` may be
|
||||
//! far in the past, but `duration_ms` reflects the actual measured worker
|
||||
//! time, so the row only contributes its real runtime, not the idle wait
|
||||
//! before force-fail.
|
||||
//!
|
||||
//! Both halves exclude **flow-orchestration kinds**
|
||||
//! (`flow, flowpreview, flownode, singlestepflow`) and **concurrency-
|
||||
//! suspended rows** (`suspend_until IS NOT NULL`) — these hold
|
||||
//! `running = true` but consume no worker slot. Same predicate as
|
||||
//! `handle_zombie_jobs` in `monitor.rs`.
|
||||
//!
|
||||
//! `WORKSPACE_FAIRNESS_MIN_TOTAL = 4` is also in worker-seconds (≈ 40 %
|
||||
//! utilization of one worker over a 10s window) — below the floor, the
|
||||
//! cluster is too quiet to bother capping anyone.
|
||||
//!
|
||||
//! ## 2. The cap is enforced stochastically, weighted by duration
|
||||
//!
|
||||
//! The pull dispatch in `jobs.rs` flips a coin on every pull: with
|
||||
//! probability `p_c` it uses the standard pull query (capped workspaces
|
||||
//! are admissible — FIFO will pick them if they're at the head), and with
|
||||
//! probability `1 − p_c` it uses the *fairness pull query* which excludes
|
||||
//! the overloaded workspaces. Doing it as a probabilistic split rather
|
||||
//! than a binary cap/uncap gate keeps victim latency flat instead of
|
||||
//! breathing in/out with each refresh cycle.
|
||||
//!
|
||||
//! The key design choice is how `p_c` is set. The natural first try is
|
||||
//! `p_c = (MAX_PERCENT + ε) / 100` — a constant. That converges the
|
||||
//! *pull-count* ratio to `MAX_PERCENT`, but only matches the worker-second
|
||||
//! ratio when capped and uncapped workspaces share the same mean job
|
||||
//! duration. The steady-state share equation is:
|
||||
//!
|
||||
//! `share = p_c · D_c / (p_c · D_c + (1 − p_c) · D_u)`
|
||||
//!
|
||||
//! where `D_c` and `D_u` are the per-job mean durations of capped and
|
||||
//! uncapped workspaces respectively. With `D_c = 34s` and `D_u = 1s` (the
|
||||
//! exact numbers observed during the lancom01-prod / jps-internal cloud
|
||||
//! incident), a constant `p_c = 0.65` (60 % + 5 % ε) yields
|
||||
//!
|
||||
//! `share = 0.65 · 34 / (0.65 · 34 + 0.35 · 1) = 22.1 / 22.45 ≈ 98%`
|
||||
//!
|
||||
//! — i.e., the "60 % cap" was in practice giving capped workspaces 98 %
|
||||
//! of worker-seconds. Victims were observed waiting 15s+ for pickup
|
||||
//! despite the cap firing on every pull.
|
||||
//!
|
||||
//! Inverting the equation for the desired share `t = (MAX_PERCENT + ε) / 100`:
|
||||
//!
|
||||
//! `p_c = t · D_u / ((1 − t) · D_c + t · D_u)`
|
||||
//!
|
||||
//! Same numbers, target 0.65: `p_c ≈ 0.054` — about 12× tighter than the
|
||||
//! count-based form. The refresh computes `p_c` and stores it in
|
||||
//! [`WORKSPACE_FAIRNESS_ADMISSION_PPM`] (parts-per-10_000, fits in an
|
||||
//! `AtomicU32`). The pull-time check is one atomic load plus one
|
||||
//! `rand::rng().random_range(0..10_000)` draw — same hot-path cost as the
|
||||
//! count-based form.
|
||||
//!
|
||||
//! ### `D_c`/`D_u` come from a separate, longer service-time window
|
||||
//!
|
||||
//! Crucially, `D_c` and `D_u` must be **true mean service times**, because
|
||||
//! the share equation above is Little's-law-based
|
||||
//! (`occupancy = arrival_rate × mean_service_time`). They are **not** taken
|
||||
//! from the occupancy aggregation: that aggregation clamps each job's
|
||||
//! contribution to the short occupancy window (`DURATION_SECS`, 10s), so a
|
||||
//! job longer than the window contributes at most 10s — fine for measuring
|
||||
//! *share*, but it would truncate `D_c` to ≤ 10s and systematically
|
||||
//! under-admit the skew exactly when capped jobs are long (the case the cap
|
||||
//! exists for: e.g. true `D_c = 34s` clamped to 10s gives `p_c ≈ 0.157`, an
|
||||
//! 86 % effective share instead of 65 %). Instead, the refresh samples true
|
||||
//! unclamped `duration_ms` of completed jobs over a longer, decoupled
|
||||
//! service-time window (`DURATION_SAMPLE_SECS`, 60s) — long enough to avoid
|
||||
//! truncation and to keep the mean stable when few jobs complete within the
|
||||
//! 10s occupancy window. So the refresh emits two per-workspace signals:
|
||||
//! windowed occupancy worker-seconds (for classification) and a 60s
|
||||
//! service-time `(Σ duration_ms, count)` (for admission), merged per
|
||||
//! workspace.
|
||||
//!
|
||||
//! ### Why we kept the fallback when the fairness pull returns empty
|
||||
//!
|
||||
//! The 100 − `p_c` % of pulls that try the fairness query (excluding
|
||||
//! capped workspaces) fall back to the standard query if the fairness
|
||||
//! query returns no row. The alternative — idle the worker, holding the
|
||||
//! slot open in case a victim shows up — was considered but rejected for
|
||||
//! the first iteration: with `p_c` correctly tightened, victims do get the
|
||||
//! slot they need *when they exist*, and absent victims, falling back to
|
||||
//! the capped pool is the right behaviour (otherwise the cluster
|
||||
//! under-utilises itself for no benefit). Adding a reserve-capacity skip
|
||||
//! is a fine-tuning lever for bursty victim arrival patterns and is left
|
||||
//! as a follow-up.
|
||||
//!
|
||||
//! ### Degenerate cases
|
||||
//!
|
||||
//! If either bucket is empty — no capped jobs, no uncapped jobs, or a
|
||||
//! capped workspace with zero completions in the 60s service-time window
|
||||
//! (all its jobs still running) — the formula is undefined. The refresh
|
||||
//! falls back to the count-based `p_c = t` in those cases — it matches
|
||||
//! the pre-refactor behaviour and is the safest thing to do when there's
|
||||
//! no service-time signal yet to weight on.
|
||||
//!
|
||||
//! ## 3. Coordinated refresh — exactly once per cycle, cluster-wide
|
||||
//!
|
||||
//! The aggregation is too expensive to run on every worker process every
|
||||
//! pull (and would produce no new information on the sub-second
|
||||
//! timescale). It runs **at most once every `refresh_interval` seconds
|
||||
//! across the entire fleet**, gated by both a per-process CAS and a
|
||||
//! DB-side row lock:
|
||||
//!
|
||||
//! 1. **Per-process gate** — `maybe_refresh_overloaded` (called from the
|
||||
//! pull path) does `LAST_REFRESH_MICROS.compare_exchange` to ensure at
|
||||
//! most one in-flight refresh per process per interval. If the CAS
|
||||
//! fails or the interval hasn't elapsed yet, the call is a no-op.
|
||||
//! Cost on the hot path: one atomic load, optionally one CAS.
|
||||
//!
|
||||
//! 2. **DB-side claim** — `refresh_overloaded` first does a cheap upsert
|
||||
//! (`INSERT ... ON CONFLICT ON background_task_state ... WHERE
|
||||
//! updated_at < NOW() − refresh_interval RETURNING true`). The `VALUES`
|
||||
//! clause is all constants, so Postgres has no expensive work to do
|
||||
//! even for losers. Only the unique winner per cycle gets `Some(true)`;
|
||||
//! losers get `None` and skip the aggregation entirely.
|
||||
//!
|
||||
//! 3. **Winner-only aggregation** — the winner runs the
|
||||
//! `v2_job_runtime ∪ v2_job_completed` worker-second aggregation
|
||||
//! returning per-workspace `(workspace_id, worker_seconds, jobs)`,
|
||||
//! classifies into overloaded/uncapped, computes `p_c`, and writes the
|
||||
//! new payload `{"overloaded": [...], "admission_ppm": N}` back to
|
||||
//! `background_task_state.workspace_fairness`.
|
||||
//!
|
||||
//! 4. **Everyone reads** — winner and losers alike then `SELECT` the
|
||||
//! current value, parse it, and update their in-process
|
||||
//! `WORKSPACE_FAIRNESS_OVERLOADED` and `WORKSPACE_FAIRNESS_ADMISSION_PPM`
|
||||
//! atomics. This is what makes losers eventually see the winner's
|
||||
//! decision; they just don't pay the aggregation cost.
|
||||
//!
|
||||
//! The refresh interval is `ACTIVE_REFRESH_SECS = 2s` when the cluster
|
||||
//! currently has a capped workspace (faster — we want the cap to lift
|
||||
//! promptly once load drops) and `IDLE_REFRESH_SECS = 5s` otherwise
|
||||
//! (slower — minimise DB load during normal operation). The DB-side guard
|
||||
//! always uses the tighter `ACTIVE_REFRESH_SECS` to bound the race
|
||||
//! window; the per-process gate enforces the idle cadence.
|
||||
//!
|
||||
//! If a refresh fails (DB error, timeout > 5s), `LAST_REFRESH_MICROS` is
|
||||
//! left set to the attempt's timestamp so the next attempt has to wait a
|
||||
//! full interval — exactly the same cooldown as a successful refresh.
|
||||
//! Resetting to `0` on failure would remove the rate limit entirely
|
||||
//! precisely when DB load is highest, which is the wrong direction.
|
||||
//!
|
||||
//! ## 4. Audit logging
|
||||
//!
|
||||
//! Workspaces entering or leaving the capped set produce
|
||||
//! `workspace_fairness.capped` / `workspace_fairness.uncapped` audit
|
||||
//! entries scoped to the `admins` workspace, with the affected workspace
|
||||
//! as the `resource` field. Emitted by the refresh winner only, so a
|
||||
//! transition produces exactly one audit row regardless of fleet size.
|
||||
//! The "previous list" diffed against is the DB value (not the per-process
|
||||
//! cache) so a freshly-restarted worker that happens to win the first
|
||||
//! claim doesn't emit spurious "newly capped" entries for workspaces that
|
||||
//! were already capped before it started.
|
||||
//!
|
||||
//! ## 5. Notable SQL performance constraints
|
||||
//!
|
||||
//! - **No `WITH params AS (...)` CTE for `window_start`.** A natural
|
||||
//! refactor would be to compute `NOW() - make_interval(secs => N)` once
|
||||
//! in a CTE and reference it in both halves of the UNION. But Postgres
|
||||
//! *materialises* the CTE and the optimiser can no longer push the
|
||||
//! `completed_at > window_start` predicate down to the
|
||||
//! `ix_job_completed_completed_at` index. On the production cloud DB
|
||||
//! (~12M `v2_job_completed` rows), that turns a 10 ms index scan into a
|
||||
//! ~47s full table scan. The query intentionally inlines `NOW()` and
|
||||
//! `NOW() - make_interval(...)` at every callsite.
|
||||
//!
|
||||
//! - **Drive running side from `v2_job_runtime`, not `v2_job_queue`.**
|
||||
//! Naive ordering ("scan v2_job_queue for `running = true`, join v2_job
|
||||
//! for the kind filter") does a Seq Scan over ~thousands of running-or-
|
||||
//! bookkeeping rows and does a PK lookup into `v2_job` for every one of
|
||||
//! them — ~10 ms in prod, but worse: bounded by *queue size*. Pivoting
|
||||
//! to drive the scan from `v2_job_runtime` filtered on
|
||||
//! `ping > NOW() - 60s` narrows to the in-flight set (small, deletes-
|
||||
//! on-completion) *before* any PK lookups: 1.3 ms, 9× less I/O,
|
||||
//! bounded by *live worker count*.
|
||||
//!
|
||||
//! ## 6. Enterprise gating
|
||||
//!
|
||||
//! The cap is an Enterprise feature. `windmill-api-settings` rejects
|
||||
//! `workspace_fairness_enabled = true` writes from non-EE builds, and on a
|
||||
//! single-tenant self-hosted deployment the default
|
||||
//! `workspace_fairness_enabled = false` keeps the pull path identical to
|
||||
//! the pre-fairness baseline. At runtime the dispatch checks the atomic
|
||||
//! only — when fairness is off, `maybe_refresh_overloaded` drains the
|
||||
//! cached state in one pull cycle (resetting `WORKSPACE_FAIRNESS_OVERLOADED`
|
||||
//! to empty and `WORKSPACE_FAIRNESS_ADMISSION_PPM` to 10_000 = "admit all"),
|
||||
//! so toggling the feature off without restarting workers is safe.
|
||||
|
||||
#[cfg(feature = "private")]
|
||||
#[allow(unused)]
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
showConflict,
|
||||
showDiff,
|
||||
extractNativeTriggerInfo,
|
||||
redactEncryptionKey,
|
||||
} from "../../types.ts";
|
||||
import { downloadZip } from "./pull.ts";
|
||||
import { runLint, printReport, checkMissingLocks } from "../lint/lint.ts";
|
||||
@@ -3095,16 +3096,22 @@ function prettyChanges(
|
||||
),
|
||||
);
|
||||
} else if (change.name === "edited") {
|
||||
const changeType = getTypeStrFromPath(change.path);
|
||||
log.info(
|
||||
colors.yellow(
|
||||
`~ ${getTypeStrFromPath(change.path)} ` +
|
||||
`~ ${changeType} ` +
|
||||
displayPath +
|
||||
colors.gray(wsNote) +
|
||||
(change.codebase ? ` (codebase changed)` : ""),
|
||||
),
|
||||
);
|
||||
if (change.before != change.after) {
|
||||
if (change.path.endsWith(".yaml")) {
|
||||
if (changeType === "encryption_key") {
|
||||
showDiff(
|
||||
redactEncryptionKey(change.before),
|
||||
redactEncryptionKey(change.after),
|
||||
);
|
||||
} else if (change.path.endsWith(".yaml")) {
|
||||
try {
|
||||
showDiff(
|
||||
yamlStringify(
|
||||
|
||||
@@ -207,13 +207,19 @@ async function reconcileIncludingFile(options: {
|
||||
}
|
||||
|
||||
function referencesIncludeLine(content: string, includeLine: string): boolean {
|
||||
// Match only when the include sits on a line by itself (allowing leading
|
||||
// and trailing whitespace). Earlier we split on `\s+`, but that
|
||||
// false-positives on commented-out includes like `<!-- @AGENTS.cli.md -->`
|
||||
// where the middle token equals the include. CRLF is handled by the
|
||||
// `\r?\n` split.
|
||||
// Match when the include appears as a whitespace-separated token on any
|
||||
// line that isn't an HTML comment. We can't require the include to be on a
|
||||
// line by itself: our own CLAUDE.md default is `Instructions are in
|
||||
// @AGENTS.md` (one sentence), and a strict equality check made `wmill
|
||||
// refresh prompts` re-prompt every run on files wmill itself wrote.
|
||||
// Skipping comment-bearing lines keeps `<!-- @AGENTS.cli.md -->` from
|
||||
// false-positiving.
|
||||
for (const line of content.split(/\r?\n/)) {
|
||||
if (line.trim() === includeLine) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.startsWith("<!--") || trimmed.endsWith("-->")) {
|
||||
continue;
|
||||
}
|
||||
if (trimmed.split(/\s+/).includes(includeLine)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
+36
-1
@@ -129,11 +129,46 @@ export function showDiff(local: string, remote: string) {
|
||||
|
||||
export function showConflict(path: string, local: string, remote: string) {
|
||||
log.info(colors.yellow(`- ${path}`));
|
||||
showDiff(local, remote);
|
||||
let isEncryptionKey = false;
|
||||
try {
|
||||
isEncryptionKey = getTypeStrFromPath(path) === "encryption_key";
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (isEncryptionKey) {
|
||||
showDiff(redactEncryptionKey(local), redactEncryptionKey(remote));
|
||||
} else {
|
||||
showDiff(local, remote);
|
||||
}
|
||||
log.info("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m");
|
||||
log.info("\n");
|
||||
}
|
||||
|
||||
// Reveal only the first 5 chars of the key so a rotation is still visible in
|
||||
// the diff (different prefixes), without leaking the whole secret to stdout.
|
||||
// The remaining chars are replaced with `*`, preserving length so the diff
|
||||
// keeps showing whether the key length changed.
|
||||
export function redactEncryptionKey(content: string): string {
|
||||
if (!content) return content;
|
||||
// The encryption_key payload is JSON-encoded (a quoted string). Parse it so
|
||||
// we redact the key value itself, then re-serialize to JSON to preserve the
|
||||
// file's shape; fall back to raw redaction if parsing fails.
|
||||
try {
|
||||
const parsed = JSON.parse(content);
|
||||
if (typeof parsed === "string") {
|
||||
return JSON.stringify(redactString(parsed));
|
||||
}
|
||||
} catch {
|
||||
// not JSON — treat content as the raw key
|
||||
}
|
||||
return redactString(content);
|
||||
}
|
||||
|
||||
function redactString(s: string): string {
|
||||
if (s.length <= 5) return s;
|
||||
return s.slice(0, 5) + "*".repeat(s.length - 5);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pushes an object to the workspace server based on its type
|
||||
* @param workspace - The workspace ID to push to
|
||||
|
||||
@@ -391,6 +391,13 @@ describe("writeAiGuidanceFiles — referencesAgentsCli (via reconciliation)", ()
|
||||
["between blank lines", "before\n\n@AGENTS.cli.md\n\nafter"],
|
||||
["leading whitespace then include", " @AGENTS.cli.md\n"],
|
||||
["CRLF line endings", "line one\r\n@AGENTS.cli.md\r\nline three"],
|
||||
// Mid-sentence include: this is how our own CLAUDE.md default looks
|
||||
// ("Instructions are in @AGENTS.md"). A strict line-equality check made
|
||||
// `wmill refresh prompts` re-prompt every run on files wmill wrote.
|
||||
["mid-sentence include", "Instructions are in @AGENTS.cli.md\n"],
|
||||
// `>` blockquote prefix doesn't disable Claude's `@`-import expansion,
|
||||
// so we treat it as a reference too.
|
||||
["blockquoted include", "> @AGENTS.cli.md"],
|
||||
])("treats %s as a reference (no append)", async (_label, content) => {
|
||||
await withTempDir(async (tempDir) => {
|
||||
await writeFile(join(tempDir, "AGENTS.md"), content, "utf8");
|
||||
@@ -406,7 +413,6 @@ describe("writeAiGuidanceFiles — referencesAgentsCli (via reconciliation)", ()
|
||||
["@AGENTS-cli-md (lookalike)", "@AGENTS-cli-md"],
|
||||
["@AGENTS.cli.md without surrounding whitespace", "foo@AGENTS.cli.md"],
|
||||
["commented-out include", "<!-- @AGENTS.cli.md -->"],
|
||||
["blockquoted include", "> @AGENTS.cli.md"],
|
||||
])("does not treat %s as a reference (append happens)", async (_label, content) => {
|
||||
await withTempDir(async (tempDir) => {
|
||||
await writeFile(join(tempDir, "AGENTS.md"), content, "utf8");
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"baseUrl": "https://pub-06154ed168a24e73a86ab84db6bf15d8.r2.dev",
|
||||
"version": "00c9834",
|
||||
"sha256": "5757e5b9cbf79c20d507dc4c588640368e84b873cb48f3806ca8c67fd1aa625f"
|
||||
"version": "fe13d03",
|
||||
"sha256": "c588569103ce065a26f334d9b625f616cd23d4204deaacd6fadcfbf16f23462c"
|
||||
}
|
||||
|
||||
@@ -681,7 +681,11 @@ export class AIChatManager {
|
||||
} else if (this.mode === AIMode.NAVIGATOR) {
|
||||
return prepareNavigatorUserMessage(pendingPrompt)
|
||||
} else if (this.mode === AIMode.GLOBAL) {
|
||||
return prepareGlobalUserMessage(pendingPrompt, this.contextManager.getSelectedContext())
|
||||
return prepareGlobalUserMessage(
|
||||
pendingPrompt,
|
||||
this.contextManager.getSelectedContext(),
|
||||
{ workspace: get(workspaceStore) }
|
||||
)
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
@@ -898,7 +902,9 @@ export class AIChatManager {
|
||||
userMessage = prepareApiUserMessage(oldInstructions)
|
||||
break
|
||||
case AIMode.GLOBAL:
|
||||
userMessage = prepareGlobalUserMessage(oldInstructions, oldSelectedContext)
|
||||
userMessage = prepareGlobalUserMessage(oldInstructions, oldSelectedContext, {
|
||||
workspace: get(workspaceStore)
|
||||
})
|
||||
break
|
||||
case AIMode.APP:
|
||||
userMessage = prepareAppUserMessage(
|
||||
|
||||
@@ -74,6 +74,8 @@ vi.mock('$lib/gen', async () => {
|
||||
}),
|
||||
AppService: wrapService(actual.AppService, {
|
||||
existsApp: vi.fn(async () => false),
|
||||
createAppRaw: vi.fn(async () => 'created'),
|
||||
updateAppRaw: vi.fn(async () => 'updated'),
|
||||
getAppByPath: vi.fn(async () => {
|
||||
throw new Error('getAppByPath mock not configured')
|
||||
}),
|
||||
@@ -96,9 +98,17 @@ vi.mock('$lib/gen', async () => {
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('./rawAppBundlerBridge', () => ({
|
||||
bundleRawAppDraft: vi.fn(async () => ({
|
||||
js: 'bundled js',
|
||||
css: 'bundled css'
|
||||
}))
|
||||
}))
|
||||
|
||||
import { globalTools, prepareGlobalSystemMessage, prepareGlobalUserMessage } from './core'
|
||||
import { UserDraft, __resetUserDraftForTesting } from '$lib/userDraft.svelte'
|
||||
import { clearGlobalDrafts } from './userDraftAdapter'
|
||||
import { bundleRawAppDraft } from './rawAppBundlerBridge'
|
||||
import {
|
||||
AppService,
|
||||
FlowService,
|
||||
@@ -960,6 +970,111 @@ describe('global AI tools', () => {
|
||||
expect(UserDraft.get('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined()
|
||||
})
|
||||
|
||||
it('deploys a new raw app draft by bundling files and creating a raw app', async () => {
|
||||
UserDraft.save(
|
||||
'raw_app',
|
||||
'f/apps/report',
|
||||
{
|
||||
summary: 'AI report',
|
||||
files: {
|
||||
'/index.tsx': 'console.log("app")',
|
||||
'/package.json': '{"dependencies":{"react":"19.0.0"}}'
|
||||
},
|
||||
runnables: {},
|
||||
data: { tables: [] }
|
||||
},
|
||||
{ workspace: WORKSPACE }
|
||||
)
|
||||
|
||||
const raw = await callGlobalTool('deploy_workspace_item', {
|
||||
type: 'app',
|
||||
path: 'f/apps/report',
|
||||
deployment_message: 'ship report'
|
||||
})
|
||||
|
||||
expect(bundleRawAppDraft).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
workspace: WORKSPACE,
|
||||
files: expect.objectContaining({
|
||||
'/index.tsx': 'console.log("app")'
|
||||
})
|
||||
})
|
||||
)
|
||||
expect(AppService.createAppRaw).toHaveBeenCalledWith({
|
||||
workspace: WORKSPACE,
|
||||
formData: {
|
||||
app: {
|
||||
path: 'f/apps/report',
|
||||
value: {
|
||||
files: {
|
||||
'/index.tsx': 'console.log("app")',
|
||||
'/package.json': '{"dependencies":{"react":"19.0.0"}}'
|
||||
},
|
||||
runnables: {},
|
||||
data: { tables: [] }
|
||||
},
|
||||
summary: 'AI report',
|
||||
policy: expect.objectContaining({ execution_mode: 'publisher' }),
|
||||
deployment_message: 'ship report',
|
||||
custom_path: undefined
|
||||
},
|
||||
js: 'bundled js',
|
||||
css: 'bundled css'
|
||||
}
|
||||
})
|
||||
expect(AppService.updateAppRaw).not.toHaveBeenCalled()
|
||||
expect(UserDraft.get('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined()
|
||||
expect(JSON.parse(raw)).toMatchObject({
|
||||
success: true,
|
||||
type: 'app',
|
||||
path: 'f/apps/report'
|
||||
})
|
||||
})
|
||||
|
||||
it('deploys an existing raw app draft by bundling files and updating the raw app', async () => {
|
||||
vi.mocked(AppService.existsApp).mockResolvedValueOnce(true)
|
||||
UserDraft.save(
|
||||
'raw_app',
|
||||
'f/apps/report',
|
||||
{
|
||||
summary: 'Updated report',
|
||||
files: { '/index.tsx': 'console.log("updated")' },
|
||||
runnables: {},
|
||||
data: { tables: ['orders'] },
|
||||
policy: { execution_mode: 'anonymous' },
|
||||
custom_path: 'kept-by-backend'
|
||||
},
|
||||
{ workspace: WORKSPACE }
|
||||
)
|
||||
|
||||
await callGlobalTool('deploy_workspace_item', {
|
||||
type: 'app',
|
||||
path: 'f/apps/report'
|
||||
})
|
||||
|
||||
expect(AppService.updateAppRaw).toHaveBeenCalledWith({
|
||||
workspace: WORKSPACE,
|
||||
path: 'f/apps/report',
|
||||
formData: {
|
||||
app: {
|
||||
path: 'f/apps/report',
|
||||
value: {
|
||||
files: { '/index.tsx': 'console.log("updated")' },
|
||||
runnables: {},
|
||||
data: { tables: ['orders'] }
|
||||
},
|
||||
summary: 'Updated report',
|
||||
policy: expect.objectContaining({ execution_mode: 'anonymous' }),
|
||||
deployment_message: undefined
|
||||
},
|
||||
js: 'bundled js',
|
||||
css: 'bundled css'
|
||||
}
|
||||
})
|
||||
expect(AppService.createAppRaw).not.toHaveBeenCalled()
|
||||
expect(UserDraft.get('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined()
|
||||
})
|
||||
|
||||
it('fills an empty rawscript module through set_flow_module_code', async () => {
|
||||
await callGlobalTool('write_flow', {
|
||||
path: 'f/flows/empty-module',
|
||||
@@ -1168,6 +1283,7 @@ describe('prepareGlobalSystemMessage', () => {
|
||||
expect(content).toContain(
|
||||
'Use discard_local_draft to remove an unsaved local draft, including the matching open editor draft'
|
||||
)
|
||||
expect(content).toContain('If the user message includes an ACTIVE EDITOR section')
|
||||
expect(content).not.toContain('AI draft')
|
||||
expect(content).not.toContain('UserDraft')
|
||||
expect(content).not.toContain('localStorage')
|
||||
@@ -1190,6 +1306,27 @@ describe('prepareGlobalSystemMessage', () => {
|
||||
})
|
||||
|
||||
describe('prepareGlobalUserMessage', () => {
|
||||
it('injects the active editor reference without contents', () => {
|
||||
__resetUserDraftForTesting()
|
||||
localStorage.clear()
|
||||
UserDraft.setLiveEditorDraft({
|
||||
workspace: WORKSPACE,
|
||||
itemKind: 'script',
|
||||
storagePath: '',
|
||||
effectivePath: 'f/scripts/live_greeting'
|
||||
})
|
||||
|
||||
const message = prepareGlobalUserMessage('Update this script', [], { workspace: WORKSPACE })
|
||||
|
||||
expect(message.content).toContain('## ACTIVE EDITOR')
|
||||
expect(message.content).toContain('type: script')
|
||||
expect(message.content).toContain('path: f/scripts/live_greeting')
|
||||
expect(message.content).toContain('isLiveDraft: true')
|
||||
expect(message.content).toContain('## INSTRUCTIONS:\nUpdate this script')
|
||||
expect(message.content).not.toContain('When the user says')
|
||||
expect(message.content).not.toContain('content')
|
||||
})
|
||||
|
||||
it('includes selected workspace item references without contents', () => {
|
||||
const message = prepareGlobalUserMessage('Update these items', [
|
||||
{
|
||||
|
||||
@@ -64,6 +64,7 @@ import {
|
||||
import type { ContextElement } from '../context'
|
||||
import { UserDraft, type UserDraftMeta } from '$lib/userDraft.svelte'
|
||||
import { emptySchema } from '$lib/utils'
|
||||
import { inferArgs } from '$lib/infer'
|
||||
import {
|
||||
resourceRequestSchema,
|
||||
scheduleRequestSchema,
|
||||
@@ -83,6 +84,7 @@ import {
|
||||
type WorkspaceItemType
|
||||
} from './workspaceItems'
|
||||
import { buildFlowDeployRequestBody, buildScriptDeployRequestBody } from './deployRequests'
|
||||
import { bundleRawAppDraft } from './rawAppBundlerBridge'
|
||||
import {
|
||||
clearEphemeralSecretVariableDraftValue,
|
||||
deleteGlobalDraft,
|
||||
@@ -111,6 +113,28 @@ const INSTRUCTION_SUBJECTS = [
|
||||
'app'
|
||||
] as const satisfies readonly WorkspaceItemType[]
|
||||
const MAX_LIST_LIMIT = 100
|
||||
type ActiveGlobalEditorType = Extract<WorkspaceItemType, 'script' | 'flow' | 'app'>
|
||||
type LiveEditorDraftKind = Parameters<typeof UserDraft.getLiveEditorDraft>[0]
|
||||
|
||||
const ACTIVE_GLOBAL_EDITOR_DRAFTS: readonly {
|
||||
itemKind: LiveEditorDraftKind
|
||||
type: ActiveGlobalEditorType
|
||||
}[] = [
|
||||
{ itemKind: 'script', type: 'script' },
|
||||
{ itemKind: 'flow', type: 'flow' },
|
||||
{ itemKind: 'raw_app', type: 'app' }
|
||||
]
|
||||
|
||||
export type GlobalActiveEditorContext = {
|
||||
type: ActiveGlobalEditorType
|
||||
path: string
|
||||
isLiveDraft: true
|
||||
}
|
||||
|
||||
export type GlobalUserMessageOptions = {
|
||||
workspace?: string
|
||||
activeEditor?: GlobalActiveEditorContext
|
||||
}
|
||||
|
||||
const itemTypeSchema = z.enum(ITEM_TYPES)
|
||||
const instructionSubjectSchema = z.enum(INSTRUCTION_SUBJECTS)
|
||||
@@ -497,7 +521,7 @@ Use tools to inspect workspace items and create local drafts for scripts, flows,
|
||||
Rules:
|
||||
- Draft tools create or update local drafts only; they do not deploy or mutate deployed workspace items.
|
||||
- Use list_workspace_items to find items and read_workspace_item before changing an existing item. For triggers, pass trigger_kind.
|
||||
- If the user refers to the open editor, use the item marked isLiveDraft=true.
|
||||
- If the user message includes an ACTIVE EDITOR section, treat it as the currently open item and use it for references like "this", "current", or "open editor".
|
||||
- Use deploy_workspace_item only after the user explicitly asks to deploy. It persists a local draft to the workspace.
|
||||
- Use discard_local_draft to remove an unsaved local draft, including the matching open editor draft. Use delete_workspace_item only to delete a deployed workspace item.
|
||||
- Variable values are never readable. For secrets, create a secret variable and reference it from resources as "$var:path/to/variable".
|
||||
@@ -516,7 +540,7 @@ Raw apps:
|
||||
- Use write_app_file, patch_app_file, and delete_app_file for frontend files.
|
||||
- Use write_app_runnable and delete_app_runnable for backend runnables.
|
||||
- Use init_app only after confirming framework, path, and summary with the user.
|
||||
- Apps cannot be deployed from chat; tell the user to open the app editor.`
|
||||
- Use deploy_workspace_item after explicit user deploy intent; raw app deploy bundles JS/CSS before saving.`
|
||||
|
||||
const DEFAULT_LIST_TYPES = ['script', 'flow'] as const satisfies readonly WorkspaceItemType[]
|
||||
|
||||
@@ -1211,7 +1235,7 @@ type InstructionSubject = (typeof INSTRUCTION_SUBJECTS)[number]
|
||||
function getAppInstructions(): string {
|
||||
return `# Global draft app instructions
|
||||
|
||||
- Global mode edits raw app drafts only; it does not save, deploy, or bundle.
|
||||
- Global mode edits raw app drafts only; it does not save or deploy unless the user explicitly asks to deploy.
|
||||
- App drafts are addressed by workspace path (e.g. \`f/folder/my_app\`). The first write tool snapshots the workspace app onto the draft, and subsequent writes accumulate.
|
||||
- To create a new app, use \`init_app\` with a path, optional summary, and a framework (\`react19\` / \`react18\` / \`svelte5\` / \`vue\`). Confirm framework + path + summary with the user before calling — do not silently default to \`react19\` even though it is the recommended choice. \`init_app\` errors if an app already exists at the path or a draft is already in flight; in that case, edit the existing one rather than re-initializing.
|
||||
- \`init_app\` seeds a starter inline runnable named \`a\` (bun, \`main(x: string) => string\`) so the React/Svelte demo button works on first render. Replace or remove it once you start building real backend runnables.
|
||||
@@ -1219,7 +1243,7 @@ function getAppInstructions(): string {
|
||||
- Backend inline runnables are addressed as \`backend/<key>/main.{ts|py}\` from the file tools, but you create or update them via \`write_app_runnable\` / \`delete_app_runnable\` (which take the runnable shape directly: \`{ name, type, inlineScript?, path?, staticInputs? }\`).
|
||||
- \`/wmill.d.ts\` (or \`wmill.ts\`) is generated automatically from the backend runnables — never write it directly.
|
||||
- Inline runnables only support \`bun\` or \`python3\` in chat. Path runnables (\`script\`/\`flow\`/\`hubscript\`) reference an existing item.
|
||||
- Apps cannot be deployed from chat. The app editor bundles JS/CSS before save; tell the user to open the app editor to deploy app drafts.
|
||||
- Use \`deploy_workspace_item\` after explicit user deploy intent. The deploy tool bundles JS/CSS before saving the raw app.
|
||||
- Use \`read_workspace_item\` with \`type: 'app'\` for a metadata summary (file paths and runnable list, no contents). Use \`read_app_file\` to read an individual file.
|
||||
- Note: the authoring reference below mentions the CLI on-disk layout (\`backend/<id>.<ext>\`, \`raw_app.yaml\`, \`sql_to_apply/\`). That layout is only relevant for the terminal workflow — in chat, apps are addressed via the tool surface above.
|
||||
|
||||
@@ -2544,7 +2568,13 @@ async function patchAppFile(
|
||||
}
|
||||
|
||||
async function recomputeAppPolicy(value: AppDraftValue): Promise<void> {
|
||||
value.policy = (await updateRawAppPolicy(value.runnables as any, value.policy as any)) as any
|
||||
const policy = (await updateRawAppPolicy(value.runnables as any, value.policy as any)) as NonNullable<
|
||||
AppDraftValue['policy']
|
||||
>
|
||||
if (!policy.execution_mode) {
|
||||
policy.execution_mode = 'publisher'
|
||||
}
|
||||
value.policy = policy
|
||||
}
|
||||
|
||||
async function writeAppRunnable(
|
||||
@@ -2719,12 +2749,6 @@ async function deployDraft(
|
||||
const { workspace, toolId, toolCallbacks } = ctx
|
||||
const { type, path, trigger_kind: triggerKind, deployment_message: deploymentMessage } = args
|
||||
|
||||
if (type === 'app') {
|
||||
throw new Error(
|
||||
'Apps cannot be deployed from chat. Open the app editor to deploy (the editor bundles JS/CSS before save).'
|
||||
)
|
||||
}
|
||||
|
||||
if (type === 'trigger' && !triggerKind) {
|
||||
throw new Error('trigger_kind is required when deploying a trigger.')
|
||||
}
|
||||
@@ -2748,10 +2772,16 @@ async function deployDraft(
|
||||
const existing = (await ScriptService.existsScriptByPath({ workspace, path }))
|
||||
? await ScriptService.getScriptByPath({ workspace, path })
|
||||
: undefined
|
||||
await ScriptService.createScript({
|
||||
workspace,
|
||||
requestBody: buildScriptDeployRequestBody(path, draft, existing, deploymentMessage)
|
||||
})
|
||||
const requestBody = buildScriptDeployRequestBody(path, draft, existing, deploymentMessage)
|
||||
// Infer the arg schema from the content so it matches the code, like the editor does.
|
||||
try {
|
||||
const schema = emptySchema()
|
||||
await inferArgs(requestBody.language, requestBody.content, schema)
|
||||
requestBody.schema = schema
|
||||
} catch (e) {
|
||||
console.error('Failed to infer script schema before deploy', e)
|
||||
}
|
||||
await ScriptService.createScript({ workspace, requestBody })
|
||||
break
|
||||
}
|
||||
case 'flow': {
|
||||
@@ -2820,6 +2850,87 @@ async function deployDraft(
|
||||
actions = [createOpenVariableAction(path)]
|
||||
break
|
||||
}
|
||||
case 'app': {
|
||||
const appDraft = draft.value as AppDraftValue
|
||||
const appValue: AppDraftValue = {
|
||||
...appDraft,
|
||||
files: { ...(appDraft.files ?? {}) },
|
||||
runnables: { ...(appDraft.runnables ?? {}) },
|
||||
data: appDraft.data ?? { ...DEFAULT_RAW_APP_DATA }
|
||||
}
|
||||
await recomputeAppPolicy(appValue)
|
||||
const policy = appValue.policy
|
||||
if (!policy) {
|
||||
throw new Error(`Draft app "${path}" has no policy to deploy.`)
|
||||
}
|
||||
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: `Bundling app "${path}"...`
|
||||
})
|
||||
const bundle = await bundleRawAppDraft({
|
||||
workspace,
|
||||
files: appValue.files,
|
||||
onLog: (delta) => {
|
||||
const lines = delta
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
const latest = lines[lines.length - 1]
|
||||
if (latest) {
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: `Bundling app "${path}"... ${latest}`
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: `Deploying app "${path}"...`
|
||||
})
|
||||
const rawAppValue = {
|
||||
files: appValue.files,
|
||||
runnables: appValue.runnables,
|
||||
data: appValue.data ?? { ...DEFAULT_RAW_APP_DATA }
|
||||
}
|
||||
const summary = appValue.summary ?? draft.summary ?? ''
|
||||
if (await AppService.existsApp({ workspace, path })) {
|
||||
// Omit custom_path on update for now. The backend preserves it when absent, while
|
||||
// sending it requires admin privileges; this chat deploy path does not yet mirror
|
||||
// the raw app editor's user/admin-specific custom_path handling.
|
||||
await AppService.updateAppRaw({
|
||||
workspace,
|
||||
path,
|
||||
formData: {
|
||||
app: {
|
||||
path,
|
||||
value: rawAppValue,
|
||||
summary,
|
||||
policy,
|
||||
deployment_message: deploymentMessage
|
||||
},
|
||||
js: bundle.js,
|
||||
css: bundle.css
|
||||
}
|
||||
})
|
||||
} else {
|
||||
await AppService.createAppRaw({
|
||||
workspace,
|
||||
formData: {
|
||||
app: {
|
||||
path,
|
||||
value: rawAppValue,
|
||||
summary,
|
||||
policy,
|
||||
deployment_message: deploymentMessage,
|
||||
custom_path: appValue.custom_path
|
||||
},
|
||||
js: bundle.js,
|
||||
css: bundle.css
|
||||
}
|
||||
})
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
deleteGlobalDraft(workspace, type, path, triggerKind, { preserveLiveDraft: true })
|
||||
@@ -2914,15 +3025,36 @@ export function prepareGlobalSystemMessage(
|
||||
}
|
||||
}
|
||||
|
||||
export function getActiveGlobalEditorContext(
|
||||
workspace: string
|
||||
): GlobalActiveEditorContext | undefined {
|
||||
for (const { itemKind, type } of ACTIVE_GLOBAL_EDITOR_DRAFTS) {
|
||||
const liveDraft = UserDraft.getLiveEditorDraft(itemKind, { workspace })
|
||||
const path = liveDraft?.effectivePath || liveDraft?.storagePath
|
||||
if (!path) continue
|
||||
return { type, path, isLiveDraft: true }
|
||||
}
|
||||
}
|
||||
|
||||
export function prepareGlobalUserMessage(
|
||||
instructions: string,
|
||||
selectedContext: ContextElement[] = []
|
||||
selectedContext: ContextElement[] = [],
|
||||
options: GlobalUserMessageOptions = {}
|
||||
): ChatCompletionUserMessageParam {
|
||||
const selectedWorkspaceItems = selectedContext.filter(
|
||||
(context) => context.type === 'workspace_script' || context.type === 'workspace_flow'
|
||||
)
|
||||
const activeEditor =
|
||||
options.activeEditor ?? (options.workspace ? getActiveGlobalEditorContext(options.workspace) : undefined)
|
||||
let content = ''
|
||||
|
||||
if (activeEditor) {
|
||||
content += '## ACTIVE EDITOR\n'
|
||||
content += `type: ${activeEditor.type}\n`
|
||||
content += `path: ${activeEditor.path}\n`
|
||||
content += `isLiveDraft: true\n\n`
|
||||
}
|
||||
|
||||
if (selectedWorkspaceItems.length > 0) {
|
||||
content += '## SELECTED CONTEXT\n'
|
||||
for (const context of selectedWorkspaceItems) {
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import { WorkspaceService } from '$lib/gen'
|
||||
|
||||
export type RawAppBundle = {
|
||||
js: string
|
||||
css: string
|
||||
}
|
||||
|
||||
type BundleRawAppFilesParams = {
|
||||
files: Record<string, string>
|
||||
sharedUiFiles?: Record<string, string>
|
||||
bundlerType?: 'esbuild' | 'rolldown'
|
||||
timeoutMs?: number
|
||||
onLog?: (delta: string) => void
|
||||
}
|
||||
|
||||
type BundleRawAppDraftParams = BundleRawAppFilesParams & {
|
||||
workspace: string
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 120_000
|
||||
|
||||
function makeRequestId(): string {
|
||||
return globalThis.crypto?.randomUUID?.() ?? Math.random().toString(36).slice(2)
|
||||
}
|
||||
|
||||
async function loadSharedUiFiles(workspace: string): Promise<Record<string, string>> {
|
||||
try {
|
||||
const res = (await WorkspaceService.getSharedUi({ workspace })) as {
|
||||
files?: Record<string, string>
|
||||
}
|
||||
return res.files ?? {}
|
||||
} catch (e) {
|
||||
console.warn('Failed to load shared UI for raw app bundling:', e)
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
export async function bundleRawAppDraft(params: BundleRawAppDraftParams): Promise<RawAppBundle> {
|
||||
const sharedUiFiles = params.sharedUiFiles ?? (await loadSharedUiFiles(params.workspace))
|
||||
return bundleRawAppFiles({
|
||||
...params,
|
||||
sharedUiFiles
|
||||
})
|
||||
}
|
||||
|
||||
export function bundleRawAppFiles({
|
||||
files,
|
||||
sharedUiFiles = {},
|
||||
bundlerType = 'esbuild',
|
||||
timeoutMs = DEFAULT_TIMEOUT_MS,
|
||||
onLog
|
||||
}: BundleRawAppFilesParams): Promise<RawAppBundle> {
|
||||
if (typeof window === 'undefined' || typeof document === 'undefined') {
|
||||
return Promise.reject(new Error('Raw app bundling requires a browser environment.'))
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const requestId = makeRequestId()
|
||||
const iframe = document.createElement('iframe')
|
||||
let settled = false
|
||||
let bundleRequestSent = false
|
||||
|
||||
const cleanup = () => {
|
||||
window.removeEventListener('message', onMessage)
|
||||
clearTimeout(timeout)
|
||||
iframe.remove()
|
||||
}
|
||||
|
||||
const settle = <T>(fn: (value: T) => void, value: T) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
cleanup()
|
||||
fn(value)
|
||||
}
|
||||
|
||||
const postToBundler = (message: Record<string, unknown>) => {
|
||||
if (!iframe.contentWindow) {
|
||||
settle(reject, new Error('Raw app bundler iframe did not initialize.'))
|
||||
return
|
||||
}
|
||||
iframe.contentWindow.postMessage(message, window.location.origin)
|
||||
}
|
||||
|
||||
const sendBundleProtocolRequest = () => {
|
||||
if (settled || bundleRequestSent) return
|
||||
bundleRequestSent = true
|
||||
postToBundler({
|
||||
type: 'bundleRawApp',
|
||||
requestId,
|
||||
files,
|
||||
sharedUiFiles,
|
||||
bundlerType
|
||||
})
|
||||
}
|
||||
|
||||
const timeout = window.setTimeout(() => {
|
||||
settle(reject, new Error('Timed out while bundling raw app.'))
|
||||
}, timeoutMs)
|
||||
|
||||
function onMessage(event: MessageEvent) {
|
||||
if (event.source !== iframe.contentWindow) return
|
||||
const data = event.data
|
||||
if (!data) return
|
||||
|
||||
if (data.type === 'bundleRawAppReady') {
|
||||
sendBundleProtocolRequest()
|
||||
return
|
||||
}
|
||||
|
||||
if (data.requestId !== requestId) return
|
||||
|
||||
if (data.type === 'appendLogs') {
|
||||
onLog?.(String(data.delta ?? ''))
|
||||
} else if (data.type === 'bundleRawAppResult') {
|
||||
const bundle = data.bundle
|
||||
if (!bundle?.js) {
|
||||
settle(reject, new Error('Raw app bundler returned an empty JavaScript bundle.'))
|
||||
return
|
||||
}
|
||||
settle(resolve, {
|
||||
js: String(bundle.js),
|
||||
css: String(bundle.css ?? '')
|
||||
})
|
||||
} else if (data.type === 'bundleRawAppError') {
|
||||
settle(reject, new Error(String(data.error ?? 'Raw app bundle failed.')))
|
||||
}
|
||||
}
|
||||
|
||||
iframe.title = 'Raw app bundler'
|
||||
iframe.tabIndex = -1
|
||||
// Windmill pages use COEP=require-corp; the static UI builder iframe must be credentialless.
|
||||
iframe.setAttribute('credentialless', '')
|
||||
iframe.style.position = 'fixed'
|
||||
iframe.style.width = '0'
|
||||
iframe.style.height = '0'
|
||||
iframe.style.border = '0'
|
||||
iframe.style.opacity = '0'
|
||||
iframe.style.pointerEvents = 'none'
|
||||
iframe.src = '/ui_builder/index.html?mode=bundle'
|
||||
|
||||
window.addEventListener('message', onMessage)
|
||||
document.body.appendChild(iframe)
|
||||
})
|
||||
}
|
||||
@@ -72,6 +72,8 @@
|
||||
let wrapperEl: HTMLDivElement | undefined = $state()
|
||||
let searchInputEl: TextInput | undefined = $state()
|
||||
|
||||
let currentValue = $derived(value ?? [])
|
||||
|
||||
$effect(() => searchInputEl?.focus())
|
||||
|
||||
let processedItems: ProcessedItem<Value>[] = $derived.by(() => {
|
||||
@@ -87,18 +89,20 @@
|
||||
})
|
||||
|
||||
let valueEntry = $derived(
|
||||
value.map((v) => processedItems.find((item) => item.value === v) ?? { value: v, label: v })
|
||||
currentValue.map(
|
||||
(v) => processedItems.find((item) => item.value === v) ?? { value: v, label: v }
|
||||
)
|
||||
)
|
||||
|
||||
function onAddValue(item: ProcessedItem<Value>) {
|
||||
if (item.__is_create && onCreateItem) {
|
||||
onCreateItem(item.value)
|
||||
} else {
|
||||
value = [...value, item.value]
|
||||
value = [...currentValue, item.value]
|
||||
}
|
||||
}
|
||||
function onRemoveValue(item: ProcessedItem<Value>) {
|
||||
value = value.filter((v) => v !== item.value)
|
||||
value = currentValue.filter((v) => v !== item.value)
|
||||
}
|
||||
|
||||
function clearValue() {
|
||||
@@ -132,7 +136,7 @@
|
||||
<!-- svelte-ignore a11y_click_events_have_key_events -->
|
||||
<!-- svelte-ignore a11y_no_noninteractive_element_interactions -->
|
||||
|
||||
{#if value.length === 0}
|
||||
{#if currentValue.length === 0}
|
||||
<span class={twMerge('text-xs h-full flex items-center flex-1 text-hint', placeholderClass)}>
|
||||
{placeholder}
|
||||
</span>
|
||||
@@ -149,7 +153,7 @@
|
||||
{allowClear}
|
||||
onRemove={onRemoveValue}
|
||||
onReorder={reorderable
|
||||
? (oldIdx, newIdx) => (value = reorder(value, oldIdx, newIdx))
|
||||
? (oldIdx, newIdx) => (value = reorder(currentValue, oldIdx, newIdx))
|
||||
: undefined}
|
||||
/>
|
||||
</ul>
|
||||
@@ -166,7 +170,7 @@
|
||||
{disablePortal}
|
||||
onSelectValue={onAddValue}
|
||||
{open}
|
||||
processedItems={processedItems.filter((item) => !value.includes(item.value))}
|
||||
processedItems={processedItems.filter((item) => !currentValue.includes(item.value))}
|
||||
value={undefined}
|
||||
{disabled}
|
||||
{filterText}
|
||||
@@ -181,7 +185,7 @@
|
||||
ulClass="options"
|
||||
>
|
||||
{#snippet header()}
|
||||
{#if processedItems.length - value.length > 0 || onCreateItem}
|
||||
{#if processedItems.length - currentValue.length > 0 || onCreateItem}
|
||||
<div class="mx-2 mb-1 mt-2 flex items-center relative">
|
||||
<TextInput
|
||||
bind:this={searchInputEl}
|
||||
|
||||
@@ -672,124 +672,124 @@
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col h-full justify-end">
|
||||
<Menubar class="flex flex-col gap-1 mb-6 md:mb-10">
|
||||
<Menubar class="flex flex-col">
|
||||
{#snippet children({ createMenu })}
|
||||
<UserMenu {isCollapsed} {createMenu} />
|
||||
<div class="flex flex-col gap-1 mb-6 md:mb-10">
|
||||
<UserMenu {isCollapsed} {createMenu} />
|
||||
|
||||
{#each secondaryMenuLinks as menuLink (menuLink.href ?? menuLink.label)}
|
||||
{#if menuLink.subItems}
|
||||
{@const notificationsCount = computeAllNotificationsCount(menuLink.subItems)}
|
||||
<Menu {createMenu} usePointerDownOutside>
|
||||
{#snippet triggr({ trigger })}
|
||||
<MenuButton
|
||||
class="!text-2xs"
|
||||
{...menuLink}
|
||||
{isCollapsed}
|
||||
{notificationsCount}
|
||||
{trigger}
|
||||
/>
|
||||
{/snippet}
|
||||
{#each secondaryMenuLinks as menuLink (menuLink.href ?? menuLink.label)}
|
||||
{#if menuLink.subItems}
|
||||
{@const notificationsCount = computeAllNotificationsCount(menuLink.subItems)}
|
||||
<Menu {createMenu} usePointerDownOutside>
|
||||
{#snippet triggr({ trigger })}
|
||||
<MenuButton
|
||||
class="!text-2xs"
|
||||
{...menuLink}
|
||||
{isCollapsed}
|
||||
{notificationsCount}
|
||||
{trigger}
|
||||
/>
|
||||
{/snippet}
|
||||
|
||||
{#snippet children({ item })}
|
||||
{#each menuLink.subItems as subItem (subItem.href ?? subItem.label)}
|
||||
<MenuItem
|
||||
class={itemClass}
|
||||
href={subItem.href}
|
||||
{item}
|
||||
onClick={() => {
|
||||
subItem?.['action']?.()
|
||||
}}
|
||||
aiId={subItem.aiId}
|
||||
aiDescription={subItem.aiDescription}
|
||||
>
|
||||
<div class="flex flex-row items-center gap-2">
|
||||
{#if subItem.icon}
|
||||
<subItem.icon size={16} />
|
||||
{/if}
|
||||
{subItem.label}
|
||||
{#if subItem?.['notificationCount']}
|
||||
<div class="ml-auto">
|
||||
<SideBarNotification notificationCount={subItem['notificationCount']} />
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</MenuItem>
|
||||
{/each}
|
||||
{/snippet}
|
||||
</Menu>
|
||||
{:else}
|
||||
<MenuSingleItem>
|
||||
{#snippet children({})}
|
||||
<MenuLink class="!text-2xs" {...menuLink} {isCollapsed} />
|
||||
{/snippet}
|
||||
</MenuSingleItem>
|
||||
{/if}
|
||||
{/each}
|
||||
{/snippet}
|
||||
</Menubar>
|
||||
|
||||
<Menubar class="flex flex-col gap-1">
|
||||
{#snippet children({ createMenu })}
|
||||
{#each thirdMenuLinks as menuLink (menuLink)}
|
||||
{#if menuLink.subItems}
|
||||
<Menu {createMenu} usePointerDownOutside>
|
||||
{#snippet triggr({ trigger })}
|
||||
<button
|
||||
class="relative w-full"
|
||||
onclick={() => {
|
||||
if (menuLink.label === 'Help') {
|
||||
openChangelogs()
|
||||
}
|
||||
}}
|
||||
>
|
||||
<MenuButton class="!text-2xs" {...menuLink} {isCollapsed} {trigger} />
|
||||
{#if menuLink.label === 'Help' && hasNewChangelogs}
|
||||
<span
|
||||
class={twMerge(
|
||||
'flex h-2 w-2 absolute',
|
||||
isCollapsed ? 'top-1 right-1' : 'right-2 top-1/2 -translate-y-1/2'
|
||||
)}
|
||||
{#snippet children({ item })}
|
||||
{#each menuLink.subItems as subItem (subItem.href ?? subItem.label)}
|
||||
<MenuItem
|
||||
class={itemClass}
|
||||
href={subItem.href}
|
||||
{item}
|
||||
onClick={() => {
|
||||
subItem?.['action']?.()
|
||||
}}
|
||||
aiId={subItem.aiId}
|
||||
aiDescription={subItem.aiDescription}
|
||||
>
|
||||
<span
|
||||
class="animate-ping absolute inline-flex h-full w-full rounded-full bg-frost-400 opacity-75"
|
||||
></span>
|
||||
<span class="relative inline-flex rounded-full h-2 w-2 bg-frost-500"></span>
|
||||
</span>
|
||||
{/if}
|
||||
</button>
|
||||
{/snippet}
|
||||
{#snippet children({ item })}
|
||||
{#each menuLink.subItems as subItem (subItem.href ?? subItem.label)}
|
||||
<MenuItem
|
||||
href={subItem.href}
|
||||
class={itemClass}
|
||||
target={subItem.external !== false ? '_blank' : undefined}
|
||||
{item}
|
||||
>
|
||||
<div class="flex flex-row items-center gap-2">
|
||||
{#if subItem.icon}
|
||||
<subItem.icon size={16} />
|
||||
{/if}
|
||||
|
||||
{subItem.label}
|
||||
</div>
|
||||
</MenuItem>
|
||||
{/each}
|
||||
{#if recentChangelogs.length > 0}
|
||||
<div class="w-full h-1 border-t"></div>
|
||||
<span class="text-xs px-4 font-bold"> Latest changelogs </span>
|
||||
{#each recentChangelogs as changelog}
|
||||
<MenuItem href={changelog.href} class={itemClass} target="_blank" {item}>
|
||||
<div class="flex flex-row items-center gap-2">
|
||||
{changelog.label}
|
||||
{#if subItem.icon}
|
||||
<subItem.icon size={16} />
|
||||
{/if}
|
||||
{subItem.label}
|
||||
{#if subItem?.['notificationCount']}
|
||||
<div class="ml-auto">
|
||||
<SideBarNotification notificationCount={subItem['notificationCount']} />
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</MenuItem>
|
||||
{/each}
|
||||
{/if}
|
||||
{/snippet}
|
||||
</Menu>
|
||||
{/if}
|
||||
{/each}
|
||||
{/snippet}
|
||||
</Menu>
|
||||
{:else}
|
||||
<MenuSingleItem>
|
||||
{#snippet children({})}
|
||||
<MenuLink class="!text-2xs" {...menuLink} {isCollapsed} />
|
||||
{/snippet}
|
||||
</MenuSingleItem>
|
||||
{/if}
|
||||
{/each}
|
||||
</div>
|
||||
|
||||
<div class="flex flex-col gap-1">
|
||||
{#each thirdMenuLinks as menuLink (menuLink)}
|
||||
{#if menuLink.subItems}
|
||||
<Menu {createMenu} usePointerDownOutside>
|
||||
{#snippet triggr({ trigger })}
|
||||
<button
|
||||
class="relative w-full"
|
||||
onclick={() => {
|
||||
if (menuLink.label === 'Help') {
|
||||
openChangelogs()
|
||||
}
|
||||
}}
|
||||
>
|
||||
<MenuButton class="!text-2xs" {...menuLink} {isCollapsed} {trigger} />
|
||||
{#if menuLink.label === 'Help' && hasNewChangelogs}
|
||||
<span
|
||||
class={twMerge(
|
||||
'flex h-2 w-2 absolute',
|
||||
isCollapsed ? 'top-1 right-1' : 'right-2 top-1/2 -translate-y-1/2'
|
||||
)}
|
||||
>
|
||||
<span
|
||||
class="animate-ping absolute inline-flex h-full w-full rounded-full bg-frost-400 opacity-75"
|
||||
></span>
|
||||
<span class="relative inline-flex rounded-full h-2 w-2 bg-frost-500"></span>
|
||||
</span>
|
||||
{/if}
|
||||
</button>
|
||||
{/snippet}
|
||||
{#snippet children({ item })}
|
||||
{#each menuLink.subItems as subItem (subItem.href ?? subItem.label)}
|
||||
<MenuItem
|
||||
href={subItem.href}
|
||||
class={itemClass}
|
||||
target={subItem.external !== false ? '_blank' : undefined}
|
||||
{item}
|
||||
>
|
||||
<div class="flex flex-row items-center gap-2">
|
||||
{#if subItem.icon}
|
||||
<subItem.icon size={16} />
|
||||
{/if}
|
||||
|
||||
{subItem.label}
|
||||
</div>
|
||||
</MenuItem>
|
||||
{/each}
|
||||
{#if recentChangelogs.length > 0}
|
||||
<div class="w-full h-1 border-t"></div>
|
||||
<span class="text-xs px-4 font-bold"> Latest changelogs </span>
|
||||
{#each recentChangelogs as changelog}
|
||||
<MenuItem href={changelog.href} class={itemClass} target="_blank" {item}>
|
||||
<div class="flex flex-row items-center gap-2">
|
||||
{changelog.label}
|
||||
</div>
|
||||
</MenuItem>
|
||||
{/each}
|
||||
{/if}
|
||||
{/snippet}
|
||||
</Menu>
|
||||
{/if}
|
||||
{/each}
|
||||
</div>
|
||||
{/snippet}
|
||||
</Menubar>
|
||||
</div>
|
||||
|
||||
@@ -696,7 +696,7 @@
|
||||
variant="default"
|
||||
startIcon={{ icon: List }}
|
||||
disabled={!allowSchedule || pathError != '' || emptyString(script_path)}
|
||||
href={`${base}/runs/${script_path}?show_schedules=true&show_future_jobs=true`}
|
||||
href={`${base}/runs/?schedule_path=${path}&job_trigger_kind=schedule&show_future_jobs=true`}
|
||||
>
|
||||
View runs
|
||||
</Button>
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
import { ArrowRight, TriangleAlert } from 'lucide-svelte'
|
||||
import type { ConfirmationModalHandle } from '../common/confirmationModal/asyncConfirmationModal.svelte'
|
||||
import type { Snippet } from 'svelte'
|
||||
import Tooltip from '../meltComponents/Tooltip.svelte'
|
||||
import { workspaceStore } from '$lib/stores'
|
||||
|
||||
type Props = {
|
||||
value: string | undefined
|
||||
@@ -37,6 +39,11 @@
|
||||
)
|
||||
)
|
||||
let open = $state(false)
|
||||
|
||||
function otherWorkspaces(dbname: string): string[] {
|
||||
const all = customInstanceDbs.current?.[dbname]?.used_by_workspaces ?? []
|
||||
return all.filter((w) => w !== $workspaceStore)
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="flex relative items-center {className}">
|
||||
@@ -52,11 +59,17 @@
|
||||
disabled={!$isCustomInstanceDbEnabled}
|
||||
>
|
||||
{#snippet endSnippet({ item })}
|
||||
{@render customInstanceDbWizardButton(item.value)}
|
||||
<div class="flex items-center gap-1">
|
||||
{@render sharedWorkspacesWarning(item.value)}
|
||||
{@render customInstanceDbWizardButton(item.value)}
|
||||
</div>
|
||||
{/snippet}
|
||||
</Select>
|
||||
{#if value}
|
||||
{@render customInstanceDbWizardButton(value, 'absolute right-1.5')}
|
||||
<div class="absolute right-1.5 flex items-center gap-1">
|
||||
{@render sharedWorkspacesWarning(value)}
|
||||
{@render customInstanceDbWizardButton(value)}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
@@ -74,12 +87,12 @@
|
||||
}
|
||||
/>
|
||||
|
||||
{#snippet customInstanceDbWizardButton(dbname: string, clazz: string = '')}
|
||||
{#snippet customInstanceDbWizardButton(dbname: string)}
|
||||
{@const status = customInstanceDbs.current?.[dbname]}
|
||||
<Button
|
||||
spacingSize="xs2"
|
||||
variant="default"
|
||||
wrapperClasses="bg-surface-input h-6 -my-2 {clazz}"
|
||||
wrapperClasses="bg-surface-input h-6 -my-2"
|
||||
onClick={() => ((openedDbNameWizard = dbname), (open = false))}
|
||||
>
|
||||
{#if !status}
|
||||
@@ -95,3 +108,21 @@
|
||||
{/if}
|
||||
</Button>
|
||||
{/snippet}
|
||||
|
||||
{#snippet sharedWorkspacesWarning(dbname: string)}
|
||||
{@const others = otherWorkspaces(dbname)}
|
||||
{#if others.length > 0}
|
||||
<Tooltip placement="top">
|
||||
<TriangleAlert
|
||||
class="text-orange-500 dark:text-orange-400"
|
||||
size={16}
|
||||
aria-label="Database is shared with other workspaces"
|
||||
/>
|
||||
{#snippet text()}
|
||||
This database is also used by workspace{others.length > 1 ? 's' : ''}
|
||||
<span class="font-semibold">{others.join(', ')}</span>. Any data written here will be shared
|
||||
with {others.length > 1 ? 'them' : 'it'}.
|
||||
{/snippet}
|
||||
</Tooltip>
|
||||
{/if}
|
||||
{/snippet}
|
||||
|
||||
@@ -462,7 +462,7 @@
|
||||
{/key}
|
||||
<div class="flex gap-2 items-center justify-end">
|
||||
<Button
|
||||
href={`${base}/runs/?schedule_path=${path}&show_schedules=true&show_future_jobs=true`}
|
||||
href={`${base}/runs/?schedule_path=${path}&job_trigger_kind=schedule&show_future_jobs=true`}
|
||||
unifiedSize="md"
|
||||
startIcon={{ icon: List }}
|
||||
variant="subtle"
|
||||
@@ -532,11 +532,7 @@
|
||||
{
|
||||
displayName: 'View runs',
|
||||
icon: List,
|
||||
href:
|
||||
base +
|
||||
'/runs/?schedule_path=' +
|
||||
path +
|
||||
'&show_schedules=true&show_future_jobs=true'
|
||||
href: `${base}/runs/?schedule_path=${path}&job_trigger_kind=schedule&show_future_jobs=true`
|
||||
},
|
||||
{
|
||||
displayName: 'Audit logs',
|
||||
|
||||
Reference in New Issue
Block a user