starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-13 00:03:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/improve-data-pipelines' into change-6607b937

Browse Source 
# Conflicts:
#	frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte
This commit is contained in:
Diego Imbert
2026-10-01 13:55:16 +02:00
parent 28cb6704f3 a0808b5b3e
commit 494e1d6d00
174 changed files with 11172 additions and 2277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ai-evals-scheduled.yml
+171
View File
@@ -0,0 +1,171 @@
name: AI Evals (scheduled)
# Full ai_evals suites on current models, posted to the AI evals dashboard in
# the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked
# over time. Weekly, since one pass of every suite at --runs 3 costs about 50M
# tokens per model: every suite runs on MODELS, and global (the mode users get)
# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Run it by
# hand to measure a branch against main.
on:
schedule:
- cron: "0 3 * * 1"
workflow_dispatch:
inputs:
modes:
description: "Space-separated modes"
default: "global flow app script cli"
models:
description: "Space-separated model aliases (bun run cli -- models)"
default: "sonnet-5.5"
global_extra_models:
description: "Extra model aliases for global mode only"
default: "gpt-6-astra gemini-3.8-flash"
runs:
description: "Runs per case"
default: "3"
reasoning:
description: "Reasoning effort for frontend modes (empty: the product default)"
default: ""
concurrency:
group: ai-evals-scheduled-${{ github.ref }}
env:
MODELS: ${{ inputs.models || 'sonnet-5.5' }}
GLOBAL_EXTRA_MODELS: ${{ inputs.global_extra_models || 'gpt-6-astra gemini-3.8-flash' }}
RUNS: ${{ inputs.runs || '3' }}
REASONING: ${{ inputs.reasoning }}
INGEST_URL: https://app.windmill.dev/api/r/f/ai/ingest_ai_eval_run
jobs:
setup:
runs-on: ubuntu-latest
outputs:
modes: ${{ steps.modes.outputs.modes }}
steps:
- id: modes
env:
MODES: ${{ inputs.modes || 'global flow app script cli' }}
run: |
echo "modes=$(jq -cn --arg m "$MODES" '$m | split(" ")
| map(select(IN("global", "flow", "app", "script", "cli")))')" >> "$GITHUB_OUTPUT"
evals:
needs: setup
runs-on: ubicloud-standard-16
timeout-minutes: 330
strategy:
fail-fast: false
matrix:
mode: ${{ fromJSON(needs.setup.outputs.modes) }}
services:
postgres:
image: postgres:16
ports:
- 5432:5432
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.97.0
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.4.0
- uses: actions/setup-node@v7
with:
node-version: "24"
- name: Build Windmill
working-directory: ./backend
env:
SQLX_OFFLINE: true
CARGO_BUILD_JOBS: 12
RUSTFLAGS: ""
run: cargo build --features quickjs
- name: Start Windmill
working-directory: ./backend
env:
DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill
RUST_LOG: info
run: |
mkdir -p ../ai_evals/logs
./target/debug/windmill > ../ai_evals/logs/windmill.log 2>&1 &
for i in $(seq 1 60); do
curl -sf http://localhost:8000/api/version > /dev/null 2>&1 && break
sleep 2
done
curl -sf http://localhost:8000/api/version > /dev/null || { tail -50 ../ai_evals/logs/windmill.log; exit 1; }
- name: Install frontend deps + generate client
working-directory: ./frontend
run: |
npm ci
npm run generate-backend-client
- name: Install CLI deps + generate CLI client
working-directory: ./cli
run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh
- name: Run ${{ matrix.mode }} evals
working-directory: ./ai_evals
env:
MODE: ${{ matrix.mode }}
WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000
WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
run: |
bun install
mkdir -p results
models="$MODELS"
[ "$MODE" = global ] && models="$models $GLOBAL_EXTRA_MODELS"
reasoning=()
[ -n "$REASONING" ] && [ "$MODE" != cli ] && reasoning=(--reasoning "$REASONING")
for m in $models; do
bun run cli -- run "$MODE" --model "$m" --runs "$RUNS" "${reasoning[@]}" \
--output "$PWD/results/$MODE-$m.json" || echo "::warning::$MODE on $m errored"
done
- name: Post results to the dashboard
if: always()
working-directory: ./ai_evals
env:
MODE: ${{ matrix.mode }}
INGEST_TOKEN: ${{ secrets.AI_EVALS_INGEST_TOKEN }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
[ -n "$INGEST_TOKEN" ] || { echo "::warning::AI_EVALS_INGEST_TOKEN is not set"; exit 0; }
shopt -s nullglob
for f in results/"$MODE"-*.json; do
# Keep what the dashboard reads; the full traces stay in the run's artifacts.
jq -c --arg ref "$GITHUB_REF" --arg trigger "$GITHUB_EVENT_NAME" --arg url "$RUN_URL" '
{result_json: (del(.cases[].prompt, .cases[].initialPath, .cases[].expectedPath)
| .cases[].attempts[] |= {attempt, passed, durationMs, toolCallCount, judgeScore,
judgeSummary, error, tokenUsage, checks: [.checks[]? | {name, passed}]}),
git_ref: $ref, trigger: $trigger, run_url: $url}' "$f" \
| curl -sf --retry 3 -X POST "$INGEST_URL" \
-H "Authorization: Bearer $INGEST_TOKEN" -H 'content-type: application/json' \
--data-binary @- && echo " <- $f" || echo "::warning::failed to post $f"
done
- name: Archive logs and results
uses: actions/upload-artifact@v4
if: always()
with:
name: ai-evals-${{ matrix.mode }}
path: |
ai_evals/logs
ai_evals/results
AGENTS.md
+2
View File
@@ -37,6 +37,8 @@ Open-source platform for internal tools, workflows, API integrations, background
- **Operator write rights**: `docs/operator-write-rights.md` — which `operator_settings` flags are
enforced rather than cosmetic, and why a right that is granted-unless-withdrawn needs `Option`
fields and a jsonb merge rather than a serde default
- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace setting that lets
operators compose flows, what the composition check must cover, and why it costs a full seat
- **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation
scope, which token labels email their owner at expiry, how OAuth login matches `login_type`, and
that every superadmin route refuses `$WM_TOKEN`. Read before designing anything that creates
ai_evals/README.md
+3 -14
View File
@@ -77,31 +77,20 @@ Public CLI surface:
- `--verbose`: stream assistant output for frontend runs
- `--skip-judge`: skip LLM judge scoring for the run
- `--execution-only`: only require the model/proxy/frontend loop to complete; skip validators, tool expectations, backend artifact validation, and judge scoring
- `--reasoning <effort>`: reasoning effort for frontend modes (`off`, `low`, `medium`, `high`, `max`, …); without it the product's default applies (`high` on models that can reason). The effort is appended to the recorded model label (`anthropic:claude-sonnet-5-5@max`)
- `--record`: append a compact tracked summary line to `ai_evals/history/<mode>.jsonl` for full-suite runs only
- `--backend-validation <mode>`: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals
## Models
Use `bun run cli -- models` to see the current aliases.
Today:
- `haiku`
- `sonnet`
- `opus`
- `4o`
- `gpt-5.5`
- `gemini-3-flash-preview`
- `gemini-3.1-pro-preview`
- `deepseek-v4-flash`
- `deepseek-v4-pro`
Use `bun run cli -- models` to see the current aliases; `core/models.ts` is the list.
Notes:
- the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5`
- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases
- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
- the judge model is separate and currently defaults to `claude-sonnet-4-6`; use `--skip-judge` for deterministic-only runs
- the judge model is separate and currently defaults to `claude-sonnet-5-5`; use `--skip-judge` for deterministic-only runs
## Case Format
ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts
+6 -4
View File
@@ -15,6 +15,7 @@ import { runEval } from "../shared";
import type { ModeRunContext } from "../../../../core/types";
import type { TokenUsage, ToolCallDetail } from "../shared/types";
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
import { evalReasoningEffort } from "../shared/providerConfig";
export interface ScriptEvalResult {
success: boolean;
@@ -41,14 +42,15 @@ export interface ScriptEvalOptions {
function resolveModelProvider(
model: string,
provider?: AIProvider,
): AIProviderModel {
): AIProviderModel & { reasoning?: string } {
const reasoning = evalReasoningEffort();
if (provider) {
return { provider, model };
return { provider, model, reasoning };
}
if (model.startsWith("claude")) {
return { provider: "anthropic", model };
return { provider: "anthropic", model, reasoning };
}
return { provider: "openai", model };
return { provider: "openai", model, reasoning };
}
export async function runScriptEval(
ai_evals/adapters/frontend/core/shared/providerConfig.ts
+13
View File
@@ -12,6 +12,12 @@ export interface EvalClients {
export interface ResolvedEvalModelProvider {
provider: FrontendEvalProvider;
model: string;
reasoning?: string;
}
/** `run --reasoning` hands the effort to the frontend runtime through the environment. */
export function evalReasoningEffort(): string | undefined {
return process.env.WMILL_AI_EVAL_REASONING || undefined;
}
export interface WindmillAiProxyClientConfig {
@@ -73,6 +79,13 @@ export function createEvalClients(input: {
export function resolveEvalModelProvider(
model: string,
provider?: FrontendEvalProvider,
): ResolvedEvalModelProvider {
return { ...resolveProvider(model, provider), reasoning: evalReasoningEffort() };
}
function resolveProvider(
model: string,
provider?: FrontendEvalProvider,
): ResolvedEvalModelProvider {
if (provider) {
return { provider, model };
ai_evals/adapters/frontend/mockBackend.ts
+332 -24
View File
@@ -1,4 +1,7 @@
import { randomUUID } from 'node:crypto'
import confluenceFixture from '../../fixtures/frontend/global/hub/confluence.json'
import servicenowFixture from '../../fixtures/frontend/global/hub/servicenow.json'
import outreachFixture from '../../fixtures/frontend/global/hub/outreach.json'
import type {
AppWithLastVersion,
CompletedJob,
@@ -250,9 +253,7 @@ export function listBenchmarkApps(workspace: string): ListableApp[] | null {
}
export function getBenchmarkAppByPath(workspace: string, path: string): AppWithLastVersion | null {
const app = benchmarkWorkspaceRunnables
.get(workspace)
?.apps?.find((entry) => entry.path === path)
const app = benchmarkWorkspaceRunnables.get(workspace)?.apps?.find((entry) => entry.path === path)
return app ? buildBenchmarkApp(app) : null
}
@@ -448,7 +449,10 @@ export function createBenchmarkCompletedJob(input: {
labels: input.label ? [input.label] : undefined
}
benchmarkJobs.set(benchmarkJobKey(input.workspace, jobId), { workspace: input.workspace, job })
benchmarkJobs.set(benchmarkJobKey(input.workspace, jobId), {
workspace: input.workspace,
job
})
return jobId
}
@@ -532,7 +536,13 @@ export function getBenchmarkFlowAllResults(workspace: string, jobId: string) {
*/
const benchmarkDrafts = new Map<
string,
{ workspace: string; kind: UserDraftItemKind; path: string; value: unknown; createdAt: string }
{
workspace: string
kind: UserDraftItemKind
path: string
value: unknown
createdAt: string
}
>()
// Counter-based timestamps: deterministic run-to-run (same event order → same
@@ -614,7 +624,9 @@ export function getBenchmarkDraftForUser(input: {
}): GetDraftForUserResponse {
const entry = benchmarkDrafts.get(benchmarkDraftKey(input.workspace, input.kind, input.path))
if (!entry) {
throw Object.assign(new Error(`no draft for "${input.path}"`), { status: 404 })
throw Object.assign(new Error(`no draft for "${input.path}"`), {
status: 404
})
}
return { value: entry.value, created_at: entry.createdAt }
}
@@ -635,7 +647,11 @@ export function getBenchmarkOwnDraft(input: {
/** Whether a deployed benchmark item exists for a draft row's kind+path —
* drives `draft_only`, which production computes against the deployed tables. */
function benchmarkDeployedExists(workspace: string, kind: UserDraftItemKind, path: string): boolean {
function benchmarkDeployedExists(
workspace: string,
kind: UserDraftItemKind,
path: string
): boolean {
if (kind === 'script') return Boolean(getBenchmarkScriptByPath(workspace, path))
if (kind === 'flow') return Boolean(getBenchmarkFlowByPath(workspace, path))
if (kind === 'app' || kind === 'raw_app') return Boolean(getBenchmarkAppByPath(workspace, path))
@@ -762,10 +778,11 @@ export function runBenchmarkDatatableSql(input: {
* shape `pollJobResult` consumes. The job is created synchronously before
* polling, so it is always present and completed.
*/
export function getBenchmarkCompletedJobResultMaybe(input: {
workspace: string
id: string
}): { success: boolean; completed: boolean; result: unknown } {
export function getBenchmarkCompletedJobResultMaybe(input: { workspace: string; id: string }): {
success: boolean
completed: boolean
result: unknown
} {
const job = getBenchmarkCompletedJob(input.workspace, input.id)
if (!job) {
throw new Error(`Job "${input.id}" not found in benchmark workspace`)
@@ -896,7 +913,9 @@ export function previewBenchmarkSchedule(input: {
}): Record<string, unknown> {
const schedule = input.requestBody?.schedule
if (typeof schedule !== 'string' || schedule.trim().split(/\s+/).length !== 6) {
throw new Error(`schedule must use a six-field cron expression, got ${JSON.stringify(schedule)}`)
throw new Error(
`schedule must use a six-field cron expression, got ${JSON.stringify(schedule)}`
)
}
return {
@@ -928,7 +947,9 @@ export function createBenchmarkHttpTrigger(input: {
typeof input.requestBody.route_path === 'string' &&
input.requestBody.route_path.startsWith('/')
) {
throw new Error(`HTTP trigger route_path must not start with /, got "${input.requestBody.route_path}"`)
throw new Error(
`HTTP trigger route_path must not start with /, got "${input.requestBody.route_path}"`
)
}
return {
path: input.requestBody.path,
@@ -1024,7 +1045,39 @@ function buildBenchmarkApp(app: BenchmarkWorkspaceApp): AppWithLastVersion {
* hub tools throw and no case can exercise hub reuse. Serving fixtures rather
* than the live hub also keeps assertions on script content stable as the real
* hub republishes new versions. */
const BENCHMARK_INTEGRATION_META_PATH = /^\/api\/integrations\/hub\/([^/]+)\/meta$/
/** One integration lifted verbatim from the content repo — its shipped scripts, its
* authored meta.json and its resource type. Hand-written fixtures make both routes to
* an integration's conventions look equally cheap; a real one is the only way to tell
* whether reading the metadata beats reading a script. */
const REAL_HUB_INTEGRATIONS = [
confluenceFixture,
servicenowFixture,
outreachFixture
] as unknown as Array<{
app: string
display_name: string
description: string
docs_url: string
curated: boolean | null
meta: unknown
resource_type: { name: string; description: string; schema: unknown }
scripts: Array<{
version_id: number
app: string
summary: string
description: string
terms: string
kind: string
language: string
content: string
schema: unknown
}>
}>
const BENCHMARK_HUB_SCRIPTS = [
...REAL_HUB_INTEGRATIONS.flatMap((integration) => integration.scripts),
{
version_id: 22235,
app: 'holded',
@@ -1112,6 +1165,87 @@ export async function main(discord_webhook: DiscordWebhook, message: string) {
message: { type: 'string' }
}
}
},
// Baremetrics carries no annotation script, on purpose: it stands for an
// integration whose conventions (an `apiKey` resource, bearer auth, the /v1
// base) are only learnable by reading a script that does something else.
{
version_id: 8995,
app: 'baremetrics',
summary: 'List Sources',
description: 'List all the sources attached to a Baremetrics account.',
terms: 'baremetrics sources list revenue metrics',
language: 'bunnative',
content: `//native
type Baremetrics = {
apiKey: string;
};
export async function main(resource: Baremetrics) {
const response = await fetch("https://api.baremetrics.com/v1/sources", {
method: "GET",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
Authorization: \`Bearer \${resource.apiKey}\`,
},
});
if (!response.ok) {
throw new Error(\`\${response.status} \${await response.text()}\`);
}
return await response.json();
}
`,
schema: {
type: 'object',
required: ['resource'],
properties: {
resource: { type: 'object', format: 'resource-baremetrics' }
}
}
},
{
version_id: 8996,
app: 'baremetrics',
summary: 'Create Customer',
description: 'Create a customer on a Baremetrics source.',
terms: 'baremetrics customer create source',
language: 'bunnative',
content: `//native
type Baremetrics = {
apiKey: string;
};
export async function main(
resource: Baremetrics,
sourceId: string,
body: { oid: string; name?: string; email?: string },
) {
const response = await fetch(
\`https://api.baremetrics.com/v1/\${sourceId}/customers\`,
{
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
Authorization: \`Bearer \${resource.apiKey}\`,
},
body: JSON.stringify(body),
},
);
if (!response.ok) {
throw new Error(\`\${response.status} \${await response.text()}\`);
}
return await response.json();
}
`,
schema: {
type: 'object',
required: ['resource', 'sourceId', 'body'],
properties: {
resource: { type: 'object', format: 'resource-baremetrics' },
sourceId: { type: 'string' },
body: { type: 'object' }
}
}
}
]
@@ -1121,20 +1255,26 @@ export async function main(discord_webhook: DiscordWebhook, message: string) {
* integration, or overlapping on three meaningful words. A looser bar answers
* "send a Slack message" with the Discord fixture, handing an unrelated case a
* plausible-looking wrong integration. */
function searchBenchmarkHubScripts(text: string) {
function searchBenchmarkHubScripts(text: string, app: string | null, kind: string | null) {
const tokens = new Set(
text
.toLowerCase()
.split(/[^a-z0-9]+/)
.filter((token) => token.length > 2)
)
return BENCHMARK_HUB_SCRIPTS.map((script) => {
const words = new Set(
`${script.app} ${script.summary} ${script.terms}`.toLowerCase().split(/[^a-z0-9]+/)
)
const score = [...tokens].filter((token) => words.has(token)).length
return { script, score, namesApp: tokens.has(script.app) }
})
// The real endpoint filters by app and kind before ranking, so honour both here —
// otherwise a narrowed search silently returns other integrations' scripts, and a
// trigger answers a search production would never have shown it to.
return BENCHMARK_HUB_SCRIPTS.filter(
(script) => (!app || script.app === app) && (!kind || (script.kind ?? 'script') === kind)
)
.map((script) => {
const words = new Set(
`${script.app} ${script.summary} ${script.terms}`.toLowerCase().split(/[^a-z0-9]+/)
)
const score = [...tokens].filter((token) => words.has(token)).length
return { script, score, namesApp: tokens.has(script.app) }
})
.filter((entry) => entry.namesApp || entry.score >= 3)
.sort((a, b) => b.score - a.score)
.map(({ script }, index) => ({
@@ -1142,12 +1282,104 @@ function searchBenchmarkHubScripts(text: string) {
id: script.version_id,
version_id: script.version_id,
summary: script.summary,
description: script.description ?? null,
app: script.app,
kind: 'script',
kind: script.kind ?? 'script',
score: 1 - index * 0.01
}))
}
/** Listing an integration is unranked and description-bearing, matching the hub's
* top-scripts endpoint — that asymmetry with the semantic search is the whole
* reason the chat browses by app when no script matches the task. */
function listBenchmarkHubScriptsByApp(app: string | null, kind: string | null) {
return BENCHMARK_HUB_SCRIPTS.filter(
(script) => (!app || script.app === app) && (!kind || (script.kind ?? 'script') === kind)
).map((script) => ({
id: script.version_id,
ask_id: script.version_id,
version_id: script.version_id,
summary: script.summary,
description: script.description ?? null,
app: script.app,
kind: script.kind ?? 'script',
views: 0,
votes: 0
}))
}
/** What `/integrations/hub/<app>/meta` serves. Carries the integration's conventions
* and none of the endpoints a case asks the model to write, so the tool shortens the
* path to that knowledge without supplying answers. */
/** The integrations a case writes against, none of which the content repo documents —
* the same majority shape as the live hub, where 18 of ~216 carry a meta.json. Their
* conventions live in their shipped scripts, which is what these cases exercise. */
const BENCHMARK_HUB_INTEGRATION_META: Record<
string,
{
display_name: string
description: string
docs_url: string
curated: boolean | null
meta: unknown
}
> = {
baremetrics: {
display_name: 'Baremetrics',
description: 'Subscription analytics for recurring-revenue businesses.',
docs_url: 'https://developers.baremetrics.com/reference',
curated: null,
meta: null
},
holded: {
display_name: 'Holded',
description: 'Invoicing, accounting and CRM for small businesses.',
docs_url: 'https://developers.holded.com/reference',
curated: null,
meta: null
}
}
/** Mirrors the hub's own derivation: hosts seen in the shipped scripts, whether they
* call the provider directly, and how many there are of each kind. */
function benchmarkDerivedFacts(app: string) {
const scripts = BENCHMARK_HUB_SCRIPTS.filter((script) => script.app === app)
const hosts = new Map<string, number>()
const languages: Record<string, number> = {}
for (const script of scripts) {
languages[script.language] = (languages[script.language] ?? 0) + 1
for (const match of script.content.matchAll(/https?:\/\/([a-zA-Z0-9._-]+)/g)) {
hosts.set(match[1], (hosts.get(match[1]) ?? 0) + 1)
}
}
return {
api_hosts: [...hosts.entries()]
.sort((a, b) => b[1] - a[1])
.map(([host, count]) => ({ host, count })),
style: 'fetch',
languages,
script_counts: {
total: scripts.length,
by_kind: scripts.reduce<Record<string, number>>((acc, script) => {
const kind = script.kind ?? 'script'
acc[kind] = (acc[kind] ?? 0) + 1
return acc
}, {})
},
top_scripts: scripts.map((script) => ({
path: `hub/${script.version_id}/${script.app}/${script.summary.toLowerCase().replaceAll(/\s+/g, '_')}`,
ask_id: script.version_id,
version_id: script.version_id,
summary: script.summary,
description: script.description ?? null,
kind: script.kind ?? 'script',
language: script.language,
views: 0,
votes: 0
}))
}
}
/** The hub keys a script by its version id; the app and slug segments that
* follow are descriptive, so match on the id exactly as the real hub does. */
function getBenchmarkHubScript(path: string) {
@@ -1190,6 +1422,9 @@ export function hasBenchmarkApiHandler(url: string): boolean {
return (
path === '/api/workers/list' ||
path === '/api/embeddings/query_hub_scripts' ||
path === '/api/scripts/hub/top' ||
path === '/api/integrations/hub/list' ||
BENCHMARK_INTEGRATION_META_PATH.test(path) ||
path.startsWith('/api/scripts/hub/get_full/') ||
BENCHMARK_AI_MODELS_PATH.test(path)
)
@@ -1213,8 +1448,81 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon
return Response.json({ data: (seed?.models ?? []).map((id) => ({ id })) })
}
if (path === '/api/embeddings/query_hub_scripts') {
const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? ''
return Response.json(searchBenchmarkHubScripts(text))
const params = new URLSearchParams(url.split('?')[1] ?? '')
return Response.json(
searchBenchmarkHubScripts(params.get('text') ?? '', params.get('app'), params.get('kind'))
)
}
if (path === '/api/scripts/hub/top') {
const params = new URLSearchParams(url.split('?')[1] ?? '')
return Response.json({
asks: listBenchmarkHubScriptsByApp(params.get('app'), params.get('kind'))
})
}
const integrationMeta = BENCHMARK_INTEGRATION_META_PATH.exec(path)
if (integrationMeta) {
const app = decodeURIComponent(integrationMeta[1])
const real = REAL_HUB_INTEGRATIONS.find((integration) => integration.app === app)
if (real) {
return Response.json({
app,
display_name: real.display_name,
description: real.description,
docs_url: real.docs_url,
curated: real.curated,
metadata_source: real.meta ? 'curated' : 'derived',
meta: real.meta,
meta_updated_at: null,
derived: benchmarkDerivedFacts(app),
resource_types: [{ id: 1, ...real.resource_type }]
})
}
const entry = BENCHMARK_HUB_INTEGRATION_META[app]
if (!entry) {
return Response.json({ error: 'integration not found' }, { status: 404 })
}
return Response.json({
app,
display_name: entry.display_name,
description: entry.description,
docs_url: entry.docs_url,
curated: entry.curated,
metadata_source: entry.meta ? 'curated' : 'derived',
meta: entry.meta,
meta_updated_at: null,
derived: benchmarkDerivedFacts(app),
resource_types: [
{
id: 1,
name: app,
description: `${entry.display_name} credentials`,
schema: {
type: 'object',
required: ['apiKey'],
properties: {
apiKey: {
type: 'string',
description: `${entry.display_name} API key`
}
}
}
}
]
})
}
if (path === '/api/integrations/hub/list') {
const apps = [...new Set(BENCHMARK_HUB_SCRIPTS.map((script) => script.app))].sort()
// Read off both fixtures the metadata endpoint serves from, or the list would
// call an integration undocumented and then hand back its authored notes.
const documented = new Set([
...REAL_HUB_INTEGRATIONS.filter((integration) => integration.meta).map(
(integration) => integration.app
),
...Object.entries(BENCHMARK_HUB_INTEGRATION_META)
.filter(([, entry]) => entry.meta)
.map(([app]) => app)
])
return Response.json(apps.map((name) => ({ name, documented: documented.has(name) })))
}
if (path.startsWith('/api/scripts/hub/get_full/')) {
const script = getBenchmarkHubScript(path)
ai_evals/adapters/frontend/mockBackendApi.test.ts
+18
View File
@@ -16,3 +16,21 @@ describe('benchmark API fetch handlers', () => {
expect(body[0]).toMatchObject({ worker: expect.any(String), worker_group: expect.any(String) })
})
})
// A benchmark whose list calls an integration undocumented while its metadata endpoint
// hands back authored notes teaches the model the flag means nothing.
describe('benchmark hub integration list', () => {
it('flags exactly the integrations whose metadata carries authored notes', async () => {
const listed = (await handleBenchmarkApiFetch('/api/integrations/hub/list').json()) as Array<{
name: string
documented: boolean
}>
expect(listed.length).toBeGreaterThan(0)
for (const { name, documented } of listed) {
const res = handleBenchmarkApiFetch(`/api/integrations/hub/${name}/meta`)
const authored = res.status === 200 && !!((await res.json()) as { meta?: unknown }).meta
expect(authored).toBe(documented)
}
})
})
ai_evals/bun.lock
+29 -91
View File
@@ -5,8 +5,8 @@
"": {
"name": "windmill-ai-evals",
"dependencies": {
"@anthropic-ai/claude-agent-sdk": "^0.2.25",
"@anthropic-ai/sdk": "^0.39.0",
"@anthropic-ai/claude-agent-sdk": "^0.3.284",
"@anthropic-ai/sdk": "^0.129.0",
"commander": "^14.0.3",
"openai": "^6.9.1",
"yaml": "^2.8.3",
@@ -18,66 +18,44 @@
},
},
"packages": {
"@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.2.87", "", { "dependencies": { "@anthropic-ai/sdk": "^0.74.0", "@modelcontextprotocol/sdk": "^1.27.1" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.34.2", "@img/sharp-darwin-x64": "^0.34.2", "@img/sharp-linux-arm": "^0.34.2", "@img/sharp-linux-arm64": "^0.34.2", "@img/sharp-linux-x64": "^0.34.2", "@img/sharp-linuxmusl-arm64": "^0.34.2", "@img/sharp-linuxmusl-x64": "^0.34.2", "@img/sharp-win32-arm64": "^0.34.2", "@img/sharp-win32-x64": "^0.34.2" }, "peerDependencies": { "zod": "^4.0.0" } }, "sha512-WWmgBPxPhBOvNT0ujI8vPTI2lK+w5YEkEZ/y1mH0EDkK/0kBnxVJNhCtG5vnueiAViwLoUOFn66pbkDiivijdA=="],
"@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.3.284", "", { "optionalDependencies": { "@anthropic-ai/claude-agent-sdk-darwin-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-darwin-x64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-x64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-x64-musl": "0.3.284", "@anthropic-ai/claude-agent-sdk-win32-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-win32-x64": "0.3.284" }, "peerDependencies": { "@anthropic-ai/sdk": ">=0.93.0", "@modelcontextprotocol/sdk": "^1.29.0", "zod": "^4.0.0" } }, "sha512-NSoJwEq6nFSf8dtaacYx37QdGgqApI3eHFUlxclMLYi8irb6ZJwEaUnPnLUCyAyg9W/tMkgZC0GXWLRCc01I0w=="],
"@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.39.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg=="],
"@anthropic-ai/claude-agent-sdk-darwin-arm64": ["@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.284", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gKY9MUjY83398uCiPLHsd87kyzu7agIM7ApqWpJkpSINep6hxx4rNoR8bUbNWg49Aoe/PW/DJkQByAQuNzR9rg=="],
"@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="],
"@anthropic-ai/claude-agent-sdk-darwin-x64": ["@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.284", "", { "os": "darwin", "cpu": "x64" }, "sha512-P+q6Z7sKeYz99uE7RJc4au1IK+4SiWe69pyHY3QXMnjZq8hSFU/5Mo8iSurLk9jRlihsMltV1rLA5naouYyCAQ=="],
"@anthropic-ai/claude-agent-sdk-linux-arm64": ["@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.284", "", { "os": "linux", "cpu": "arm64" }, "sha512-LDpuYDaz+pCdG29iy1pw4P1D2YVtpZOb4rWOPxxSpg2fFyUeQ5WrfnuszOxhDWpNh1k1ekDiKm3A9OzGcrkdtA=="],
"@anthropic-ai/claude-agent-sdk-linux-arm64-musl": ["@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.284", "", { "os": "linux", "cpu": "arm64" }, "sha512-U3/uAv1TS8sMWsgWQSxYbdND5AaRXXpP5RxHU1oYC5ZizlZE8/xAVum9dj1QX3rVZJg8EzsqZveq/AkEv5ABfQ=="],
"@anthropic-ai/claude-agent-sdk-linux-x64": ["@anthropic-ai/claude-agent-sdk-linux-x64@0.3.284", "", { "os": "linux", "cpu": "x64" }, "sha512-yGytBCCwJvWeg1FzFpgnLM9BOM2vKVExtv6pMJHMtF82J5yhKHcxodRaYlQVd/HzJJ2nruKb9EnrhRBZ//4yRw=="],
"@anthropic-ai/claude-agent-sdk-linux-x64-musl": ["@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.284", "", { "os": "linux", "cpu": "x64" }, "sha512-4x0Q8CFkCNbeENE7pRhuUmkwh/sQfTq1yWi4ZE56stve9pgoKzu6ZDa1SEtvK1YIooGE/qM5rVS8xFLE1gbQ0w=="],
"@anthropic-ai/claude-agent-sdk-win32-arm64": ["@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.284", "", { "os": "win32", "cpu": "arm64" }, "sha512-zOXkdPHhElyxyFJ6eY5R+YGUswqBDZ+ZHNaRmQGJ28RkQYgwG5imNy/nfFR+/CPGlMnH9ZzbGc2rKYApm1K4Tg=="],
"@anthropic-ai/claude-agent-sdk-win32-x64": ["@anthropic-ai/claude-agent-sdk-win32-x64@0.3.284", "", { "os": "win32", "cpu": "x64" }, "sha512-VGaFRDCOPloj5IjvJTyy5JsSl9sE2HR3W+A6f/cjgh23/7Y5vK7/ka+1JQD+AkoGO9tbHqP2O6god3IBrLvOaw=="],
"@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.129.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1", "standardwebhooks": "^1.0.0" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-MH7LB20kNpGLUpPTe2OG5XvL/KhX8HUO9XyBaFjgFk6TLS4Xjt0VPIuMKywe7POKKPslX+QxSlVok9e+8kG5Nw=="],
"@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="],
"@hono/node-server": ["@hono/node-server@1.19.12", "", { "peerDependencies": { "hono": "^4" } }, "sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw=="],
"@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="],
"@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="],
"@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="],
"@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="],
"@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="],
"@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="],
"@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="],
"@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="],
"@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="],
"@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="],
"@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="],
"@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="],
"@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="],
"@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="],
"@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="],
"@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="],
"@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="],
"@stablelib/base64": ["@stablelib/base64@1.0.1", "", {}, "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ=="],
"@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="],
"@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
"@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="],
"abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="],
"@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
"accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
"agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="],
"ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="],
"ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
"body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="],
"bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
@@ -88,8 +66,6 @@
"call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="],
"combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="],
"commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="],
"content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="],
@@ -106,8 +82,6 @@
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
"delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],
"depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
@@ -122,14 +96,10 @@
"es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
"es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="],
"escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
"etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="],
"event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="],
"eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="],
"eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
@@ -140,16 +110,12 @@
"fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
"fast-sha256": ["fast-sha256@1.3.0", "", {}, "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ=="],
"fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="],
"finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="],
"form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="],
"form-data-encoder": ["form-data-encoder@1.7.2", "", {}, "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="],
"formdata-node": ["formdata-node@4.4.1", "", { "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" } }, "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ=="],
"forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="],
"fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="],
@@ -164,16 +130,12 @@
"has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="],
"has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="],
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
"hono": ["hono@4.12.9", "", {}, "sha512-wy3T8Zm2bsEvxKZM5w21VdHDDcwVS1yUFFY6i8UobSsKfFceT7TOwhbhfKsDyx7tYQlmRM5FLpIuYvNFyjctiA=="],
"http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="],
"humanize-ms": ["humanize-ms@1.2.1", "", { "dependencies": { "ms": "^2.0.0" } }, "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ=="],
"iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
@@ -208,10 +170,6 @@
"negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="],
"node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
"node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="],
"object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="],
"object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="],
@@ -262,30 +220,24 @@
"side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="],
"standardwebhooks": ["standardwebhooks@1.1.1", "", { "dependencies": { "@stablelib/base64": "^1.0.0", "fast-sha256": "^1.3.0" } }, "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ=="],
"statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="],
"toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
"tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
"ts-algebra": ["ts-algebra@2.0.0", "", {}, "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw=="],
"type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
"undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
"unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="],
"vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="],
"web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="],
"webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
"whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
"which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
"wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
@@ -295,19 +247,5 @@
"zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
"zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="],
"@anthropic-ai/claude-agent-sdk/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.74.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-srbJV7JKsc5cQ6eVuFzjZO7UR3xEPJqPamHFIe29bs38Ij2IripoAhC0S5NslNbaFUYqBKypmmpzMTpqfHEUDw=="],
"@types/node-fetch/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
"bun-types/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
"form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
"@types/node-fetch/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
"bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
"form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="],
}
}
ai_evals/cases/global.yaml
+171
View File
@@ -2312,6 +2312,105 @@
- email
skipJudge: true
# `List Pages` takes a cursor without ever following one, so where the next cursor
# comes from (`_links.next`) is stated in the authored metadata and in no script body.
# Paginating is therefore the one task where reading scripts is not an equal route.
- id: global-hub4-confluence-paginate-all-pages
prompt: |-
I need a script that returns every page in one of my Confluence spaces, not just the
first batch — it should keep fetching until there are no more.
Save it as a draft at `f/evals/global/confluence_all_pages` and leave it as an AI
draft; do not deploy it.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
runtime:
maxTurns: 14
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
path: f/evals/global/confluence_all_pages
valueIncludes:
- /wiki/
- Basic
toolExpect:
requiredToolsUsed:
- write_script
# Either route to the integration's conventions is fine; the checklist is where the
# difference between them has to show up.
requiredToolsAnyOf:
- - search_hub_scripts
- get_hub_integration
forbiddenToolsUsed:
- deploy_workspace_item
judgeChecklist:
- the script loops until the API stops returning a next link, rather than making a single request
- the next cursor is read from the response's `_links.next` or the Link header, not from an invented field such as `next_cursor`, `nextPageToken` or `offset`
- it authenticates with HTTP Basic built from the Confluence resource's email and API token rather than a bearer token
- the script takes a Confluence resource as an input parameter rather than loose credential strings
- the result stays an AI draft and is not deployed
# Unlike the Confluence case, both facts this needs — the {"result"} envelope and
# PATCH-not-PUT — are visible in ServiceNow's own shipped scripts. That makes it the
# control for whether the metadata earns its place when the scripts already answer.
- id: global-hub5-servicenow-assign-incident
prompt: |-
Write me a script that assigns an existing ServiceNow incident to a person, looking
that person up by their email address.
Save it as a draft at `f/evals/global/servicenow_assign_incident` and leave it as an
AI draft; do not deploy it.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
runtime:
maxTurns: 14
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
path: f/evals/global/servicenow_assign_incident
toolExpect:
requiredToolsUsed:
- write_script
requiredToolsAnyOf:
- - search_hub_scripts
- get_hub_integration
forbiddenToolsUsed:
- deploy_workspace_item
judgeChecklist:
- it looks the person up in the sys_user table by email rather than assuming a sys_id was supplied
- it reads records out of the `result` envelope the ServiceNow API wraps responses in
- it sets assigned_to to the user's sys_id rather than to the raw email address
- it updates the incident with a partial update (PATCH) rather than a full replace that would clear the fields it did not send
- the result stays an AI draft and is not deployed
# Outreach is JSON:API. Its content type and payload envelope are both stated in the
# metadata and visible in the shipped update scripts, so this is a second control.
- id: global-hub6-outreach-reschedule-task
prompt: |-
Write me a script that moves an existing Outreach task to a new due date.
Save it as a draft at `f/evals/global/outreach_reschedule_task` and leave it as an AI
draft; do not deploy it.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
runtime:
maxTurns: 14
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
path: f/evals/global/outreach_reschedule_task
toolExpect:
requiredToolsUsed:
- write_script
requiredToolsAnyOf:
- - search_hub_scripts
- get_hub_integration
forbiddenToolsUsed:
- deploy_workspace_item
judgeChecklist:
- the request sends the JSON:API content type application/vnd.api+json for its body
- the payload wraps the change as data.type and data.attributes rather than sending bare top-level fields
- it identifies the task by id inside the payload rather than only in the URL, and treats the id as a number
- it updates the existing task with PATCH rather than creating a new one
- the result stays an AI draft and is not deployed
# --- Windmill Hub reuse (search_hub_scripts + read_workspace_item on a hub/ path) ---
# Holded's API is obscure enough that a model writing from memory cannot reproduce
# its endpoint and `key` auth header — so the draft's fidelity to the published
@@ -2358,6 +2457,78 @@
- the document type, document id, and recipient emails are inputs to the script
- the result stays an AI draft and is not deployed
# Same task as global-hub1 with the hub never mentioned: this is what tells us the
# model reaches for the hub on its own rather than only when pointed at it.
- id: global-hub2-checks-hub-unprompted
prompt: |-
I want to email one of my Holded invoices to a customer from Windmill.
Save it as a draft script at `f/evals/global/holded_send_document` and leave it as
an AI draft; do not deploy it.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
runtime:
maxTurns: 12
validate:
draftCountExactly: 1
requiredDrafts:
# No language assertion: the Holded script's body is `//native`, so a faithful
# fork lands on bun or bunnative depending on how literally it is copied.
- type: script
path: f/evals/global/holded_send_document
valueIncludes:
- api.holded.com/api/invoicing/v1/documents
- mailTemplateId
toolExpect:
requiredToolsUsed:
- search_hub_scripts
- write_script
forbiddenToolsUsed:
- deploy_workspace_item
judgeChecklist:
- the draft sends an existing Holded document by email rather than creating one
- the request targets Holded's document send endpoint, not an invented URL
- authentication uses Holded's own key header rather than a bearer token
- the result stays an AI draft and is not deployed
# The fixture hub carries two Baremetrics scripts, neither for annotations, so nothing
# does what is asked. Its conventions — an `apiKey` resource, bearer auth, the
# api.baremetrics.com/v1 base — are only knowable by reading one of those near-misses.
- id: global-hub3-integration-as-example
prompt: |-
Write me a script that adds an annotation to my Baremetrics account.
Save it as a draft at `f/evals/global/baremetrics_create_annotation` and leave it as
an AI draft; do not deploy it.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
runtime:
maxTurns: 14
validate:
draftCountExactly: 1
requiredDrafts:
# No language assertion: matching the hub script's own dialect is the point,
# and Baremetrics' scripts are `//native`, so both bun and bunnative are right.
- type: script
path: f/evals/global/baremetrics_create_annotation
valueIncludes:
- api.baremetrics.com/v1
- apiKey
toolExpect:
requiredToolsUsed:
- search_hub_scripts
- write_script
# Reading a near-miss script and reading the integration's own metadata are two
# routes to the same conventions, so requiring either keeps the case about the
# outcome rather than about which route the model took.
requiredToolsAnyOf:
- - read_workspace_item
- get_hub_integration
forbiddenToolsUsed:
- deploy_workspace_item
judgeChecklist:
- the script takes a Baremetrics resource as an input parameter rather than a bare API key string
- the resource type it declares carries an apiKey field, matching how other Baremetrics hub scripts take credentials
- the request authenticates with a bearer token built from that resource
- the script posts an annotation rather than reading data back
- the result stays an AI draft and is not deployed
# The value of a secret variable is unreadable, so a metadata-only edit must leave
# it untouched: passing any `value` here means inventing one, which silently
# replaces the real secret at deploy.
ai_evals/cli/index.ts
+18
View File
@@ -108,6 +108,10 @@ async function main() {
"--record",
"append a compact summary line to ai_evals/history/<mode>.jsonl",
)
.option(
"--reasoning <effort>",
"reasoning effort for frontend modes (e.g. off, low, medium, high, max); default: the product's",
)
.option(
"--backend-validation <mode>",
`backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`,
@@ -126,6 +130,7 @@ async function main() {
executionOnly?: boolean;
record?: boolean;
backendValidation?: string;
reasoning?: string;
},
) => {
await handleRun({
@@ -140,6 +145,7 @@ async function main() {
executionOnly: options.executionOnly ?? false,
record: options.record ?? false,
backendValidation: options.backendValidation,
reasoning: options.reasoning,
});
},
);
@@ -190,6 +196,7 @@ async function handleRun(input: {
executionOnly: boolean;
record: boolean;
backendValidation?: string;
reasoning?: string;
}) {
if (input.record && input.caseIds.length > 0) {
throw new Error(
@@ -217,6 +224,13 @@ async function handleRun(input: {
"--backend-validation currently supports only flow and script modes",
);
}
if (input.reasoning) {
if (input.mode === "cli") {
throw new Error("--reasoning only applies to frontend modes");
}
// The frontend runtime runs in a child process, which inherits it.
process.env.WMILL_AI_EVAL_REASONING = input.reasoning;
}
if (input.mode !== "cli") {
await assertWindmillBackendReachable(resolveWindmillBackendSettings());
}
@@ -257,6 +271,10 @@ async function handleRun(input: {
backendValidation,
});
if (input.reasoning && result.runModel) {
result.runModel = `${result.runModel}@${input.reasoning}`;
}
const resolvedOutputPath =
models.length === 1
? resolveRunOutputPath(input.mode, input.outputPath)
ai_evals/core/judge.ts
+6 -8
View File
@@ -1,7 +1,7 @@
import Anthropic from "@anthropic-ai/sdk";
import type { EvalMode, JudgeResult } from "./types";
export const DEFAULT_JUDGE_MODEL = "claude-sonnet-4-6";
export const DEFAULT_JUDGE_MODEL = "claude-sonnet-5-5";
const JUDGE_TOOL_NAME = "submit_judgement";
@@ -29,6 +29,7 @@ export async function judgeOutput(input: {
const system = [
"You evaluate benchmark outputs for Windmill AI generation.",
`Always answer by calling the ${JUDGE_TOOL_NAME} tool.`,
"Deterministic checks already run separately. Focus on whether the final output satisfies the user request.",
"If expected state is provided, treat it as a valid example and reward semantically equivalent outputs.",
"If a checklist is provided, treat it as the explicit acceptance criteria for this case.",
@@ -69,8 +70,8 @@ export async function judgeOutput(input: {
try {
const response = await client.messages.create({
model,
max_tokens: 1024,
temperature: 0,
// The judge thinks by default, and thinking shares this budget with the verdict.
max_tokens: 16000,
system,
messages: [{ role: "user", content: user }],
tools: [
@@ -93,11 +94,8 @@ export async function judgeOutput(input: {
},
},
],
tool_choice: {
type: "tool",
name: JUDGE_TOOL_NAME,
disable_parallel_tool_use: true,
},
// Current models refuse a forced tool_choice ("tool"/"any").
tool_choice: { type: "auto", disable_parallel_tool_use: true },
});
const toolUseBlock = response.content.find(
ai_evals/core/models.ts
+71
View File
@@ -78,6 +78,32 @@ export const EVAL_MODELS: EvalModelSpec[] = [
model: "opus",
},
},
{
id: "sonnet-5.5",
label: "Claude Sonnet 5.5",
aliases: ["sonnet-5.5", "claude-sonnet-5.5", "claude-sonnet-5-5"],
frontend: {
provider: "anthropic",
model: "claude-sonnet-5-5",
},
cli: {
provider: "anthropic",
model: "claude-sonnet-5-5",
},
},
{
id: "opus-5.5",
label: "Claude Opus 5.5",
aliases: ["opus-5.5", "claude-opus-5.5", "claude-opus-5-5"],
frontend: {
provider: "anthropic",
model: "claude-opus-5-5",
},
cli: {
provider: "anthropic",
model: "claude-opus-5-5",
},
},
{
id: "4o",
label: "GPT-4o",
@@ -96,6 +122,51 @@ export const EVAL_MODELS: EvalModelSpec[] = [
model: "gpt-5.5",
},
},
{
id: "gpt-5.6-sol",
label: "GPT-5.6 Sol",
aliases: ["gpt-5.6-sol"],
frontend: {
provider: "openai",
model: "gpt-5.6-sol",
},
},
{
id: "gpt-6-astra",
label: "GPT-6 Astra",
aliases: ["gpt-6-astra", "gpt-6"],
frontend: {
provider: "openai",
model: "gpt-6-astra",
},
},
{
id: "gpt-6-sol",
label: "GPT-6 Sol",
aliases: ["gpt-6-sol"],
frontend: {
provider: "openai",
model: "gpt-6-sol",
},
},
{
id: "gpt-6-luna",
label: "GPT-6 Luna",
aliases: ["gpt-6-luna"],
frontend: {
provider: "openai",
model: "gpt-6-luna",
},
},
{
id: "gemini-3.8-flash",
label: "Gemini 3.8 Flash",
aliases: ["gemini-3.8-flash"],
frontend: {
provider: "googleai",
model: "gemini-3.8-flash",
},
},
{
id: "gemini-3-flash-preview",
label: "Gemini 3 Flash Preview",
ai_evals/fixtures/frontend/global/hub/confluence.json
+983
View File
@@ -0,0 +1,983 @@
{
"app": "confluence",
"display_name": "Confluence",
"description": "Atlassian's team workspace: pages, blog posts and spaces.",
"docs_url": "https://developer.atlassian.com/cloud/confluence/rest/v2/intro/",
"curated": true,
"meta": {
"api_docs": "https://developer.atlassian.com/cloud/confluence/rest/v2/intro/",
"shipped_surfaces": [
"pages",
"blogposts",
"spaces",
"search",
"users"
],
"validation": {
"status": "live_validated",
"method": "smoke test via `bun --env-file=.env.test -e` (+ `bun test` with the windmill-client fake for the trigger) against a free Confluence Cloud site (a test site) with an API token. All 13 actions + the trigger returned 2xx: full page lifecycle (create -> get -> update -> list -> delete), blog post create/update/delete, list_spaces, search_content (CQL), get_current_user, and the new_or_updated_page polling trigger.",
"sources": {
"pages": "HIGH \u2014 Pipedream actions + official v2 OpenAPI; create/get/update/list/delete all run live (2xx)",
"blogposts": "HIGH \u2014 Pipedream actions + official v2 OpenAPI; create/update/delete all run live (2xx)",
"spaces": "HIGH \u2014 Pipedream list-spaces + official v2 OpenAPI; list_spaces run live (2xx)",
"search": "HIGH \u2014 Pipedream search-content + official v1 REST; search_content (CQL) run live (2xx)",
"users": "HIGH \u2014 v1 /wiki/rest/api/user/current run live (2xx) with Basic auth",
"trigger": "HIGH \u2014 new_or_updated_page e2e-tested under bun test with the auto-fake; first run sets the watermark, a subsequent run emits a freshly created page"
},
"low_confidence_fields": []
},
"pagination": {
"pattern": "cursor",
"request_params": {
"cursor": "cursor",
"limit": "limit"
},
"response_fields": {
"next": "_links.next"
},
"default_limit": 25,
"notes": "v2 endpoints return an opaque cursor inside the `_links.next` relative URL (?cursor=...) and also in the Link response header. The v1 /search endpoint paginates the same way."
},
"gotchas": [
"Two API versions are in play: pages / blog posts / spaces use the v2 REST API (/wiki/api/v2); CQL search and current-user use the v1 REST API (/wiki/rest/api).",
"Auth is HTTP Basic with the Atlassian account email as the username and an API token (NOT the account password) as the password.",
"v2 query params are kebab-case (body-format, created-date, get-draft, root-level); v2 JSON request-body fields are camelCase (spaceId, parentId).",
"Updating a page or blog post requires version.number = currentVersion + 1; the update_* scripts fetch the current version and increment it automatically.",
"spaceId / parentId in v2 request bodies are numeric IDs passed as strings, not space keys.",
"DELETE returns 204 with an empty body and trashes by default; pass purge=true to permanently remove an item that is already in the trash."
],
"enums": {
"page_status": [
"current",
"archived",
"deleted",
"trashed",
"draft",
"historical"
],
"body_representation": [
"storage",
"atlas_doc_format"
],
"body_format_read": [
"storage",
"atlas_doc_format",
"view",
"export_view",
"anonymous_export_view",
"styled_view",
"editor"
],
"space_type": [
"global",
"collaboration",
"knowledge_base",
"personal"
],
"page_sort": [
"id",
"-id",
"created-date",
"-created-date",
"modified-date",
"-modified-date",
"title",
"-title"
]
},
"errors": {
"401": "Authentication failed \u2014 verify the email + API token pair and that the token has not been revoked.",
"404": "Resource not found, or the authenticating account lacks permission to view it (Confluence often returns 404 rather than 403 for content the user cannot access)."
}
},
"resource_type": {
"name": "confluence",
"description": "Confluence Cloud credentials. Authenticate with an Atlassian account email and an API token (HTTP Basic auth) against your site base URL.",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"apiToken": {
"description": "API token for your Atlassian account. Create one at https://id.atlassian.com/manage-profile/security/api-tokens",
"type": "string"
},
"baseUrl": {
"description": "Base URL of your Confluence Cloud site, e.g. https://your-domain.atlassian.net (no trailing slash, no /wiki).",
"type": "string"
},
"email": {
"description": "Email of the Atlassian account used to authenticate.",
"type": "string"
}
},
"required": [
"email",
"apiToken",
"baseUrl"
],
"type": "object"
}
},
"scripts": [
{
"version_id": 28765,
"app": "confluence",
"summary": "Create Blog Post",
"description": "Create a new blog post in a space.",
"terms": "confluence create blog post create a new blog post in a space.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * Create Blog Post\n * Create a new blog post in a space. The body is supplied as a content string in the chosen representation (default \"storage\", i.e. Confluence storage HTML).\n */\nexport async function main(\n auth: RT.Confluence,\n space_id: string,\n title: string,\n body: string,\n status: \"current\" | \"draft\" | undefined,\n representation: \"storage\" | \"atlas_doc_format\" | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/api/v2/blogposts`)\n\n const payload = {\n spaceId: space_id,\n status: status ?? \"current\",\n title,\n body: {\n representation: representation ?? \"storage\",\n value: body,\n },\n }\n\n const response = await fetch(url, {\n method: \"POST\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify(payload),\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"body": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"representation": {
"default": null,
"description": "",
"enum": [
"storage",
"atlas_doc_format"
],
"originalType": "enum",
"type": "string"
},
"space_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"status": {
"default": null,
"description": "",
"enum": [
"current",
"draft"
],
"originalType": "enum",
"type": "string"
},
"title": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth",
"space_id",
"title",
"body"
],
"type": "object"
}
},
{
"version_id": 28766,
"app": "confluence",
"summary": "Create Page",
"description": "Create a new page in a space.",
"terms": "confluence create page create a new page in a space.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * Create Page\n * Create a new page in a space. The body is supplied as a content string in the chosen representation (default \"storage\", i.e. Confluence storage HTML).\n */\nexport async function main(\n auth: RT.Confluence,\n space_id: string,\n title: string,\n body: string,\n status: \"current\" | \"draft\" | undefined,\n parent_id: string | undefined,\n representation: \"storage\" | \"atlas_doc_format\" | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/api/v2/pages`)\n\n const payload = {\n spaceId: space_id,\n status: status ?? \"current\",\n title,\n parentId: parent_id,\n body: {\n representation: representation ?? \"storage\",\n value: body,\n },\n }\n\n const response = await fetch(url, {\n method: \"POST\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify(payload),\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"body": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"parent_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"representation": {
"default": null,
"description": "",
"enum": [
"storage",
"atlas_doc_format"
],
"originalType": "enum",
"type": "string"
},
"space_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"status": {
"default": null,
"description": "",
"enum": [
"current",
"draft"
],
"originalType": "enum",
"type": "string"
},
"title": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth",
"space_id",
"title",
"body"
],
"type": "object"
}
},
{
"version_id": 28767,
"app": "confluence",
"summary": "Delete Blog Post",
"description": "Delete a blog post by its ID (moved to trash by default; purge to permanently remove).",
"terms": "confluence delete blog post delete a blog post by its id (moved to trash by default; purge to permanently remove).",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * Delete Blog Post\n * Delete a blog post by its ID. By default the post is moved to the trash; set purge to permanently remove an already-trashed post.\n */\nexport async function main(\n auth: RT.Confluence,\n blog_post_id: string,\n purge: boolean | undefined,\n draft: boolean | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/api/v2/blogposts/${blog_post_id}`)\n if (purge !== undefined) url.searchParams.append(\"purge\", String(purge))\n if (draft !== undefined) url.searchParams.append(\"draft\", String(draft))\n\n const response = await fetch(url, {\n method: \"DELETE\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n if (response.status === 204) return { success: true }\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"blog_post_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"draft": {
"default": null,
"description": "",
"type": "boolean"
},
"purge": {
"default": null,
"description": "",
"type": "boolean"
}
},
"required": [
"auth",
"blog_post_id"
],
"type": "object"
}
},
{
"version_id": 28768,
"app": "confluence",
"summary": "Delete Page",
"description": "Delete a page by its ID (moved to trash by default; purge to permanently remove).",
"terms": "confluence delete page delete a page by its id (moved to trash by default; purge to permanently remove).",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * Delete Page\n * Delete a page by its ID. By default the page is moved to the trash; set purge to permanently remove an already-trashed page.\n */\nexport async function main(\n auth: RT.Confluence,\n page_id: string,\n purge: boolean | undefined,\n draft: boolean | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/api/v2/pages/${page_id}`)\n if (purge !== undefined) url.searchParams.append(\"purge\", String(purge))\n if (draft !== undefined) url.searchParams.append(\"draft\", String(draft))\n\n const response = await fetch(url, {\n method: \"DELETE\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n if (response.status === 204) return { success: true }\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"draft": {
"default": null,
"description": "",
"type": "boolean"
},
"page_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"purge": {
"default": null,
"description": "",
"type": "boolean"
}
},
"required": [
"auth",
"page_id"
],
"type": "object"
}
},
{
"version_id": 28769,
"app": "confluence",
"summary": "Get Current User",
"description": "Returns the profile of the user that the API token authenticates as.",
"terms": "confluence get current user returns the profile of the user that the api token authenticates as.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * Get Current User\n * Returns the profile of the user that the API token authenticates as (account ID, display name, email).\n */\nexport async function main(auth: RT.Confluence) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/rest/api/user/current`)\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
}
},
"required": [
"auth"
],
"type": "object"
}
},
{
"version_id": 28770,
"app": "confluence",
"summary": "Get Page by ID",
"description": "Retrieve a single page by its ID, optionally returning its body in a specific format or a previous version.",
"terms": "confluence get page by id retrieve a single page by its id, optionally returning its body in a specific format or a previous version.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * Get Page by ID\n * Retrieve a single page by its ID, optionally returning its body in a specific format or a previous version.\n */\nexport async function main(\n auth: RT.Confluence,\n page_id: string,\n body_format:\n | \"storage\"\n | \"atlas_doc_format\"\n | \"view\"\n | \"export_view\"\n | \"anonymous_export_view\"\n | \"styled_view\"\n | \"editor\"\n | undefined,\n version: number | undefined,\n get_draft: boolean | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/api/v2/pages/${page_id}`)\n if (body_format !== undefined)\n url.searchParams.append(\"body-format\", body_format)\n if (version !== undefined) url.searchParams.append(\"version\", String(version))\n if (get_draft !== undefined)\n url.searchParams.append(\"get-draft\", String(get_draft))\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"body_format": {
"default": null,
"description": "",
"enum": [
"storage",
"atlas_doc_format",
"view",
"export_view",
"anonymous_export_view",
"styled_view",
"editor"
],
"originalType": "enum",
"type": "string"
},
"get_draft": {
"default": null,
"description": "",
"type": "boolean"
},
"page_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"version": {
"default": null,
"description": "",
"type": "number"
}
},
"required": [
"auth",
"page_id"
],
"type": "object"
}
},
{
"version_id": 28771,
"app": "confluence",
"summary": "Get Pages in Space",
"description": "Retrieve a paginated list of pages within a given space.",
"terms": "confluence get pages in space retrieve a paginated list of pages within a given space.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * Get Pages in Space\n * Retrieve a paginated list of pages within a given space.\n */\nexport async function main(\n auth: RT.Confluence,\n space_id: string,\n status: (\"current\" | \"archived\" | \"deleted\" | \"trashed\")[] | undefined,\n title: string | undefined,\n sort:\n | \"id\"\n | \"-id\"\n | \"created-date\"\n | \"-created-date\"\n | \"modified-date\"\n | \"-modified-date\"\n | \"title\"\n | \"-title\"\n | undefined,\n body_format: \"storage\" | \"atlas_doc_format\" | undefined,\n cursor: string | undefined,\n limit: number | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/api/v2/spaces/${space_id}/pages`)\n for (const s of status ?? []) url.searchParams.append(\"status\", s)\n if (title !== undefined && title !== \"\")\n url.searchParams.append(\"title\", title)\n if (sort !== undefined) url.searchParams.append(\"sort\", sort)\n if (body_format !== undefined)\n url.searchParams.append(\"body-format\", body_format)\n if (cursor !== undefined && cursor !== \"\")\n url.searchParams.append(\"cursor\", cursor)\n if (limit !== undefined) url.searchParams.append(\"limit\", String(limit))\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"body_format": {
"default": null,
"description": "",
"enum": [
"storage",
"atlas_doc_format"
],
"originalType": "enum",
"type": "string"
},
"cursor": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"limit": {
"default": null,
"description": "",
"type": "number"
},
"sort": {
"default": null,
"description": "",
"enum": [
"id",
"-id",
"created-date",
"-created-date",
"modified-date",
"-modified-date",
"title",
"-title"
],
"originalType": "enum",
"type": "string"
},
"space_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"status": {
"default": null,
"description": "",
"items": {
"enum": [
"current",
"archived",
"deleted",
"trashed"
],
"type": "string"
},
"originalType": "enum[]",
"type": "array"
},
"title": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth",
"space_id"
],
"type": "object"
}
},
{
"version_id": 28772,
"app": "confluence",
"summary": "List Blog Posts",
"description": "Retrieve a paginated list of blog posts on the site, with optional filtering and sorting.",
"terms": "confluence list blog posts retrieve a paginated list of blog posts on the site, with optional filtering and sorting.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * List Blog Posts\n * Retrieve a paginated list of blog posts on the site, with optional filtering and sorting.\n */\nexport async function main(\n auth: RT.Confluence,\n status: (\"current\" | \"archived\" | \"deleted\" | \"trashed\")[] | undefined,\n title: string | undefined,\n sort:\n | \"id\"\n | \"-id\"\n | \"created-date\"\n | \"-created-date\"\n | \"modified-date\"\n | \"-modified-date\"\n | undefined,\n body_format: \"storage\" | \"atlas_doc_format\" | undefined,\n cursor: string | undefined,\n limit: number | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/api/v2/blogposts`)\n for (const s of status ?? []) url.searchParams.append(\"status\", s)\n if (title !== undefined && title !== \"\")\n url.searchParams.append(\"title\", title)\n if (sort !== undefined) url.searchParams.append(\"sort\", sort)\n if (body_format !== undefined)\n url.searchParams.append(\"body-format\", body_format)\n if (cursor !== undefined && cursor !== \"\")\n url.searchParams.append(\"cursor\", cursor)\n if (limit !== undefined) url.searchParams.append(\"limit\", String(limit))\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"body_format": {
"default": null,
"description": "",
"enum": [
"storage",
"atlas_doc_format"
],
"originalType": "enum",
"type": "string"
},
"cursor": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"limit": {
"default": null,
"description": "",
"type": "number"
},
"sort": {
"default": null,
"description": "",
"enum": [
"id",
"-id",
"created-date",
"-created-date",
"modified-date",
"-modified-date"
],
"originalType": "enum",
"type": "string"
},
"status": {
"default": null,
"description": "",
"items": {
"enum": [
"current",
"archived",
"deleted",
"trashed"
],
"type": "string"
},
"originalType": "enum[]",
"type": "array"
},
"title": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth"
],
"type": "object"
}
},
{
"version_id": 28773,
"app": "confluence",
"summary": "List Pages",
"description": "Retrieve a paginated list of pages on the site, with optional filtering and sorting.",
"terms": "confluence list pages retrieve a paginated list of pages on the site, with optional filtering and sorting.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * List Pages\n * Retrieve a paginated list of pages on the site, with optional filtering and sorting.\n */\nexport async function main(\n auth: RT.Confluence,\n status: (\"current\" | \"archived\" | \"deleted\" | \"trashed\")[] | undefined,\n title: string | undefined,\n sort:\n | \"id\"\n | \"-id\"\n | \"created-date\"\n | \"-created-date\"\n | \"modified-date\"\n | \"-modified-date\"\n | \"title\"\n | \"-title\"\n | undefined,\n body_format: \"storage\" | \"atlas_doc_format\" | undefined,\n cursor: string | undefined,\n limit: number | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/api/v2/pages`)\n for (const s of status ?? []) url.searchParams.append(\"status\", s)\n if (title !== undefined && title !== \"\")\n url.searchParams.append(\"title\", title)\n if (sort !== undefined) url.searchParams.append(\"sort\", sort)\n if (body_format !== undefined)\n url.searchParams.append(\"body-format\", body_format)\n if (cursor !== undefined && cursor !== \"\")\n url.searchParams.append(\"cursor\", cursor)\n if (limit !== undefined) url.searchParams.append(\"limit\", String(limit))\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"body_format": {
"default": null,
"description": "",
"enum": [
"storage",
"atlas_doc_format"
],
"originalType": "enum",
"type": "string"
},
"cursor": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"limit": {
"default": null,
"description": "",
"type": "number"
},
"sort": {
"default": null,
"description": "",
"enum": [
"id",
"-id",
"created-date",
"-created-date",
"modified-date",
"-modified-date",
"title",
"-title"
],
"originalType": "enum",
"type": "string"
},
"status": {
"default": null,
"description": "",
"items": {
"enum": [
"current",
"archived",
"deleted",
"trashed"
],
"type": "string"
},
"originalType": "enum[]",
"type": "array"
},
"title": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth"
],
"type": "object"
}
},
{
"version_id": 28774,
"app": "confluence",
"summary": "List Spaces",
"description": "Retrieve a paginated list of spaces on the site, with optional filtering by key, type, and status.",
"terms": "confluence list spaces retrieve a paginated list of spaces on the site, with optional filtering by key, type, and status.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * List Spaces\n * Retrieve a paginated list of spaces on the site, with optional filtering by key, type, and status.\n */\nexport async function main(\n auth: RT.Confluence,\n keys: string[] | undefined,\n type: \"global\" | \"collaboration\" | \"knowledge_base\" | \"personal\" | undefined,\n status: \"current\" | \"archived\" | undefined,\n sort: \"id\" | \"-id\" | \"key\" | \"-key\" | \"name\" | \"-name\" | undefined,\n cursor: string | undefined,\n limit: number | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/api/v2/spaces`)\n if (keys !== undefined && keys.length > 0)\n url.searchParams.append(\"keys\", keys.join(\",\"))\n if (type !== undefined) url.searchParams.append(\"type\", type)\n if (status !== undefined) url.searchParams.append(\"status\", status)\n if (sort !== undefined) url.searchParams.append(\"sort\", sort)\n if (cursor !== undefined && cursor !== \"\")\n url.searchParams.append(\"cursor\", cursor)\n if (limit !== undefined) url.searchParams.append(\"limit\", String(limit))\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"cursor": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"keys": {
"default": null,
"description": "",
"items": {
"type": "string"
},
"originalType": "string[]",
"type": "array"
},
"limit": {
"default": null,
"description": "",
"type": "number"
},
"sort": {
"default": null,
"description": "",
"enum": [
"id",
"-id",
"key",
"-key",
"name",
"-name"
],
"originalType": "enum",
"type": "string"
},
"status": {
"default": null,
"description": "",
"enum": [
"current",
"archived"
],
"originalType": "enum",
"type": "string"
},
"type": {
"default": null,
"description": "",
"enum": [
"global",
"collaboration",
"knowledge_base",
"personal"
],
"originalType": "enum",
"type": "string"
}
},
"required": [
"auth"
],
"type": "object"
}
},
{
"version_id": 28775,
"app": "confluence",
"summary": "Search Content (CQL)",
"description": "Search for content using the Confluence Query Language (CQL).",
"terms": "confluence search content (cql) search for content using the confluence query language (cql).",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * Search Content (CQL)\n * Search for content using the Confluence Query Language (CQL), e.g. `type=page AND space=ENG AND text ~ \"release notes\"`.\n */\nexport async function main(\n auth: RT.Confluence,\n cql: string,\n cursor: string | undefined,\n limit: number | undefined,\n expand: string | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const url = new URL(`${base}/wiki/rest/api/search`)\n url.searchParams.append(\"cql\", cql)\n if (cursor !== undefined && cursor !== \"\")\n url.searchParams.append(\"cursor\", cursor)\n if (limit !== undefined) url.searchParams.append(\"limit\", String(limit))\n if (expand !== undefined && expand !== \"\")\n url.searchParams.append(\"expand\", expand)\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"cql": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"cursor": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"expand": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"limit": {
"default": null,
"description": "",
"type": "number"
}
},
"required": [
"auth",
"cql"
],
"type": "object"
}
},
{
"version_id": 28776,
"app": "confluence",
"summary": "Update Blog Post",
"description": "Update a blog post's title, body, or status. The current version is fetched automatically and incremented.",
"terms": "confluence update blog post update a blog post's title, body, or status. the current version is fetched automatically and incremented.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * Update Blog Post\n * Update a blog post's title, body, or status. The current version is fetched automatically and incremented; omitted fields keep their existing values.\n */\nexport async function main(\n auth: RT.Confluence,\n blog_post_id: string,\n title: string | undefined,\n body: string | undefined,\n status: \"current\" | \"draft\" | \"archived\" | undefined,\n representation: \"storage\" | \"atlas_doc_format\" | undefined,\n version_message: string | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const repr = representation ?? \"storage\"\n const authHeader = \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`)\n\n // Fetch the current blog post to obtain the version number and any fields not being changed.\n const currentRes = await fetch(\n `${base}/wiki/api/v2/blogposts/${blog_post_id}?body-format=${repr}`,\n {\n method: \"GET\",\n headers: { Authorization: authHeader, Accept: \"application/json\" },\n }\n )\n if (!currentRes.ok) {\n throw new Error(`${currentRes.status} ${await currentRes.text()}`)\n }\n const current = (await currentRes.json()) as {\n title: string\n status: string\n version: { number: number }\n body?: { [key: string]: { value?: string } }\n }\n\n const payload = {\n id: blog_post_id,\n status: status ?? current.status,\n title: title ?? current.title,\n body: {\n representation: repr,\n value: body ?? current.body?.[repr]?.value ?? \"\",\n },\n version: {\n number: current.version.number + 1,\n message: version_message,\n },\n }\n\n const response = await fetch(\n `${base}/wiki/api/v2/blogposts/${blog_post_id}`,\n {\n method: \"PUT\",\n headers: {\n Authorization: authHeader,\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify(payload),\n }\n )\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"blog_post_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"body": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"representation": {
"default": null,
"description": "",
"enum": [
"storage",
"atlas_doc_format"
],
"originalType": "enum",
"type": "string"
},
"status": {
"default": null,
"description": "",
"enum": [
"current",
"draft",
"archived"
],
"originalType": "enum",
"type": "string"
},
"title": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"version_message": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth",
"blog_post_id"
],
"type": "object"
}
},
{
"version_id": 28777,
"app": "confluence",
"summary": "Update Page",
"description": "Update a page's title, body, or status. The current version is fetched automatically and incremented.",
"terms": "confluence update page update a page's title, body, or status. the current version is fetched automatically and incremented.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\n/**\n * Update Page\n * Update a page's title, body, or status. The current version is fetched automatically and incremented; omitted fields keep their existing values.\n */\nexport async function main(\n auth: RT.Confluence,\n page_id: string,\n title: string | undefined,\n body: string | undefined,\n status: \"current\" | \"draft\" | \"archived\" | undefined,\n representation: \"storage\" | \"atlas_doc_format\" | undefined,\n version_message: string | undefined\n) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const repr = representation ?? \"storage\"\n const authHeader = \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`)\n\n // Fetch the current page to obtain the version number and any fields not being changed.\n const currentRes = await fetch(\n `${base}/wiki/api/v2/pages/${page_id}?body-format=${repr}`,\n {\n method: \"GET\",\n headers: { Authorization: authHeader, Accept: \"application/json\" },\n }\n )\n if (!currentRes.ok) {\n throw new Error(`${currentRes.status} ${await currentRes.text()}`)\n }\n const current = (await currentRes.json()) as {\n title: string\n status: string\n version: { number: number }\n body?: { [key: string]: { value?: string } }\n }\n\n const payload = {\n id: page_id,\n status: status ?? current.status,\n title: title ?? current.title,\n body: {\n representation: repr,\n value: body ?? current.body?.[repr]?.value ?? \"\",\n },\n version: {\n number: current.version.number + 1,\n message: version_message,\n },\n }\n\n const response = await fetch(`${base}/wiki/api/v2/pages/${page_id}`, {\n method: \"PUT\",\n headers: {\n Authorization: authHeader,\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify(payload),\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"body": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"page_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"representation": {
"default": null,
"description": "",
"enum": [
"storage",
"atlas_doc_format"
],
"originalType": "enum",
"type": "string"
},
"status": {
"default": null,
"description": "",
"enum": [
"current",
"draft",
"archived"
],
"originalType": "enum",
"type": "string"
},
"title": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"version_message": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth",
"page_id"
],
"type": "object"
}
},
{
"version_id": 28778,
"app": "confluence",
"summary": "New or Updated Page",
"description": "Emits pages created or updated since the last poll, tracked via Windmill state. Optionally restrict to a single space.",
"terms": "confluence new or updated page emits pages created or updated since the last poll, tracked via windmill state. optionally restrict to a single space.",
"kind": "trigger",
"language": "bunnative",
"content": "//native\n\nimport * as wmill from \"windmill-client\"\n\n/**\n * New or Updated Page\n * Emits pages created or updated since the last poll, tracked via Windmill state. Optionally restrict to a single space.\n */\nexport async function main(auth: RT.Confluence, space_id: string | undefined) {\n const base = auth.baseUrl.replace(/\\/$/, \"\")\n const lastChecked: number = (await wmill.getState()) ?? 0\n\n const path = space_id\n ? `/wiki/api/v2/spaces/${space_id}/pages`\n : `/wiki/api/v2/pages`\n const url = new URL(`${base}${path}`)\n url.searchParams.append(\"sort\", \"-modified-date\")\n url.searchParams.append(\"limit\", \"50\")\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: \"Basic \" + btoa(`${auth.email}:${auth.apiToken}`),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n const data = (await response.json()) as {\n results?: { version?: { createdAt?: string } }[]\n }\n const pages = data.results ?? []\n\n const ts = (p: { version?: { createdAt?: string } }) =>\n p.version?.createdAt ? Date.parse(p.version.createdAt) : 0\n const maxTs = pages.reduce((m, p) => Math.max(m, ts(p)), 0)\n\n // First run: set the watermark to the newest page seen and don't emit a backlog.\n if (!lastChecked) {\n if (maxTs) await wmill.setState(maxTs)\n return []\n }\n\n const fresh = pages\n .filter((p) => ts(p) > lastChecked)\n .sort((a, b) => ts(a) - ts(b))\n\n if (maxTs > lastChecked) {\n await wmill.setState(maxTs)\n }\n\n return fresh\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-confluence",
"type": "object"
},
"space_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth"
],
"type": "object"
}
}
]
}
ai_evals/fixtures/frontend/global/hub/outreach.json
+1055
View File
File diff suppressed because it is too large Load Diff
ai_evals/fixtures/frontend/global/hub/servicenow.json
+672
View File
@@ -0,0 +1,672 @@
{
"app": "servicenow",
"display_name": "ServiceNow",
"description": "IT service management: incidents, records and attachments.",
"docs_url": "https://www.servicenow.com/docs/bundle/zurich-api-reference/page/integrate/inbound-rest/concept/c_RESTAPI.html",
"curated": true,
"meta": {
"api_docs": "https://www.servicenow.com/docs/bundle/zurich-api-reference/page/integrate/inbound-rest/concept/c_RESTAPI.html",
"shipped_surfaces": [
"table",
"stats",
"attachment",
"import_set",
"user"
],
"validation": {
"status": "live_validated",
"method": "All surfaces run live against a ServiceNow PDI (a developer instance, Zurich) on 2026-06-04 with Basic auth: Table CRUD (create / get / update PATCH+PUT / delete on incident, delete verified by 404), list_records (encoded query + fields + pagination), aggregate (Stats group-by count), get_current_user, list_tables, and the table/fields/group_by dynselect resolvers. Attachment upload (binary /file) -> list -> download -> delete round-trip with the downloaded bytes byte-for-byte matching the upload. insert_import_set + get_import_set_result against imp_user (transform status surfaced). Trigger e2e-tested via bun test with the windmill-client fake. All created records cleaned up.",
"sources": {
"table_crud": "HIGH - run live (create/get/update PATCH & PUT/delete on incident)",
"aggregate_stats": "HIGH - run live (/api/now/stats group-by count on incident)",
"attachment": "HIGH - run live (binary /file upload -> list -> download -> delete round-trip; downloaded bytes verified equal)",
"import_set": "HIGH - run live (POST /api/now/import/imp_user + GET result; top-level {import_set,staging_table,result} envelope, transform status surfaced)",
"user": "HIGH - run live (/api/now/ui/user/current_user -> admin sys_user)",
"trigger": "HIGH - e2e-tested live via bun test (first run sets watermark + no backlog, second run emits the new record)",
"dropdowns": "HIGH - run live (sys_db_object table list; sys_dictionary super_class inheritance walk confirmed - incident resolves 106 fields incl. task-inherited short_description/priority/state)"
},
"low_confidence_fields": [
"trigger datetime comparison assumes the integration user's timezone is UTC/GMT (validated with the PDI admin user, whose TZ is UTC); a non-UTC integration user could shift the sys_updated_on> watermark boundary by the offset"
]
},
"pagination": {
"pattern": "offset",
"request_params": {
"offset": "sysparm_offset",
"limit": "sysparm_limit"
},
"response_fields": {
"next": "Link (response header, rel=\"next\")",
"total": "X-Total-Count (response header)"
},
"default_limit": 10000,
"notes": "Table API pages with sysparm_offset + sysparm_limit. Each page response carries a Link header (RFC 5988) with rel=\"first\"/\"prev\"/\"next\"/\"last\" full URLs, plus X-Total-Count with the total match count. Add sysparm_no_count=true to skip the count on huge tables. The list_records action exposes limit+offset and returns one page; iterate offset until fewer than limit rows return."
},
"gotchas": [
"Base path is {instance_url}/api/now; instance_url lives on the resource. Auth is Basic (username:password) OR Bearer token \u2014 scripts send Bearer when auth.token is set, otherwise Basic. Both hit identical paths.",
"Table/Attachment/Stats reads/writes return a {\"result\": ...} envelope: list/stats -> result is an array, get/create/update -> result is an object. EXCEPTION: the Import Set API (POST /api/now/import/{staging}, GET .../{sys_id}) returns a TOP-LEVEL object {import_set, staging_table, result:[...]} with NO outer 'result' wrapper. Scripts return the raw JSON (envelope intact).",
"Datetimes are 'YYYY-MM-DD HH:MM:SS' in UTC with NO trailing 'Z'. Used directly in encoded queries, e.g. sys_updated_on>2026-01-01 00:00:00.",
"sysparm_query is an ENCODED query string: '^' = AND clause separator, '^OR' = OR, operators =, !=, LIKE, >=, <=, IN, STARTSWITH, ORDERBY, ORDERBYDESC. A literal '^' inside a value must be doubled to '^^'. Example: active=true^priority=1^ORDERBYDESCsys_updated_on.",
"System fields sys_id, sys_created_on, sys_updated_on, sys_created_by are auto-managed; do not send them on create/update.",
"DELETE (record and attachment) returns 204 No Content with an empty body \u2014 guard with `if (response.status === 204) return { success: true }` before response.json().",
"Update: PATCH partial-updates; PUT replaces the whole record (clears omitted fields). update_record sends PUT when replace=true, PATCH otherwise.",
"Attachment upload uses POST /api/now/attachment/file with table_name, table_sys_id, file_name in the QUERY STRING and the raw file bytes as the body (Content-Type = the file's MIME type), NOT multipart. Returns 201. Download is GET /api/now/attachment/{sys_id}/file (raw bytes).",
"Import Set transform runs SYNCHRONOUSLY. A result[].status of 'error' is an application-level failure even on HTTP 200/201 \u2014 inspect result[].status / error_message, don't trust the HTTP code alone.",
"Table-name DROPDOWN comes from GET /api/now/table/sys_db_object?sysparm_fields=name,label (use the 'name' as the value). A table's COLUMNS come from sys_dictionary \u2014 but sys_dictionary stores each field once, under the table where it is DEFINED, so name=<table> returns ONLY that table's own fields. Extended tables (incident/change_request/sc_task all extend task) inherit columns like short_description/priority/assigned_to/state that live on the parent. To get the full column set, walk sys_db_object.super_class up the hierarchy (GET sys_db_object?sysparm_query=name=<t>&sysparm_fields=super_class.name, repeat) and query GET sys_dictionary?sysparm_query=nameIN<table>,<ancestors>^elementISNOTEMPTY&sysparm_fields=element,column_label (elementISNOTEMPTY drops the collection/meta row).",
"Table API enforces row- and field-level ACLs: a 200 can omit fields the user can't read; inaccessible tables/ops return 403."
],
"enums": {
"sysparm_display_value": [
"true",
"false",
"all"
],
"sysparm_view": [
"desktop",
"mobile",
"both"
],
"import_set_result_status": [
"inserted",
"updated",
"ignored",
"error"
],
"incident_state_note": "incident.state and priority/urgency/impact are instance-configurable; defaults: state 1=New,2=In Progress,3=On Hold,6=Resolved,7=Closed,8=Canceled; priority/urgency/impact 1-5. On task/incident, priority is CALCULATED from impact+urgency (a priority value sent on create/update is recomputed/ignored) \u2014 set impact+urgency instead. Never hardcode for generic actions."
},
"errors": {
"401": "Authentication failed or OAuth token expired/invalid. Check username/password or refresh the bearer token.",
"403": "ACL denies the table or operation, or the user lacks the required role (e.g. rest_api_explorer / table ACL). Response: {\"error\":{\"message\":\"User Not Authorized\"}}.",
"404": "No Record found (bad sys_id) or Invalid table (bad/inaccessible table name, common Import Set staging-table error).",
"400": "Bad encoded query, invalid field name, or malformed request body."
}
},
"resource_type": {
"name": "servicenow",
"description": "ServiceNow instance credential. Set instance_url to your instance base (https://<instance>.service-now.com). Authenticate with EITHER Basic auth (username + password) OR an OAuth 2.0 bearer token \u2014 scripts send a Bearer header when token is set, otherwise Basic auth.",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"order": [
"instance_url",
"username",
"password",
"token"
],
"properties": {
"instance_url": {
"default": "",
"description": "Instance base URL, e.g. https://dev12345.service-now.com (no trailing slash). Every REST call is made against this host.",
"placeholder": "https://dev12345.service-now.com",
"type": "string"
},
"password": {
"default": "",
"description": "Basic auth password. Leave blank when using an OAuth bearer token.",
"password": true,
"type": "string"
},
"token": {
"default": "",
"description": "OAuth 2.0 access token (Bearer). Populated automatically when you connect via Windmill's ServiceNow OAuth flow (admin configures the instance + client id/secret in instance settings), or paste one manually. Provide this instead of username/password to authenticate with OAuth.",
"password": true,
"type": "string"
},
"username": {
"default": "",
"description": "Basic auth username. Leave blank when using an OAuth bearer token.",
"type": "string"
}
},
"required": [
"instance_url"
],
"type": "object"
}
},
"scripts": [
{
"version_id": 28665,
"app": "servicenow",
"summary": "Aggregate Records",
"description": "Count records on a table (Aggregate API), optionally filtered by an encoded query and grouped by a field.",
"terms": "servicenow aggregate records count records on a table (aggregate api), optionally filtered by an encoded query and grouped by a field.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nexport type DynSelect_table = string\nexport type DynSelect_group_by = string\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n// Dropdown of the instance's tables (sys_db_object).\nexport async function table(auth: RT.Servicenow) {\n const response = await fetch(\n `${auth.instance_url}/api/now/table/sys_db_object?sysparm_fields=name,label&sysparm_limit=10000`,\n {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n const { result } = (await response.json()) as {\n result: { name: string; label: string }[]\n }\n return result\n .filter((t) => t.name)\n .map((t) => ({ value: t.name, label: `${t.label} (${t.name})` }))\n .sort((a, b) => a.label.localeCompare(b.label))\n}\n\n// Walk the table's inheritance chain (sys_db_object.super_class) so the group-by\n// options include fields inherited from parent tables, not just fields defined\n// on the chosen table (sys_dictionary stores each field under its defining\n// table \u2014 e.g. incident extends task, so priority/state live on task).\n// Best-effort: any failure falls back to just the table itself.\nasync function tableChain(auth: RT.Servicenow, table: string) {\n const chain: string[] = []\n let current: string | undefined = table\n while (current && !chain.includes(current)) {\n chain.push(current)\n const response = await fetch(\n `${auth.instance_url}/api/now/table/sys_db_object?sysparm_query=name=${current}&sysparm_fields=super_class.name&sysparm_limit=1`,\n {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!response.ok) break\n const { result } = (await response.json()) as {\n result: { \"super_class.name\"?: string }[]\n }\n current = result[0]?.[\"super_class.name\"] || undefined\n }\n return chain\n}\n\n// Dependent select: the chosen table's columns (sys_dictionary) to group by,\n// including columns inherited from parent tables.\nexport async function group_by(auth: RT.Servicenow, table: DynSelect_table) {\n if (!table) return []\n const chain = await tableChain(auth, table)\n const url = new URL(`${auth.instance_url}/api/now/table/sys_dictionary`)\n url.searchParams.append(\n \"sysparm_query\",\n `nameIN${chain.join(\",\")}^elementISNOTEMPTY`\n )\n url.searchParams.append(\"sysparm_fields\", \"element,column_label\")\n url.searchParams.append(\"sysparm_limit\", \"10000\")\n\n const response = await fetch(url, {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n const { result } = (await response.json()) as {\n result: { element: string; column_label: string }[]\n }\n return result\n .filter((f) => f.element)\n .map((f) => ({\n value: f.element,\n label: `${f.column_label} (${f.element})`,\n }))\n .sort((a, b) => a.label.localeCompare(b.label))\n}\n\n/**\n * Aggregate Records\n * Count records on a table (Aggregate API), optionally filtered by an encoded query and grouped by a field. Returns ServiceNow's stats envelope with the count per group.\n */\nexport async function main(\n auth: RT.Servicenow,\n table: DynSelect_table,\n query: string | undefined,\n group_by: DynSelect_group_by | undefined\n) {\n const url = new URL(`${auth.instance_url}/api/now/stats/${table}`)\n url.searchParams.append(\"sysparm_count\", \"true\")\n if (query !== undefined && query !== \"\") {\n url.searchParams.append(\"sysparm_query\", query)\n }\n if (group_by !== undefined && group_by !== \"\") {\n url.searchParams.append(\"sysparm_group_by\", group_by)\n }\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"group_by": {
"default": null,
"description": "",
"format": "dynselect-group_by",
"type": "object"
},
"query": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"table": {
"default": null,
"description": "",
"format": "dynselect-table",
"type": "object"
}
},
"required": [
"auth",
"table"
],
"type": "object"
}
},
{
"version_id": 28666,
"app": "servicenow",
"summary": "Create Record",
"description": "Insert a record into any table (incident, change_request, sys_user, cmdb_ci, or a custom table). Pick the table from a live dropdown and pass a column/value map.",
"terms": "servicenow create record insert a record into any table (incident, change_request, sys_user, cmdb_ci, or a custom table). pick the table from a live dropdown and pass a column/value map.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nexport type DynSelect_table = string\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n// Dropdown of the instance's tables (sys_db_object).\nexport async function table(auth: RT.Servicenow) {\n const response = await fetch(\n `${auth.instance_url}/api/now/table/sys_db_object?sysparm_fields=name,label&sysparm_limit=10000`,\n {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n const { result } = (await response.json()) as {\n result: { name: string; label: string }[]\n }\n return result\n .filter((t) => t.name)\n .map((t) => ({ value: t.name, label: `${t.label} (${t.name})` }))\n .sort((a, b) => a.label.localeCompare(b.label))\n}\n\n/**\n * Create Record\n * Insert a record into any table (incident, change_request, sys_user, cmdb_ci, or a custom table). Pick the table from a live dropdown and pass a column/value map.\n */\nexport async function main(\n auth: RT.Servicenow,\n table: DynSelect_table,\n body: { [key: string]: any }\n) {\n const url = new URL(`${auth.instance_url}/api/now/table/${table}`)\n\n const response = await fetch(url, {\n method: \"POST\",\n headers: {\n Authorization: authHeader(auth),\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify(body),\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"body": {
"default": null,
"description": "",
"properties": {},
"type": "object"
},
"table": {
"default": null,
"description": "",
"format": "dynselect-table",
"type": "object"
}
},
"required": [
"auth",
"table",
"body"
],
"type": "object"
}
},
{
"version_id": 28667,
"app": "servicenow",
"summary": "Delete Attachment",
"description": "Delete an attachment by sys_id. Returns {success, sys_id} (ServiceNow replies 204 No Content).",
"terms": "servicenow delete attachment delete an attachment by sys_id. returns {success, sys_id} (servicenow replies 204 no content).",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n/**\n * Delete Attachment\n * Delete an attachment by its sys_id. ServiceNow returns 204 No Content on success, so this returns {success, sys_id}.\n */\nexport async function main(auth: RT.Servicenow, attachment_sys_id: string) {\n const response = await fetch(\n `${auth.instance_url}/api/now/attachment/${attachment_sys_id}`,\n {\n method: \"DELETE\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return { success: true, sys_id: attachment_sys_id }\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"attachment_sys_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
}
},
"required": [
"auth",
"attachment_sys_id"
],
"type": "object"
}
},
{
"version_id": 28668,
"app": "servicenow",
"summary": "Delete Record",
"description": "Delete a record by sys_id. Returns {success, sys_id} (ServiceNow replies 204 No Content).",
"terms": "servicenow delete record delete a record by sys_id. returns {success, sys_id} (servicenow replies 204 no content).",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nexport type DynSelect_table = string\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n// Dropdown of the instance's tables (sys_db_object).\nexport async function table(auth: RT.Servicenow) {\n const response = await fetch(\n `${auth.instance_url}/api/now/table/sys_db_object?sysparm_fields=name,label&sysparm_limit=10000`,\n {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n const { result } = (await response.json()) as {\n result: { name: string; label: string }[]\n }\n return result\n .filter((t) => t.name)\n .map((t) => ({ value: t.name, label: `${t.label} (${t.name})` }))\n .sort((a, b) => a.label.localeCompare(b.label))\n}\n\n/**\n * Delete Record\n * Delete a record by sys_id. ServiceNow returns 204 No Content on success, so this returns {success, sys_id}.\n */\nexport async function main(\n auth: RT.Servicenow,\n table: DynSelect_table,\n sys_id: string\n) {\n const url = new URL(`${auth.instance_url}/api/now/table/${table}/${sys_id}`)\n\n const response = await fetch(url, {\n method: \"DELETE\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return { success: true, sys_id }\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"sys_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"table": {
"default": null,
"description": "",
"format": "dynselect-table",
"type": "object"
}
},
"required": [
"auth",
"table",
"sys_id"
],
"type": "object"
}
},
{
"version_id": 28670,
"app": "servicenow",
"summary": "Get Current User",
"description": "Return the sys_user record of the authenticated user.",
"terms": "servicenow get current user return the sys_user record of the authenticated user.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n/**\n * Get Current User\n * Return the sys_user record of the authenticated user (resolves current_user, then fetches the full record from sys_user).\n */\nexport async function main(auth: RT.Servicenow) {\n const currentResponse = await fetch(\n `${auth.instance_url}/api/now/ui/user/current_user`,\n {\n method: \"GET\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!currentResponse.ok) {\n throw new Error(`${currentResponse.status} ${await currentResponse.text()}`)\n }\n const { result } = (await currentResponse.json()) as {\n result: { user_sys_id: string }\n }\n\n const userResponse = await fetch(\n `${auth.instance_url}/api/now/table/sys_user/${result.user_sys_id}`,\n {\n method: \"GET\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!userResponse.ok) {\n throw new Error(`${userResponse.status} ${await userResponse.text()}`)\n }\n\n return await userResponse.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
}
},
"required": [
"auth"
],
"type": "object"
}
},
{
"version_id": 28671,
"app": "servicenow",
"summary": "Get Import Set Result",
"description": "Fetch the outcome of a prior import row by the staging-row sys_id returned by Insert Into Import Set (result.sys_id).",
"terms": "servicenow get import set result fetch the outcome of a prior import row by the staging-row sys_id returned by insert into import set (result.sys_id).",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n/**\n * Get Import Set Result\n * Fetch the outcome of a prior import row by the staging-row sys_id returned by Insert Into Import Set (result.sys_id), returning the transform result (target table, status, target record link).\n */\nexport async function main(\n auth: RT.Servicenow,\n staging_table: string,\n sys_id: string\n) {\n const url = new URL(\n `${auth.instance_url}/api/now/import/${staging_table}/${sys_id}`\n )\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"staging_table": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"sys_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth",
"staging_table",
"sys_id"
],
"type": "object"
}
},
{
"version_id": 28672,
"app": "servicenow",
"summary": "Get Record",
"description": "Retrieve a single record by sys_id. Optionally restrict the returned columns, and choose raw vs. display values.",
"terms": "servicenow get record retrieve a single record by sys_id. optionally restrict the returned columns, and choose raw vs. display values.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nexport type DynSelect_table = string\nexport type DynMultiselect_fields = string[]\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n// Dropdown of the instance's tables (sys_db_object).\nexport async function table(auth: RT.Servicenow) {\n const response = await fetch(\n `${auth.instance_url}/api/now/table/sys_db_object?sysparm_fields=name,label&sysparm_limit=10000`,\n {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n const { result } = (await response.json()) as {\n result: { name: string; label: string }[]\n }\n return result\n .filter((t) => t.name)\n .map((t) => ({ value: t.name, label: `${t.label} (${t.name})` }))\n .sort((a, b) => a.label.localeCompare(b.label))\n}\n\n// Walk the table's inheritance chain (sys_db_object.super_class) so the column\n// list includes fields inherited from parent tables, not just fields defined on\n// the chosen table (sys_dictionary stores each field under its defining table \u2014\n// e.g. incident extends task, so short_description/priority live on task).\n// Best-effort: any failure falls back to just the table itself.\nasync function tableChain(auth: RT.Servicenow, table: string) {\n const chain: string[] = []\n let current: string | undefined = table\n while (current && !chain.includes(current)) {\n chain.push(current)\n const response = await fetch(\n `${auth.instance_url}/api/now/table/sys_db_object?sysparm_query=name=${current}&sysparm_fields=super_class.name&sysparm_limit=1`,\n {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!response.ok) break\n const { result } = (await response.json()) as {\n result: { \"super_class.name\"?: string }[]\n }\n current = result[0]?.[\"super_class.name\"] || undefined\n }\n return chain\n}\n\n// Dependent multi-select: the chosen table's columns (sys_dictionary), including\n// columns inherited from parent tables.\nexport async function fields(auth: RT.Servicenow, table: DynSelect_table) {\n if (!table) return []\n const chain = await tableChain(auth, table)\n const url = new URL(`${auth.instance_url}/api/now/table/sys_dictionary`)\n url.searchParams.append(\n \"sysparm_query\",\n `nameIN${chain.join(\",\")}^elementISNOTEMPTY`\n )\n url.searchParams.append(\"sysparm_fields\", \"element,column_label\")\n url.searchParams.append(\"sysparm_limit\", \"10000\")\n\n const response = await fetch(url, {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n const { result } = (await response.json()) as {\n result: { element: string; column_label: string }[]\n }\n return result\n .filter((f) => f.element)\n .map((f) => ({\n value: f.element,\n label: `${f.column_label} (${f.element})`,\n }))\n .sort((a, b) => a.label.localeCompare(b.label))\n}\n\n/**\n * Get Record\n * Retrieve a single record by sys_id. Optionally restrict the returned columns, and choose raw vs. display values.\n */\nexport async function main(\n auth: RT.Servicenow,\n table: DynSelect_table,\n sys_id: string,\n fields: DynMultiselect_fields | undefined,\n display_value: \"false\" | \"true\" | \"all\" | undefined\n) {\n const url = new URL(`${auth.instance_url}/api/now/table/${table}/${sys_id}`)\n if (fields && fields.length > 0) {\n url.searchParams.append(\"sysparm_fields\", fields.join(\",\"))\n }\n if (display_value !== undefined && display_value !== \"\") {\n url.searchParams.append(\"sysparm_display_value\", display_value)\n }\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"display_value": {
"default": null,
"description": "",
"enum": [
"false",
"true",
"all"
],
"originalType": "enum",
"type": "string"
},
"fields": {
"default": null,
"description": "",
"format": "dynmultiselect-fields",
"type": "object"
},
"sys_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"table": {
"default": null,
"description": "",
"format": "dynselect-table",
"type": "object"
}
},
"required": [
"auth",
"table",
"sys_id"
],
"type": "object"
}
},
{
"version_id": 28673,
"app": "servicenow",
"summary": "Insert Into Import Set",
"description": "Push a row into an import set staging table; the transform map runs synchronously. Inspect result[].status.",
"terms": "servicenow insert into import set push a row into an import set staging table; the transform map runs synchronously. inspect result[].status.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n/**\n * Insert Into Import Set\n * Push a row into an import set staging table; the associated transform map runs synchronously. Inspect result[].status (inserted/updated/ignored/error) \u2014 a transform error returns 200/201 with status \"error\".\n */\nexport async function main(\n auth: RT.Servicenow,\n staging_table: string,\n body: { [key: string]: any }\n) {\n const url = new URL(`${auth.instance_url}/api/now/import/${staging_table}`)\n\n const response = await fetch(url, {\n method: \"POST\",\n headers: {\n Authorization: authHeader(auth),\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify(body),\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"body": {
"default": null,
"description": "",
"properties": {},
"type": "object"
},
"staging_table": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth",
"staging_table",
"body"
],
"type": "object"
}
},
{
"version_id": 28674,
"app": "servicenow",
"summary": "List Attachments",
"description": "List attachment metadata, optionally filtered with an encoded query (e.g. table_name=incident^table_sys_id=<id>).",
"terms": "servicenow list attachments list attachment metadata, optionally filtered with an encoded query (e.g. table_name=incident^table_sys_id=<id>).",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n/**\n * List Attachments\n * List attachment metadata (sys_id, file_name, content_type, size, download_link). Filter with an encoded query, e.g. table_name=incident^table_sys_id=<record sys_id>.\n */\nexport async function main(\n auth: RT.Servicenow,\n query: string | undefined,\n limit: number | undefined\n) {\n const url = new URL(`${auth.instance_url}/api/now/attachment`)\n if (query !== undefined && query !== \"\") {\n url.searchParams.append(\"sysparm_query\", query)\n }\n url.searchParams.append(\"sysparm_limit\", String(limit ?? 100))\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"limit": {
"default": null,
"description": "",
"type": "number"
},
"query": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth"
],
"type": "object"
}
},
{
"version_id": 28675,
"app": "servicenow",
"summary": "List Records",
"description": "Query records from any table with an encoded query (sysparm_query), pagination, column selection, and raw/display values.",
"terms": "servicenow list records query records from any table with an encoded query (sysparm_query), pagination, column selection, and raw/display values.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nexport type DynSelect_table = string\nexport type DynMultiselect_fields = string[]\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n// Dropdown of the instance's tables (sys_db_object).\nexport async function table(auth: RT.Servicenow) {\n const response = await fetch(\n `${auth.instance_url}/api/now/table/sys_db_object?sysparm_fields=name,label&sysparm_limit=10000`,\n {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n const { result } = (await response.json()) as {\n result: { name: string; label: string }[]\n }\n return result\n .filter((t) => t.name)\n .map((t) => ({ value: t.name, label: `${t.label} (${t.name})` }))\n .sort((a, b) => a.label.localeCompare(b.label))\n}\n\n// Walk the table's inheritance chain (sys_db_object.super_class) so the column\n// list includes fields inherited from parent tables, not just fields defined on\n// the chosen table (sys_dictionary stores each field under its defining table \u2014\n// e.g. incident extends task, so short_description/priority live on task).\n// Best-effort: any failure falls back to just the table itself.\nasync function tableChain(auth: RT.Servicenow, table: string) {\n const chain: string[] = []\n let current: string | undefined = table\n while (current && !chain.includes(current)) {\n chain.push(current)\n const response = await fetch(\n `${auth.instance_url}/api/now/table/sys_db_object?sysparm_query=name=${current}&sysparm_fields=super_class.name&sysparm_limit=1`,\n {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!response.ok) break\n const { result } = (await response.json()) as {\n result: { \"super_class.name\"?: string }[]\n }\n current = result[0]?.[\"super_class.name\"] || undefined\n }\n return chain\n}\n\n// Dependent multi-select: the chosen table's columns (sys_dictionary), including\n// columns inherited from parent tables.\nexport async function fields(auth: RT.Servicenow, table: DynSelect_table) {\n if (!table) return []\n const chain = await tableChain(auth, table)\n const url = new URL(`${auth.instance_url}/api/now/table/sys_dictionary`)\n url.searchParams.append(\n \"sysparm_query\",\n `nameIN${chain.join(\",\")}^elementISNOTEMPTY`\n )\n url.searchParams.append(\"sysparm_fields\", \"element,column_label\")\n url.searchParams.append(\"sysparm_limit\", \"10000\")\n\n const response = await fetch(url, {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n const { result } = (await response.json()) as {\n result: { element: string; column_label: string }[]\n }\n return result\n .filter((f) => f.element)\n .map((f) => ({\n value: f.element,\n label: `${f.column_label} (${f.element})`,\n }))\n .sort((a, b) => a.label.localeCompare(b.label))\n}\n\n/**\n * List Records\n * Query records from any table with an encoded query (sysparm_query), pagination, column selection, and raw/display values. Returns one page; iterate offset until fewer than limit rows return.\n */\nexport async function main(\n auth: RT.Servicenow,\n table: DynSelect_table,\n query: string | undefined,\n limit: number | undefined,\n offset: number | undefined,\n fields: DynMultiselect_fields | undefined,\n display_value: \"false\" | \"true\" | \"all\" | undefined\n) {\n const url = new URL(`${auth.instance_url}/api/now/table/${table}`)\n if (query !== undefined && query !== \"\") {\n url.searchParams.append(\"sysparm_query\", query)\n }\n url.searchParams.append(\"sysparm_limit\", String(limit ?? 100))\n if (offset !== undefined) {\n url.searchParams.append(\"sysparm_offset\", String(offset))\n }\n if (fields && fields.length > 0) {\n url.searchParams.append(\"sysparm_fields\", fields.join(\",\"))\n }\n if (display_value !== undefined && display_value !== \"\") {\n url.searchParams.append(\"sysparm_display_value\", display_value)\n }\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"display_value": {
"default": null,
"description": "",
"enum": [
"false",
"true",
"all"
],
"originalType": "enum",
"type": "string"
},
"fields": {
"default": null,
"description": "",
"format": "dynmultiselect-fields",
"type": "object"
},
"limit": {
"default": null,
"description": "",
"type": "number"
},
"offset": {
"default": null,
"description": "",
"type": "number"
},
"query": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"table": {
"default": null,
"description": "",
"format": "dynselect-table",
"type": "object"
}
},
"required": [
"auth",
"table"
],
"type": "object"
}
},
{
"version_id": 28676,
"app": "servicenow",
"summary": "List Tables",
"description": "List the instance tables (sys_db_object) with API name, label, and sys_id. Optionally filter by a name/label substring.",
"terms": "servicenow list tables list the instance tables (sys_db_object) with api name, label, and sys_id. optionally filter by a name/label substring.",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n/**\n * List Tables\n * List the instance's tables (sys_db_object) with their API name, label, and sys_id. Optionally filter by a substring matched against name or label.\n */\nexport async function main(\n auth: RT.Servicenow,\n name_filter: string | undefined\n) {\n const url = new URL(`${auth.instance_url}/api/now/table/sys_db_object`)\n url.searchParams.append(\"sysparm_fields\", \"name,label,sys_id\")\n if (name_filter !== undefined && name_filter !== \"\") {\n url.searchParams.append(\n \"sysparm_query\",\n `nameLIKE${name_filter}^ORlabelLIKE${name_filter}`\n )\n }\n url.searchParams.append(\"sysparm_limit\", \"10000\")\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"name_filter": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
}
},
"required": [
"auth"
],
"type": "object"
}
},
{
"version_id": 28677,
"app": "servicenow",
"summary": "Update Record",
"description": "Update a record by sys_id with a column/value map. PATCH (partial) by default; set replace=true to PUT (full overwrite).",
"terms": "servicenow update record update a record by sys_id with a column/value map. patch (partial) by default; set replace=true to put (full overwrite).",
"kind": "script",
"language": "bunnative",
"content": "//native\n\nexport type DynSelect_table = string\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n// Dropdown of the instance's tables (sys_db_object).\nexport async function table(auth: RT.Servicenow) {\n const response = await fetch(\n `${auth.instance_url}/api/now/table/sys_db_object?sysparm_fields=name,label&sysparm_limit=10000`,\n {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n const { result } = (await response.json()) as {\n result: { name: string; label: string }[]\n }\n return result\n .filter((t) => t.name)\n .map((t) => ({ value: t.name, label: `${t.label} (${t.name})` }))\n .sort((a, b) => a.label.localeCompare(b.label))\n}\n\n/**\n * Update Record\n * Update a record by sys_id with a column/value map. PATCH (partial update) by default; set replace=true to PUT (full overwrite that clears omitted fields).\n */\nexport async function main(\n auth: RT.Servicenow,\n table: DynSelect_table,\n sys_id: string,\n body: { [key: string]: any },\n replace: boolean | undefined\n) {\n const url = new URL(`${auth.instance_url}/api/now/table/${table}/${sys_id}`)\n\n const response = await fetch(url, {\n method: replace ? \"PUT\" : \"PATCH\",\n headers: {\n Authorization: authHeader(auth),\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify(body),\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n return await response.json()\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"body": {
"default": null,
"description": "",
"properties": {},
"type": "object"
},
"replace": {
"default": null,
"description": "",
"type": "boolean"
},
"sys_id": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"table": {
"default": null,
"description": "",
"format": "dynselect-table",
"type": "object"
}
},
"required": [
"auth",
"table",
"sys_id",
"body"
],
"type": "object"
}
},
{
"version_id": 28679,
"app": "servicenow",
"summary": "New or Updated Record",
"description": "Emits records whose timestamp is newer than the last poll, tracked via Windmill state. Defaults to sys_updated_on.",
"terms": "servicenow new or updated record emits records whose timestamp is newer than the last poll, tracked via windmill state. defaults to sys_updated_on.",
"kind": "trigger",
"language": "bunnative",
"content": "//native\n\nimport * as wmill from \"windmill-client\"\n\nexport type DynSelect_table = string\n\nfunction authHeader(auth: RT.Servicenow) {\n return auth.token\n ? `Bearer ${auth.token}`\n : `Basic ${btoa(`${auth.username}:${auth.password}`)}`\n}\n\n// Dropdown of the instance's tables (sys_db_object).\nexport async function table(auth: RT.Servicenow) {\n const response = await fetch(\n `${auth.instance_url}/api/now/table/sys_db_object?sysparm_fields=name,label&sysparm_limit=10000`,\n {\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n }\n )\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n const { result } = (await response.json()) as {\n result: { name: string; label: string }[]\n }\n return result\n .filter((t) => t.name)\n .map((t) => ({ value: t.name, label: `${t.label} (${t.name})` }))\n .sort((a, b) => a.label.localeCompare(b.label))\n}\n\n/**\n * New or Updated Record\n * Emits records from a table whose timestamp is newer than the last poll, tracked via Windmill state. Defaults to sys_updated_on (set timestamp_field to sys_created_on for new-only). Optionally narrow with an extra encoded query.\n */\nexport async function main(\n auth: RT.Servicenow,\n table: DynSelect_table,\n timestamp_field: \"sys_updated_on\" | \"sys_created_on\" | undefined,\n query: string | undefined\n) {\n const tsField =\n timestamp_field && timestamp_field !== \"\"\n ? timestamp_field\n : \"sys_updated_on\"\n const lastChecked: string | undefined = await wmill.getState()\n\n // ServiceNow datetimes are 'YYYY-MM-DD HH:MM:SS' in UTC (no trailing Z).\n const nowUtc = new Date().toISOString().slice(0, 19).replace(\"T\", \" \")\n\n // First run: set the watermark to now and don't emit a backlog.\n if (!lastChecked) {\n await wmill.setState(nowUtc)\n return []\n }\n\n let encoded = `${tsField}>${lastChecked}`\n if (query !== undefined && query !== \"\") {\n encoded += `^${query}`\n }\n encoded += `^ORDERBY${tsField}`\n\n const url = new URL(`${auth.instance_url}/api/now/table/${table}`)\n url.searchParams.append(\"sysparm_query\", encoded)\n url.searchParams.append(\"sysparm_limit\", \"200\")\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n Authorization: authHeader(auth),\n Accept: \"application/json\",\n },\n })\n\n if (!response.ok) {\n throw new Error(`${response.status} ${await response.text()}`)\n }\n\n const { result } = (await response.json()) as {\n result: Array<{ [key: string]: any }>\n }\n\n if (result.length > 0) {\n await wmill.setState(result[result.length - 1][tsField])\n }\n\n return result\n}\n",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"auth": {
"default": null,
"description": "",
"format": "resource-servicenow",
"type": "object"
},
"query": {
"default": null,
"description": "",
"originalType": "string",
"type": "string"
},
"table": {
"default": null,
"description": "",
"format": "dynselect-table",
"type": "object"
},
"timestamp_field": {
"default": null,
"description": "",
"enum": [
"sys_updated_on",
"sys_created_on"
],
"originalType": "enum",
"type": "string"
}
},
"required": [
"auth",
"table"
],
"type": "object"
}
}
]
}
ai_evals/package.json
+2 -2
View File
@@ -8,8 +8,8 @@
"test:frontend-graph": "cd ../frontend && node_modules/.bin/vitest run --project server --config ../ai_evals/adapters/frontend/vitest.unit.config.ts"
},
"dependencies": {
"@anthropic-ai/claude-agent-sdk": "^0.2.25",
"@anthropic-ai/sdk": "^0.39.0",
"@anthropic-ai/claude-agent-sdk": "^0.3.284",
"@anthropic-ai/sdk": "^0.129.0",
"commander": "^14.0.3",
"openai": "^6.9.1",
"yaml": "^2.8.3"
backend/.sqlx/query-094a07eaa5714ec739786717f110ea539e84c33de23476e042a4a1a7cb529dac.json
Generated
+34
View File
@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\",\n COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\",\n COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\"\n FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "flows!",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "schedules!",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "triggers!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null,
null,
null
]
},
"hash": "094a07eaa5714ec739786717f110ea539e84c33de23476e042a4a1a7cb529dac"
}
backend/.sqlx/query-298e0e30afdc973c517a9b4ea99f5dd1616a62c6ecbaaea312c686038d21e6fd.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND path = $2 AND hash = $3)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Int8"
]
},
"nullable": [
null
]
},
"hash": "298e0e30afdc973c517a9b4ea99f5dd1616a62c6ecbaaea312c686038d21e6fd"
}
backend/.sqlx/query-3dccc8e745f4a0973541088f66172e74af6828c1ab52cf7a6fd10b305deade85.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "3dccc8e745f4a0973541088f66172e74af6828c1ab52cf7a6fd10b305deade85"
}
backend/.sqlx/query-4bc47050c74a02ab3169c3165898b2af07e995de71564d867b171f97f719fbde.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND path = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "4bc47050c74a02ab3169c3165898b2af07e995de71564d867b171f97f719fbde"
}
backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT bool_and(operator) FROM (\n SELECT operator FROM usr WHERE email = $1 AND is_service_account IS false\n UNION ALL\n SELECT operator FROM workspace_invite WHERE email = $1\n ) t",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "bool_and",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72"
}
backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json
Generated
+35
View File
@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "runnable_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "scope_kind",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "launched_by_app",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
null,
null,
null
]
},
"hash": "7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec"
}
backend/.sqlx/query-84fcddaf5bc61d607a6e6e5e31de7436b203a3baa7ef0509cb8e6c52270ae3a9.json
Generated
-35
View File
@@ -1,35 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT j.runnable_path,\n CASE\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id\n WHERE j.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "runnable_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "scope_kind",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "launched_by_app",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
true,
null,
null
]
},
"hash": "84fcddaf5bc61d607a6e6e5e31de7436b203a3baa7ef0509cb8e6c52270ae3a9"
}
backend/.sqlx/query-4afdc63af51e599b9d6fe6cedcd7980fd761a29df0f2a6820f3c8d7a29c3c20a.json → backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg",
"query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT COALESCE((ws.operator_settings->>'builder_flows')::boolean, false)) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg",
"describe": {
"columns": [
{
@@ -34,5 +34,5 @@
null
]
},
"hash": "4afdc63af51e599b9d6fe6cedcd7980fd761a29df0f2a6820f3c8d7a29c3c20a"
"hash": "cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b"
}
backend/.sqlx/query-f3a740394038ddfced97254d8dcd4de49f63fb6dc4c8aab126dbe7cd611a862c.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value AS \"value: sqlx::types::Json<serde_json::Value>\"\n FROM resource WHERE workspace_id = $1 AND path = $2 AND resource_type = 'ai_agent'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value: sqlx::types::Json<serde_json::Value>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true
]
},
"hash": "f3a740394038ddfced97254d8dcd4de49f63fb6dc4c8aab126dbe7cd611a862c"
}
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
5a2b6b8527250bd12cf856d8a667a9ef3106ec60
40ac1c5f8cbce3843b582d9b392d3f3cc7eca3e6
backend/tests/fixtures/jobs_read_auth.sql
Vendored
+26
View File
@@ -65,6 +65,32 @@ INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, resu
('15151515-1515-1515-1515-151515151515', 'test-workspace', 1000, 'success'::job_status,
'{"wrapped": "WRAPPED_FLOW_RESULT"}');
-- Scoped to an agent. An agent run is a preview of the one-step flow the run endpoint builds
-- (step `__wm_agent_root`), filed under the agent's path, or `<path>.chat` for a chat turn.
INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES (
encode(sha256('RUN_SCOPED_AGENT_TOKEN'::bytea), 'hex'), 'RUN_AGENT', 'RUN_SCOPED_AGENT_TOKEN',
'test2@windmill.dev', 'agent token', false,
ARRAY['jobs:run:agents:f/shared/agent1']
);
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner, raw_flow
) VALUES
('17171717-1717-1717-1717-171717171717', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'flowpreview', 'deno', 'f/shared/agent1', 'flow', true,
'{"modules": [{"id": "__wm_agent_root", "value": {"type": "aiagent", "tools": []}}]}'),
('18181818-1818-1818-1818-181818181818', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'flowpreview', 'deno', 'f/shared/agent1.chat', 'flow', true,
'{"modules": [{"id": "__wm_agent_root", "value": {"type": "aiagent", "tools": []}}]}');
INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES
('17171717-1717-1717-1717-171717171717', 'test-workspace', 1000, 'success'::job_status,
'{"agent": "AGENT_RESULT"}'),
('18181818-1818-1818-1818-181818181818', 'test-workspace', 1000, 'success'::job_status,
'{"agent": "AGENT_CHAT_RESULT"}');
-- A token pairing an app scope with a run scope, as someone driving an app's components
-- programmatically would build. `APP_INLINE_JOB` is an inline-script component run: no
-- `jobs:run` scope can name its kind, so only the `apps:run` half puts it in reach.
backend/tests/inline_preview_auth.rs
+9 -9
View File
@@ -18,7 +18,7 @@
//!
//! This test pins down:
//! - an Operator's own token is rejected by the operator guard (the core fix;
//! pre-fix this reached the inline executor instead of returning 401),
//! pre-fix this reached the inline executor instead of returning 403),
//! - a regular non-operator passes the guard (the fix must not over-block the
//! legitimate inline preview flow): in the test harness the worker inline
//! utils are not registered, so a caller past the guard gets the distinct
@@ -121,10 +121,10 @@ async fn test_inline_preview_authorization(db: Pool<Postgres>) -> anyhow::Result
// 1. CORE REGRESSION: an Operator must be rejected by the operator guard.
// Pre-fix this fell through to the inline executor (arbitrary code
// execution); post-fix it returns 401 with the operator guard message.
// execution); post-fix it returns 403 with the operator guard message.
let (status, body) = post(&url, "OPERATOR_TOKEN", &inline_preview_body()).await;
assert_eq!(
status, 401,
status, 403,
"Operator must be rejected from inline preview (got {status}): {body}"
);
assert!(
@@ -139,7 +139,7 @@ async fn test_inline_preview_authorization(db: Pool<Postgres>) -> anyhow::Result
// the operator guard did not reject it.
let (status, body) = post(&url, "SECRET_TOKEN_2", &inline_preview_body()).await;
assert_ne!(
status, 401,
status, 403,
"non-operator must not be blocked by the operator guard (got {status}): {body}"
);
assert!(
@@ -155,7 +155,7 @@ async fn test_inline_preview_authorization(db: Pool<Postgres>) -> anyhow::Result
operator_job_token(uuid::Uuid::parse_str(RUNNING_JOB_ID).unwrap()).await;
let (status, body) = post(&url, &running_job_token, &datatable_query_body()).await;
assert_ne!(
status, 401,
status, 403,
"operator job token of a running job must pass the guard for a datatable query (got {status}): {body}"
);
assert!(
@@ -188,7 +188,7 @@ async fn test_inline_preview_authorization(db: Pool<Postgres>) -> anyhow::Result
] {
let (status, body) = post(&url, &running_job_token, &payload).await;
assert_eq!(
status, 401,
status, 403,
"operator job token must be rejected for a {label} payload (got {status}): {body}"
);
assert!(
@@ -208,7 +208,7 @@ async fn test_inline_preview_authorization(db: Pool<Postgres>) -> anyhow::Result
let token = operator_job_token(job_id).await;
let (status, body) = post(&url, &token, &datatable_query_body()).await;
assert_eq!(
status, 401,
status, 403,
"operator job token of a {label} job must be rejected (got {status}): {body}"
);
assert!(
@@ -229,7 +229,7 @@ async fn test_inline_preview_authorization(db: Pool<Postgres>) -> anyhow::Result
);
let (status, body) = post(&fallback_url, "OPERATOR_TOKEN", &datatable_query_body()).await;
assert_eq!(
status, 401,
status, 403,
"Operator must be rejected from the preview fallback (got {status}): {body}"
);
assert!(
@@ -246,7 +246,7 @@ async fn test_inline_preview_authorization(db: Pool<Postgres>) -> anyhow::Result
let deferred_url = format!("{fallback_url}?{deferral}");
let (status, body) = post(&deferred_url, &running_job_token, &datatable_query_body()).await;
assert_eq!(
status, 401,
status, 403,
"operator job token must not schedule a deferred preview with {deferral} (got {status}): {body}"
);
assert!(
backend/tests/jobs_read_auth.rs
+20
View File
@@ -456,6 +456,26 @@ async fn test_single_job_read_authorization(db: Pool<Postgres>) -> anyhow::Resul
);
}
// An agent run and a chat turn of it belong to the agent, readable by the token scoped to
// it and by no flow-scoped one.
for (job, expected) in [
("17171717-1717-1717-1717-171717171717", "AGENT_RESULT"),
("18181818-1818-1818-1818-181818181818", "AGENT_CHAT_RESULT"),
] {
let path = format!("completed/get_result/{job}");
let (status, body) = get(&base, &path, Some("RUN_SCOPED_AGENT_TOKEN")).await;
assert!(
status.is_success() && body.contains(expected),
"agent-scoped token must read its agent's run {job} (got {status}): {body}"
);
let (status, body) = get(&base, &path, Some("RUN_SCOPED_TOKEN")).await;
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"a flow-scoped token must not read an agent run {job} (got {status}): {body}"
);
}
// An `apps:run:<app>` scope is a start grant too: the inline-script component run it
// launched — a kind no `jobs:run` scope can name — stays readable to a token scoped
// to that app, and stays out of reach for one that is only scoped to run jobs.
backend/tests/python_jobs.rs
+74
View File
@@ -465,6 +465,37 @@ def main(x: int):
);
}
#[test]
fn test_python_execd_async() {
let script = r#"
import asyncio
_import_loop = asyncio.new_event_loop()
asyncio.set_event_loop(_import_loop)
async def preprocessor(x: int):
return {"x": x * 2}
async def main(x: int):
return {"x": x + 100, "same_loop": asyncio.get_running_loop() is _import_loop}
"#;
let results = run_py_raw_protocol_test(
&[("f/test/async", script)],
vec![
ProtocolCmd::ExecdPreprocess { args: serde_json::json!({"x": 5}) },
ProtocolCmd::Execd { args: serde_json::json!({"x": 7}) },
],
);
assert_eq!(
results,
vec![
DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 10})),
DedicatedWorkerResult::Success(serde_json::json!({"x": 110, "same_loop": true})),
DedicatedWorkerResult::Success(serde_json::json!({"x": 107, "same_loop": true})),
]
);
}
// ==================== Argument Transformation Tests ====================
#[test]
@@ -760,6 +791,49 @@ def main():
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base"))]
async fn test_python_async_main(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let content = r#"
import asyncio
async def main(x: int):
await asyncio.sleep(0)
return x + 1
"#
.to_owned();
let job = JobPayload::Code(RawCode {
hash: None,
content,
path: None,
language: ScriptLang::Python3,
lock: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
modules: None,
tag: None,
});
let result = RunJob::from(job)
.arg("x", serde_json::json!(41))
.run_until_complete(&db, false, port)
.await
.json_result()
.unwrap();
assert_eq!(result, serde_json::json!(42));
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base"))]
async fn test_python_result_preserves_infinity_in_string(db: Pool<Postgres>) -> anyhow::Result<()> {
backend/windmill-ai/src/providers/mod.rs
+164
View File
@@ -14,6 +14,130 @@ use std::time::{Duration, Instant};
/// its `thinking` param, Gemini to a zero budget or the model's floor).
pub(crate) const REASONING_OFF_SENTINEL: &str = "none";
/// The effort to send for a model, dropping the off sentinel on a model that rejects every
/// disable: the model then reasons at its default instead of failing the request. The UI
/// never offers off on these, so this guards an agent step saved before it stopped, or an
/// effort passed in as a flow input.
pub fn effective_reasoning_effort<'a>(model: &str, effort: Option<&'a str>) -> Option<&'a str> {
match effort {
Some(effort)
if effort == REASONING_OFF_SENTINEL
&& reasoning_rule(model).is_some_and(|rule| !rule.can_disable) =>
{
None
}
effort => effort,
}
}
/// Whether a request with function tools must send the off sentinel on Chat Completions:
/// the model refuses tools there while it reasons, and does accept being turned off.
pub(crate) fn completions_tools_need_reasoning_off(model: &str) -> bool {
reasoning_rule(model).is_some_and(|rule| rule.completions_tools_need_off && rule.can_disable)
}
/// What the backend needs to know about a model's reasoning: the rows of `REASONING_RULES`
/// in the frontend's `reasoningRegistry.ts`, cut down to the two facts the wire needs.
/// `reasoningParity.json` next to that file is checked by both sides' tests.
struct ReasoningRule {
matches: fn(&str) -> bool,
/// False when the provider rejects every disable, so the off sentinel must not be sent.
can_disable: bool,
/// Chat Completions refuses function tools while the model reasons, even with the
/// effort omitted (live-verified); the Responses API has no such limit.
completions_tools_need_off: bool,
}
/// Matched in order against the lowercased model id, first match wins. A model no row
/// matches keeps whatever effort it was given.
const REASONING_RULES: &[ReasoningRule] = &[
// Live-verified: Fable, Mythos and the 5.x point releases reject `thinking: disabled`.
ReasoningRule {
matches: |m| m.contains("claude-fable") || m.contains("claude-mythos"),
can_disable: false,
completions_tools_need_off: false,
},
ReasoningRule {
matches: is_claude_5_point_release,
can_disable: false,
completions_tools_need_off: false,
},
// Live-verified: astra takes low..max only, where sol and luna also take `none`.
ReasoningRule {
matches: |m| base_id(m).starts_with("gpt-6-astra"),
can_disable: false,
completions_tools_need_off: true,
},
ReasoningRule {
matches: |m| gpt_version(m).is_some_and(|(major, _)| major >= 6),
can_disable: true,
completions_tools_need_off: true,
},
ReasoningRule {
matches: |m| matches!(gpt_version(m), Some((5, Some(minor))) if minor >= 5),
can_disable: true,
completions_tools_need_off: true,
},
ReasoningRule {
matches: |m| matches!(gpt_version(m), Some((5, Some(_)))),
can_disable: true,
completions_tools_need_off: false,
},
// gpt-5 and the o-series reject `none`.
ReasoningRule {
matches: |m| matches!(gpt_version(m), Some((5, None))),
can_disable: false,
completions_tools_need_off: false,
},
ReasoningRule {
matches: |m| {
let base = base_id(m);
base.starts_with('o') && base[1..].starts_with(|c: char| c.is_ascii_digit())
},
can_disable: false,
completions_tools_need_off: false,
},
];
fn reasoning_rule(model: &str) -> Option<&'static ReasoningRule> {
let model = model.to_lowercase();
REASONING_RULES.iter().find(|rule| (rule.matches)(&model))
}
/// The id after a gateway's `vendor/` and before a `:variant`.
fn base_id(model: &str) -> &str {
let last = model.rsplit('/').next().unwrap_or(model);
last.split(':').next().unwrap_or(last)
}
/// `(major, minor)` of a `gpt-` id. The major is one digit then a separator or the end,
/// since Azure names gpt-3.5 `gpt-35-turbo`.
fn gpt_version(model: &str) -> Option<(u32, Option<u32>)> {
let rest = base_id(model).strip_prefix("gpt-")?;
let mut chars = rest.chars();
let major = chars.next()?.to_digit(10)?;
match chars.next() {
None | Some('-') => Some((major, None)),
Some('.') => {
let minor: String = chars.take_while(char::is_ascii_digit).collect();
Some((major, minor.parse().ok()))
}
_ => None,
}
}
/// Sonnet or Opus 5.x with x >= 1. The version match stops at one digit so a dated id
/// (`claude-sonnet-5-20260101`) stays Sonnet 5.
fn is_claude_5_point_release(model: &str) -> bool {
let model = model.replace('.', "-");
["claude-opus-5-", "claude-sonnet-5-"].iter().any(|prefix| {
model.split(prefix).skip(1).any(|rest| {
let mut chars = rest.chars();
matches!(chars.next(), Some('1'..='9')) && !matches!(chars.next(), Some('0'..='9'))
})
})
}
/// Whether a Claude model removed the sampling params (`temperature`, `top_p`,
/// `top_k`). On these, any value is a hard 400 — `temperature is deprecated for
/// this model` — whatever the thinking mode, so the param has to be dropped on
@@ -119,6 +243,46 @@ pub fn remember_chat_completions_only(base_url: &str, model: &str) {
CHAT_COMPLETIONS_ONLY.insert((base_url.to_string(), model.to_string()), Instant::now());
}
#[cfg(test)]
mod reasoning_rule_tests {
use super::*;
/// The frontend registry's test reads the same file. These rules see the model id
/// alone, so it holds only rows whose answer doesn't depend on the provider.
#[test]
fn agrees_with_the_frontend_registry() {
let rows: Vec<serde_json::Value> = serde_json::from_str(include_str!(
"../../../../frontend/src/lib/components/copilot/reasoningParity.json"
))
.unwrap();
for row in rows {
let model = row["model"].as_str().unwrap();
let can_disable = row["canDisable"].as_bool().unwrap();
let tools_need_off = row["completionsToolsNeedOff"].as_bool().unwrap();
let sent = effective_reasoning_effort(model, Some(REASONING_OFF_SENTINEL));
assert_eq!(sent.is_some(), can_disable, "{model}");
assert_eq!(
effective_reasoning_effort(model, Some("low")),
Some("low"),
"{model}"
);
assert_eq!(
completions_tools_need_reasoning_off(model),
tools_need_off && can_disable,
"{model}"
);
}
}
#[test]
fn reads_ids_the_frontend_resolves_first() {
// Azure's gpt-3.5 is not major 35, and a gateway prefix is not part of the id.
assert_eq!(gpt_version("gpt-35-turbo"), None);
assert!(completions_tools_need_reasoning_off("openai/gpt-6-sol"));
assert_eq!(effective_reasoning_effort("openai/o3", Some("none")), None);
}
}
#[cfg(test)]
mod chat_completions_only_tests {
use super::*;
backend/windmill-ai/src/providers/other.rs
+10
View File
@@ -1,3 +1,4 @@
use super::{completions_tools_need_reasoning_off, REASONING_OFF_SENTINEL};
use crate::{
ai_providers::AIProvider,
image_handler::prepare_messages_for_api,
@@ -143,6 +144,15 @@ impl OtherQueryBuilder {
let (reasoning_effort, thinking, temperature) =
provider_reasoning_fields(&self.provider_kind, args.reasoning_effort, args.temperature);
// With tools, gpt-5.5+ only run here with reasoning off: turn it off where the model
// can, rather than failing every turn.
let reasoning_effort = if args.tools.is_some_and(|tools| !tools.is_empty())
&& completions_tools_need_reasoning_off(args.model)
{
Some(REASONING_OFF_SENTINEL)
} else {
reasoning_effort
};
// Build request with stream_options for usage tracking
let request_with_usage = OpenAICompletionRequest {
backend/windmill-ai/src/types.rs
+4 -1
View File
@@ -294,7 +294,10 @@ impl ProviderWithResource {
/// The reasoning effort to thread to the provider, treating an empty string
/// (e.g. a cleared flow input) as unset.
pub fn get_reasoning_effort(&self) -> Option<&str> {
self.reasoning_effort.as_deref().filter(|s| !s.is_empty())
crate::providers::effective_reasoning_effort(
&self.model,
self.reasoning_effort.as_deref().filter(|s| !s.is_empty()),
)
}
pub async fn get_base_url(&self, db: &DB) -> Result<String, Error> {
backend/windmill-api-auth/src/scopes.rs
+16 -4
View File
@@ -621,12 +621,17 @@ const FLOW_JOBS: [&'static str; 6] = [
"jobs/run_and_stream/f",
];
/// Runs of a saved AI agent: the server builds the run from the stored agent, so the route is
/// scoped by the agent's path like a deployed runnable's, as `jobs:run:agents:<path>`.
const AGENT_JOBS: [&'static str; 1] = ["jobs/run/agent"];
lazy_static::lazy_static! {
static ref RUN_PATH_ACTIONS: Vec<&'static str> = {
let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies", "apps_u/execute_component", "apps_u/upload_s3_file"];
v.extend(SCRIPT_JOBS);
v.extend(FLOW_JOBS);
v.extend(AGENT_JOBS);
v
};
}
@@ -646,13 +651,13 @@ fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction {
}
}
/// Checks the route path to determine the runnable kind (either "flows" or "scripts").
/// Checks the route path to determine the runnable kind ("agents", "flows" or "scripts").
///
/// The order of checks is important:
/// - Flow-related paths are checked first to avoid false positives, as some flow paths
/// (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`).
///
/// Returns `"flows"` or `"scripts"` based on the match, or `None` if no match is found.
/// Returns the kind based on the match, or `None` if no match is found.
fn determine_kind_from_route(route_path: &str) -> Option<String> {
if route_path.starts_with("jobs") {
// Preview/bundle runs execute arbitrary code with no deployed path, so
@@ -666,6 +671,9 @@ fn determine_kind_from_route(route_path: &str) -> Option<String> {
{
return None;
}
if AGENT_JOBS.iter().any(|path| route_path.starts_with(path)) {
return Some("agents".to_string());
}
if FLOW_JOBS.iter().any(|path| route_path.starts_with(path)) {
return Some("flows".to_string());
} else if SCRIPT_JOBS.iter().any(|path| route_path.starts_with(path)) {
@@ -1103,9 +1111,9 @@ fn is_global_read_open_to_job_token(route_path: &str) -> bool {
| "/api/settings/global/hub_base_url"
| "/api/docs/search"
| "/api/docs/page"
| "/api/integrations/hub/list"
| "/api/embeddings/query_hub_scripts"
) || route_path.starts_with("/api/scripts/hub/")
) || route_path.starts_with("/api/integrations/hub/")
|| route_path.starts_with("/api/scripts/hub/")
|| route_path.starts_with("/api/flows/hub/")
|| route_path.starts_with("/api/apps/hub/")
}
@@ -1725,6 +1733,10 @@ mod tests {
scope_for_route("POST", "/api/w/ws/jobs/run/f/u/x/y").as_deref(),
Some("jobs:run:flows")
);
assert_eq!(
scope_for_route("POST", "/api/w/ws/jobs/run/agent/u/x/y").as_deref(),
Some("jobs:run:agents")
);
// Preview/bundle runs have no deployed path and their handlers require the
// broad `jobs:run` scope, so the derived scope must not carry a kind.
backend/windmill-api-embeddings/src/lib.rs
+38
View File
@@ -78,6 +78,8 @@ pub struct HubScriptResult {
id: i64,
version_id: i64,
summary: String,
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
app: String,
kind: String,
score: f32,
@@ -163,6 +165,9 @@ struct HubScript {
id: i64,
version_id: i64,
summary: String,
// Nearly a fifth of hub scripts carry an explicit `"description": null`; a bare
// String here fails the whole blob and takes hub search down with it.
description: Option<String>,
app: String,
kind: String,
embedding: Vec<f32>,
@@ -360,6 +365,10 @@ impl EmbeddingsDb {
let mut hm = HashMap::new();
hm.insert("ask_id".to_string(), script.ask_id.clone().to_string());
hm.insert("summary".to_string(), script.summary.clone());
hm.insert(
"description".to_string(),
script.description.clone().unwrap_or_default(),
);
hm.insert("app".to_string(), script.app.clone());
hm.insert("kind".to_string(), script.kind.clone());
hm.insert("id".to_string(), script.id.clone().to_string());
@@ -517,6 +526,10 @@ impl EmbeddingsDb {
.get("summary")
.ok_or(Error::msg("no summary"))?
.to_owned(),
description: metadata
.get("description")
.filter(|d| !d.is_empty())
.map(|d| d.to_owned()),
app: metadata.get("app").ok_or(Error::msg("no app"))?.to_owned(),
kind: metadata
.get("kind")
@@ -707,6 +720,31 @@ pub fn global_service() -> Router {
mod tests {
use super::trim_to_top_score;
// The blob carries an explicit `"description": null` for roughly a fifth of hub
// scripts, and an older hub omits the key entirely. A bare String here fails the
// whole 155 MB array and takes hub search down with it.
#[test]
fn reads_a_hub_script_whether_or_not_it_has_a_description() {
let present = r#"{"ask_id":1,"id":2,"version_id":3,"summary":"s","description":"d","app":"a","kind":"script","embedding":[]}"#;
let null = r#"{"ask_id":1,"id":2,"version_id":3,"summary":"s","description":null,"app":"a","kind":"script","embedding":[]}"#;
let missing = r#"{"ask_id":1,"id":2,"version_id":3,"summary":"s","app":"a","kind":"script","embedding":[]}"#;
assert_eq!(
serde_json::from_str::<super::HubScript>(present)
.unwrap()
.description,
Some("d".to_string())
);
for without in [null, missing] {
assert_eq!(
serde_json::from_str::<super::HubScript>(without)
.unwrap()
.description,
None
);
}
}
#[test]
fn trims_scores_more_than_5pct_below_top() {
// top=1.0, cutoff at 0.95: 0.96 stays (0.04 drop), 0.93 is the first
backend/windmill-api-flows/src/flows.rs
+150 -28
View File
@@ -16,10 +16,12 @@ use axum::{
};
use windmill_api_auth::{
auth::{list_tokens_internal, TruncatedTokenWithEmail},
build_scope_path_predicate, check_scopes, maybe_refresh_folders, require_owner_of_path,
ApiAuthed,
build_scope_path_predicate, check_scopes, get_scope_tags, maybe_refresh_folders,
require_owner_of_path, ApiAuthed,
};
use windmill_common::workspaces::{
check_deploy_rules, check_operator_can_build_flows, operator_can_build_flows, RuleCheckResult,
};
use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult};
use windmill_common::{
user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay},
utils::HTTP_CLIENT,
@@ -35,7 +37,7 @@ use sqlx::{FromRow, Postgres, Transaction};
use windmill_audit::audit_oss::{audit_log, AuditAuthorable};
use windmill_audit::ActionKind;
use windmill_common::assets::{clear_static_asset_usage, AssetUsageKind};
use windmill_common::flows::FlowModule;
use windmill_common::flows::{FlowModule, FlowValue};
use windmill_common::min_version::{
MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2,
MIN_VERSION_SUPPORTS_NODE_DEBOUNCING,
@@ -241,7 +243,7 @@ async fn list_flows(
// Append the authed user's drafts at paths with no deployed flow; see scripts.rs.
if lq.include_draft_only.unwrap_or(false)
&& !authed.is_operator
&& (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?)
&& offset == 0
&& lq.path_start.is_none()
&& lq.path_exact.is_none()
@@ -570,7 +572,13 @@ async fn list_paths_linking_agent(
Ok(Json(flows))
}
async fn validate_flow(new_flow: &NewFlow) -> error::Result<()> {
async fn validate_flow(
new_flow: &NewFlow,
authed: &ApiAuthed,
db: &DB,
user_db: &UserDB,
w_id: &str,
) -> error::Result<()> {
#[cfg(not(feature = "enterprise"))]
if new_flow.ws_error_handler_muted.is_some_and(|val| val) {
return Err(Error::BadRequest(
@@ -581,9 +589,139 @@ async fn validate_flow(new_flow: &NewFlow) -> error::Result<()> {
guard_flow_from_debounce_data(new_flow).await?;
if authed.is_operator {
validate_operator_flow(
&new_flow.parse_flow_value()?,
&new_flow.tag,
new_flow.schema.as_ref().map(|s| s.0.get()),
authed,
db,
user_db,
w_id,
)
.await?;
}
return Ok(());
}
/// What an operator with builder rights must pass to store a flow, deployed or as a draft: a
/// developer who loads a builder's draft in the editor runs its code as themselves.
pub async fn validate_operator_flow(
value: &FlowValue,
flow_tag: &Option<String>,
schema: Option<&str>,
authed: &ApiAuthed,
db: &DB,
user_db: &UserDB,
w_id: &str,
) -> error::Result<()> {
// Dynamic dropdown code runs as whoever loads the flow's form: it is code like a step's.
if let Some(schema) = schema {
let schema: serde_json::Value = serde_json::from_str(schema)?;
if schema.get("x-windmill-dyn-select-code").is_some() {
return Err(Error::PermissionDenied(
"This flow has dynamic dropdown code, so only a developer can edit it".to_string(),
));
}
}
validate_operator_composed_flow(value, flow_tag, authed, db, user_db, w_id).await
}
/// Runs on every write and every preview of a flow authored by an operator with builder rights.
/// The walk in `check_flow_is_composition_only` only sees the value; what it collects is
/// authorized here against the caller's own permissions.
pub async fn validate_operator_composed_flow(
value: &FlowValue,
flow_tag: &Option<String>,
authed: &ApiAuthed,
db: &DB,
user_db: &UserDB,
w_id: &str,
) -> error::Result<()> {
let mut refs = windmill_common::flows::check_flow_is_composition_only(value)?;
// A tag is how a step picks the worker group it runs on: unauthorized, a builder could route
// a job onto a privileged one.
refs.tags.extend(flow_tag.clone().filter(|t| !t.is_empty()));
if !refs.tags.is_empty() {
// Job-aware: a WM_TOKEN running as a superadmin must not unlock restricted tags.
let is_super_admin = windmill_api_auth::is_super_admin_authed(db, authed).await?;
for tag in &refs.tags {
windmill_common::jobs::check_tag_available_for_workspace_internal(
db,
w_id,
tag,
None,
std::future::ready(w_id.to_string()),
is_super_admin,
get_scope_tags(authed),
)
.await?;
}
}
if refs.runnables.is_empty() && refs.pinned_scripts.is_empty() {
return Ok(());
}
// A flow can step through the same script thirty times; this runs on every write, preview and
// dependency job.
refs.runnables.sort();
refs.runnables.dedup();
refs.pinned_scripts
.sort_by_key(|(path, hash)| (path.clone(), hash.0));
refs.pinned_scripts
.dedup_by_key(|(path, hash)| (path.clone(), hash.0));
// The worker resolves a step's path with the root DB handle and runs it as that runnable's
// `on_behalf_of`, so composing an unreadable path would run code the builder cannot see. RLS
// on this transaction is the check.
let mut tx = user_db.clone().begin(authed).await?;
for (is_flow, path) in &refs.runnables {
let readable = if *is_flow {
sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2)",
w_id,
path,
)
} else {
sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND path = $2)",
w_id,
path,
)
}
.fetch_one(&mut *tx)
.await?
.unwrap_or(false);
if !readable {
return Err(Error::PermissionDenied(format!(
"{} {path} does not exist or is not readable by you",
if *is_flow { "Flow" } else { "Script" }
)));
}
}
// A pinned step is dispatched by its hash alone, ignoring the path beside it, so a readable
// path paired with another script's hash would still run that other script.
for (path, hash) in &refs.pinned_scripts {
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND path = $2 AND hash = $3)",
w_id,
path,
hash.0,
)
.fetch_one(&mut *tx)
.await?
.unwrap_or(false);
if !exists {
return Err(Error::PermissionDenied(format!(
"Version {hash} is not a readable version of {path}"
)));
}
}
tx.commit().await?;
Ok(())
}
async fn create_flow(
authed: ApiAuthed,
Extension(db): Extension<DB>,
@@ -592,11 +730,7 @@ async fn create_flow(
Path(w_id): Path<String>,
Json(mut nf): Json<NewFlow>,
) -> Result<(StatusCode, String)> {
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot create flows for security reasons".to_string(),
));
}
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "create flows").await?;
check_scopes(&authed, || format!("flows:write:{}", nf.path))?;
// A `<= 0` flow timeout is "unset", not a 0-second limit that kills every run instantly.
@@ -616,7 +750,7 @@ async fn create_flow(
return Err(Error::PermissionDenied(msg));
}
validate_flow(&nf).await?;
validate_flow(&nf, &authed, &db, &user_db, &w_id).await?;
if *CLOUD_HOSTED {
let nb_flows =
sqlx::query_scalar!("SELECT COUNT(*) FROM flow WHERE workspace_id = $1", &w_id)
@@ -1187,11 +1321,7 @@ async fn update_flow(
Path((w_id, flow_path)): Path<(String, StripPath)>,
Json(ef): Json<EditFlow>,
) -> Result<String> {
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot update flows for security reasons".to_string(),
));
}
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "update flows").await?;
let flow_path = flow_path.to_path();
// The URL identifies the flow being updated; the body path is only needed to rename.
let mut nf = ef.into_new_flow(flow_path);
@@ -1218,7 +1348,7 @@ async fn update_flow(
return Err(Error::PermissionDenied(msg));
}
validate_flow(&nf).await?;
validate_flow(&nf, &authed, &db, &user_db, &w_id).await?;
let authed = maybe_refresh_folders(&flow_path, &w_id, authed, &db).await;
let mut tx = user_db.clone().begin(&authed).await?;
@@ -1855,11 +1985,7 @@ async fn archive_flow_by_path(
Path((w_id, path)): Path<(String, StripPath)>,
Json(archived): Json<Archived>,
) -> Result<String> {
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot archive flows for security reasons".to_string(),
));
}
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "archive flows").await?;
let path = path.to_path();
check_scopes(&authed, || format!("flows:write:{}", path))?;
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
@@ -2000,11 +2126,7 @@ async fn delete_flow_by_path(
Path((w_id, path)): Path<(String, StripPath)>,
Query(query): Query<DeleteFlowQuery>,
) -> Result<String> {
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot delete flows for security reasons".to_string(),
));
}
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "delete flows").await?;
let path = path.to_path();
check_scopes(&authed, || format!("flows:write:{}", path))?;
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
backend/windmill-api-integration-tests/tests/agent_runs.rs
+60
View File
@@ -0,0 +1,60 @@
use serde_json::json;
use sqlx::{Pool, Postgres};
use windmill_test_utils::*;
const AGENT: &str = "u/test-user-3/agent";
async fn run(base_url: &str, token: &str) -> anyhow::Result<(u16, String)> {
let resp = reqwest::Client::new()
.post(format!(
"{base_url}/w/test-workspace/jobs/run/agent/{AGENT}"
))
.header("Authorization", format!("Bearer {token}"))
.json(&json!({ "user_message": "hi" }))
.send()
.await?;
Ok((resp.status().as_u16(), resp.text().await?))
}
/// Reading an agent is what allows running it, and the run is the caller's own: `test-user-2`, a
/// plain member, runs `test-user-3`'s agent once it may read it, lending nobody's permissions.
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_agent_run_needs_read_and_runs_as_caller(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let base_url = format!("http://localhost:{}/api", server.addr.port());
sqlx::query(
"INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by)
VALUES ('test-workspace', $1, $2, 'ai_agent', '{}', 'test-user-3')",
)
.bind(AGENT)
.bind(json!({ "system_prompt": "hi" }))
.execute(&db)
.await?;
let (status, body) = run(&base_url, "SECRET_TOKEN_2").await?;
assert_eq!(status, 404, "an unreadable agent: {body}");
sqlx::query(
"UPDATE resource SET extra_perms = '{\"u/test-user-2\": false}'
WHERE workspace_id = 'test-workspace' AND path = $1",
)
.bind(AGENT)
.execute(&db)
.await?;
let (status, body) = run(&base_url, "SECRET_TOKEN_2").await?;
assert_eq!(status, 201, "a readable agent: {body}");
let (permissioned_as, created_by) = sqlx::query_as::<_, (String, String)>(
"SELECT permissioned_as, created_by FROM v2_job WHERE id = $1::uuid",
)
.bind(&body)
.fetch_one(&db)
.await?;
assert_eq!(permissioned_as, "u/test-user-2");
assert_eq!(created_by, "test-user-2");
Ok(())
}
backend/windmill-api-integration-tests/tests/drafts.rs
+1 -1
View File
@@ -165,7 +165,7 @@ async fn test_update_draft_write_authorization(db: Pool<Postgres>) -> anyhow::Re
.json(&json!({ "value": { "a": 1 } }))
.send()
.await?;
assert_eq!(r.status(), 401, "operator rejected");
assert_eq!(r.status(), 403, "operator rejected");
Ok(())
}
backend/windmill-api-integration-tests/tests/operator_builder_flows.rs
+317
View File
@@ -0,0 +1,317 @@
use serde_json::json;
use sqlx::{Pool, Postgres};
use windmill_common::workspaces::invalidate_operator_rights_cache;
use windmill_test_utils::*;
const WS: &str = "test-workspace";
fn operator_client() -> reqwest::Client {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
reqwest::header::AUTHORIZATION,
reqwest::header::HeaderValue::from_str("Bearer OPERATOR_TOKEN_1").unwrap(),
);
reqwest::ClientBuilder::new()
.default_headers(headers)
.build()
.unwrap()
}
async fn set_builder(db: &Pool<Postgres>, flows: bool) -> anyhow::Result<()> {
sqlx::query(
"UPDATE workspace_settings SET operator_settings = $1::text::jsonb WHERE workspace_id = $2",
)
.bind(format!(r#"{{"builder_flows": {flows}}}"#))
.bind(WS)
.execute(db)
.await?;
// The right is read through a process-global 60s cache keyed by workspace id.
invalidate_operator_rights_cache(WS);
Ok(())
}
async fn add_script(db: &Pool<Postgres>, hash: i64, path: &str, owner: &str) -> anyhow::Result<()> {
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema,
summary, description, lock, extra_perms)
VALUES ($1, $2, $3, 'x', 'bun', 'script', $4, '{}', '', '', '', '{}')",
)
.bind(WS)
.bind(hash)
.bind(path)
.bind(owner)
.execute(db)
.await?;
Ok(())
}
fn composition_flow_at(path: &str, step_path: &str) -> serde_json::Value {
json!({
"path": path,
"summary": "",
"description": "",
"schema": {},
"value": {"modules": [{
"id": "a",
"value": {"type": "script", "path": step_path, "input_transforms": {}}
}]}
})
}
fn inline_code_flow(path: &str) -> serde_json::Value {
json!({
"path": path,
"summary": "",
"description": "",
"schema": {},
"value": {"modules": [{
"id": "a",
"value": {
"type": "rawscript",
"content": "export async function main() { return 1 }",
"language": "bun",
"input_transforms": {}
}
}]}
})
}
/// The whole boundary in one pass: the builder right lets an operator compose runnables that are
/// already deployed and nothing more, and the endpoints that author code stay shut whether or not
/// it is granted.
#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))]
async fn test_operator_builder_flows_boundary(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let api = format!("http://localhost:{port}/api/w/{WS}");
let c = operator_client();
// A composition-only flow references a runnable that exists and the builder can read, so the
// fixture needs one.
add_script(&db, 4241, "u/operator/some_script", "operator").await?;
set_builder(&db, false).await?;
let resp = c
.post(format!("{api}/flows/create"))
.json(&composition_flow_at(
"u/operator/f1",
"u/operator/some_script",
))
.send()
.await?;
assert_eq!(
resp.status(),
403,
"an operator without the builder right must not create a flow"
);
set_builder(&db, true).await?;
// A payload that omits the key, like a git-sync file that predates it, keeps the right.
let resp = reqwest::Client::new()
.post(format!("{api}/workspaces/operator_settings"))
.header("Authorization", "Bearer SECRET_TOKEN")
.json(&json!({"runs": true}))
.send()
.await?;
assert_eq!(resp.status(), 200, "{}", resp.text().await?);
let resp = c
.post(format!("{api}/flows/create"))
.json(&composition_flow_at(
"u/operator/f1",
"u/operator/some_script",
))
.send()
.await?;
assert!(
resp.status().is_success(),
"a builder must be able to create a composition-only flow: {}",
resp.text().await?
);
let mut tagged = composition_flow_at("u/operator/f5", "u/operator/some_script");
tagged["tag"] = json!("privileged_group");
let resp = c
.post(format!("{api}/flows/create"))
.json(&tagged)
.send()
.await?;
assert_eq!(
resp.status(),
400,
"a builder must not route a flow onto a worker tag the workspace cannot use"
);
let mut with_dyn_code = composition_flow_at("u/operator/f1", "u/operator/some_script");
with_dyn_code["schema"] =
json!({"x-windmill-dyn-select-code": "x", "x-windmill-dyn-select-lang": "bun"});
let resp = c
.post(format!("{api}/flows/update/u/operator/f1"))
.json(&with_dyn_code)
.send()
.await?;
assert_eq!(
resp.status(),
403,
"a builder must not save a flow carrying dropdown code"
);
let resp = c
.post(format!("{api}/jobs/run/dynamic_select"))
.json(
&json!({"entrypoint_function": "f", "runnable_ref": {"source": "inline", "code": "x"}}),
)
.send()
.await?;
assert_eq!(
resp.status(),
403,
"an operator's inline dropdown refusal must not be a 401"
);
// A dependency job rewrites bookkeeping stored under the path it names, so a builder must not
// aim one at a path it cannot write.
let resp = c
.post(format!("{api}/jobs/run/flow_dependencies"))
.json(&json!({"path": "u/alice/private_flow", "flow_value": {"modules": []}}))
.send()
.await?;
assert_eq!(
resp.status(),
403,
"a builder must not run a dependency job on a path it cannot write"
);
// A legacy draft has no owner, so it lists as the builder's own, and a script draft is one
// the builder cannot write: that row must read as not writable, not fail the whole list.
sqlx::query(
"INSERT INTO draft (workspace_id, path, typ, value) VALUES ($1, 'u/operator/some_script', 'script', '{}')",
)
.bind(WS)
.execute(&db)
.await?;
let resp = c.get(format!("{api}/drafts/list")).send().await?;
assert_eq!(resp.status(), 200, "{}", resp.text().await?);
let drafts: serde_json::Value = resp.json().await?;
assert_eq!(drafts[0]["can_write"], false, "{drafts}");
let resp = c
.post(format!("{api}/flows/create"))
.json(&inline_code_flow("u/operator/f2"))
.send()
.await?;
assert_eq!(
resp.status(),
403,
"a builder must not deploy a flow carrying inline code"
);
// Draft storage strips a NUL, turning these keys into the plain ones the check reads.
let mut nul_value = inline_code_flow("u/operator/f1");
let v = nul_value.as_object_mut().unwrap().remove("value").unwrap();
nul_value["value\0"] = v;
let mut nul_dyn = composition_flow_at("u/operator/f1", "u/operator/some_script");
nul_dyn["schema"] = json!({"x-windmill-dyn-select-code\0": "x"});
for (draft, expected) in [
(
composition_flow_at("u/operator/f1", "u/operator/some_script"),
200,
),
(inline_code_flow("u/operator/f1"), 403),
(with_dyn_code.clone(), 403),
(nul_value, 403),
(nul_dyn, 403),
] {
let resp = c
.post(format!("{api}/drafts/update/flow/u/operator/f1"))
.json(&json!({ "value": draft, "force": true }))
.send()
.await?;
let status = resp.status();
assert_eq!(
status,
expected,
"flow draft {draft}: {}",
resp.text().await?
);
}
// Same for the preview path, which runs a request-supplied flow value rather than a stored one.
let resp = c
.post(format!("{api}/jobs/run/preview_flow"))
.json(&json!({"value": inline_code_flow("u/operator/f2")["value"], "args": {}}))
.send()
.await?;
assert_eq!(
resp.status(),
403,
"a builder must not preview a flow carrying inline code"
);
// Authoring code directly stays shut with the right granted.
let resp = c
.post(format!("{api}/scripts/create"))
.json(&json!({
"path": "u/operator/s1",
"summary": "",
"description": "",
"content": "export async function main() { return 1 }",
"language": "bun",
"is_template": false
}))
.send()
.await?;
assert!(
!resp.status().is_success(),
"a builder must not create a script"
);
// `permissions_test` gives the operator fixture no rights on `u/alice/**`.
add_script(&db, 4243, "u/alice/private", "alice").await?;
let resp = c
.post(format!("{api}/flows/create"))
.json(&composition_flow_at("u/operator/f4", "u/alice/private"))
.send()
.await?;
assert_eq!(
resp.status(),
403,
"a builder must not compose a runnable it cannot read"
);
add_script(&db, 4242, "u/operator/pinned", "operator").await?;
let pinned = |hash: &str| {
json!({
"path": "u/operator/f3", "summary": "", "description": "", "schema": {},
"value": {"modules": [{
"id": "a",
"value": {
"type": "script", "path": "u/operator/pinned", "hash": hash,
"input_transforms": {}
}
}]}
})
};
let resp = c
.post(format!("{api}/flows/create"))
.json(&pinned("0000000000000000"))
.send()
.await?;
assert_eq!(
resp.status(),
403,
"a builder must not pin a hash that is not a version of the step's path"
);
let resp = c
.post(format!("{api}/flows/create"))
.json(&pinned("0000000000001092"))
.send()
.await?;
assert!(
resp.status().is_success(),
"a builder must be able to pin the real version of a readable script: {}",
resp.text().await?
);
Ok(())
}
backend/windmill-api-workspaces/src/workspaces.rs
+35 -10
View File
@@ -3471,7 +3471,6 @@ fn apply_pg_tls_env(
Ok(None)
}
#[cfg(test)]
mod pg_tls_env_tests {
use super::apply_pg_tls_env;
@@ -4242,7 +4241,9 @@ async fn edit_datatable_config(
// the other managed cluster, one whose role ids name nothing in that cluster's catalog.
// Refuse it instead: turning roles off first is one step, and it keeps discarding an access
// decision something somebody chose rather than a side effect of moving a database.
let old_kind = old.and_then(|old| old.database.as_ref()).map(|d| d.resource_type);
let old_kind = old
.and_then(|old| old.database.as_ref())
.map(|d| d.resource_type);
if dt.permissions.is_some()
&& dt
.database
@@ -10871,8 +10872,12 @@ async fn invite_user(
nu.email = nu.email.to_lowercase();
#[cfg(feature = "enterprise")]
if let Some(msg) =
windmill_common::ee_oss::check_seat_cap_for_new_user(&db, &nu.email, nu.operator).await?
if let Some(msg) = windmill_common::ee_oss::check_seat_cap_for_new_user(
&db,
&nu.email,
windmill_common::workspaces::consumes_operator_seat(&db, &w_id, nu.operator).await?,
)
.await?
{
return Err(Error::BadRequest(msg));
}
@@ -11025,8 +11030,12 @@ async fn add_user(
};
#[cfg(feature = "enterprise")]
if let Some(msg) =
windmill_common::ee_oss::check_seat_cap_for_new_user(&db, &nu.email, nu.operator).await?
if let Some(msg) = windmill_common::ee_oss::check_seat_cap_for_new_user(
&db,
&nu.email,
windmill_common::workspaces::consumes_operator_seat(&db, &w_id, nu.operator).await?,
)
.await?
{
return Err(Error::BadRequest(msg));
}
@@ -11589,10 +11598,15 @@ struct ChangeOperatorSettings {
folders: bool,
#[serde(default)]
workers: bool,
/// Writes operators may perform unless withdrawn, so `None` (key absent) must mean "leave as
/// stored" rather than a value: the row is merged, not overwritten, and this endpoint takes
/// whole-object payloads from git-sync files that predate the key. Defaulting either way here
/// would make an older file silently withdraw or restore the right on every pull.
/// Write rights, so `None` (key absent) must mean "leave as stored" rather than a value: the
/// row is merged, not overwritten, and this endpoint takes whole-object payloads from git-sync
/// files that predate the key. Defaulting either way here would make an older file silently
/// withdraw or grant the right on every push.
///
/// `builder_flows` lets every operator of this workspace compose flows out of already-deployed
/// runnables, and makes each of them consume a full author seat instead of half of one.
#[serde(default, skip_serializing_if = "Option::is_none")]
builder_flows: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
manage_schedules: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -11607,6 +11621,17 @@ async fn update_operator_settings(
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
// Every operator of the workspace turns into a full seat, which an offline license may not
// cover. It is a no-op delta when the right is already on.
#[cfg(feature = "enterprise")]
if settings.builder_flows == Some(true) {
if let Some(msg) =
windmill_common::ee_oss::check_seat_cap_for_operator_builder(&db, &w_id).await?
{
return Err(Error::BadRequest(msg));
}
}
let mut tx = db.begin().await?;
let settings_json = serde_json::json!(settings);
backend/windmill-api/openapi.yaml
+226
View File
@@ -9909,6 +9909,13 @@ paths:
properties:
name:
type: string
documented:
description: >-
whether the integration carries provider knowledge checked
against the live API, on top of the resource type and example
scripts /integrations/hub/{app}/meta returns for any
integration. Absent on a hub predating the flag.
type: boolean
picks:
description: how often the integration has been picked, absent on a hub that does not count picks
type: integer
@@ -9919,6 +9926,162 @@ paths:
required:
- name
/integrations/hub/{app}/meta:
get:
summary: get hub integration metadata
operationId: getHubIntegrationMeta
tags:
- integration
parameters:
- name: app
description: integration slug
in: path
required: true
schema:
type: string
responses:
"200":
description: integration metadata
content:
application/json:
schema:
type: object
properties:
app:
type: string
display_name:
type: string
description:
type: string
nullable: true
docs_url:
type: string
nullable: true
curated:
description: >-
whether the script set was pruned to idiomatic actions rather
than generated one per endpoint; null when nobody has assessed
it, which is distinct from a stated false
type: boolean
nullable: true
metadata_source:
description: >-
whether the provider knowledge was authored (curated) or
inferred from the shipped scripts (derived)
type: string
enum:
- curated
- derived
meta:
description: >-
the integration's authored meta.json verbatim; the content
repo owns its schema, so it is passed through unvalidated
type: object
nullable: true
additionalProperties: true
meta_updated_at:
type: string
nullable: true
derived:
description: facts computed from the integration's shipped scripts
type: object
properties:
api_hosts:
type: array
items:
type: object
properties:
host:
type: string
count:
type: number
required:
- host
- count
style:
type: string
enum:
- fetch
- sdk
- mixed
- unknown
languages:
type: object
additionalProperties:
type: number
script_counts:
type: object
properties:
total:
type: number
by_kind:
type: object
additionalProperties:
type: number
required:
- total
- by_kind
top_scripts:
type: array
items:
type: object
properties:
path:
type: string
ask_id:
type: number
version_id:
type: number
summary:
type: string
description:
type: string
nullable: true
kind:
type: string
language:
type: string
nullable: true
views:
type: number
votes:
type: number
required:
- path
- summary
- kind
required:
- api_hosts
- style
- languages
- script_counts
- top_scripts
resource_types:
type: array
items:
type: object
properties:
id:
type: number
name:
type: string
description:
type: string
nullable: true
schema:
description: >-
the resource type's JSON schema; the hub stores it as text
and returns it as a JSON string, so a client must be ready
for either that or an object
required:
- name
- schema
# `derived` and `resource_types` are omitted by a hub older than the
# client, which reads every section as optional.
required:
- app
- display_name
/flows/hub/list:
get:
summary: list all hub flows
@@ -10247,6 +10410,9 @@ paths:
type: number
summary:
type: string
description:
type: string
nullable: true
app:
type: string
version_id:
@@ -10316,6 +10482,8 @@ paths:
type: number
summary:
type: string
description:
type: string
app:
type: string
kind:
@@ -16180,6 +16348,47 @@ paths:
a message. The body is JSON: `{ "error": string, "running_turn": { "job_id",
"user_seq" } }`.
/w/{workspace}/jobs/run/agent/{path}:
post:
summary: run a saved AI agent
description: |
Runs the `ai_agent` resource at `path` as stored, for any caller who can read it
(operators included), as the caller, who needs access to the agent's AI resource and to
whatever its tools use. With `memory_id`, the run is a turn
of the caller's conversation with the agent; without, a single run.
operationId: runAgent
tags:
- job
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
- name: memory_id
description: The conversation this turn belongs to. A uuid is used as is; any other string is hashed within the workspace and agent.
in: query
schema:
type: string
requestBody:
description: the run's inputs
required: true
content:
application/json:
schema:
type: object
properties:
user_message:
type: string
user_attachments:
type: array
items: {}
responses:
"201":
description: job created
content:
text/plain:
schema:
type: string
format: uuid
/w/{workspace}/jobs/run_wait_result/preview_flow:
post:
summary: run flow preview and wait for result
@@ -31991,6 +32200,17 @@ components:
(over a deployed row or a synthesized draft-only row).
Frontend appends a `*` to the displayed name.
type: boolean
agent_memory:
description: |
For an `ai_agent` resource only, its `memory` setting: the one part
of the value a listing returns, since it decides whether the agent
keeps a conversation.
draft_path:
type: string
description: |
On a draft-only row, the path its editor has staged when it
differs from the storage path (e.g. a never-deployed item parked at
`u/{user}/draft_{uuid}`).
required:
- path
- resource_type
@@ -36183,6 +36403,9 @@ components:
True when the authed user has a draft for this app — either no
deployed row exists at this path (draft-only) or the user has
saved a per-user draft on top of the deployed row.
draft_only:
type: boolean
description: No deployed counterpart exists at this path — the draft is the whole item.
draft_path:
type: string
description: |
@@ -37546,6 +37769,9 @@ components:
workers:
type: boolean
description: Whether operators can view workers page
builder_flows:
type: boolean
description: Whether operators can compose flows out of existing runnables (consumes a full seat). Omitting the field leaves the stored value unchanged.
manage_schedules:
type: boolean
description: Whether operators can create, edit and delete schedules. Granted unless withdrawn; omitting the field leaves the stored value unchanged.
backend/windmill-api/src/agent_runs.rs
+184
View File
@@ -0,0 +1,184 @@
//! Running a saved AI agent from its own page, and the one-step flow its runs and evals share.
//!
//! The run is built here from the stored agent rather than sent by the client, which is what lets
//! anyone who can read the agent run it, operators included, the way a deployed flow runs by path:
//! a preview takes arbitrary code, this takes only the run's inputs. It runs as the caller, who
//! needs access to the agent's AI resource and to whatever its tools use.
use std::collections::HashMap;
use axum::{
extract::{Path, Query},
Extension, Json,
};
use hyper::StatusCode;
use serde_json::value::RawValue;
use windmill_api_auth::check_scopes;
use windmill_common::{
db::UserDB,
error::{Error, Result},
jobs::JobPayload,
users::username_to_permissioned_as,
utils::{not_found_if_none, StripPath},
};
use windmill_queue::{push, PushArgs, PushIsolationLevel};
use crate::ai_evals::run::config_to_draft;
use crate::ai_evals::subject::AgentDraft;
use crate::db::{ApiAuthed, DB};
use crate::jobs::{handle_chat_conversation_messages, set_flow_memory_id, RunJobQuery};
/// The id the agent's step carries in the run, as the editor's chat names it. A token scoped to
/// `jobs:run:agents:<path>` reads its runs back by it (`require_job_within_run_scope`).
const AGENT_NODE_ID: &str = "__wm_agent_root";
/// A turn of the agent's chat when `memory_id` names a conversation, a single run otherwise.
/// Returns the job id.
pub(crate) async fn run_agent(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
Json(args): Json<HashMap<String, Box<RawValue>>>,
) -> Result<(StatusCode, String)> {
#[cfg(feature = "enterprise")]
crate::jobs::check_license_key_valid().await?;
let path = path.to_path();
check_scopes(&authed, || format!("jobs:run:agents:{path}"))?;
// Read through the caller's own permissions: reading the agent is what allows running it.
let mut tx = user_db.clone().begin(&authed).await?;
let value = sqlx::query_scalar!(
"SELECT value AS \"value: sqlx::types::Json<serde_json::Value>\"
FROM resource WHERE workspace_id = $1 AND path = $2 AND resource_type = 'ai_agent'",
w_id,
path
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let Some(sqlx::types::Json(value)) = not_found_if_none(value, "Agent", path)? else {
return Err(Error::BadRequest(format!(
"Agent {path} has no configuration"
)));
};
let config = config_to_draft(value)?;
// A chat turn is filed where the agent's chat lists its conversations, which a flow cannot
// name (flow paths carry no `.`), so the two never share a conversation list.
let chat = run_query.memory_id.is_some();
let run_path = if chat {
format!("{path}.chat")
} else {
path.to_string()
};
// A step memory id would replace the conversation's, and the agent would forget the turns
// before; history comes from the conversation alone.
let flow_value = agent_step_flow(
&config,
AGENT_NODE_ID,
chat,
&["memory_id", "previous_messages"],
)?;
let push_authed = authed.clone().into();
let (uuid, mut tx) = push(
&db,
PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()),
&w_id,
JobPayload::RawFlow {
value: flow_value,
path: Some(run_path.clone()),
restarted_from: None,
},
PushArgs::from(&args),
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
authed.token_prefix.as_deref(),
authed.username_override.as_deref(),
None,
None,
None,
None,
None,
None,
false,
false,
None,
true,
None,
None,
None,
None,
Some(&push_authed),
false,
None,
authed.trigger_or_fallback(None),
None,
)
.await?;
if let Some(memory_id) = run_query.memory_key(&w_id, &run_path) {
set_flow_memory_id(&mut tx, uuid, memory_id).await?;
}
if chat {
// A real conversation, not a test one: this is the agent as deployed, not a draft.
handle_chat_conversation_messages(
&mut tx,
&authed,
&w_id,
&run_path,
&run_query,
args.get("user_message"),
uuid,
false,
&args,
)
.await?;
}
tx.commit().await?;
Ok((StatusCode::CREATED, uuid.to_string()))
}
/// The agent as a one-step flow, validated by deserializing through `FlowValue` rather than
/// trusted as raw JSON. Its own transforms are the step's, minus `without`, and the run's inputs
/// supply the message and the attachments over the top. Always inlined, never a link to the
/// resource: a linked step would resolve the agent when it runs, not as it was read here.
pub(crate) fn agent_step_flow(
config: &AgentDraft,
id: &str,
chat: bool,
without: &[&str],
) -> Result<windmill_common::flows::FlowValue> {
let mut input_transforms = match &config.input_transforms {
serde_json::Value::Object(map) => map.clone(),
_ => serde_json::Map::new(),
};
for key in without {
input_transforms.remove(*key);
}
for key in ["user_message", "user_attachments"] {
input_transforms.insert(
key.to_string(),
serde_json::json!({ "type": "javascript", "expr": format!("flow_input.{}", key) }),
);
}
let mut flow = serde_json::json!({
"modules": [{
"id": id,
"value": {
"type": "aiagent",
"tools": config.tools,
"input_transforms": serde_json::Value::Object(input_transforms),
}
}]
});
if chat {
flow["chat_input_enabled"] = serde_json::json!(true);
}
Ok(serde_json::from_value(flow)?)
}
backend/windmill-api/src/ai_evals/run.rs
+2 -32
View File
@@ -248,7 +248,7 @@ fn build_run_flow(
/// The agent step, reading its case from the iteration rather than from the flow's arguments.
fn agent_module(config: &AgentDraft) -> Result<serde_json::Value> {
let flow = build_case_flow(config)?;
let flow = crate::agent_runs::agent_step_flow(config, AGENT_NODE_ID, false, &[])?;
let mut value = serde_json::to_value(&flow.modules[0].value)?;
if let Some(map) = value.as_object_mut() {
let transforms = map
@@ -269,36 +269,6 @@ fn agent_module(config: &AgentDraft) -> Result<serde_json::Value> {
Ok(serde_json::json!({ "id": AGENT_NODE_ID, "value": value }))
}
/// The agent step as a one-module flow, so the module shape is validated by deserializing
/// through `FlowValue` rather than trusted as raw JSON.
fn build_case_flow(config: &AgentDraft) -> Result<windmill_common::flows::FlowValue> {
// The configuration runs exactly as authored: its own brain transforms are the module's, and
// the case supplies the message and the attachments over the top.
let mut input_transforms = match &config.input_transforms {
serde_json::Value::Object(map) => map.clone(),
_ => serde_json::Map::new(),
};
for key in ["user_message", "user_attachments"] {
input_transforms.insert(
key.to_string(),
serde_json::json!({ "type": "javascript", "expr": format!("flow_input.{}", key) }),
);
}
// Always inlined, never a link to the resource: a linked step would resolve the agent when
// each case runs, which is the one thing a run of a named version must not do.
let mut agent_value = serde_json::Map::new();
agent_value.insert("type".to_string(), serde_json::json!("aiagent"));
agent_value.insert("tools".to_string(), serde_json::json!(config.tools));
agent_value.insert(
"input_transforms".to_string(),
serde_json::Value::Object(input_transforms),
);
Ok(serde_json::from_value(serde_json::json!({
"modules": [{ "id": AGENT_NODE_ID, "value": serde_json::Value::Object(agent_value) }]
}))?)
}
/// How many times the agent has been saved, not the identity of the row holding that value: runs
/// are named by it and compared by it, so it has to be the resource's own count rather than a
/// sequence the whole instance shares.
@@ -348,7 +318,7 @@ pub(crate) async fn require_agent(
/// transforms, its tools the module's tools. The same conversion for a draft and for what is
/// deployed, so the two hash comparably — which is what lets a draft run be recognised as the
/// version it became.
fn config_to_draft(value: serde_json::Value) -> Result<AgentDraft> {
pub(crate) fn config_to_draft(value: serde_json::Value) -> Result<AgentDraft> {
let mut config = match value {
serde_json::Value::Object(map) => map,
_ => return Err(Error::BadRequest("The agent is not an object".to_string())),
backend/windmill-api/src/drafts.rs
+55 -17
View File
@@ -14,13 +14,16 @@ use axum::{
Json, Router,
};
use serde::{Deserialize, Serialize};
use windmill_api_flows::flows::validate_operator_flow;
use windmill_common::{
db::UserDB,
error::{Error, Result},
flows::FlowValue,
user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX},
users::resolve_username_to_email,
utils::{check_proper_path, strip_json_nul},
variables::{build_crypt, encrypt},
workspaces::operator_can_build_flows,
};
pub fn workspaced_service() -> Router {
@@ -101,10 +104,10 @@ async fn list_drafts(
Path(w_id): Path<String>,
Query(query): Query<ListDraftsQuery>,
) -> Result<Json<Vec<DraftListItem>>> {
// Operators have no drafts of their own (they can't write any, see
// `require_can_write_path`), so this list is always empty for them. They
// can still READ some collaborators' drafts via `/drafts/get`.
if authed.is_operator {
// Without builder rights an operator has no drafts of their own (they can't write any, see
// `require_can_write_path`), so this list is always empty for them. They can still READ some
// collaborators' drafts via `/drafts/get`.
if authed.is_operator && !operator_can_build_flows(&db, &w_id).await? {
return Ok(Json(vec![]));
}
let all_users = query.all_users.unwrap_or(false);
@@ -154,10 +157,13 @@ async fn list_drafts(
{
Ok(()) => true,
// A stored draft can sit at an unwritable path — unauthorized,
// or malformed (`BadRequest`; the `draft` table has no path
// constraint). Either way it's not writable, and one bad row
// must not 400 the whole listing.
Err(Error::NotAuthorized(_)) | Err(Error::BadRequest(_)) => false,
// refused to an operator (`PermissionDenied`), or malformed
// (`BadRequest`; the `draft` table has no path constraint).
// Either way it's not writable, and one bad row must not fail
// the whole listing.
Err(Error::NotAuthorized(_))
| Err(Error::PermissionDenied(_))
| Err(Error::BadRequest(_)) => false,
Err(e) => return Err(e),
};
out.push(row);
@@ -464,6 +470,32 @@ async fn update_draft(
}
}
if authed.is_operator && kind == UserDraftItemKind::Flow {
if let Some(value) = &req.value {
#[derive(Deserialize)]
struct FlowDraft {
#[serde(default)]
value: FlowValue,
schema: Option<Box<serde_json::value::RawValue>>,
tag: Option<String>,
}
// The text stored below, NULs stripped: a stripped NUL can rename a key into one
// this check reads.
let draft: FlowDraft = serde_json::from_str(&strip_json_nul(value.0.get()))
.map_err(|e| Error::BadRequest(format!("Invalid flow draft: {e}")))?;
validate_operator_flow(
&draft.value,
&draft.tag,
draft.schema.as_deref().map(|s| s.get()),
&authed,
&db,
&user_db,
&w_id,
)
.await?;
}
}
let applied = if let Some(value) = &req.value {
// Secret variable values must never sit in `draft.value` in plaintext
// (see `encrypt_secret_variable_value`).
@@ -1111,13 +1143,14 @@ fn table_for_kind(kind: UserDraftItemKind) -> Option<&'static str> {
}
/// Resolves to `Ok(())` if `authed` may SAVE a draft at `path`. Operators are
/// rejected outright. Two layers:
/// rejected, except for flow drafts in a workspace that granted them builder
/// rights. Two layers:
/// 1. Claim-based namespace rules (admin, own `u/`, member `g/`, writable
/// `f/`) — mirror what RLS reads from the same JWT claims, and are the
/// ENTIRE check for draft-only paths (no deployed row for RLS to use).
/// 2. An RLS write-probe on the deployed row (`SELECT ... FOR UPDATE`) for
/// what the path can't answer, above all item-level extra_perms grants.
async fn require_can_write_path(
pub(crate) async fn require_can_write_path(
authed: &ApiAuthed,
db: &DB,
user_db: &UserDB,
@@ -1128,14 +1161,19 @@ async fn require_can_write_path(
if authed.is_admin {
return Ok(());
}
// Operators are read-only and never WRITE drafts. Read access is
// deliberately asymmetric: `require_can_read_path` has no operator block,
// so an operator can still READ a draft they can read via `/drafts/get`,
// mirroring their read access to deployed content. Intended.
// Operators are read-only and never WRITE drafts, except a flow draft where the workspace
// granted the builder right: the kind has to be checked, or the right would open drafts of
// kinds it says nothing about. Read access is deliberately asymmetric:
// `require_can_read_path` has no operator block, so an operator can still READ a draft they
// can read via `/drafts/get`, mirroring their read access to deployed content. Intended.
if authed.is_operator {
return Err(Error::NotAuthorized(
"operators cannot save drafts".to_string(),
));
let granted =
matches!(kind, UserDraftItemKind::Flow) && operator_can_build_flows(db, w_id).await?;
if !granted {
return Err(Error::PermissionDenied(
"operators cannot save drafts".to_string(),
));
}
}
// Cheap claim-based namespace checks first: they evaluate the same JWT
// claims RLS reads, so the outcome matches the policies while sparing the
backend/windmill-api/src/integration.rs
+66 -2
View File
@@ -1,9 +1,16 @@
use crate::{db::DB, HTTP_CLIENT};
use axum::{extract::Query, response::IntoResponse, routing::get, Extension, Router};
use axum::{
extract::{Path, Query},
response::IntoResponse,
routing::get,
Extension, Router,
};
use windmill_common::{error::Error, utils::query_elems_from_hub, HUB_BASE_URL};
pub fn global_service() -> Router {
Router::new().route("/hub/list", get(list_hub_integrations))
Router::new()
.route("/hub/list", get(list_hub_integrations))
.route("/hub/{app}/meta", get(get_hub_integration_meta))
}
#[derive(serde::Deserialize)]
@@ -29,3 +36,60 @@ async fn list_hub_integrations(
.await?;
Ok::<_, Error>((status_code, headers, response))
}
/// Axum percent-decodes a path parameter, so an interpolated slug carrying `..`, `?`
/// or `#` re-targets the proxied GET at another path on the hub origin — with the
/// instance's hub credentials attached. Slugs are `[A-Za-z0-9_-]`; reject the rest.
fn is_hub_integration_slug(app: &str) -> bool {
!app.is_empty()
&& app.len() <= 64
&& app
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
}
/// Everything a caller needs to write code against one integration: its resource
/// types, the provider knowledge the content repo authored, and facts derived from
/// the shipped scripts. A hub older than the endpoint answers 404, which passes
/// through as-is.
async fn get_hub_integration_meta(
Path(app): Path<String>,
Extension(db): Extension<DB>,
) -> impl IntoResponse {
if !is_hub_integration_slug(&app) {
return Err(Error::BadRequest(format!(
"Not a valid integration name: {app}"
)));
}
let (status_code, headers, response) = query_elems_from_hub(
&HTTP_CLIENT,
&format!("{}/integrations/{}/meta", **HUB_BASE_URL.load(), app),
None,
&db,
)
.await?;
Ok::<_, Error>((status_code, headers, response))
}
#[cfg(test)]
mod tests {
use super::is_hub_integration_slug;
#[test]
fn rejects_slugs_that_would_re_target_the_proxied_request() {
assert!(is_hub_integration_slug("confluence"));
assert!(is_hub_integration_slug("aws-ses"));
assert!(is_hub_integration_slug("bamboo_hr"));
assert!(is_hub_integration_slug("RSS"));
for escape in [
"../../scripts/top",
"confluence?foo=bar",
"confluence#frag",
"confluence/meta",
"",
] {
assert!(!is_hub_integration_slug(escape), "accepted {escape}");
}
}
}
backend/windmill-api/src/jobs.rs
+60 -18
View File
@@ -26,6 +26,7 @@ use std::time::Instant;
use tokio::io::AsyncReadExt;
use tower::ServiceBuilder;
use url::Url;
use windmill_api_flows::flows::validate_operator_composed_flow;
use windmill_common::assets::AssetUsageAccessType;
use windmill_common::auth::TOKEN_PREFIX_LEN;
#[cfg(feature = "run_inline")]
@@ -53,7 +54,9 @@ use windmill_common::worker::{Connection, CLOUD_HOSTED, WINDMILL_DIR};
use windmill_common::workspace_dependencies::{
RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES,
};
use windmill_common::workspaces::{check_user_against_rule, ProtectionRuleKind, RuleCheckResult};
use windmill_common::workspaces::{
check_operator_can_build_flows, check_user_against_rule, ProtectionRuleKind, RuleCheckResult,
};
use windmill_common::DYNAMIC_INPUT_CACHE;
#[cfg(all(feature = "enterprise", feature = "instance_smtp"))]
use windmill_common::{email_oss::send_email_html, server::load_smtp_config};
@@ -282,6 +285,7 @@ pub fn workspaced_service() -> Router {
)
.route("/add_batch_jobs/{n}", post(add_batch_jobs))
.route("/run/preview_flow", post(run_preview_flow_job))
.route("/run/agent/{*path}", post(crate::agent_runs::run_agent))
.route(
"/run_wait_result/preview_flow",
post(run_wait_result_preview_flow),
@@ -1894,7 +1898,10 @@ async fn require_job_within_run_scope(
// NULL for a job no such scope reaches directly (previews, dependency jobs,
// flow-inlined scripts) — those are still readable as a step of a matching flow,
// through their ancestors. A `singlestepflow` wraps either a script or a flow, so it
// projects onto the wrapped runnable the same way the batch-rerun query does.
// projects onto the wrapped runnable the same way the batch-rerun query does. An agent
// run is a preview of the one-step flow `agent_runs::agent_step_flow` builds, filed under the
// agent's path (`<path>.chat` for a chat turn); the editor's own runs of it look the same,
// and are runs of that agent too.
let chain = sqlx::query!(
r#"WITH RECURSIVE chain(id, parent_job) AS (
SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2
@@ -1902,8 +1909,11 @@ async fn require_job_within_run_scope(
SELECT j.id, j.parent_job FROM v2_job j
JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2
)
SELECT j.runnable_path,
SELECT
CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\.chat$', '')
ELSE j.runnable_path END AS runnable_path,
CASE
WHEN a.agent THEN 'agents'
WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'
WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'
WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN
@@ -1916,7 +1926,10 @@ async fn require_job_within_run_scope(
) = 'flow' THEN 'flows' ELSE 'scripts' END
END AS scope_kind,
CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app
FROM v2_job j JOIN chain c ON c.id = j.id
FROM v2_job j JOIN chain c ON c.id = j.id,
LATERAL (SELECT j.kind = 'flowpreview'
AND j.raw_flow->'modules'->1 IS NULL
AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a
WHERE j.workspace_id = $2"#,
job_id,
w_id,
@@ -8600,7 +8613,9 @@ fn operator_preview_refusal(job_id: Option<Uuid>) -> error::Error {
} else {
"Operators cannot run preview jobs for security reasons"
};
error::Error::NotAuthorized(reason.to_string())
// 403, not 401: the frontend reads a 401 as a dead session and logs the user out, and the
// flow editor builders open reaches this route.
error::Error::PermissionDenied(reason.to_string())
}
async fn run_preview_script(
@@ -9282,14 +9297,32 @@ pub struct RunFlowDependenciesResponse {
async fn push_flow_dependencies_job(
authed: &ApiAuthed,
db: &DB,
user_db: &UserDB,
w_id: &str,
req: RunFlowDependenciesRequest,
) -> error::Result<Uuid> {
check_scopes(authed, || format!("jobs:run"))?;
check_operator_can_build_flows(db, w_id, authed.is_operator, "run dependencies jobs").await?;
// The dependency job locks whatever inline code this request carries, on a worker. A
// composition-only flow has none, so validating here costs a builder nothing and keeps the
// lock step from becoming the way to run code the write path refuses.
if authed.is_operator {
return Err(error::Error::NotAuthorized(
"Operators cannot run dependencies jobs for security reasons".to_string(),
));
validate_operator_composed_flow(&req.flow_value, &None, authed, db, user_db, w_id).await?;
// The job rewrites bookkeeping stored under `req.path` (dependency map, asset usages,
// lock error) even with `skip_flow_update`, so a builder must be able to write there.
crate::drafts::require_can_write_path(
authed,
db,
user_db,
w_id,
windmill_common::user_drafts::UserDraftItemKind::Flow,
&req.path,
)
.await
.map_err(|e| match e {
error::Error::NotAuthorized(msg) => error::Error::PermissionDenied(msg),
e => e,
})?;
}
if req.raw_deps.is_some() {
@@ -9355,20 +9388,22 @@ async fn push_flow_dependencies_job(
async fn run_flow_dependencies_job(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(req): Json<RunFlowDependenciesRequest>,
) -> error::Result<Response> {
let uuid = push_flow_dependencies_job(&authed, &db, &w_id, req).await?;
let uuid = push_flow_dependencies_job(&authed, &db, &user_db, &w_id, req).await?;
run_wait_result(&db, uuid, &w_id, None, false, &authed.username).await
}
async fn run_flow_dependencies_job_async(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(req): Json<RunFlowDependenciesRequest>,
) -> error::Result<(StatusCode, String)> {
let uuid = push_flow_dependencies_job(&authed, &db, &w_id, req).await?;
let uuid = push_flow_dependencies_job(&authed, &db, &user_db, &w_id, req).await?;
Ok((StatusCode::CREATED, uuid.to_string()))
}
@@ -9669,15 +9704,24 @@ async fn run_preview_flow_job(
Query(run_query): Query<RunJobQuery>,
Json(raw_flow): Json<PreviewFlow>,
) -> error::Result<(StatusCode, String)> {
if authed.is_operator {
return Err(error::Error::NotAuthorized(
"Operators cannot run preview jobs for security reasons".to_string(),
));
}
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "run preview jobs").await?;
// Flow preview runs an arbitrary, request-supplied flow definition; require the broad
// jobs:run scope so a narrowly-scoped token cannot escape its scope. See run_preview_script.
check_scopes(&authed, || format!("jobs:run"))?;
require_path_read_access_for_preview(&authed, &raw_flow.path)?;
// A builder must be able to test what it composes, but the submitted value is not the stored
// one: without this the preview is a way to run inline code the write path refuses.
if authed.is_operator {
validate_operator_composed_flow(
&raw_flow.value,
&raw_flow.tag,
&authed,
&db,
&user_db,
&w_id,
)
.await?;
}
// Restarting copies the source runs' step results into the new run, and the queue resolves
// them with the service pool, so every run the request names must be readable as the caller.
let mut level = raw_flow.restarted_from.as_ref();
@@ -9811,9 +9855,7 @@ async fn run_dynamic_select(
DynamicSelectRunnableRef::Inline { .. }
) && authed.is_operator
{
return Err(error::Error::NotAuthorized(
"Operators cannot run preview jobs for security reasons".to_string(),
));
return Err(operator_preview_refusal(None));
}
if !is_valid_entrypoint_name(&request.entrypoint_function) {
backend/windmill-api/src/lib.rs
+1
View File
@@ -124,6 +124,7 @@ pub mod storage_list_ee;
mod storage_list_oss;
mod workspace_dependencies;
mod agent_runs;
mod ai_evals;
mod approvals;
#[cfg(all(feature = "enterprise", feature = "private"))]
backend/windmill-api/src/runnables.rs
+14 -5
View File
@@ -34,6 +34,7 @@ use std::collections::HashMap;
use windmill_common::{
db::UserDB,
error::{Error, JsonResult},
workspaces::operator_can_build_flows,
};
use windmill_types::scripts::ScriptHash;
use windmill_types::user_drafts::DraftUserRef;
@@ -397,7 +398,7 @@ fn draft_branch_sql(kind: &str) -> String {
async fn list_runnables(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Extension(_db): Extension<DB>,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Query(q): Query<ListRunnablesQuery>,
) -> JsonResult<ListRunnablesResponse> {
@@ -571,9 +572,9 @@ async fn list_runnables(
// Draft-only rows are the caller's own work in progress: never archived, so they
// have no place in the archived view, and carrying no labels of their own they are
// out of scope of a label filter (as in the per-kind endpoints). Operators don't
// see other people's drafts and have none of their own to see.
// see other people's drafts and have none of their own, except a builder's flows.
let include_drafts = q.include_draft_only.unwrap_or(false)
&& !authed.is_operator
&& (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?)
&& !show_archived
&& q.label.as_ref().filter(|s| !s.is_empty()).is_none();
let draft_extras_for = |kind: &str| -> Vec<String> {
@@ -659,7 +660,7 @@ async fn list_runnables(
keyset: Option<&str>,
limit: Option<usize>|
-> Option<String> {
if !include_drafts || !kinds.contains(&kind) {
if !include_drafts || !kinds.contains(&kind) || (authed.is_operator && kind != "flow") {
return None;
}
// `fav` is ignored: with no favorite join there is nothing to filter on, and the
@@ -1004,9 +1005,17 @@ async fn add_draft_counts(
q: &CountRunnablesQuery,
counts: &mut HashMap<String, i64>,
) -> Result<(), Error> {
if !q.include_draft_only.unwrap_or(false) || authed.is_operator {
if !q.include_draft_only.unwrap_or(false) {
return Ok(());
}
// An operator has no drafts of their own, except a builder's flows.
let kinds: &[&str] = if !authed.is_operator {
kinds
} else if kinds.contains(&"flow") && operator_can_build_flows(db, w_id).await? {
&["flow"]
} else {
return Ok(());
};
// $1 = workspace, $2 = the caller's email.
let mut binds: Vec<String> = vec![];
let branches: Vec<String> = kinds
backend/windmill-common/src/ee_oss.rs
+8
View File
@@ -71,6 +71,14 @@ pub async fn check_seat_cap_for_reactivation(
Ok(None)
}
#[cfg(all(feature = "enterprise", not(feature = "private")))]
pub async fn check_seat_cap_for_operator_builder(
_db: &DB,
_w_id: &str,
) -> anyhow::Result<Option<String>> {
Ok(None)
}
#[cfg(all(feature = "enterprise", not(feature = "private")))]
pub async fn compute_instance_hash(_db: &DB) -> anyhow::Result<Option<String>> {
// Implementation is not open source
backend/windmill-common/src/flows.rs
+240
View File
@@ -17,6 +17,7 @@ use crate::{
cache::{self, FlowExtras},
db::DB,
error::{to_anyhow, Error},
scripts::ScriptHash,
utils::{http_get_from_hub, StripPath},
worker::{to_raw_value, Connection},
DEFAULT_HUB_BASE_URL, HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION,
@@ -246,6 +247,156 @@ pub async fn resolve_modules(
Ok(())
}
/// Checks a flow value contains nothing but composition of runnables that already exist, which is
/// all an operator with builder rights may author. Walks the modules, the preprocessor and failure
/// modules, every branch, and the `tools` of an AI agent step.
///
/// Returns what the caller still has to authorize against its own permissions, which this
/// value-only walk cannot: every runnable the steps reference, the worker tags they pin, and the
/// `(path, hash)` pairs of version-pinned steps. See [`ComposedFlowRefs`] for why each one is not
/// already settled by the walk.
pub fn check_flow_is_composition_only(value: &FlowValue) -> Result<ComposedFlowRefs, Error> {
let mut refs = ComposedFlowRefs::default();
for module in value
.modules
.iter()
.chain(value.preprocessor_module.as_deref())
.chain(value.failure_module.as_deref())
{
check_module_is_composition_only(module, &mut refs)?;
}
Ok(refs)
}
/// What [`check_flow_is_composition_only`] collects for the caller to authorize.
#[derive(Default)]
pub struct ComposedFlowRefs {
pub tags: Vec<String>,
/// Every workspace runnable a step references, as `(is_flow, path)`.
pub runnables: Vec<(bool, String)>,
/// Version-pinned script steps, as `(path, hash)`.
pub pinned_scripts: Vec<(String, ScriptHash)>,
}
fn check_module_is_composition_only(
module: &FlowModule,
refs: &mut ComposedFlowRefs,
) -> Result<(), Error> {
let value = module
.get_value()
.map_err(|e| Error::BadRequest(format!("Step {} could not be read: {e}", module.id)))?;
check_module_value_is_composition_only(&value, &module.id, refs)
}
fn check_module_value_is_composition_only(
value: &FlowModuleValue,
id: &str,
refs: &mut ComposedFlowRefs,
) -> Result<(), Error> {
let refuse = |what: &str| Err(refused(id, what));
// A node id points at code in a `flow_node` row, which only the dependency job produces by
// hoisting a step's code, so an authored value carrying one names another flow's code and slips
// past this walk. Editor payloads come from the un-hoisted `flow_version.value`, so refusing
// them costs nothing legitimate.
let refuse_node = |node: &Option<FlowNodeId>| match node {
Some(_) => refuse("references code stored outside the flow"),
None => Ok(()),
};
let mut push_tag = |tag: &Option<String>| {
if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) {
refs.tags.push(tag.to_string());
}
};
match value {
FlowModuleValue::RawScript { .. } => return refuse("has inline code"),
FlowModuleValue::FlowScript { .. } => {
return refuse("references code stored outside the flow")
}
FlowModuleValue::Identity => {}
FlowModuleValue::Script { path, hash, tag_override, .. } => {
check_composable_path(path, id)?;
push_tag(tag_override);
refs.runnables.push((false, path.clone()));
if let Some(hash) = hash {
refs.pinned_scripts.push((path.clone(), *hash));
}
}
FlowModuleValue::Flow { path, .. } => {
check_composable_path(path, id)?;
refs.runnables.push((true, path.clone()));
}
FlowModuleValue::ForloopFlow { modules, modules_node, .. }
| FlowModuleValue::WhileloopFlow { modules, modules_node, .. } => {
refuse_node(modules_node)?;
for module in modules {
check_module_is_composition_only(module, refs)?;
}
}
FlowModuleValue::BranchOne { branches, default, default_node } => {
refuse_node(default_node)?;
for module in default {
check_module_is_composition_only(module, refs)?;
}
check_branches_are_composition_only(branches, id, refs)?;
}
FlowModuleValue::BranchAll { branches, .. } => {
check_branches_are_composition_only(branches, id, refs)?
}
FlowModuleValue::AIAgent { tools, tag, agent, .. } => {
// A linked agent resolves its tools from an `ai_agent` resource at run time, and
// operators may write resources, so those tools are outside this check: the list can
// be swapped for a raw script after the flow is deployed.
if agent.is_some() {
return refuse("links an AI agent resource, whose tools live outside the flow");
}
push_tag(tag);
for tool in tools {
if let ToolValue::FlowModule(value) = &tool.value {
check_module_value_is_composition_only(value, &tool.id, refs)?;
}
}
}
}
Ok(())
}
fn check_branches_are_composition_only(
branches: &[Branch],
id: &str,
refs: &mut ComposedFlowRefs,
) -> Result<(), Error> {
for branch in branches {
if branch.modules_node.is_some() {
return Err(refused(
id,
"has a branch that references code stored outside the flow",
));
}
for module in &branch.modules {
check_module_is_composition_only(module, refs)?;
}
}
Ok(())
}
fn refused(id: &str, what: &str) -> Error {
Error::PermissionDenied(format!(
"Step {id} {what}, so only a developer can edit this flow. Builders can compose scripts \
and flows that are already deployed."
))
}
fn check_composable_path(path: &str, id: &str) -> Result<(), Error> {
if path.starts_with("hub/") {
return Err(Error::PermissionDenied(format!(
"Step {id}: hub runnables are not available to operators with builder rights. Deploy \
it to the workspace first."
)));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -281,4 +432,93 @@ mod tests {
let err = extract_hub_flow_id_from_path("hub/flows/0").unwrap_err();
assert!(matches!(err, Error::BadRequest(_)));
}
fn flow(value: serde_json::Value) -> FlowValue {
serde_json::from_value(value).unwrap()
}
#[test]
fn composition_check_accepts_a_composed_flow_and_collects_its_tags() {
let refs = check_flow_is_composition_only(&flow(serde_json::json!({"modules": [{
"id": "a",
"value": {"type": "forloopflow", "iterator": {"type": "static", "value": []},
"parallel": false, "modules": [
{"id": "b", "value": {"type": "script", "path": "f/x/s", "tag_override": "gpu"}},
{"id": "c", "value": {"type": "flow", "path": "f/x/f"}},
{"id": "d", "value": {"type": "aiagent", "input_transforms": {}, "tag": "ai",
"tools": [{"id": "t", "value": {"tool_type": "flowmodule",
"type": "script", "path": "f/x/tool"}}]}}
]}
}]})))
.unwrap();
assert_eq!(refs.tags, vec!["gpu".to_string(), "ai".to_string()]);
}
/// The walk covers `modules`, so a node reference is a way past it: it names code hoisted
/// into a `flow_node` row, possibly another flow's.
#[test]
fn composition_check_rejects_node_references() {
for value in [
serde_json::json!({"modules": [{"id": "a", "value": {"type": "forloopflow",
"iterator": {"type": "static", "value": []}, "parallel": false,
"modules": [], "modules_node": 7}}]}),
serde_json::json!({"modules": [{"id": "a", "value": {"type": "branchone",
"branches": [], "default": [], "default_node": 7}}]}),
serde_json::json!({"modules": [{"id": "a", "value": {"type": "branchall",
"branches": [{"expr": "true", "modules": [], "modules_node": 7}]}}]}),
serde_json::json!({"modules": [{"id": "a", "value": {"type": "flowscript",
"id": 7, "language": "bun"}}]}),
] {
assert!(check_flow_is_composition_only(&flow(value)).is_err());
}
}
#[test]
fn composition_check_rejects_code_reachable_through_an_ai_agent() {
for value in [
serde_json::json!({"modules": [{"id": "a", "value": {"type": "aiagent",
"input_transforms": {}, "tools": [{"id": "t", "value": {"tool_type": "flowmodule",
"type": "rawscript", "content": "x", "language": "bun"}}]}}]}),
serde_json::json!({"modules": [{"id": "a", "value": {"type": "aiagent",
"input_transforms": {}, "tools": [], "agent": "$res:f/x/agent"}}]}),
] {
assert!(check_flow_is_composition_only(&flow(value)).is_err());
}
}
#[test]
fn composition_check_rejects_code_in_every_module_slot() {
let inline = serde_json::json!({"type": "rawscript", "content": "x", "language": "bun"});
for value in [
serde_json::json!({"modules": [{"id": "a", "value": inline}]}),
serde_json::json!({"modules": [], "failure_module": {"id": "f", "value": inline}}),
serde_json::json!({"modules": [], "preprocessor_module": {"id": "p", "value": inline}}),
] {
assert!(check_flow_is_composition_only(&flow(value)).is_err());
}
}
/// A step carrying a `hash` dispatches on that hash alone: the caller must verify the pair
/// exists and is readable, so the walk has to surface it rather than pass it through.
#[test]
fn composition_check_reports_version_pinned_steps() {
let refs = check_flow_is_composition_only(&flow(serde_json::json!({"modules": [
{"id": "a", "value": {"type": "script", "path": "f/x/s", "hash": "000000000000007b"}},
{"id": "b", "value": {"type": "script", "path": "f/x/t"}}
]})))
.unwrap();
assert_eq!(
refs.pinned_scripts,
vec![("f/x/s".to_string(), ScriptHash(123))]
);
}
#[test]
fn composition_check_rejects_hub_runnables() {
for kind in ["script", "flow"] {
let value = serde_json::json!({"modules": [{"id": "a",
"value": {"type": kind, "path": "hub/1234/thing"}}]});
assert!(check_flow_is_composition_only(&flow(value)).is_err());
}
}
}
backend/windmill-common/src/workspaces.rs
+64 -11
View File
@@ -1044,7 +1044,8 @@ pub async fn guest_app_admits<'c, E: sqlx::Executor<'c, Database = sqlx::Postgre
}
/// Billable members of `w_id` and the seats they cost, as `ceil(developers + operators/2)`. Service
/// accounts cannot log in and do not take a seat; a disabled member is not billed either.
/// accounts cannot log in and do not take a seat; a disabled member is not billed either. In a
/// workspace that granted operators builder rights, every operator counts as a developer.
///
/// The workspace is invoiced by a job outside this codebase that counts the same rows with its own
/// SQL. The two must be changed together: this rule disagreeing with that one is what bills a
@@ -1063,10 +1064,15 @@ pub async fn billable_seats(db: &crate::DB, w_id: &str) -> Result<BillableSeats>
.fetch_one(db)
.await
.map_err(|e| Error::internal_err(format!("counting billable seats of {w_id}: {e:#}")))?;
let (developers, operators) = if operator_can_build_flows(db, w_id).await? {
(row.developers + row.operators, 0)
} else {
(row.developers, row.operators)
};
Ok(BillableSeats {
developers: row.developers,
operators: row.operators,
seats: ((row.developers as f64) + 0.5 * (row.operators as f64)).ceil() as i64,
developers,
operators,
seats: ((developers as f64) + 0.5 * (operators as f64)).ceil() as i64,
})
}
@@ -1189,7 +1195,17 @@ pub fn invalidate_protection_rules_cache(workspace_id: &str) {
// Operator rights cache
lazy_static::lazy_static! {
static ref OPERATOR_RIGHTS_CACHE: Cache<String, (OperatorManageRights, i64)> = Cache::new(1000);
static ref OPERATOR_RIGHTS_CACHE: Cache<String, (OperatorRights, i64)> = Cache::new(1000);
}
/// Every operator right of a workspace, read and cached together because they share one
/// `operator_settings` column and one invalidation. The two groups have opposite polarity:
/// `builder_flows` is granted on request and costs a seat, the `manage` rights are held by
/// default and cost nothing.
#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
pub struct OperatorRights {
pub builder_flows: bool,
pub manage: OperatorManageRights,
}
/// Writes an operator may perform unless the workspace withdraws them. Unlike the visibility
@@ -1256,7 +1272,7 @@ impl OperatorManageRights {
///
/// Call it before opening an RLS transaction: it takes a connection from the root pool, and a
/// second pooled connection held alongside a transaction self-deadlocks on a one-connection pool.
async fn operator_manage_rights(db: &DB, workspace_id: &str) -> Result<OperatorManageRights> {
async fn operator_rights(db: &DB, workspace_id: &str) -> Result<OperatorRights> {
let now = chrono::Utc::now().timestamp();
if let Some((rights, expiry)) = OPERATOR_RIGHTS_CACHE.get(workspace_id) {
@@ -1265,10 +1281,12 @@ async fn operator_manage_rights(db: &DB, workspace_id: &str) -> Result<OperatorM
}
}
// Coalesced to true, matching `OperatorManageRights::default`: an absent key means the
// workspace never configured the right, not that it withdrew it.
// The builder key defaults to false, a right the workspace has to grant; the manage keys to
// true, rights it has to withdraw. An absent key means "never configured" for both, so the
// defaults have to differ here rather than at the call sites.
let row = sqlx::query!(
"SELECT COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\",
"SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\",
COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\",
COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\"
FROM workspace_settings WHERE workspace_id = $1",
workspace_id
@@ -1282,7 +1300,10 @@ async fn operator_manage_rights(db: &DB, workspace_id: &str) -> Result<OperatorM
})?;
let rights = row
.map(|r| OperatorManageRights { schedules: r.schedules, triggers: r.triggers })
.map(|r| OperatorRights {
builder_flows: r.flows,
manage: OperatorManageRights { schedules: r.schedules, triggers: r.triggers },
})
.unwrap_or_default();
OPERATOR_RIGHTS_CACHE.insert(workspace_id.to_string(), (rights, now + 60));
@@ -1290,6 +1311,38 @@ async fn operator_manage_rights(db: &DB, workspace_id: &str) -> Result<OperatorM
Ok(rights)
}
/// Whether operators of this workspace may compose flows out of already-deployed runnables. Per
/// workspace, not per user: every operator gets it, and consumes a full seat for it.
pub async fn operator_can_build_flows(db: &DB, workspace_id: &str) -> Result<bool> {
Ok(operator_rights(db, workspace_id).await?.builder_flows)
}
/// Gate for a write only a workspace that granted builder rights lets operators perform. `action`
/// completes "Operators cannot {action} for security reasons".
pub async fn check_operator_can_build_flows(
db: &DB,
workspace_id: &str,
is_operator: bool,
action: &str,
) -> Result<()> {
if is_operator && !operator_can_build_flows(db, workspace_id).await? {
return Err(Error::PermissionDenied(format!(
"Operators cannot {action} for security reasons"
)));
}
Ok(())
}
/// Whether a membership consumes an operator (half) seat rather than an author seat. A builder
/// right makes an operator an author of deployable artifacts, so it weighs a full seat.
pub async fn consumes_operator_seat(
db: &DB,
workspace_id: &str,
is_operator: bool,
) -> Result<bool> {
Ok(is_operator && !operator_can_build_flows(db, workspace_id).await?)
}
/// Invalidate the operator rights cache for a workspace
pub fn invalidate_operator_rights_cache(workspace_id: &str) {
OPERATOR_RIGHTS_CACHE.remove(workspace_id);
@@ -1306,7 +1359,7 @@ pub async fn check_operator_can_manage(
is_operator: bool,
kind: ManageKind,
) -> Result<()> {
if is_operator && !operator_manage_rights(db, workspace_id).await?.has(kind) {
if is_operator && !operator_rights(db, workspace_id).await?.manage.has(kind) {
// 403, not 401: the caller is authenticated and simply lacks the right. The frontend reads
// an uncaught 401 as a dead session and logs the user out, so `NotAuthorized` here would
// eject an operator from the app instead of telling them why.
backend/windmill-common/tests/billing_workspace.rs
+15
View File
@@ -118,6 +118,21 @@ async fn paid_seats_and_fork_count(db: Pool<Postgres>) {
(2, 2, 3)
);
sqlx::query(
"INSERT INTO workspace_settings (workspace_id, operator_settings)
VALUES ('seat-root', '{\"builder_flows\": true}')",
)
.execute(&db)
.await
.unwrap();
windmill_common::workspaces::invalidate_operator_rights_cache("seat-root");
let breakdown = billable_seats(&db, "seat-root").await.unwrap();
assert_eq!(
(breakdown.developers, breakdown.operators, breakdown.seats),
(4, 0, 4),
"builder rights bill every operator as a developer"
);
insert_ws(&db, "seat-fork1", Some("seat-root"), false).await;
insert_ws(&db, "seat-fork2", Some("seat-root"), false).await;
// A deleted fork itself is not counted...
backend/windmill-store/src/resources.rs
+41 -11
View File
@@ -208,6 +208,25 @@ pub struct ListableResource {
pub is_draft: Option<bool>,
}
/// A row of the resource listing: the resource as a single read returns it, plus what only the
/// listing carries.
#[derive(FromRow, Serialize)]
pub struct ListedResource {
#[sqlx(flatten)]
#[serde(flatten)]
pub resource: ListableResource,
/// An `ai_agent`'s `memory` setting, the one part of its value a listing shows: whether it
/// keeps a conversation decides how it is offered. `value` stays unlisted for every type.
#[serde(skip_serializing_if = "Option::is_none")]
#[sqlx(default)]
pub agent_memory: Option<serde_json::Value>,
/// On a draft-only row, the path its editor has staged when it differs from the storage path
/// (a new item parked at `u/{user}/draft_{uuid}`), as the other kinds' listings report it.
#[serde(skip_serializing_if = "Option::is_none")]
#[sqlx(default)]
pub draft_path: Option<String>,
}
#[derive(Deserialize)]
pub struct CreateResource {
pub path: String,
@@ -324,7 +343,7 @@ async fn list_resources(
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<ListableResource>> {
) -> JsonResult<Vec<ListedResource>> {
let (per_page, offset) = paginate(pagination);
let mut sqlb = SqlBuilder::select_from("resource")
@@ -346,6 +365,7 @@ async fn list_resources(
"resource.labels",
"folder_labels(resource.workspace_id, resource.path) as inherited_labels",
"ws_specific.path IS NOT NULL as ws_specific",
"CASE WHEN resource.resource_type = 'ai_agent' THEN resource.value->'memory' END as agent_memory",
])
// Scalar EXISTS flags the authed user's per-user draft without fanning rows out.
.field(
@@ -430,11 +450,11 @@ async fn list_resources(
let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let allowed = build_scope_path_predicate(&authed, "resources", "read");
let mut rows = sqlx::query_as::<_, ListableResource>(&sql)
let mut rows = sqlx::query_as::<_, ListedResource>(&sql)
.fetch_all(&mut *tx)
.await?
.into_iter()
.filter(|r| allowed(&r.path))
.filter(|r| allowed(&r.resource.path))
.collect::<Vec<_>>();
tx.commit().await?;
@@ -469,14 +489,18 @@ async fn list_resources(
let v: serde_json::Value =
serde_json::from_str(row.value.0.get()).unwrap_or(serde_json::Value::Null);
// ResourceEditor's `ResourceState`: { path, description, args, labels?, wsSpecific, resource_type? }
let path = v
.get("path")
.and_then(|s| s.as_str())
.unwrap_or("")
.to_string();
if path.is_empty() || !allowed(&path) {
// Listed at the draft's key rather than its `path`: an editor keys the draft on the path
// it opened, which is the only one a draft read can find, while `path` moves on rename.
// The staged `path` is reported beside it as `draft_path`.
let path = row.path;
if !allowed(&path) {
continue;
}
let draft_path = v
.get("path")
.and_then(|s| s.as_str())
.filter(|p| !p.is_empty() && *p != path)
.map(str::to_string);
let description = v
.get("description")
.and_then(|x| x.as_str())
@@ -507,8 +531,13 @@ async fn list_resources(
})
});
let ws_specific = v.get("wsSpecific").and_then(|x| x.as_bool());
let agent_memory = if resource_type == "ai_agent" {
value.as_ref().and_then(|a| a.get("memory")).cloned()
} else {
None
};
rows.push(ListableResource {
let resource = ListableResource {
workspace_id: w_id.clone(),
path,
value,
@@ -530,7 +559,8 @@ async fn list_resources(
draft_only: Some(true),
// Synthesized rows are the authed user's draft.
is_draft: Some(true),
});
};
rows.push(ListedResource { resource, agent_memory, draft_path });
}
}
backend/windmill-worker/src/python_executor.rs
+37 -8
View File
@@ -828,7 +828,7 @@ pub async fn handle_python_job(
for k, v in list(pre_args.items()):
if v == '<function call>':
del pre_args[k]
kwargs = inner_script.preprocessor(**pre_args)
kwargs = _await(inner_script.preprocessor(**pre_args))
kwrags_json = res_to_json(kwargs, type(kwargs))
with open("args.json", 'w', encoding="utf-8") as f:
f.write(kwrags_json)"#
@@ -969,6 +969,8 @@ result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.j
def res_to_json(res, typ):
{res_to_json_body}
{PY_AWAIT_HELPER}
try:
{preprocessor}
{spread}
@@ -977,7 +979,7 @@ try:
del args[k]
if inner_script.{main_override} is None or not callable(inner_script.{main_override}):
raise ValueError("{main_override} function is missing")
res = inner_script.{main_override}(**args)
res = _await(inner_script.{main_override}(**args))
typ = type(res)
if hasattr(res, '__iter__') and not isinstance(res, (str, dict, list, bytes, tuple, set, frozenset, range, memoryview, bytearray)) and typ.__name__ != 'DataFrame':
for chunk in res:
@@ -1513,6 +1515,8 @@ _fix=lambda s:s if 'Infinity' not in s and 'NaN' not in s and '\\u0000' not in s
def res_to_json(res, typ):
{res_to_json_body}
{PY_AWAIT_HELPER}
{functions}
{registrations}
@@ -1546,11 +1550,11 @@ for line in sys.stdin:
continue
kwargs = json.loads(args_json, strict=False)
pre_args = entry['pre_transform'](kwargs)
preprocessed = mod.preprocessor(**pre_args)
preprocessed = _await(mod.preprocessor(**pre_args))
preprocessed_json = json.dumps(preprocessed, separators=(',', ':'), default=str).replace('\n', '')
sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n")
main_args = entry['transform'](preprocessed if preprocessed else {{}})
res = mod.main(**main_args)
res = _await(mod.main(**main_args))
typ = type(res)
res_json = res_to_json(res, typ)
sys.stdout.write("wm_res[success]:" + res_json + "\n")
@@ -1574,11 +1578,11 @@ for line in sys.stdin:
continue
kwargs = json.loads(args_json, strict=False)
pre_args = entry['pre_transform'](kwargs)
preprocessed = mod.preprocessor(**pre_args)
preprocessed = _await(mod.preprocessor(**pre_args))
preprocessed_json = json.dumps(preprocessed, separators=(',', ':'), default=str).replace('\n', '')
sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n")
main_args = entry['transform'](preprocessed if preprocessed else {{}})
res = mod.main(**main_args)
res = _await(mod.main(**main_args))
typ = type(res)
res_json = res_to_json(res, typ)
sys.stdout.write("wm_res[success]:" + res_json + "\n")
@@ -1596,7 +1600,7 @@ for line in sys.stdin:
entry = next(iter(scripts.values()))
kwargs = json.loads(args_json, strict=False)
args = entry['transform'](kwargs)
res = entry['mod'].main(**args)
res = _await(entry['mod'].main(**args))
typ = type(res)
res_json = res_to_json(res, typ)
sys.stdout.write("wm_res[success]:" + res_json + "\n")
@@ -1624,7 +1628,7 @@ for line in sys.stdin:
kwargs = json.loads(args_json, strict=False)
args = entry['transform'](kwargs)
res = entry['mod'].main(**args)
res = _await(entry['mod'].main(**args))
typ = type(res)
res_json = res_to_json(res, typ)
sys.stdout.write("wm_res[success]:" + res_json + "\n")
@@ -3330,6 +3334,31 @@ This is not normal behavior, please make sure all workers have enough memory.\n
};
}
/// Python helper running what an `async def` main or preprocessor returns.
/// One loop per process: a dedicated worker's module-level async clients and
/// locks are bound to the loop they were first used on, so `asyncio.run`'s
/// loop-per-call would break them from the second job on. A loop the script
/// set while importing (`asyncio.set_event_loop`) is reused. asyncio is
/// imported lazily so sync jobs don't pay its import time.
const PY_AWAIT_HELPER: &str = r#"_loop = None
def _await(r):
global _loop
if not hasattr(r, '__await__'):
return r
import asyncio
if _loop is None or _loop.is_closed():
import warnings
try:
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
_loop = asyncio.get_event_loop()
if _loop.is_closed():
raise RuntimeError
except RuntimeError:
_loop = asyncio.new_event_loop()
asyncio.set_event_loop(_loop)
return _loop.run_until_complete(r)"#;
/// Python function body for `res_to_json(res, typ)`.
/// Handles DataFrame, bytes, dict coercion + JSON serialization.
fn python_res_to_json_body(postprocessor: &str) -> String {
cli/src/core/settings.ts
+1 -1
View File
@@ -413,7 +413,7 @@ export async function pushWorkspaceSettings(
const localOperatorSettings = localSettings.operator_settings && {
...localSettings.operator_settings,
};
for (const key of ["manage_schedules", "manage_triggers"] as const) {
for (const key of ["builder_flows", "manage_schedules", "manage_triggers"] as const) {
const remote = settings.operator_settings?.[key];
if (localOperatorSettings && localOperatorSettings[key] === undefined && remote !== undefined) {
localOperatorSettings[key] = remote;
cli/src/guidance/skills.gen.ts
Generated
+1 -1
View File
@@ -5788,7 +5788,7 @@ needs becomes unreachable.
},
"user_message": { "type": "javascript", "expr": "flow_input.user_message" },
"user_attachments": { "type": "javascript", "expr": "flow_input.files" },
"memory": { "type": "static", "value": { "kind": "window", "context_length": 10 } },
"memory": { "type": "static", "value": { "kind": "compaction" } },
"streaming": { "type": "static", "value": true },
"output_type": { "type": "static", "value": "text" }
},
docs/operator-builder-rights.md
+76
View File
@@ -0,0 +1,76 @@
# Operator builder rights
`operator_settings.builder_flows` lets every operator of a workspace compose flows out of runnables
that already exist. It does not make them authors: the boundary the operator role draws is
**authoring code and running arbitrary code**, and this does not move it.
It is a write right, unlike the visibility flags beside it, and unlike the withdrawable rights in
`docs/operator-write-rights.md` it is granted on request and costs a seat. Read it with
`windmill_common::workspaces::operator_can_build_flows` (60s cache, shared with the withdrawable
rights) and gate a write with `check_operator_can_build_flows`.
## What the check has to cover
`check_flow_is_composition_only` (`windmill-common/src/flows.rs`) walks a `FlowValue` and refuses
anything that carries code. Three of its rules exist because the obvious walk misses them:
- **`FlowScript` and any populated `modules_node` / `default_node`.** These name code hoisted into
a `flow_node` row. Only the dependency job produces them, so an authored value carrying one
names code stored under some other flow. The walk covers `modules`, so a node reference is a way
past it.
- **An AI agent step's `tools`.** `ToolValue::FlowModule` wraps a whole `FlowModuleValue`, so a
tool can be a raw script.
- **An AI agent step's `agent` link.** A linked agent resolves its tools from an `ai_agent`
resource at run time, and operators may write resources, so the tool list is outside this check
and can be swapped for a raw script after the flow is approved.
It also returns what a value-only walk cannot authorize, for the caller to check against its own
permissions:
- **the worker tags the steps pin**, or a builder routes a job onto a privileged worker group;
- **every runnable a step references**. `script_to_payload` resolves a step's path with the root DB
handle (`db_authed = None`) and returns the referenced runnable's `on_behalf_of`, which
`worker_flow` then applies to the step job. So composing a path is enough to run it, and to run
it as whoever it runs as: `validate_operator_composed_flow` re-checks each path under the
caller's RLS. This is the general case; the one below is on top of it, not instead of it.
- **the `(path, hash)` of every version-pinned step**. A step carrying a `hash` is dispatched by
that hash alone, with the path beside it never consulted, so a readable path paired with another
script's hash still runs that other script.
The value is not the only code a flow carries: the schema's `x-windmill-dyn-select-code` fills its
dynamic dropdowns, run as whoever loads the form, so `validate_operator_flow` refuses it like a
step's code. A builder therefore cannot save a developer's flow that has dropdowns; its editor
still previews them through the deployed flow rather than the inline route operators are refused.
Which developer flows a builder may edit at all is left to permissions: write access to the flow
or its folder.
Call it on every write **and** every preview: `run_preview_flow_job` and
`push_flow_dependencies_job` both take a request-supplied flow value, so leaving either out makes
it the way to run what the write path refuses. A flow draft goes through `validate_operator_flow`
too: a developer who loads a builder's draft in the editor runs its code as themselves.
## Billing
An operator of a builder workspace consumes a full author seat: composing deployable artifacts
makes them an author, and there is no half-author. `consumes_operator_seat` is the seat-role
helper; the EE counting queries share `OPERATOR_SEAT_SQL` so the displayed, enforced and reported
numbers agree. The one exception is `get_user_usage` in `stats_ee.rs`: it is a compile-checked
`query!`, which cannot interpolate the constant, so it spells the predicate out. Change both
together.
On cloud, `billable_seats` applies the same rule; see its doc for the out-of-repo invoice it must
match.
Granting the right runs `check_seat_cap_for_operator_builder`, which prices the change by counting
seats twice rather than by counting the workspace's operators: an operator who already authors
elsewhere must not be charged again, so re-saving settings that already have the right on is a
zero delta and never blocks.
## Accepted risks
- All-or-nothing per workspace: there is no per-user builder role.
- `operator_settings` is git-synced, so a pull can flip every operator's class in a workspace and
the billed seat count with it.
- A builder writes JavaScript expressions: step inputs, branch predicates, `stop_after_if` and
`suspend`. They run in QuickJS with no filesystem or network access, and forbidding them would
leave nothing to compose with.
docs/operator-write-rights.md
+1 -1
View File
@@ -79,7 +79,7 @@ authorizing writes on every other replica until its own entry expires.
These name capabilities operators already hold, so absence has to mean "never configured", not a
value. That is easy to get wrong in two places, and the obvious implementation gets both wrong:
- The read coalesces to **true** (`operator_manage_rights`), including for a workspace with no
- The read coalesces to **true** (`operator_rights`), including for a workspace with no
`workspace_settings` row, which is what `OperatorManageRights::default` is for. Coalescing to
false instead revokes the right on upgrade for every workspace that ever saved operator settings,
since those rows carry explicit keys and none of them is this one.
frontend/src/lib/attachments/newTabModifier.dom.test.ts
-106
View File
@@ -1,106 +0,0 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { newTabModifier } from './newTabModifier.svelte'
const onPlatform = (userAgent: string) => vi.stubGlobal('navigator', { userAgent })
const LINUX = 'Mozilla/5.0 (X11; Linux x86_64)'
const MAC = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'
const attached: (() => void)[] = []
/** Attach to a fresh element and return it with its cleanup, as `{@attach}` would. */
function pill() {
const node = document.createElement('span')
document.body.append(node)
const modifier = newTabModifier()
const cleanup = modifier.attach(node) as () => void
attached.push(cleanup)
const hover = (init: MouseEventInit = {}) =>
node.dispatchEvent(new MouseEvent('mouseenter', init))
const move = (init: MouseEventInit = {}) => node.dispatchEvent(new MouseEvent('mousemove', init))
const unhover = () => node.dispatchEvent(new MouseEvent('mouseleave'))
return { modifier, hover, move, unhover, cleanup }
}
const keydown = (init: KeyboardEventInit) =>
window.dispatchEvent(new KeyboardEvent('keydown', init))
describe('newTabModifier', () => {
// The window listeners outlive the DOM, so every case has to be torn down through the
// attachment rather than by emptying the body.
afterEach(() => {
attached.splice(0).forEach((cleanup) => cleanup())
document.body.replaceChildren()
vi.unstubAllGlobals()
})
// The hover event carries the live modifier state, so a modifier pressed before the pointer
// arrived (or while this window was unfocused) is picked up rather than read as false.
it('seeds from the hover event, per platform', () => {
onPlatform(LINUX)
const linux = pill()
linux.hover({ ctrlKey: true })
expect(linux.modifier.held).toBe(true)
onPlatform(MAC)
const mac = pill()
// macOS ctrl+click is a secondary click, so it must not read as a new-tab modifier.
mac.hover({ ctrlKey: true })
expect(mac.modifier.held).toBe(false)
mac.hover({ metaKey: true })
expect(mac.modifier.held).toBe(true)
})
// Editors and menus stop keydown propagation to keep their own shortcuts, so a bubble-phase
// listener would go blind whenever focus sits in one.
it('sees a keydown that a focused element stops from propagating', () => {
onPlatform(LINUX)
const { modifier, hover } = pill()
hover()
const input = document.createElement('input')
input.addEventListener('keydown', (e) => e.stopPropagation())
document.body.append(input)
input.dispatchEvent(
new KeyboardEvent('keydown', { key: 'Control', ctrlKey: true, bubbles: true })
)
expect(modifier.held).toBe(true)
})
// A modifier held across a keyboard app switch is cleared by the blur and delivers no keydown
// on the way back, while the pointer parked on the pill fires no fresh mouseenter either.
it('re-seeds from pointer movement after the window lost focus', () => {
onPlatform(LINUX)
const { modifier, hover, move } = pill()
hover({ ctrlKey: true })
window.dispatchEvent(new Event('blur'))
expect(modifier.held).toBe(false)
move({ ctrlKey: true })
expect(modifier.held).toBe(true)
})
it('stops tracking once unhovered', () => {
onPlatform(LINUX)
const { modifier, hover, unhover } = pill()
hover({ ctrlKey: true })
unhover()
expect(modifier.held).toBe(false)
keydown({ key: 'Control', ctrlKey: true })
expect(modifier.held).toBe(false)
})
it('stops tracking when the element is destroyed while hovered', () => {
onPlatform(LINUX)
const { modifier, hover, cleanup } = pill()
hover({ ctrlKey: true })
// Hovering again without leaving must not strand the first hover's listeners, which nothing
// would then hold a reference to.
hover({ ctrlKey: true })
cleanup()
expect(modifier.held).toBe(false)
keydown({ key: 'Control', ctrlKey: true })
expect(modifier.held).toBe(false)
})
})
frontend/src/lib/attachments/newTabModifier.svelte.ts
-66
View File
@@ -1,66 +0,0 @@
import type { Attachment } from 'svelte/attachments'
import { isMac } from '$lib/utils'
/**
* Tracks whether the modifier that turns a click into a new browser tab is held, but only while
* the attached element is hovered, which is the only moment the answer is used.
*/
export function newTabModifier() {
let held = $state(false)
// Only the modifier that actually yields a tab: shift opens a window, alt can start a
// download, and on macOS ctrl+click is a secondary click.
// Taken from each event rather than accumulated across keydown/keyup pairs, so a keyup lost to
// a focus change cannot strand the flag on.
const sync = (event: KeyboardEvent | MouseEvent) => {
held = isMac() ? event.metaKey : event.ctrlKey
}
const clear = () => {
held = false
}
const attach: Attachment<HTMLElement> = (node) => {
// One controller per hover: a mirrored remove list leaks any listener whose options drift.
let hover: AbortController | undefined
const leave = () => {
hover?.abort()
hover = undefined
clear()
}
const enter = (event: MouseEvent) => {
// Seeded from the hover itself: mouse events carry the same modifier flags as key events,
// so a modifier already held before the pointer arrived reads correctly.
sync(event)
// Re-entering without an intervening leave would strand the previous controller: nothing
// else references it, so its listeners could never be removed.
hover?.abort()
hover = new AbortController()
const { signal } = hover
// Same reason the hover seeds: a modifier held across a keyboard app switch delivers no
// keydown on the way back, so the pointer is all that is left to re-read it from.
node.addEventListener('mousemove', sync, { signal })
// Capture: editors and menus stopPropagation the keys they handle, hiding the modifier
// from a bubble-phase listener whenever focus sits in one.
window.addEventListener('keydown', sync, { capture: true, signal })
window.addEventListener('keyup', sync, { capture: true, signal })
// Not capture, unlike the two above: blur does not bubble but does reach the window while
// capturing, so it would fire for every element that loses focus.
window.addEventListener('blur', clear, { signal })
}
const life = new AbortController()
node.addEventListener('mouseenter', enter, { signal: life.signal })
node.addEventListener('mouseleave', leave, { signal: life.signal })
return () => {
life.abort()
leave()
}
}
return {
get held() {
return held
},
attach
}
}
frontend/src/lib/common.ts
+2
View File
@@ -49,6 +49,8 @@ export interface SchemaProperty {
hideWhenChatEnabled?: boolean
/** Why the oneOf variant is chat mode's to pick. Set = selector disabled, reason shown. */
lockOneOfWhenChatEnabled?: string
/** One line under the oneOf selector explaining the selected variant, keyed by its title. */
oneOfHints?: Record<string, string>
password?: boolean
order?: string[]
nullable?: boolean
frontend/src/lib/components/ArgInput.svelte
+4
View File
@@ -1162,6 +1162,10 @@
{/each}
{/snippet}
</ToggleButtonGroup>
{@const oneOfHint = extra?.['oneOfHints']?.[effectiveOneOfSelected ?? '']}
{#if oneOfHint}
<div class="-mt-4 mb-2 text-2xs text-hint">{oneOfHint}</div>
{/if}
{#if effectiveOneOfSelected}
{@const objIdx = oneOf.findIndex((o) => o.title === effectiveOneOfSelected)}
{@const obj = oneOf[objIdx]}
frontend/src/lib/components/DraftConflictAlert.svelte
+11 -16
View File
@@ -1,5 +1,5 @@
<script lang="ts">
import { Alert, Button } from '$lib/components/common'
import { Alert } from '$lib/components/common'
interface Props {
/** Take the draft the server holds, replacing what is on screen. */
@@ -13,19 +13,14 @@
let { onReload, onOverwrite, busy = false }: Props = $props()
</script>
<Alert type="warning" title="Your draft changed elsewhere">
<div class="flex flex-col items-start gap-2">
<div>
It was saved from another tab or session since this one read it, so your changes here are no
longer being saved.
</div>
<div class="flex flex-row gap-2">
<Button unifiedSize="sm" variant="default" disabled={busy} onClick={onReload}>
Load the other version
</Button>
<Button unifiedSize="sm" variant="default" disabled={busy} onClick={onOverwrite}>
Keep mine
</Button>
</div>
</div>
<Alert
type="warning"
title="Your draft changed elsewhere"
actions={[
{ label: 'Load the other version', variant: 'default', disabled: busy, onClick: onReload },
{ label: 'Keep mine', variant: 'default', disabled: busy, onClick: onOverwrite }
]}
>
It was saved from another tab or session since this one read it, so your changes here are no longer
being saved.
</Alert>
frontend/src/lib/components/EditableSchemaForm.svelte
+19 -9
View File
@@ -19,6 +19,7 @@
import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
import Label from './Label.svelte'
import { sendUserToast } from '$lib/toast'
import { useOperatorBuilderFlows } from '$lib/operatorWriteRights'
import Toggle from './Toggle.svelte'
import {
DynamicInput,
@@ -35,9 +36,14 @@
import Section from '$lib/components/Section.svelte'
import Editor from './Editor.svelte'
import AddPropertyV2 from './schema/AddPropertyV2.svelte'
import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
import {
useOperatingUser,
useOperatingWorkspace
} from '$lib/components/operatingWorkspace.svelte'
const operatingWorkspace = useOperatingWorkspace()
const operatingUser = useOperatingUser()
const operatorBuilderFlows = useOperatorBuilderFlows()
// export let openEditTab: () => void = () => {}
const dispatch = createEventDispatcher()
@@ -88,6 +94,8 @@
schemaFormClassName?: string
onChange?: (args: Record<string, any>) => void
workspace?: string | undefined
/** The deployed flow an operator's dropdown previews read their options from. */
deployedFlowPath?: string
}
let {
@@ -127,7 +135,8 @@
extraTab,
schemaFormClassName = undefined,
onChange = undefined,
workspace = undefined
workspace = undefined,
deployedFlowPath = undefined
}: Props = $props()
let ws = $derived(workspace ?? $operatingWorkspace)
@@ -469,11 +478,12 @@
order: e.detail
}
}}
helperScript={{
source: 'inline',
code: dynCode!,
lang: dynLang!
}}
helperScript={DynamicInput.flowHelperScript(
dynCode,
dynLang,
deployedFlowPath,
operatingUser.current?.operator
)}
prettifyHeader={isAppInput}
disabled={!!previewSchema}
{diff}
@@ -486,7 +496,7 @@
{@render runButton?.()}
{#if dynamicFunctions.length > 0}
{#if dynamicFunctions.length > 0 && !$operatorBuilderFlows}
<Section
label="Dynamic input functions"
collapsable={true}
@@ -815,7 +825,7 @@
{#each typeOptions as x}
<ToggleButton value={x[1]} label={x[0]} {item} />
{/each}
{#if showDynOpt}
{#if showDynOpt && !$operatorBuilderFlows}
{#each DYNAMIC_OPTIONS as x}
<ToggleButton value={x[1]} label={x[0]} {item} />
{/each}
frontend/src/lib/components/EditorHeader.svelte
+20 -94
View File
@@ -1,10 +1,7 @@
<script lang="ts">
import { ChevronRight, Pencil } from 'lucide-svelte'
import { Alert, Button } from '$lib/components/common'
import { ChevronRight } from 'lucide-svelte'
import EditableInput from '$lib/components/common/EditableInput.svelte'
import Popover from '$lib/components/meltComponents/Popover.svelte'
import Path from '$lib/components/Path.svelte'
import Label from '$lib/components/Label.svelte'
import PathEditPopover from '$lib/components/PathEditPopover.svelte'
import {
dirKey,
KIND_LABEL_LOWER,
@@ -15,17 +12,6 @@
} from '$lib/components/workspacePicker'
import BreadcrumbSegment from '$lib/components/BreadcrumbSegment.svelte'
import { splitItemPath } from '$lib/components/breadcrumbPath'
import { isOwner } from '$lib/utils'
import { userStore } from '$lib/stores'
import {
useOperatingUser,
useOperatingWorkspace
} from '$lib/components/operatingWorkspace.svelte'
const operatingWorkspace = useOperatingWorkspace()
const operatingUser = useOperatingUser()
const actingUser = $derived(operatingUser.current)
interface Props {
summary?: string
path?: string
@@ -76,23 +62,10 @@
workspaceId
}: Props = $props()
let pathPopoverOpen = $state(false)
/** Snapshot of `path` taken when the pen popover opens; cleared on close.
* While set, the breadcrumb derives from it instead of `path` so the pen
* anchor doesn't reflow as the user types in the popover (which would drag
* the popover with it via floating-ui's auto-update). */
/** Held by the pen popover while it is open, so the trail below — and the pen anchored to its
* end — does not reflow as the user types. */
let snapshotPath = $state<string | undefined>(undefined)
function setPathPopoverOpen(open: boolean) {
pathPopoverOpen = open
// Only snapshot when there's a path to freeze. New flows / scripts open
// the popover with `path === ''` and rely on `Path.reset()` to seed a
// path; if we snapshot the empty value the breadcrumb stays blank for
// the whole popover lifetime. Leaving `snapshotPath` undefined lets
// `displayPath` track the live seeded path instead.
snapshotPath = open && path ? path : undefined
}
// Path segments. e.g. "f/demo/weather_report" → scope "f/demo", slug "weather_report".
// `snapshotPath` keeps the breadcrumb frozen while the pen popover is open
// so the trigger doesn't drift mid-edit. Outside of that, the breadcrumb
@@ -101,11 +74,6 @@
let displayPath = $derived(snapshotPath ?? path ?? '')
let segments = $derived(splitItemPath(displayPath) ?? null)
// Treat an empty path as ownable so the pen popover lets a user pick the
// path for a brand-new item. `Path.reset()` then synthesizes a default
// under their own user/folder scope.
let own = $derived(!path || isOwner(path, actingUser, $operatingWorkspace))
// Virtual entry for the picker: surfaces the currently-edited item at its
// live path (which may differ from `savedPath` mid-rename, so the picker
// otherwise has no leaf to expand).
@@ -156,6 +124,8 @@
onPick={handlePickerSelect}
/>
{/each}
{/if}
{#if segments?.leaf}
{@const leafKey = leafKeyFor(kind, segments.leaf.fullPath)}
{@const leafParent = segments.dirs[segments.dirs.length - 1]?.fullPath}
<ChevronRight size={10} class="shrink-0" />
@@ -172,65 +142,21 @@
{/if}
</nav>
<!-- Pen → path-edit popover. Skipped entirely when path editing is
disabled so the user doesn't see an inert button. -->
<!-- Skipped entirely when path editing is disabled so the user doesn't see an inert
button. -->
{#if pathEditable}
<Popover
placement="bottom-start"
contentClasses="p-4"
usePointerDownOutside
excludeSelectors=".drawer"
disableFocusTrap
closeOnOtherPopoverOpen
bind:isOpen={() => pathPopoverOpen, setPathPopoverOpen}
>
{#snippet trigger()}
<Button
variant="subtle"
unifiedSize="xs"
iconOnly
startIcon={{ icon: Pencil }}
title="Edit path"
aria-label="Edit path"
btnClasses={penVisibility === 'hover' && !pathPopoverOpen
? 'opacity-0 transition-opacity group-hover:opacity-100 focus-visible:opacity-100'
: ''}
/>
{/snippet}
{#snippet content()}
<div class="flex flex-col gap-6 w-[480px]">
{#if own}
<Path
autofocus
bind:path
initialPath={snapshotPath ?? path ?? ''}
namePlaceholder={kind}
{kind}
size="sm"
drawerOffset={4000}
workspaceOverride={workspaceId}
/>
{#if savedPath && path && path !== savedPath}
<Alert
type="info"
size="xs"
title="Deploy the {kind} to make the path change effective."
/>
{/if}
{#if onBehalfOfEmail}
<Alert type="info" title="Run on behalf of" size="xs">
This flow will be redeployed on behalf of you ({$userStore?.email}) instead of {onBehalfOfEmail}
</Alert>
{/if}
{:else}
<Label label="Path">
<span class="text-xs font-mono text-secondary">{path}</span>
<p class="text-2xs text-tertiary mt-1">Only the owner can change the path</p>
</Label>
{/if}
</div>
{/snippet}
</Popover>
<!-- Path only: this bar carries its own summary field below, and `hidePath` would take
the pen with it, leaving a condensed pane no way to name the item. -->
<PathEditPopover
bind:path
bind:snapshotPath
summaryEditable={false}
{savedPath}
{kind}
{workspaceId}
{onBehalfOfEmail}
{penVisibility}
/>
{/if}
</div>
{/if}
frontend/src/lib/components/FlowBuilder.svelte
+61 -24
View File
@@ -28,7 +28,8 @@
userPathPrefix,
type Item,
type StateStore,
type Value
type Value,
emptyString
} from '$lib/utils'
import { sendUserToast } from '$lib/toast'
import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
@@ -38,6 +39,7 @@
import { getContext, onDestroy, setContext, untrack } from 'svelte'
import PageHeaderContent from '$lib/components/PageHeaderContent.svelte'
import PathEditPopover from '$lib/components/PathEditPopover.svelte'
import { pageHeader, PHONE_BAR } from '$lib/components/pageHeaderRegistry.svelte'
import { writable } from 'svelte/store'
import CenteredPage from './CenteredPage.svelte'
@@ -70,7 +72,6 @@
import FlowHistory from './flows/FlowHistory.svelte'
import EditorHeader from './EditorHeader.svelte'
import AutosaveIndicator from './AutosaveIndicator.svelte'
import EditableInput from './common/EditableInput.svelte'
import type { FlowBuilderWhitelabelCustomUi } from './custom_ui'
import FlowYamlEditor from './flows/header/FlowYamlEditor.svelte'
import { type TriggerContext, type ScheduleTrigger } from './triggers'
@@ -109,12 +110,13 @@
import { UserDraft } from '$lib/userDraft.svelte'
import { setOpenInSessionHandoff } from './sessions/openInSessionContext'
import { getEditorStoragePath, setEditorStoragePath } from './editorStoragePathContext'
import { useTriggerLock } from '$lib/operatorWriteRights'
import { useOperatorBuilderFlows, useTriggerLock } from '$lib/operatorWriteRights'
import {
useOperatingUser,
useOperatingWorkspace
} from '$lib/components/operatingWorkspace.svelte'
const triggerLock = useTriggerLock()
const operatorBuilderFlows = useOperatorBuilderFlows()
const operatingWorkspace = useOperatingWorkspace()
const operatingUser = useOperatingUser()
@@ -222,8 +224,19 @@
// Top-bar responsive collapse. Measured via bind:clientWidth — we can't
// rely on viewport `md:` because the editor lives inside other panes
// (session pane, drawer, etc.) where the viewport stays wide.
/** Held by the pen's popover while it is open; the band's trail reads it so the path does not
* reflow under the pointer as the user types. */
let pathSnapshot = $state<string | undefined>(undefined)
let topbarWidth = $state(0)
const compactTopbar = $derived(topbarWidth > 0 && topbarWidth < 720)
// In the page header the buttons share the row with the breadcrumb, so they collapse sooner:
// the trail and the summary take ~560px before either truncates, and the full group — the
// dropdown, Diff, the preview trio and Deploy — is ~610px.
const compactBelow = $derived(ownsPageHeader ? 1200 : 720)
const compactTopbar = $derived.by(() => {
const w = ownsPageHeader ? pageHeader.barWidth : topbarWidth
return w > 0 && w < compactBelow
})
// A phone's bar holds the trail and Deploy: the diff joins the menu and the preview buttons
// stand down, since there is no room to run a flow beside its own name.
const phoneTopbar = $derived(
@@ -1541,7 +1554,7 @@
<AIChangesWarningModal bind:open={aiChangesWarningOpen} onConfirm={aiChangesConfirmCallback} />
{#key renderCount}
{#if !actingUser?.operator}
{#if !actingUser?.operator || $operatorBuilderFlows}
{#if $pathStore}
<FlowHistory bind:this={flowHistory} path={$pathStore} {onHistoryRestore} />
{/if}
@@ -1556,7 +1569,12 @@
<!-- The editor's own top bar is the page header on this route: its breadcrumb and
summary are the header's, and its buttons are the header's actions. -->
<PageHeaderContent
item={{ kind: 'flow', path: $pathStore, summaryContent: flowSummary }}
item={{
// Frozen while the pen's popover is open so the trail holds still as the user types.
kind: 'flow',
path: pathSnapshot ?? $pathStore,
summaryContent: flowSummary
}}
actions={flowHeaderActions}
contexts={headerContexts}
/>
@@ -1636,18 +1654,31 @@
{/if}
{#snippet flowSummary()}
<!-- The summary stays editable where the editor's own bar had it: the breadcrumb
beside it is the path, which this editor renames from its settings tab. -->
<EditableInput
value={flowStore.val.summary ?? ''}
placeholder="Add a summary..."
editable={customUi?.topBar?.editableSummary != false}
commitOnInput
size="sm"
onSave={(v) => (flowStore.val.summary = v)}
textClass="text-xs font-medium text-emphasis leading-tight"
class="max-w-full min-w-0"
/>
<!-- Not edited in place: the pen beside it opens the summary and the path together,
so the band reads as a name rather than a form. `title` for one it truncates. -->
<div class="group flex items-center gap-1 min-w-0">
<span
class="min-w-0 truncate text-xs {emptyString(flowStore.val.summary)
? 'text-tertiary italic font-normal'
: 'font-medium text-emphasis'}"
title={flowStore.val.summary}
>{emptyString(flowStore.val.summary) ? 'Add a summary...' : flowStore.val.summary}</span
>
{#if customUi?.topBar?.editablePath != false || customUi?.topBar?.editableSummary != false}
<PathEditPopover
penVisibility={emptyString(flowStore.val.summary) ? 'always' : 'hover'}
bind:summary={flowStore.val.summary}
summaryEditable={customUi?.topBar?.editableSummary != false}
pathEditable={customUi?.topBar?.editablePath != false}
bind:path={$pathStore}
bind:snapshotPath={pathSnapshot}
savedPath={initialPath}
kind="flow"
onBehalfOfEmail={$savedOnBehalfOfEmail}
workspaceId={autosaveWorkspace}
/>
{/if}
</div>
{/snippet}
{#snippet flowHeaderActions()}
@@ -1701,9 +1732,11 @@
/>
{/snippet}
<!-- Rendered either inline in the top bar (wide) or as a graph overlay
(compactTopbar). Crossing the 720px threshold remounts
FlowPreviewButtons; any open preview state will reset. -->
<!-- Rendered either inline in the top bar (wide) or as a graph overlay. The two
conditions are exact complements: testing the flow is reachable at every width,
and FlowPreviewButtons — which owns the preview state and the handles the graph
calls into — is mounted exactly once. Crossing a threshold remounts it, so any
open preview resets. -->
{#snippet previewButtons()}
<FlowPreviewButtons
{suspendStatus}
@@ -1733,7 +1766,7 @@
{#if flowStateStore.val}
<FlowEditor
bind:this={flowEditor}
graphOverlay={compactTopbar ? previewButtons : undefined}
graphOverlay={compactTopbar || phoneTopbar ? previewButtons : undefined}
{disabledFlowInputs}
disableAi={disableAi || customUi?.stepInputs?.ai == false}
disableSettings={customUi?.settingsPanel === false}
@@ -1772,7 +1805,9 @@
{forceTestTab}
{highlightArg}
aiChatOpen={aiChatManager.open}
showFlowAiButton={!disableAi && customUi?.topBar?.aiBuilder != false}
showFlowAiButton={!disableAi &&
customUi?.topBar?.aiBuilder != false &&
!$operatorBuilderFlows}
toggleAiChat={() => aiChatManager.toggleOpen()}
{sessionOpen}
onOpenPreview={flowPreviewButtons?.openPreview}
@@ -1801,7 +1836,9 @@
{/if}
</div>
{:else}
Flow Builder not available to operators
<div class="h-full w-full center-center text-sm text-secondary">
Flow builder not available to operators
</div>
{/if}
{/key}
frontend/src/lib/components/FlowPreviewContent.svelte
+14 -10
View File
@@ -28,7 +28,7 @@
RefreshCw,
X
} from 'lucide-svelte'
import { sendUserToast, type StateStore } from '$lib/utils'
import { DynamicInput, sendUserToast, type StateStore } from '$lib/utils'
import { dfs } from './flows/dfs'
import { sliceModules } from './flows/flowStateUtils.svelte'
import InputSelectedBadge from './schema/InputSelectedBadge.svelte'
@@ -44,9 +44,13 @@
import FlowRestartButton from './FlowRestartButton.svelte'
import { useNestedRestartState } from './useNestedRestartState.svelte'
import { buildFlowRecording, downloadRecordingJson } from './recording/runRecording'
import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
import {
useOperatingUser,
useOperatingWorkspace
} from '$lib/components/operatingWorkspace.svelte'
const operatingWorkspace = useOperatingWorkspace()
const operatingUser = useOperatingUser()
interface Props {
previewMode: 'upTo' | 'whole'
@@ -552,14 +556,14 @@
savedArgs = $state.snapshot(previewArgs.val)
}}
bind:isValid
helperScript={flowStore.val.schema?.['x-windmill-dyn-select-code'] &&
flowStore.val.schema?.['x-windmill-dyn-select-lang']
? {
source: 'inline',
code: flowStore.val.schema['x-windmill-dyn-select-code'] as string,
lang: flowStore.val.schema['x-windmill-dyn-select-lang'] as ScriptLang
}
: undefined}
helperScript={DynamicInput.flowHelperScript(
flowStore.val.schema?.['x-windmill-dyn-select-code'] as string | undefined,
flowStore.val.schema?.['x-windmill-dyn-select-lang'] as
| ScriptLang
| undefined,
$initialPathStore,
operatingUser.current?.operator
)}
/>
</div>
{/key}
frontend/src/lib/components/ForkWorkspaceBanner.svelte
+1 -1
View File
@@ -109,7 +109,7 @@
if (comparisonFor !== ws) {
comparison = undefined
comparisonFor = undefined
}
}
const seq = ++requestSeq
loading = true
error = undefined
frontend/src/lib/components/NavBreadcrumb.svelte
+4 -2
View File
@@ -148,7 +148,9 @@ same weight, size and icon size, so the line reads as one control rather than fo
{/snippet}
{#snippet itemKindIcon()}
{#if item}<WorkspaceItemKindIcon kind={item.kind} size={ICON} />{/if}
<!-- No kind, no glyph: WorkspaceItemKindIcon falls through to the script icon, which would
label an agent a script. -->
{#if item?.kind}<WorkspaceItemKindIcon kind={item.kind} size={ICON} />{/if}
{/snippet}
{#snippet sectionKindIcon()}
@@ -304,7 +306,7 @@ same weight, size and icon size, so the line reads as one control rather than fo
</span>
{/if}
</div>
{:else}
{:else if item.kind}
<span class="{SEGMENT} hover:bg-transparent" aria-current="page">
{@render itemKindIcon()}
<span class="truncate">{KIND_LABEL_LOWER[item.kind]}</span>
frontend/src/lib/components/PageHeaderBar.svelte
+63 -24
View File
@@ -7,7 +7,7 @@ section name — and the route's own buttons at the far end.
The row's height matches the sidebar's own header row, so the two read as one band.
-->
<script lang="ts">
import { PanelLeft, PanelTopDashed } from 'lucide-svelte'
import { PanelLeft, PanelTop, PanelTopDashed } from 'lucide-svelte'
import { navDetached } from './sidebar/navDetached.svelte'
import { navHandleSlot } from './sidebar/navHandlePlacement.svelte'
import NavBreadcrumb from './NavBreadcrumb.svelte'
@@ -20,6 +20,7 @@ The row's height matches the sidebar's own header row, so the two read as one ba
navHidden = false,
hideNavHandle = false,
panelled = false,
onPin,
onUnpin
}: {
navHidden?: boolean
@@ -29,6 +30,10 @@ The row's height matches the sidebar's own header row, so the two read as one ba
/** The sidebar is a panel over the page rather than a rail beside it — detached, or a
* window too narrow to seat one. Either way this bar carries the switch that opens it. */
panelled?: boolean
/** Keeps the band on a page that owns the viewport. The corner handle that called the band
* down is underneath it by the time the pointer arrives, so this bar has to carry the pin
* itself — it lands on the same pixels, in place of the sidebar handle it hides. */
onPin?: () => void
/** Sends the band back behind its handle, on a page that owns the viewport. */
onUnpin?: () => void
} = $props()
@@ -59,12 +64,19 @@ The row's height matches the sidebar's own header row, so the two read as one ba
if (!el || el.nodeType !== 1) return
const box = document.getElementById('content')
if (!box || !box.contains(el)) return
// The answer this event would give, before measuring anything. While it matches what the
// band already draws there is nothing to decide — which is the steady state of a scroll,
// and the two rects below each force a synchronous layout. This listener captures every
// scroller in the app (Monaco, the audit table, a run's logs), about once a frame while any
// of them moves, so the common path has to stay off the layout.
const next = el.scrollTop > 0
if (next === scrolledUnder) return
// Only a box that starts where the content does has anything passing under the band. The
// audit page's list begins 285px lower, behind its own chart and column headers, and a
// select's option list floats wherever it opens: neither moves anything under the header.
const top = el.getBoundingClientRect().top - box.getBoundingClientRect().top
if (top > AT_THE_TOP_PX) return
scrolledUnder = el.scrollTop > 0
scrolledUnder = next
}
// A page left while scrolled would otherwise hand its edge to the next one, which may have
// nothing to scroll at all.
@@ -87,6 +99,36 @@ The row's height matches the sidebar's own header row, so the two read as one ba
'shadow-[inset_0_-1px_0_0_rgb(var(--color-border-light))] dark:shadow-[inset_0_-1px_0_0_#374151] [html.github-dark_&]:shadow-[inset_0_-1px_0_0_rgb(var(--color-border-light))]'
)}
>
<!-- Pin before the sidebar's handle, so it holds the first slot whether or not the handle is
there. Pinning brings the handle back — the band is no longer hidden — and from behind the
handle the unpin button would sit one slot over from where the pin was just clicked, which
for an operator (detached by default) means the second click opens the sidebar instead. -->
{#if onPin}
<!-- Same box and offsets as the corner handle it comes to rest over, so the pointer that
called the band down is already on this button. -->
<button
class="flex items-center p-1.5 rounded hover:bg-surface-hover"
aria-label="Pin the header on deployed apps"
title="Pin the header on deployed apps"
onclick={() => onPin?.()}
>
<PanelTop size={16} class="flex-shrink-0 text-hint" />
</button>
{/if}
{#if onUnpin}
<button
class="flex items-center p-1.5 rounded hover:bg-surface-hover"
aria-label="Unpin the header from deployed apps"
title="Unpin the header from deployed apps"
onclick={() => onUnpin?.()}
>
<!-- Dashed while the band is shown, solid while it is away: the pair the sidebar's own
toggle uses, so the two controls read as one idea. -->
<PanelTopDashed size={16} class="flex-shrink-0 text-hint" />
</button>
{/if}
{#if (navDetached.val || panelled) && !navHidden && !hideNavHandle}
<!-- Reveals the hidden sidebar, and only that: hovering slides the card in, clicking holds
it there. Attaching it for good belongs to the toggle in the sidebar's own footer, where
@@ -110,20 +152,6 @@ The row's height matches the sidebar's own header row, so the two read as one ba
</div>
{/if}
{#if onUnpin}
<!-- After the sidebar's handle: the sidebar is the outer thing, the band sits inside it. -->
<button
class="flex items-center p-1.5 rounded hover:bg-surface-hover"
aria-label="Unpin the header from deployed apps"
title="Unpin the header from deployed apps"
onclick={() => onUnpin?.()}
>
<!-- Dashed while the band is shown, solid while it is away: the pair the sidebar's own
toggle uses, so the two controls read as one idea. -->
<PanelTopDashed size={16} class="flex-shrink-0 text-hint" />
</button>
{/if}
<!-- The breadcrumb yields width grudgingly (shrink-[0.1]): when a page fills the bar with
controls, they are what should narrow, not the name of where the user is. It still gives
way rather than pushing them off the bar once there is nothing left to take.
@@ -134,13 +162,20 @@ The row's height matches the sidebar's own header row, so the two read as one ba
navigate the host's workspace. -->
{#if !navHidden}
<div class={twMerge('flex min-w-0', phone ? 'shrink min-w-[10rem]' : 'shrink-[0.1]')}>
<NavBreadcrumb
{item}
{section}
{narrow}
afterName={content?.afterName}
actingWorkspaceId={content?.actingWorkspaceId}
/>
<!-- Bridged like the actions below: what a page hangs off its own name renders here, out
of the tree that named it, and the pen in there asks that tree who the acting user is
before it offers to rename anything. -->
{#key content?.contexts}
<ContextBridge contexts={content?.contexts}>
<NavBreadcrumb
{item}
{section}
{narrow}
afterName={content?.afterName}
actingWorkspaceId={content?.actingWorkspaceId}
/>
</ContextBridge>
{/key}
</div>
{/if}
@@ -153,7 +188,11 @@ The row's height matches the sidebar's own header row, so the two read as one ba
carries it down with the summary instead of stranding it on the line above. -->
<span class="shrink-0 text-hint/40 text-xs px-0.5" aria-hidden="true">·</span>
{#if item.summaryContent}
{@render item.summaryContent()}
{#key content?.contexts}
<ContextBridge contexts={content?.contexts}>
{@render item.summaryContent()}
</ContextBridge>
{/key}
{:else}
<span class="min-w-0 truncate text-xs font-medium text-emphasis">{item.summary}</span>
{/if}
frontend/src/lib/components/PathEditPopover.svelte
+165
View File
@@ -0,0 +1,165 @@
<!--
@component
The pen that opens an item's summary and path for editing, beside the breadcrumb that shows them.
A caller that draws the path itself should render it from `snapshotPath ?? path`: while the
popover is open this holds the path as it was when it opened, so the trail — and the pen anchored
to it — does not reflow under the pointer as the user types, which floating-ui would follow.
-->
<script lang="ts">
import { Pencil } from 'lucide-svelte'
import { Alert, Button } from '$lib/components/common'
import Popover from '$lib/components/meltComponents/Popover.svelte'
import Path from '$lib/components/Path.svelte'
import Label from '$lib/components/Label.svelte'
import TextInput from '$lib/components/text_input/TextInput.svelte'
import type { WorkspaceItemKind } from '$lib/components/workspacePicker'
import { isOwner } from '$lib/utils'
import { userStore } from '$lib/stores'
import {
useOperatingUser,
useOperatingWorkspace
} from '$lib/components/operatingWorkspace.svelte'
const operatingWorkspace = useOperatingWorkspace()
const operatingUser = useOperatingUser()
const actingUser = $derived(operatingUser.current)
interface Props {
/** The item's human name. Committed trimmed, like the bar's own field did. */
summary?: string
/** When false the summary is shown but not editable, for a host whose `customUi` says so. */
summaryEditable?: boolean
/** When false the path is shown but not editable, the same way. A host may allow one and
* refuse the other, and the pen offers whichever it is given. */
pathEditable?: boolean
path?: string
/** The item's *saved* path on the server, so the popover can say a rename needs deploying. */
savedPath?: string
kind?: WorkspaceItemKind
/** Workspace the path picker scopes to; defaults to the operating workspace. */
workspaceId?: string
/** When set, warns that a redeploy happens on behalf of the current user instead. */
onBehalfOfEmail?: string | undefined
penVisibility?: 'hover' | 'always'
/** The path as it was when the popover opened, undefined while it is closed. See the
* component note: the caller draws its trail from this so it holds still mid-rename. */
snapshotPath?: string | undefined
}
let {
summary = $bindable(),
summaryEditable = true,
pathEditable = true,
path = $bindable(),
savedPath,
kind = 'flow',
workspaceId,
onBehalfOfEmail,
penVisibility = 'hover',
snapshotPath = $bindable()
}: Props = $props()
let open = $state(false)
const penLabel = $derived(
summaryEditable && pathEditable
? 'Edit summary and path'
: summaryEditable
? 'Edit summary'
: 'Edit path'
)
// Treat an empty path as ownable so the popover lets a user pick the path for a brand-new
// item. `Path.reset()` then synthesizes a default under their own user/folder scope.
const own = $derived(pathEditable && (!path || isOwner(path, actingUser, $operatingWorkspace)))
function setOpen(v: boolean) {
open = v
// Only snapshot when there is a path to freeze. A new flow or script opens with `path === ''`
// and relies on `Path.reset()` to seed one; snapshotting the empty value would leave the
// trail blank for the popover's whole lifetime.
snapshotPath = v && path ? path : undefined
}
</script>
<Popover
placement="bottom-start"
contentClasses="p-4"
usePointerDownOutside
excludeSelectors=".drawer"
disableFocusTrap
closeOnOtherPopoverOpen
bind:isOpen={() => open, setOpen}
>
{#snippet trigger()}
<Button
variant="subtle"
unifiedSize="xs"
iconOnly
startIcon={{ icon: Pencil }}
title={penLabel}
aria-label={penLabel}
btnClasses={penVisibility === 'hover' && !open
? 'opacity-0 transition-opacity group-hover:opacity-100 focus-visible:opacity-100'
: ''}
/>
{/snippet}
{#snippet content()}
<div class="flex flex-col gap-6 w-[480px]">
<!-- The summary first: it is the name a reader sees, and the path below is where that
name lives. Not autofocused — the pen's own job is the path, which takes the cursor. -->
{#if summaryEditable}
<Label label="Summary">
<TextInput
bind:value={summary}
inputProps={{
placeholder: 'Add a summary...',
// The popover sits in a page that binds keys of its own (the editors all do),
// and a summary is prose — it must not reach them.
onkeydown: (e: KeyboardEvent) => e.stopPropagation(),
onblur: () => (summary = summary?.trim())
}}
size="sm"
/>
</Label>
{:else if summary}
<Label label="Summary">
<span class="text-xs text-secondary">{summary}</span>
</Label>
{/if}
{#if own}
<Path
autofocus
bind:path
initialPath={snapshotPath ?? path ?? ''}
namePlaceholder={kind}
{kind}
size="sm"
drawerOffset={4000}
workspaceOverride={workspaceId}
/>
{#if savedPath && path && path !== savedPath}
<Alert
type="info"
size="xs"
title="Deploy the {kind} to make the path change effective."
/>
{/if}
{#if onBehalfOfEmail}
<Alert type="info" title="Run on behalf of" size="xs">
This flow will be redeployed on behalf of you ({$userStore?.email}) instead of {onBehalfOfEmail}
</Alert>
{/if}
{:else}
<Label label="Path">
<span class="text-xs font-mono text-secondary">{path}</span>
<p class="text-2xs text-tertiary mt-1">
{pathEditable
? 'Only the owner can change the path'
: 'This path cannot be changed here'}
</p>
</Label>
{/if}
</div>
{/snippet}
</Popover>
frontend/src/lib/components/RunForm.svelte
+24 -4
View File
@@ -116,13 +116,16 @@
loading?: boolean
noVariablePicker?: boolean
viewKeybinding?: boolean
scheduledForStr: string | undefined
invisible_to_owner: boolean | undefined
overrideTag: string | undefined
scheduledForStr?: string | undefined
invisible_to_owner?: boolean | undefined
overrideTag?: string | undefined
overrideTagNote?: string
args?: Record<string, any>
jsonView?: boolean
isValid?: boolean
/** Controls beside the Run button of a form that cannot schedule, in the row a
* schedulable one gives its Advanced options. */
actions?: import('svelte').Snippet
}
let {
@@ -141,7 +144,8 @@
overrideTagNote = undefined,
args = $bindable(),
jsonView = false,
isValid = $bindable(true)
isValid = $bindable(true),
actions = undefined
}: Props = $props()
let showPsCommonParams = $derived(
@@ -385,6 +389,22 @@
</div>
{/if}
</div>
{:else if actions}
<!-- The schedulable row's layout, with the caller's controls where Advanced sits. -->
<div class="flex-row-reverse flex-wrap flex w-full gap-4 mt-2 md:mt-6">
<Button
{loading}
variant="accent"
unifiedSize="md"
btnClasses="!inline-flex"
disabled={!isValid && !jsonView}
on:click={() => run(null)}
shortCut={{ Icon: CornerDownLeft, hide: !viewKeybinding }}
>
{buttonText}
</Button>
<div>{@render actions()}</div>
</div>
{:else}
<Button
btnClasses="!px-6 !py-1 w-full"
frontend/src/lib/components/RunsPage.svelte
+2 -2
View File
@@ -875,8 +875,8 @@
</div>
{:else}
<div class="w-full h-full flex flex-col" bind:clientWidth={innerWidth}>
<!-- Everything that used to sit in a row above the graph — the page's name, its queue, the
filters and the timeframe — is in the page header. -->
<!-- `actionsFlexible`: the filter row holds a search field that can give width back, so the
bar lets this box shrink on a phone instead of holding its content width. -->
<PageHeaderContent afterName={runsHint} actions={headerActions} actionsFlexible />
<!-- Graph -->
frontend/src/lib/components/ScriptBuilder.svelte
+35 -14
View File
@@ -94,8 +94,8 @@
import DefaultScripts from './DefaultScripts.svelte'
import { getContext, onDestroy, onMount, setContext, tick, untrack } from 'svelte'
import EditorHeader from './EditorHeader.svelte'
import EditableInput from './common/EditableInput.svelte'
import PageHeaderContent from '$lib/components/PageHeaderContent.svelte'
import PathEditPopover from '$lib/components/PathEditPopover.svelte'
import { pageHeader, PHONE_BAR } from '$lib/components/pageHeaderRegistry.svelte'
import ScriptSettingsBadges from './ScriptSettingsBadges.svelte'
import Badge from './common/badge/Badge.svelte'
@@ -200,6 +200,10 @@
// right group (hide the ~200px tag select, icon-only Diff/Settings) before the
// full group crowds the path into heavy truncation; ~900 is where the path
// keeps a usable width given the right group's natural ~440px.
/** Held by the pen's popover while it is open; the band's trail reads it so the path does not
* reflow under the pointer as the user types. */
let pathSnapshot = $state<string | undefined>(undefined)
let topbarWidth = $state(0)
// In the page header the buttons share the row with the breadcrumb, so they collapse sooner:
// the trail and the summary take ~560px before either truncates, and the full button group is
@@ -2280,7 +2284,12 @@
<!-- The editor's own top bar is the page header on this route: the script's path and
summary are the breadcrumb's, and its buttons are the header's actions. -->
<PageHeaderContent
item={{ kind: 'script', path: script.path, summaryContent: scriptSummary }}
item={{
// Frozen while the pen's popover is open so the trail holds still as the user types.
kind: 'script',
path: pathSnapshot ?? script.path,
summaryContent: scriptSummary
}}
afterName={scriptMarks}
actions={scriptHeaderActions}
/>
@@ -2329,18 +2338,30 @@
{/if}
{#snippet scriptSummary()}
<!-- The summary stays editable where the editor's own bar had it; the path beside it is
the breadcrumb, which this editor renames from its settings drawer. -->
<EditableInput
value={script.summary ?? ''}
placeholder="Add a summary..."
editable={customUi?.topBar?.editableSummary != false}
commitOnInput
size="sm"
onSave={(v) => (script.summary = v.trim())}
textClass="text-xs font-medium text-emphasis leading-tight"
class="max-w-full min-w-0"
/>
<!-- Not edited in place: the pen beside it opens the summary and the path together,
so the band reads as a name rather than a form. `title` for one it truncates. -->
<div class="group flex items-center gap-1 min-w-0">
<span
class="min-w-0 truncate text-xs {emptyString(script.summary)
? 'text-tertiary italic font-normal'
: 'font-medium text-emphasis'}"
title={script.summary}
>{emptyString(script.summary) ? 'Add a summary...' : script.summary}</span
>
{#if customUi?.topBar?.editablePath != false || customUi?.topBar?.editableSummary != false}
<PathEditPopover
penVisibility={emptyString(script.summary) ? 'always' : 'hover'}
bind:summary={script.summary}
summaryEditable={customUi?.topBar?.editableSummary != false}
pathEditable={customUi?.topBar?.editablePath != false}
bind:path={script.path}
bind:snapshotPath={pathSnapshot}
savedPath={initialPath}
kind="script"
workspaceId={autosaveWorkspace}
/>
{/if}
</div>
{/snippet}
{#snippet scriptMarks()}
frontend/src/lib/components/SummaryPathDisplay.svelte
+3 -3
View File
@@ -143,9 +143,9 @@
{/if}
<div class="flex items-center gap-3 max-w-full">
<span
class="{compact ? 'text-xs font-medium' : 'text-sm font-semibold'} truncate {emptyString(
summary
)
class="{compact
? 'text-xs font-medium'
: 'text-sm font-semibold'} truncate {emptyString(summary)
? 'text-tertiary italic font-normal'
: 'text-emphasis'}"
>
frontend/src/lib/components/WorkspaceDraftsBanner.svelte
+6 -1
View File
@@ -41,7 +41,12 @@
On a phone the sentence goes and the badge becomes the button: the count is the part that
has to be seen, and the bar has room for one thing. -->
{#if phone}
<Button variant="subtle" unifiedSize="sm" onclick={openDraftCompare} title="Review & deploy drafts">
<Button
variant="subtle"
unifiedSize="sm"
onclick={openDraftCompare}
title="Review & deploy drafts"
>
<Badge color="blue" small>{draftCount}</Badge>
</Button>
{:else}
frontend/src/lib/components/apps/editor/AppEditorHeader.svelte
+83 -28
View File
@@ -8,7 +8,7 @@
import { discardDraftAfterDeploy } from '$lib/userDraftToast'
import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
import { enterpriseLicense, userStore, userWorkspaces, workspaceStore } from '$lib/stores'
import { isMac, type Item, userPathPrefix } from '$lib/utils'
import { isMac, type Item, userPathPrefix, emptyString } from '$lib/utils'
import { random_adj } from '$lib/components/random_positive_adjetive'
import {
AlignHorizontalSpaceAround,
@@ -52,8 +52,8 @@
import DebugPanel from './contextPanel/DebugPanel.svelte'
import EditorHeader from '$lib/components/EditorHeader.svelte'
import EditableInput from '$lib/components/common/EditableInput.svelte'
import PageHeaderContent from '$lib/components/PageHeaderContent.svelte'
import PathEditPopover from '$lib/components/PathEditPopover.svelte'
import { pageHeader, PHONE_BAR } from '$lib/components/pageHeaderRegistry.svelte'
import AutosaveIndicator from '$lib/components/AutosaveIndicator.svelte'
import { editPathFor } from '$lib/components/workspacePicker'
@@ -232,6 +232,10 @@
// The header's buttons render under the page header, not under this component, so the contexts
// they look up have to travel with them. The app's own two carry everything the canvas toggles,
// the panel buttons, Debug runs and the preview switch read.
/** Held by the pen's popover while it is open; the band's trail reads it so the path does not
* reflow under the pointer as the user types. */
let pathSnapshot = $state<string | undefined>(undefined)
const headerContexts = new Map<any, any>([
['AppViewerContext', getContext('AppViewerContext')],
['AppEditorContext', getContext('AppEditorContext')]
@@ -260,7 +264,12 @@
// In the page header the buttons share the row with the breadcrumb, so they shed the canvas
// toggles and fold Debug runs into the menu sooner: the whole group is ~840px and the trail
// with the summary takes ~430px of the same row.
const compactBelow = $derived(ownsPageHeader ? 1350 : 720)
// Measured on the band rather than estimated: with the canvas toggles shown the actions alone
// are 826-911px and the trail with the summary another ~500, so the row overflows the bar at
// every width up to about 1470 — 1350 was not high enough either. Below this the toggles stand
// down and the menu carries all three, which is what keeps them reachable rather than merely
// out of the way.
const compactBelow = $derived(ownsPageHeader ? 1500 : 720)
const compactTopbar = $derived.by(() => {
const w = ownsPageHeader ? pageHeader.barWidth : topbarWidth
return w > 0 && w < compactBelow
@@ -616,7 +625,33 @@
selectedJobId = $jobs[$jobs.length - 1]
}
$jobsDrawerOpen = true
},
}
},
// The canvas toggles stand down with the bar, and the breakpoint among them is the
// only way onto the mobile layout — so each one keeps an entry here.
{
displayName: $breakpoint === 'sm' ? 'Computer view' : 'Mobile view',
icon: $breakpoint === 'sm' ? Laptop2 : Smartphone,
action: () => ($breakpoint = $breakpoint === 'sm' ? 'lg' : 'sm')
},
{
displayName: $app?.fullscreen ? 'Centered canvas' : 'Full-width canvas',
icon: $app?.fullscreen ? AlignHorizontalSpaceAround : Expand,
action: () => {
if ($app) $app.fullscreen = !$app.fullscreen
}
},
{
displayName:
$app?.darkMode === undefined
? 'Theme: automatic'
: $app.darkMode
? 'Theme: dark'
: 'Theme: light',
icon: $app?.darkMode === undefined ? SunMoon : $app.darkMode ? Moon : Sun,
// Cycles auto → light → dark, the order the toggle group reads in.
action: () =>
setTheme($app?.darkMode === undefined ? false : $app.darkMode ? undefined : true),
separatorBottom: true
}
]
@@ -943,7 +978,15 @@
<!-- The editor's own top bar is the page header on this route: the app's path and summary are
the breadcrumb's, and everything else the bar carried rides along as the header's actions. -->
<PageHeaderContent
item={{ kind: 'app', path: $appPath || newPath || undefined, summaryContent: appSummary }}
item={{
// The path being edited, not the stored one: a brand-new app is parked at a
// `draft_<uuid>` placeholder, and the trail would name that instead of the path Deploy
// will create. Same fallback chain the rename sites read.
// Frozen while the pen's popover is open so the trail holds still as the user types.
kind: 'app',
path: pathSnapshot ?? (newEditedPath || $appPath || newPath || undefined),
summaryContent: appSummary
}}
actions={appHeaderActions}
contexts={headerContexts}
/>
@@ -978,29 +1021,42 @@
{/if}
{#snippet appSummary()}
<!-- The summary stays editable where the editor's own bar had it; the path beside it is the
breadcrumb, which this editor renames from its deploy drawer. -->
<EditableInput
value={$summary ?? ''}
placeholder="Add a summary..."
commitOnInput
size="sm"
onSave={(v) => ($summary = v.trim())}
textClass="text-xs font-medium text-emphasis leading-tight"
class="max-w-full min-w-0"
/>
<!-- Not edited in place: the pen beside it opens the summary and the path together,
so the band reads as a name rather than a form. `title` for one it truncates. -->
<div class="group flex items-center gap-1 min-w-0">
<span
class="min-w-0 truncate text-xs {emptyString($summary)
? 'text-tertiary italic font-normal'
: 'font-medium text-emphasis'}"
title={$summary}>{emptyString($summary) ? 'Add a summary...' : $summary}</span
>
<PathEditPopover
penVisibility={emptyString($summary) ? 'always' : 'hover'}
bind:summary={$summary}
bind:path={newEditedPath}
bind:snapshotPath={pathSnapshot}
savedPath={$appPath || newPath || undefined}
kind="app"
/>
</div>
{/snippet}
{#snippet appHeaderActions()}
<!-- The bar's own order kept: what the canvas looks like, how the draft is doing, which panels
are open, then what to do with the app. -->
{@render canvasToggles()}
{@render autosaveIndicator()}
{#if !phoneTopbar}
{@render panelButtons()}
{@render awarenessMark()}
{/if}
{@render appActions()}
are open, then what to do with the app. The set scrolls rather than pushing past the end of
the bar — this editor carries the widest action row of any page, and the row it replaced
scrolled for the same reason. -->
<div
class="flex flex-row items-center gap-2 min-w-0 overflow-x-auto scrollbar-hidden whitespace-nowrap"
>
{@render canvasToggles()}
{@render autosaveIndicator()}
{#if !phoneTopbar}
{@render panelButtons()}
{@render awarenessMark()}
{/if}
{@render appActions()}
</div>
{/snippet}
{#snippet canvasToggles()}
@@ -1213,11 +1269,10 @@
</div>
</Button>
</div>
<!-- The export is in the menu on a phone; the preview switch keeps its place there, since
<!-- Draws nothing in the bar: it is the export drawer, opened from the menu's Export
entries through this instance. The preview switch keeps its place on a phone, since
seeing the app is most of what an app editor is good for on one. -->
<div class={phoneTopbar ? 'hidden' : 'contents'}>
<AppExportButton bind:this={appExport} />
</div>
<AppExportButton bind:this={appExport} />
<PreviewToggle loading={loading.save} iconOnly={phoneTopbar} />
<Button
variant="accent"
frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte
+22 -6
View File
@@ -20,6 +20,7 @@
import { getUserExt } from '$lib/user'
import { Pen } from 'lucide-svelte'
import { page } from '$app/state'
import { isMenuHidden } from '$lib/components/sessions/sessionMode.svelte'
let {
workspace,
@@ -53,6 +54,11 @@
const hideEditBtn = page.url.searchParams.get('hideEditBtn') === 'true'
const hideRefreshBar = page.url.searchParams.get('hideRefreshBar') === 'true'
const showEdit = $derived(canWriteApp && !hideEditBtn)
// Decides where Edit goes, not whether it appears: without the workspace navigation there is
// no band to put it in. Same rule the layout hides the sidebar by.
const menuHidden = $derived(isMenuHidden(page.url))
// Embedder side: mint a scoped embed token (by path) from the member's session.
async function fetchEmbedToken(opts?: { sdkConsent?: boolean }): Promise<{ token?: string }> {
const headers: Record<string, string> = {}
@@ -149,12 +155,22 @@
</PublicAppFrame>
</div>
{#if canWriteApp && !hideEditBtn}
<!-- The page header carries it, with the app's name: a floating button over the app's own
canvas lands on whatever the app draws there. -->
<!-- Edit sits with the app's name rather than at the far end of the bar: on a page whose header
is only there while hovered, the far end is a journey across the window. -->
<PageHeaderContent item={{ kind: 'app', path }} afterName={editAction} fullBleed />
<!-- The band names the app for whoever opened it, write access or not: the route alone registers
no item, and the breadcrumb would fall back to the section name "Apps". -->
<PageHeaderContent
item={{ kind: 'app', path }}
afterName={showEdit && !menuHidden ? editAction : undefined}
fullBleed
/>
<!-- With a band, Edit sits with the app's name rather than at the far end of the bar: on a page
whose header is only there while hovered, the far end is a journey across the window. An
embed has no band — it would cost the app 44px of the iframe to carry one button — so Edit
floats over the canvas there instead. -->
{#if showEdit && menuHidden}
<div class="absolute bottom-4 right-4 z-50">
{@render editAction()}
</div>
{/if}
{#snippet editAction()}
frontend/src/lib/components/apps/editor/PublicApp.svelte
+12 -7
View File
@@ -150,13 +150,18 @@
{:else if app}
{#key app}
{#if app.raw_app && effectiveWorkspace}
<RawAppPreview
workspace={effectiveWorkspace}
user={$userStore}
secret={app.bundle_secret}
path={app.path}
runnables={(app.value?.runnables ?? {}) as Record<string, Runnable>}
/>
<!-- The bundle's iframe is `h-full`, so something has to give it a height to resolve
against. In the workspace the layout hands one down; the share, embed and custom-path
routes have nothing above them, and an unfloored iframe falls back to 150px. -->
<div class={inWorkspace ? 'h-full w-full' : 'h-screen w-full'}>
<RawAppPreview
workspace={effectiveWorkspace}
user={$userStore}
secret={app.bundle_secret}
path={app.path}
runnables={(app.value?.runnables ?? {}) as Record<string, Runnable>}
/>
</div>
{:else if app.raw_app && !effectiveWorkspace}
<div class="px-4 mt-20">
<Alert type="error" title="Configuration error">
frontend/src/lib/components/breadcrumbPath.test.ts
+27
View File
@@ -0,0 +1,27 @@
import { describe, it, expect } from 'vitest'
import { splitItemPath } from './breadcrumbPath'
describe('splitItemPath', () => {
it('walks the scope, each subfolder, then the item, each with its cumulative path', () => {
expect(splitItemPath('f/demo/sub/weather_report')).toEqual({
dirs: [
{ name: 'f/demo', fullPath: 'f/demo' },
{ name: 'sub', fullPath: 'f/demo/sub' }
],
leaf: { name: 'weather_report', fullPath: 'f/demo/sub/weather_report' }
})
})
it('gives up before there is a scope', () => {
expect(splitItemPath('f/demo')).toBeUndefined()
})
it('keeps the folders while the name is still being typed', () => {
// The editor binds the path live, so every keystroke reaches the breadcrumb: the trail must
// not drop out and come back on the first character of the name.
expect(splitItemPath('f/demo/')).toEqual({
dirs: [{ name: 'f/demo', fullPath: 'f/demo' }],
leaf: undefined
})
})
})
frontend/src/lib/components/breadcrumbPath.ts
+10 -3
View File
@@ -6,9 +6,12 @@ export type PathDir = { name: string; fullPath: string }
* `u/<user>`), then each subfolder, then the item itself. Each level carries the cumulative path,
* which is what the picker opens and highlights.
*
* Returns undefined for anything shorter than scope + name, such as a path still being typed.
* Returns undefined before there is even a scope. The leaf is undefined while the path names a
* folder but no item yet (`f/demo/`) — the levels it does name still come back.
*/
export function splitItemPath(path: string): { dirs: PathDir[]; leaf: PathDir } | undefined {
export function splitItemPath(
path: string
): { dirs: PathDir[]; leaf: PathDir | undefined } | undefined {
const parts = path.split('/')
if (parts.length < 3) return undefined
const scope = parts.slice(0, 2).join('/')
@@ -19,5 +22,9 @@ export function splitItemPath(path: string): { dirs: PathDir[]; leaf: PathDir }
acc = `${acc}/${slug[i]}`
dirs.push({ name: slug[i], fullPath: acc })
}
return { dirs, leaf: { name: slug[slug.length - 1], fullPath: path } }
// A leaf with an empty name draws a blank segment. The editor binds the path live, so the
// breadcrumb sees every keystroke: the folders stay put while the name is being typed rather
// than the whole trail dropping out and coming back.
const name = slug[slug.length - 1]
return { dirs, leaf: name ? { name, fullPath: path } : undefined }
}
frontend/src/lib/components/common/alert/Alert.svelte
+25 -2
View File
@@ -1,5 +1,6 @@
<script lang="ts">
import { type AlertType, classes, icons } from './model'
import { type AlertAction, type AlertType, classes, icons } from './model'
import Button from '../button/Button.svelte'
import Tooltip from '$lib/components/Tooltip.svelte'
import { ChevronDown, ChevronUp } from 'lucide-svelte'
import { slide } from 'svelte/transition'
@@ -25,6 +26,9 @@
descriptionStyle?: string | undefined
class?: string | undefined
isCollapsed?: boolean
/** Buttons under the body, colored to match the alert's type. They stay visible while a
* collapsible alert is collapsed. */
actions?: AlertAction[]
children?: import('svelte').Snippet
}
@@ -47,6 +51,7 @@
descriptionStyle = undefined,
class: classNames = undefined,
isCollapsed = $bindable(true),
actions = [],
children
}: Props = $props()
@@ -66,7 +71,7 @@
<div
class={twMerge(
notRounded ? '' : 'rounded-md',
size === 'sm' ? 'p-4' : 'p-3',
size === 'sm' ? 'px-3 py-2' : 'px-2.5 py-1.5',
classes[type].bgClass,
bgClass,
classNames
@@ -123,6 +128,24 @@
{@render children?.()}
</div>
{/if}
{#if actions.length > 0}
<div class={twMerge('flex flex-wrap gap-2', hasTitleRow || children ? 'mt-2' : '')}>
{#each actions as action}
<Button
unifiedSize={size === 'sm' ? 'sm' : 'xs'}
variant={action.variant ?? 'default'}
tone={type}
startIcon={action.startIcon}
disabled={action.disabled}
loading={action.loading}
onClick={action.onClick}
>
{action.label}
</Button>
{/each}
</div>
{/if}
</div>
</div>
</div>
frontend/src/lib/components/common/alert/model.ts
+12 -1
View File
@@ -1,6 +1,17 @@
import { AlertCircle, AlertTriangle, CheckCircle2, Info } from 'lucide-svelte'
export type AlertType = 'success' | 'error' | 'warning' | 'info'
import type { ButtonType } from '../button/model'
export type AlertType = ButtonType.Tone
export type AlertAction = {
label: string
onClick: () => void
variant?: 'accent' | 'default' | 'subtle'
startIcon?: ButtonType.Icon
disabled?: boolean
loading?: boolean
}
export const classes: Record<AlertType, Record<string, string>> = {
info: {
frontend/src/lib/components/common/button/Button.svelte
+16 -4
View File
@@ -80,6 +80,9 @@
startIcon?: ButtonType.Icon | undefined
endIcon?: ButtonType.Icon | undefined
destructive?: boolean
/** Colors the `accent`, `default` and `subtle` variants to match an Alert of this type.
* Ignored when `dropdownItems` is set: split buttons keep the neutral palette. */
tone?: ButtonType.Tone
shortCut?: { key?: string; hide?: boolean; Icon?: any; withoutModifier?: boolean } | undefined
tooltipPopover?:
| {
@@ -130,6 +133,7 @@
startIcon = undefined,
endIcon = undefined,
destructive = false,
tone = undefined,
shortCut = undefined,
tooltipPopover = undefined,
dropdownBtnClasses = '',
@@ -143,6 +147,8 @@
...rest
}: Props = $props()
const activeTone = $derived(dropdownItems ? undefined : tone)
function computeDropdowns(menuItems: MenuItem[] | (() => MenuItem[])): Item[] {
const items = typeof menuItems === 'function' ? menuItems() : menuItems
return items.map((item) => ({
@@ -185,7 +191,11 @@
if (['accent-secondary', 'accent', 'default', 'subtle'].includes(variant)) {
let style = destructive
? ButtonType.DestructiveVariantStyles[variant]
: ButtonType.VariantStyles[variant]
: activeTone
? ButtonType.ToneVariantStyles[activeTone][
variant === 'accent-secondary' ? 'accent' : variant
]
: ButtonType.VariantStyles[variant]
// For default variant with dropdowns, remove border from button since it's on wrapper
if (
variant === 'default' &&
@@ -272,9 +282,11 @@
'justify-center items-center text-center inline-flex gap-2',
'active:opacity-80 transition-[background-color,opacity] duration-150',
disabled
? ['default', 'subtle'].includes(variant)
? '!text-disabled'
: '!bg-surface-disabled !text-disabled'
? activeTone && !destructive
? 'disabled:opacity-50'
: ['default', 'subtle'].includes(variant)
? '!text-disabled'
: '!bg-surface-disabled !text-disabled'
: '',
loading ? 'cursor-wait' : '',
selected && ['default', 'subtle'].includes(variant)
frontend/src/lib/components/common/button/model.ts
+41
View File
@@ -199,6 +199,47 @@ export namespace ButtonType {
'bg-transparent hover:bg-red-500 hover:text-white dark:hover:bg-red-600 text-primary focus-visible:bg-red-100 dark:focus-visible:bg-red-900/30 focus-visible:ring-red-300'
}
export type Tone = 'info' | 'warning' | 'error' | 'success'
// Colors a button to match the Alert it sits in: text and borders take the alert's hue so a
// gray button never lands on a tinted background. Hovers are gated on `enabled:` because
// `:hover` still matches a disabled button, and a disabled one must not react to the cursor.
export const ToneVariantStyles: Record<Tone, Record<'accent' | 'default' | 'subtle', string>> =
{
info: {
accent:
'bg-blue-500 enabled:hover:bg-blue-600 focus-visible:bg-blue-600 text-white focus-visible:ring-blue-300',
default:
'border border-blue-300 dark:border-blue-400/40 bg-transparent enabled:hover:bg-blue-500/10 dark:enabled:hover:bg-blue-400/15 text-blue-700 dark:text-blue-200 focus-visible:ring-blue-300',
subtle:
'bg-transparent enabled:hover:bg-blue-500/10 dark:enabled:hover:bg-blue-400/15 text-blue-700 dark:text-blue-200 focus-visible:ring-blue-300'
},
warning: {
accent:
'bg-yellow-400 enabled:hover:bg-yellow-500 focus-visible:bg-yellow-500 text-yellow-950 focus-visible:ring-yellow-300',
default:
'border border-yellow-300 dark:border-yellow-400/40 bg-transparent enabled:hover:bg-yellow-500/15 dark:enabled:hover:bg-yellow-400/15 text-yellow-800 dark:text-yellow-300 focus-visible:ring-yellow-300',
subtle:
'bg-transparent enabled:hover:bg-yellow-500/15 dark:enabled:hover:bg-yellow-400/15 text-yellow-800 dark:text-yellow-300 focus-visible:ring-yellow-300'
},
error: {
accent:
'bg-red-500 enabled:hover:bg-red-600 focus-visible:bg-red-600 text-white focus-visible:ring-red-300',
default:
'border border-red-300 dark:border-red-400/40 bg-transparent enabled:hover:bg-red-500/10 dark:enabled:hover:bg-red-400/15 text-red-800 dark:text-red-300 focus-visible:ring-red-300',
subtle:
'bg-transparent enabled:hover:bg-red-500/10 dark:enabled:hover:bg-red-400/15 text-red-800 dark:text-red-300 focus-visible:ring-red-300'
},
success: {
accent:
'bg-green-600 enabled:hover:bg-green-700 focus-visible:bg-green-700 text-white focus-visible:ring-green-300',
default:
'border border-green-300 dark:border-green-400/40 bg-transparent enabled:hover:bg-green-500/10 dark:enabled:hover:bg-green-400/15 text-green-800 dark:text-green-300 focus-visible:ring-green-300',
subtle:
'bg-transparent enabled:hover:bg-green-500/10 dark:enabled:hover:bg-green-400/15 text-green-800 dark:text-green-300 focus-visible:ring-green-300'
}
}
export const VariantSpacingClasses: Record<ButtonType.Size, string> = {
xs3: 'px-0.5 py-[1px]',
xs2: 'px-2 py-[4px]',
frontend/src/lib/components/common/table/AgentRow.svelte
+122
View File
@@ -0,0 +1,122 @@
<script lang="ts">
import { base } from '$lib/base'
import Dropdown from '$lib/components/DropdownV2.svelte'
import SharedBadge from '$lib/components/SharedBadge.svelte'
import DraftBadge from '$lib/components/DraftBadge.svelte'
import type ShareModal from '$lib/components/ShareModal.svelte'
import type DeployWorkspaceDrawer from '$lib/components/DeployWorkspaceDrawer.svelte'
import InheritedLabels from '$lib/components/InheritedLabels.svelte'
import type { ListableResource } from '$lib/gen'
import { userStore, workspaceStore } from '$lib/stores'
import { createEventDispatcher } from 'svelte'
import { Pen } from 'lucide-svelte'
import Button from '../button/Button.svelte'
import ChatFlowBadge from '$lib/components/flows/ChatFlowBadge.svelte'
import { keepsManagedMemory } from '$lib/components/flows/agentFormFields'
import { agentMenuItems, deleteAgent } from '$lib/components/flows/agentActions'
import { getDeployUiSettings } from '$lib/components/home/deploy_ui'
import Row from './Row.svelte'
/**
* A home-page row for a saved agent. The agent is an `ai_agent` resource, so the row opens the
* agent editor the resources page hosts, and its menu carries the resource actions.
*/
interface Props {
agent: ListableResource & { canWrite: boolean }
marked: string | undefined
shareModal: ShareModal
deploymentDrawer: DeployWorkspaceDrawer
deleteConfirmedCallback?: (() => void) | undefined
depth?: number
menuOpen: boolean
keyboardSelected?: boolean
}
let {
agent,
marked,
shareModal,
deploymentDrawer,
deleteConfirmedCallback = $bindable(),
depth = 0,
menuOpen = $bindable(),
keyboardSelected = false
}: Props = $props()
const dispatch = createEventDispatcher()
async function remove(path: string) {
if (await deleteAgent($workspaceStore!, path)) dispatch('change')
}
let editHref = $derived(`${base}/agents/edit/${agent.path}`)
// A draft-only agent has nothing deployed for its page to run, so it opens in the editor.
let rowHref = $derived(agent.draft_only ? editHref : `${base}/agents/get/${agent.path}`)
</script>
{#snippet chatBadge()}
<ChatFlowBadge title="Managed memory on: this agent opens as a conversation" />
{/snippet}
<Row
href={rowHref}
titleBadge={keepsManagedMemory(agent.agent_memory) ? chatBadge : undefined}
kind="agent"
{keyboardSelected}
{marked}
path={agent.draft_path ?? agent.path}
summary={agent.description}
workspaceId={agent.workspace_id ?? $workspaceStore ?? ''}
canFavorite={false}
{depth}
>
{#snippet badges()}
<SharedBadge canWrite={agent.canWrite} extraPerms={agent.extra_perms} />
<DraftBadge
is_draft={agent.is_draft}
draft_only={agent.draft_only}
currentUsername={$userStore?.username}
workspace={$workspaceStore ?? undefined}
itemKind="resource"
path={agent.path}
onMigrated={() => dispatch('change')}
/>
<InheritedLabels labels={agent.inherited_labels} />
{/snippet}
{#snippet actions()}
{#if agent.canWrite}
<span class="hidden md:inline-flex">
<Button
variant="subtle"
wrapperClasses="w-20"
unifiedSize="md"
startIcon={{ icon: Pen }}
href={editHref}
>
Edit
</Button>
</span>
{/if}
<Dropdown
items={async () =>
agentMenuItems({
deployUiSettings: await getDeployUiSettings(),
path: agent.path,
canWrite: agent.canWrite,
draftOnly: Boolean(agent.draft_only),
wsSpecific: agent.ws_specific,
onPermissions: () => shareModal.openDrawer?.(agent.path, 'resource'),
onDeploy: () => deploymentDrawer.openDrawer(agent.path, 'resource'),
onDelete: (event) => {
const { path } = agent
if (event?.shiftKey) remove(path)
else deleteConfirmedCallback = () => remove(path)
}
})}
on:open={() => {
menuOpen = true
}}
/>
{/snippet}
</Row>
frontend/src/lib/components/common/table/FlowRow.svelte
+12 -8
View File
@@ -9,6 +9,7 @@
import type ShareModal from '$lib/components/ShareModal.svelte'
import { FlowService, type Flow } from '$lib/gen'
import { userStore, userWorkspaces, workspaceStore } from '$lib/stores'
import { useOperatorBuilderFlows } from '$lib/operatorWriteRights'
import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
import { createEventDispatcher } from 'svelte'
import Badge from '../badge/Badge.svelte'
@@ -42,6 +43,8 @@
import EditInForkButton from './EditInForkButton.svelte'
import { isCloudHosted } from '$lib/cloud'
const operatorBuilderFlows = useOperatorBuilderFlows()
interface Props {
flow: Flow & {
draft_only?: boolean
@@ -217,6 +220,7 @@
let { draft_only, path, archived } = flow
let owner = isOwner(path, $userStore, $workspaceStore)
const canEdit = flow.canWrite && showEditButton
const hideForOperator = $userStore?.operator && !$operatorBuilderFlows
if (draft_only) {
return [
...selectMenuItems(rowSelection),
@@ -251,7 +255,7 @@
// list endpoint only surfaces own/legacy draft-only rows), so
// discarding it never requires write permission on the path.
disabled: !showEditButton,
hide: $userStore?.operator
hide: hideForOperator
}
]
}
@@ -267,7 +271,7 @@
icon: GitFork,
href: `${base}/flows/add?template=${path}`,
disabled: !showEditButton,
hide: $userStore?.operator
hide: hideForOperator
},
{
displayName: editInForkLabel($workspaceStore, $userWorkspaces),
@@ -293,7 +297,7 @@
moveDrawer.openDrawer(path, flow.summary, 'flow')
},
disabled: !owner || archived || !canEdit,
hide: $userStore?.operator
hide: hideForOperator
},
{
displayName: 'Copy path',
@@ -321,7 +325,7 @@
action: () => {
flowHistory?.open()
},
hide: $userStore?.operator
hide: hideForOperator
},
{
displayName: 'Schedule',
@@ -330,7 +334,7 @@
scheduleEditor?.openNew(true, path)
},
disabled: archived,
hide: $userStore?.operator
hide: hideForOperator
},
{
displayName: 'Permissions',
@@ -338,7 +342,7 @@
action: () => {
shareModal.openDrawer && shareModal.openDrawer(path, 'flow')
},
hide: $userStore?.operator
hide: hideForOperator
},
{
displayName: archived ? 'Unarchive' : 'Archive',
@@ -348,7 +352,7 @@
},
type: 'delete',
disabled: !owner || !canEdit,
hide: $userStore?.operator
hide: hideForOperator
},
{
displayName: 'Delete',
@@ -365,7 +369,7 @@
},
type: 'delete',
disabled: !owner || !canEdit,
hide: $userStore?.operator
hide: hideForOperator
}
]
}}
frontend/src/lib/components/common/table/Row.svelte
+1
View File
@@ -39,6 +39,7 @@
| 'flow'
| 'app'
| 'raw_app'
| 'agent'
| 'resource'
| 'variable'
| 'resource_type'
frontend/src/lib/components/common/table/RowIcon.svelte
+4
View File
@@ -8,6 +8,7 @@
import AzureIcon from '$lib/components/icons/AzureIcon.svelte'
import GoogleCloudIcon from '$lib/components/icons/GoogleCloudIcon.svelte'
import {
Bot,
Boxes,
Calendar,
Code2,
@@ -29,6 +30,7 @@
| 'app'
| 'raw_app'
| 'raw_app_file'
| 'agent'
| 'resource'
| 'variable'
| 'resource_type'
@@ -99,6 +101,8 @@
<FileIcon name={path ?? ''} {size} />
{:else if effectiveKind === 'script'}
<Code2 {size} class="text-blue-500" />
{:else if effectiveKind === 'agent'}
<Bot {size} class="text-violet-500" />
{:else if effectiveKind === 'variable'}
<DollarSign {size} class="text-gray-400" />
{:else if effectiveKind === 'resource'}
frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte
+35 -2
View File
@@ -50,6 +50,8 @@
import { getChatViewHost } from './chatViewHost'
import { getAiChatManager } from './aiChatManagerContext'
import ChatTypingIndicator from './ChatTypingIndicator.svelte'
import ToolGroupDisplay from './ToolGroupDisplay.svelte'
import { chatItemKey, groupToolRuns } from './toolGroups'
import AIChatInput from './AIChatInput.svelte'
import AttachedFilesBar from './files/AttachedFilesBar.svelte'
import QueuedMessageChip from './QueuedMessageChip.svelte'
@@ -237,6 +239,7 @@
const mcpMenu = new McpMenu(aiChatManager, () => assistantSettings?.open('mcp'))
let plusMenuOpen = $state(false)
let editingMessageIndex = $state<number | null>(null)
const chatItems = $derived(groupToolRuns(messages))
// Escape stops the generation when focus is on the chat (or parked on
// body), but stays with other widgets (e.g. the session's Monaco editor).
@@ -872,15 +875,45 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
onscroll={onScroll}
>
<div class="{columnClass} flex flex-col pb-2" bind:clientHeight={height}>
{#each messages as message, messageIndex (messageIndex)}
{#snippet messageRow(
message: DisplayMessage,
messageIndex: number,
isLast: boolean,
inGroup: boolean
)}
<AIChatMessage
{message}
{messageIndex}
{availableContext}
bind:editingMessageIndex
isLast={messageIndex === messages.length - 1}
{isLast}
{inGroup}
showAnswerActions={showsAnswerActions[messageIndex]}
/>
{/snippet}
{#snippet groupRow(message: DisplayMessage, messageIndex: number)}
{@render messageRow(message, messageIndex, false, true)}
{/snippet}
{#each chatItems as item (chatItemKey(item))}
{#if item.kind === 'group'}
<div
class={twMerge(
'mb-1 min-w-0 text-sm px-2 text-primary',
item.entries.at(-1)?.index === messages.length - 1 && '!mb-12'
)}
>
<div class="px-[1px]">
<ToolGroupDisplay group={item} entry={groupRow} />
</div>
</div>
{:else}
{@render messageRow(
item.message,
item.index,
item.index === messages.length - 1,
false
)}
{/if}
{/each}
{#if freeTierExhausted}
{@render freeTierExhaustedBanner()}
frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts
+12 -2
View File
@@ -168,6 +168,7 @@ import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRout
import { appendAttachedFilesRoster } from './files/fileTools'
import { ENTER_PLAN_MODE_TOOL, EXIT_PLAN_MODE_TOOL } from './planMode'
import { PlanModeController, type PlanModeHost } from './planModeController.svelte'
import { navigationOperatorBuilderFlows } from '$lib/operatorWriteRights'
// Compaction of the stored history: once the projected request size
// (contextTokens — the provider's report when current, a fresh chars/4
@@ -300,6 +301,8 @@ export function supportsPlanMode(mode: AIMode): boolean {
return PLAN_MODES.has(mode)
}
const isOperatorBuilderFlows = fromStore(navigationOperatorBuilderFlows)
export function isAIModeVisible(mode: AIMode): boolean {
return mode !== AIMode.GLOBAL || isGlobalAiEnabled()
}
@@ -1445,12 +1448,19 @@ export class AIChatManager implements ChatViewHost {
.map((s) => ({ ...s, kind: 'skill' as const }))
])
// The flow and script builders both write code, which the backend refuses from an operator
// with the builder right: leaving them reachable would only produce work that cannot be
// deployed.
allowedModes: Record<AIMode, boolean> = $derived({
script:
this.flowAiChatHelpers === undefined &&
this.scriptEditorOptions !== undefined &&
!this.disabledModes.script,
flow: this.flowAiChatHelpers !== undefined && !this.disabledModes.flow,
!this.disabledModes.script &&
!isOperatorBuilderFlows.current,
flow:
this.flowAiChatHelpers !== undefined &&
!this.disabledModes.flow &&
!isOperatorBuilderFlows.current,
app: this.appAiChatHelpers !== undefined && !this.disabledModes.app,
navigator: !this.disabledModes.navigator,
ask: !this.disabledModes.ask,
frontend/src/lib/components/copilot/chat/AIChatMessage.svelte
+10 -3
View File
@@ -47,6 +47,8 @@
editingMessageIndex: number | null
isLast?: boolean
showAnswerActions?: boolean
/** Rendered inside a tool group, which owns the horizontal padding and the preview chip. */
inGroup?: boolean
}
let {
@@ -55,7 +57,8 @@
availableContext,
editingMessageIndex = $bindable(null),
isLast = false,
showAnswerActions = true
showAnswerActions = true,
inGroup = false
}: Props = $props()
// The edit box edits a copy of THIS message's original context, not the live
@@ -146,7 +149,8 @@
class={twMerge(
'text-sm px-2',
message.role === 'user' && 'py-1',
message.role === 'tool' && 'text-primary'
message.role === 'tool' && 'text-primary',
inGroup && 'px-0'
)}
>
{#if message.role === 'assistant'}
@@ -159,7 +163,10 @@
>
{:else if message.role === 'tool'}
<div class="px-[1px]"
><ToolExecutionDisplay message={message as ToolDisplayMessage} /></div
><ToolExecutionDisplay
message={message as ToolDisplayMessage}
hidePreviewChip={inGroup}
/></div
>
{:else}
{#if message.role === 'user' && message.images && message.images.length > 0}
frontend/src/lib/components/copilot/chat/AiChatLayout.svelte
-1
View File
@@ -77,7 +77,6 @@
aiChatManager.cancel('aiChatLayout destroyed')
historyManager.close()
})
</script>
{#if !disableAi}
frontend/src/lib/components/copilot/chat/AssistantToolsSection.svelte
+5 -5
View File
@@ -21,10 +21,13 @@ they follow the mode, the connected servers and the workspace's AI settings.
tools,
providerTools,
active,
blocksClose = $bindable()
blocksClose = $bindable(),
description = "What the assistant can call in this session: the built-in tools, whatever the connected MCP servers expose, and what the model's provider runs itself."
}: {
tools: ToolSummary[]
providerTools: ProviderToolSummary[]
/** What the list is, under its heading. */
description?: string
/** Whether this is the panel on screen. Gates the detail page's build, which pulls
* in the schema table and its syntax highlighter. */
active: boolean
@@ -145,10 +148,7 @@ they follow the mode, the connected servers and the workspace's AI settings.
onkeydown={highlight.onKeydown}
onpointermove={highlight.pointerMoved}
>
<Section
label="Tools"
description="What the assistant can call in this session: the built-in tools, whatever the connected MCP servers expose, and what the model's provider runs itself."
>
<Section label="Tools" {description}>
<!-- Sticks to the top of the scrolling panel so a 70-row list stays searchable. -->
<div class="sticky top-0 z-10 bg-surface pb-2">
<TextInput
frontend/src/lib/components/copilot/chat/ChatCollapsibleCard.svelte
+47 -12
View File
@@ -1,8 +1,9 @@
<script lang="ts">
import { ChevronRight } from 'lucide-svelte'
import { twMerge } from 'tailwind-merge'
import { slide } from 'svelte/transition'
import { fade, slide } from 'svelte/transition'
import type { Snippet } from 'svelte'
import { HeldValue, LABEL_MIN_MS } from './heldValue.svelte'
interface Props {
label: string
@@ -18,6 +19,13 @@
toggleable?: boolean
// Sweeps a highlight across the label while the row is in progress.
shimmer?: boolean
/** For a label rewritten at each stage of a running call, sometimes several times a
* second: each label (with its shimmer) stays up at least 700 ms, one replaced sooner is
* skipped, and the new one fades in. */
settleLabel?: boolean
/** The step in progress, shown under the header while collapsed. Held with the label, so
* the two change together. */
liveLine?: string
// Ahead of the label, inside the toggle button: a status that reads as part of the
// row rather than as another control, leaving the chevron next to the label it opens.
headerLeft?: Snippet
@@ -39,6 +47,8 @@
onToggle,
toggleable = true,
shimmer = false,
settleLabel = false,
liveLine,
headerLeft,
headerRight,
belowHeader,
@@ -48,6 +58,17 @@
labelClass,
contentClass
}: Props = $props()
// The shimmer and live line are held with the label, in one value: held apart, a call settling
// right after its last status would show the running label, shimmer or line beside the
// settled others.
const held = new HeldValue(
() => ({ prefix: labelPrefix, label, shimmer, liveLine }),
() => (settleLabel ? LABEL_MIN_MS : 0)
)
const shown = $derived(
settleLabel ? held.current : { prefix: labelPrefix, label, shimmer, liveLine }
)
</script>
<div class={twMerge('font-mono text-xs', className)}>
@@ -59,8 +80,8 @@
highlight && 'text-emphasis'
)}
>
{#if labelPrefix}<span class="font-normal text-secondary">{labelPrefix}</span
>&nbsp;{/if}{label}
{#if shown.prefix}<span class="font-normal text-secondary">{shown.prefix}</span
>&nbsp;{/if}{shown.label}
</span>
{/snippet}
@@ -75,16 +96,27 @@
aria-expanded={toggleable ? expanded : undefined}
>
{@render headerLeft?.()}
{#if shimmer}
<span class="shimmer inline-flex items-center min-w-0">
{@render labelText(false)}
<span class="shimmer-band inline-flex items-center min-w-0" aria-hidden="true">
{@render labelText(true)}
</span>
<!-- A settled label fades in. The key sits outside the shimmer branch because a label
change often comes with a shimmer change, and a transition inside a branch being
swapped out does not play. In only: an outgoing copy would widen the row. Unsettled
labels (an agent trace's ticking duration) keep one key and update in place. -->
{#key settleLabel ? `${shown.prefix ?? ''}\n${shown.label}` : ''}
<span
class="inline-flex items-center min-w-0"
in:fade={{ duration: settleLabel ? 400 : 0 }}
>
{#if shown.shimmer}
<span class="shimmer inline-flex items-center min-w-0">
{@render labelText(false)}
<span class="shimmer-band inline-flex items-center min-w-0" aria-hidden="true">
{@render labelText(true)}
</span>
</span>
{:else}
{@render labelText(false)}
{/if}
</span>
{:else}
{@render labelText(false)}
{/if}
{/key}
{#if toggleable}
<ChevronRight
class={twMerge(
@@ -105,6 +137,9 @@
{@render headerButton()}
{/if}
{#if shown.liveLine && !expanded}
<div class="pl-3 text-2xs text-tertiary truncate">{shown.liveLine}</div>
{/if}
{@render belowHeader?.()}
{#if expanded && children}
frontend/src/lib/components/copilot/chat/LinkRenderer.svelte
+55 -66
View File
@@ -3,7 +3,7 @@
import { ExternalLink, PanelRight } from 'lucide-svelte'
import { Button } from '$lib/components/common'
import RowIcon from '$lib/components/common/table/RowIcon.svelte'
import { newTabModifier } from '$lib/attachments/newTabModifier.svelte'
import Tooltip from '$lib/components/meltComponents/Tooltip.svelte'
import {
hasToolDisplayActionHandler,
runToolDisplayAction
@@ -40,87 +40,76 @@
const action = workspaceItemAction(wmKind, wmPath, wmTargetKind, wmRawApp === 'true')
return action && hasToolDisplayActionHandler(action.type) ? action : undefined
})
// Only the preview panel takes the plain click. A drawer keeps its own button beside an
// Only the preview panel takes the plain click. A drawer stays in the hover menu beside the
// outbound link: the docked chat mounts drawer handlers on nearly every page, so claiming
// that click would redirect these pills far outside the sessions page.
const previewAction = $derived(available?.type === 'open_item_preview' ? available : undefined)
const drawerAction = $derived(available?.type === 'open_created_resource' ? available : undefined)
const allowedHref = $derived(safeHref(href, window.location.href))
const modifier = newTabModifier()
const hint = $derived(
previewAction ? `Open ${wmPath} in the preview panel` : `Open ${wmPath} in a new tab`
)
async function openDrawer(event?: Event) {
event?.preventDefault()
event?.stopPropagation()
if (drawerAction) {
await runToolDisplayAction(drawerAction)
}
async function openAvailable() {
if (available) await runToolDisplayAction(available)
}
async function onclick(event: MouseEvent) {
// Modifier clicks are the only remaining route to the tab once the plain click is
// spoken for, so leave them to the browser.
// Modifier clicks still reach the tab from the pill itself, so leave them to the browser.
if (!previewAction || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) return
event.preventDefault()
await runToolDisplayAction(previewAction)
}
const menuItemClass = 'w-full justify-start no-underline'
</script>
{#if allowedHref}
{#if wmKind}
<!-- Only a preview pill can change icon, so only it is worth tracking the modifier for. -->
<span
class="group inline-flex items-baseline"
{@attach previewAction ? modifier.attach : undefined}
>
<a
href={allowedHref}
target={previewAction ? undefined : '_blank'}
rel={previewAction ? undefined : 'noopener noreferrer'}
title={title || hint}
{onclick}
class="inline-flex items-baseline gap-1 px-1 rounded hover:bg-surface-hover text-primary no-underline font-mono text-[0.9em] align-baseline"
>
<!-- Kind icon and action icon share one fixed 12px box, so the pill is the same
width at rest and on hover and the surrounding sentence never reflows. -->
<span class="relative inline-flex self-center shrink-0 w-3 h-3">
<span class="absolute inset-0 transition-opacity group-hover:opacity-0">
<RowIcon kind={wmKind} size={12} />
</span>
<span
class="absolute inset-0 flex items-center justify-center text-tertiary opacity-0 transition-opacity group-hover:opacity-100"
>
<!-- Narrower than the modifier list `onclick` bails on: only the modifier that
really opens a tab may show the icon for one. -->
{#if previewAction && !modifier.held}
<PanelRight size={12} />
{:else}
<ExternalLink size={11} />
{/if}
</span>
</span>
{@render children?.()}
</a>
{#if drawerAction}
<Button
type="button"
size="xs3"
variant="subtle"
iconOnly
startIcon={{ icon: PanelRight }}
title="Open in editor"
aria-label="Open {wmPath} in editor"
wrapperClasses="ml-0.5 inline-flex self-center shrink-0 opacity-0 group-hover:opacity-100 transition-opacity"
btnClasses="!w-auto !rounded !p-0.5 !text-tertiary"
onClick={openDrawer}
/>
{/if}
{#snippet pill()}
<a
href={allowedHref}
target={previewAction ? undefined : '_blank'}
rel={previewAction ? undefined : 'noopener noreferrer'}
title={available ? undefined : title || `Open ${wmPath} in a new tab`}
{onclick}
class="inline-flex items-baseline gap-1 px-1 rounded hover:bg-surface-hover text-primary no-underline font-mono align-baseline"
>
<span class="inline-flex self-center shrink-0">
<RowIcon kind={wmKind!} size={12} />
</span>
{@render children?.()}
</a>
{/snippet}
{#if allowedHref}
{#if wmKind && available}
<!-- A Tooltip rather than Menu/Popover: those wrap their trigger in a <button>, which
cannot hold the pill's <a>, and the pill must stay a real link for cmd/middle-click.
The menu floats over the text, so opening it never reflows the sentence. -->
<Tooltip placement="top-start" openDelay={200} closeDelay={150} customBgClass="bg-surface p-1">
{@render pill()}
{#snippet text()}
<div class="flex flex-col min-w-36">
<Button
variant="subtle"
unifiedSize="sm"
startIcon={{ icon: PanelRight }}
btnClasses={menuItemClass}
onClick={openAvailable}
>
{previewAction ? 'Open in panel' : 'Open in editor'}
</Button>
<Button
variant="subtle"
unifiedSize="sm"
startIcon={{ icon: ExternalLink }}
btnClasses={menuItemClass}
href={allowedHref}
target="_blank"
>
Open in new tab
</Button>
</div>
{/snippet}
</Tooltip>
{:else if wmKind}
{@render pill()}
{:else}
<a href={allowedHref} target="_blank" rel="noopener noreferrer" {title}>
{@render children?.()}
frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte
+8 -2
View File
@@ -51,9 +51,10 @@
interface Props {
message: ToolDisplayMessage
hidePreviewChip?: boolean
}
let { message }: Props = $props()
let { message, hidePreviewChip = false }: Props = $props()
// Recorded by the call itself, from the connected-server list rather than from the
// model's arguments — which is what lets a reloaded transcript still resolve it, and
@@ -142,7 +143,11 @@
// shown once the tool settled, never while loading/erroring/awaiting confirmation.
const showPreviewChip = $derived(
Boolean(
message.previewCard && !message.isLoading && !message.error && !message.needsConfirmation
!hidePreviewChip &&
message.previewCard &&
!message.isLoading &&
!message.error &&
!message.needsConfirmation
)
)
@@ -317,6 +322,7 @@
onToggle={() => (isExpanded = !isExpanded)}
toggleable={detailsAvailable || message.isStreamingArguments === true}
shimmer={isRunning}
settleLabel
class={message.isQueued && !message.error
? 'opacity-60 hover:opacity-100 transition-opacity'
: ''}
frontend/src/lib/components/copilot/chat/ToolGroupDisplay.svelte
+87
View File
@@ -0,0 +1,87 @@
<script lang="ts">
import type { Snippet } from 'svelte'
import ChatCollapsibleCard from './ChatCollapsibleCard.svelte'
import ToolPreviewCard from './ToolPreviewCard.svelte'
import type { DisplayMessage, ToolDisplayMessage } from './shared'
import { callFailed, groupHeader, type ToolGroup } from './toolGroups'
import McpServerIcon from '$lib/components/mcp/McpServerIcon.svelte'
import { resolveMcpServerMark } from '$lib/components/mcp/serverMark'
interface Props {
group: ToolGroup
entry: Snippet<[DisplayMessage, number]>
}
let { group, entry }: Props = $props()
let expanded = $state(false)
const calls = $derived(
group.entries.map((e) => e.message).filter((m): m is ToolDisplayMessage => m.role === 'tool')
)
// Same states as a single row: a group whose calls are all still waiting their turn is faded
// like a queued row. Once one call has run, the group is in progress until every call has,
// including the moments between two calls when none is executing.
const executing = $derived(calls.some((m) => m.isLoading || m.isStreamingArguments))
const waiting = $derived(calls.some((m) => m.isQueued))
const queued = $derived(!executing && waiting && calls.every((m) => m.isQueued))
const running = $derived(executing || (waiting && !queued))
const header = $derived(groupHeader(group, running ? 'running' : queued ? 'queued' : 'settled'))
const failedCount = $derived(calls.filter(callFailed).length)
// Every edit of one item carries the same chip, so the group shows it once.
const previewCard = $derived(calls.findLast((m) => m.previewCard && !m.error)?.previewCard)
// An explore header has a prefix only when every call is an MCP read on one server; that
// group is marked with the server's icon, as a single MCP row is.
const mcpServer = $derived(
group.groupKind === 'explore' && header.prefix
? calls.find((m) => m.mcpServer)?.mcpServer
: undefined
)
// What the group is doing right now, so a collapsed run still names its current step.
const liveLine = $derived(
running
? (
calls.findLast((m) => m.isLoading || m.isStreamingArguments) ??
calls.find((m) => m.isQueued)
)?.content
: undefined
)
</script>
{#snippet serverMark()}
{#if mcpServer}
{#await resolveMcpServerMark(mcpServer.workspace, mcpServer.path) then mark}
<McpServerIcon icon={mark.icon} size={14} />
{/await}
{/if}
{/snippet}
{#snippet status()}
<div class="flex items-center gap-2 shrink-0">
{#if failedCount > 0}
<span class="font-main text-2xs text-red-600 dark:text-red-400">{failedCount} failed</span>
{/if}
{#if previewCard}
<ToolPreviewCard card={previewCard} />
{/if}
</div>
{/snippet}
<ChatCollapsibleCard
labelPrefix={header.prefix}
label={header.label}
{expanded}
onToggle={() => (expanded = !expanded)}
shimmer={running}
settleLabel
{liveLine}
class={queued ? 'opacity-60 hover:opacity-100 transition-opacity' : ''}
labelClass="truncate"
headerRight={failedCount > 0 || previewCard ? status : undefined}
headerLeft={mcpServer ? serverMark : undefined}
contentClass="border-0 border-l rounded-none bg-transparent p-0 pl-2 ml-1.5 mt-0.5"
>
{#each group.entries as { message, index } (index)}
{@render entry(message, index)}
{/each}
</ChatCollapsibleCard>
frontend/src/lib/components/copilot/chat/app/core.ts
+1 -1
View File
@@ -974,7 +974,7 @@ Use \`patch_file\` for small, localized edits when you can copy an exact snippet
### Discovery
- \`search_workspace(query, type)\`: Search workspace scripts and flows
- \`get_runnable_details(path, type)\`: Get details (summary, description, schema, content) of a specific script or flow
- \`search_hub_scripts(query)\`: Search hub scripts
- \`search_hub_scripts(query, integration)\`: Search hub scripts, or list one integration's scripts by slug
### Data Tables
- \`list_datatables()\`: List configured datatables with schema and table names only. Does not include columns. Use this directly for table-list or available-tables summaries.
frontend/src/lib/components/copilot/chat/draftWriteResults.ts
+4
View File
@@ -0,0 +1,4 @@
/** `result` of a draft write that did not land. The write reports it through `result` alone, not
* `error`, so a tool group reads these to count the call as failed. */
export const DRAFT_CONFLICT_RESULT = 'Conflict'
export const DRAFT_SAVE_FAILED_RESULT = 'Save failed'

Some files were not shown because too many files have changed in this diff Show More