diff --git a/.release-please-manifest.json b/.release-please-manifest.json index f4fac9d2e0..4100cf6bbc 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.805.0" + ".": "1.809.0" } diff --git a/AGENTS.md b/AGENTS.md index 2b204a65db..f8c83ec465 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,7 +37,7 @@ Open-source platform for internal tools, workflows, API integrations, background - **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill. - **Frontend UUIDs**: do not call `crypto.randomUUID()` in frontend code. Import `randomUUID` from `$lib/utils/uuid` instead. - **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy and `xhigh` reasoning, on `gpt-6-astra` rather than the action's `gpt-5.6-sol`; requires the `codex` CLI >= 0.153.4. -- **Domain guides**: `.claude/skills/native-trigger/` and `frontend/tutorial-system-guide.mdc` +- **Domain guides**: `.claude/skills/native-trigger/` - **Brand/UI guidelines**: `frontend/brand-guidelines.md` - **Domain vocabulary**: `CONTEXT.md` — the words this codebase uses for its own concepts (step, step setting, trigger step, …). Name things the way it does. - **CLI commands**: when adding/modifying/removing a command, subcommand, option, or description in `cli/src/commands/`, run `python system_prompts/generate.py` to refresh `system_prompts/auto-generated/` and `cli/src/guidance/skills.gen.ts`. The CLI docs the agents use to operate `wmill` are derived from the source — stale generated files give agents the wrong flags. diff --git a/CHANGELOG.md b/CHANGELOG.md index f431b9fd96..d82cc8a185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,93 @@ # Changelog +## [1.809.0](https://github.com/windmill-labs/windmill/compare/v1.808.0...v1.809.0) (2026-09-10) + + +### Features + +* add a minimal skin for the approval page and slack/teams ([#11061](https://github.com/windmill-labs/windmill/issues/11061)) ([63cb46d](https://github.com/windmill-labs/windmill/commit/63cb46d7bb9db1d996aa38e06bd3afebc60111bd)) +* live queue status per tag and bounded queue metric charts ([#11067](https://github.com/windmill-labs/windmill/issues/11067)) ([569adb8](https://github.com/windmill-labs/windmill/commit/569adb85c1885d289e80a70a166f0f74e6d5ba83)) +* **otel:** read the OTLP metrics temporality preference ([#11064](https://github.com/windmill-labs/windmill/issues/11064)) ([2f88769](https://github.com/windmill-labs/windmill/commit/2f8876908719b3640d7cfc9364a7b0f7145fc356)) +* **otel:** support standard OTEL resource attribute env vars ([#10974](https://github.com/windmill-labs/windmill/issues/10974)) ([0a40eea](https://github.com/windmill-labs/windmill/commit/0a40eea37a7dbde5fc4760d6333d81186dfee255)) +* report script metadata with no content file in wmill lint ([#11053](https://github.com/windmill-labs/windmill/issues/11053)) ([8820b9f](https://github.com/windmill-labs/windmill/commit/8820b9fc644c6620c50517cd0d902015e2e670e2)) +* show the workspace an operator is in, and let them switch ([#11059](https://github.com/windmill-labs/windmill/issues/11059)) ([385086f](https://github.com/windmill-labs/windmill/commit/385086ffc21c72cd07624584932a4a301c23a732)) +* tuck other users' spaces into a collapsible home tree row ([#11073](https://github.com/windmill-labs/windmill/issues/11073)) ([d87f089](https://github.com/windmill-labs/windmill/commit/d87f089288996af9ea7e3b017a5ef35d4ded880d)) + + +### Bug Fixes + +* **ai-chat:** test_run_flow could test a different flow than the one asked ([#11066](https://github.com/windmill-labs/windmill/issues/11066)) ([fa73539](https://github.com/windmill-labs/windmill/commit/fa73539839071491fb2cbad9242f52ad22b975bf)) +* bound list_jobs runtime and paginate runs on the sorted column ([#11072](https://github.com/windmill-labs/windmill/issues/11072)) ([f517402](https://github.com/windmill-labs/windmill/commit/f51740253871960b55ab2aa8989e8df3fbde0351)) +* **frontend:** clear the flow graph selection through xyflow's store ([#11056](https://github.com/windmill-labs/windmill/issues/11056)) ([b4be8bc](https://github.com/windmill-labs/windmill/commit/b4be8bc5354fbd3a47c267c331aaf603c0f90e6e)) +* **frontend:** recompute dataflow edges when selecting a step ([#11070](https://github.com/windmill-labs/windmill/issues/11070)) ([08d876a](https://github.com/windmill-labs/windmill/commit/08d876aebf32ebb995a8c1839aa87794c0176bed)) +* **frontend:** restore heading sizes in note markdown and keep group notes on id change ([#11047](https://github.com/windmill-labs/windmill/issues/11047)) ([e63072c](https://github.com/windmill-labs/windmill/commit/e63072c216383700a23504be89782f4c69657174)) +* give every table a primary key so the db can be logically replicated ([#11036](https://github.com/windmill-labs/windmill/issues/11036)) ([e62bfdc](https://github.com/windmill-labs/windmill/commit/e62bfdcd8c6f1389601ccd2b5809c5eff0ff262e)) +* keep an app's deployed policy on wmill push ([#11049](https://github.com/windmill-labs/windmill/issues/11049)) ([0af7675](https://github.com/windmill-labs/windmill/commit/0af7675588300863883a99c7f14bc53cafec8a7e)) +* refuse cross-site GET requests that run Hub scripts ([#11054](https://github.com/windmill-labs/windmill/issues/11054)) ([ab9efc8](https://github.com/windmill-labs/windmill/commit/ab9efc897cc94d61a67263772806b15e9225cece)) +* skip the deploy PR when the git sync push committed nothing ([#11076](https://github.com/windmill-labs/windmill/issues/11076)) ([8ecbd33](https://github.com/windmill-labs/windmill/commit/8ecbd339eef7314a93d599fcea4377299d4c493d)) +* space the trailing AI settings cards ([#11044](https://github.com/windmill-labs/windmill/issues/11044)) ([5d7eed1](https://github.com/windmill-labs/windmill/commit/5d7eed1c02b0966289cc8cca00a15d76dad187a5)) +* stop uv pip compile emitting lockfile annotations ([#11042](https://github.com/windmill-labs/windmill/issues/11042)) ([8aa8b7e](https://github.com/windmill-labs/windmill/commit/8aa8b7ee6c23f859f169637c0bfd3f8d964509e9)) +* surface why a private or untrusted git host is unreachable ([#11068](https://github.com/windmill-labs/windmill/issues/11068)) ([c57b18e](https://github.com/windmill-labs/windmill/commit/c57b18e46fcdd319213fe0a537cb18de418ca688)) + + +### Performance Improvements + +* index the FK columns that cascade on workspace delete ([#11052](https://github.com/windmill-labs/windmill/issues/11052)) ([9a563f6](https://github.com/windmill-labs/windmill/commit/9a563f6d72da28fe09b785cd0683e9698df72bba)) +* only write queue metrics when a tag's backlog changes ([#11055](https://github.com/windmill-labs/windmill/issues/11055)) ([9d75929](https://github.com/windmill-labs/windmill/commit/9d75929247ea2ec39286971fcbebf95d886194f3)) + +## [1.808.0](https://github.com/windmill-labs/windmill/compare/v1.807.0...v1.808.0) (2026-09-09) + + +### Features + +* run and test scripts from the AI chat through an argument form ([#11001](https://github.com/windmill-labs/windmill/issues/11001)) ([a6abf2c](https://github.com/windmill-labs/windmill/commit/a6abf2c8a744e9ee6acf4830cdfbb84f2f95cb36)) + +## [1.807.0](https://github.com/windmill-labs/windmill/compare/v1.806.0...v1.807.0) (2026-09-09) + + +### Features + +* add a dismissible instance-wide announcement banner ([#11037](https://github.com/windmill-labs/windmill/issues/11037)) ([abf4c6c](https://github.com/windmill-labs/windmill/commit/abf4c6c2348014ea4401b9be62b5b15e5800e879)) +* batch chained DDL statements into a single migration ([#11038](https://github.com/windmill-labs/windmill/issues/11038)) ([656e609](https://github.com/windmill-labs/windmill/commit/656e609595833bc854f845d35cf43157e76d732f)) +* create the cloud workspace in onboarding, and teach the empty home ([#10959](https://github.com/windmill-labs/windmill/issues/10959)) ([fd35b47](https://github.com/windmill-labs/windmill/commit/fd35b4765843879cb2254f402c142fd7510f1916)) +* link from the public run view to the authenticated run page ([#11041](https://github.com/windmill-labs/windmill/issues/11041)) ([09b81a9](https://github.com/windmill-labs/windmill/commit/09b81a9294bed2795a1a7b688d0b02e0a61957ea)) +* make guest access unavailable on the shared cloud ([#11040](https://github.com/windmill-labs/windmill/issues/11040)) ([0b63e0a](https://github.com/windmill-labs/windmill/commit/0b63e0a6929088ff25def4fd6547cf61668251a5)) + + +### Bug Fixes + +* ignore comments and continuations in python lockfiles ([#11035](https://github.com/windmill-labs/windmill/issues/11035)) ([90c4e10](https://github.com/windmill-labs/windmill/commit/90c4e1020a2ff896977648dd68b572413cea7709)) +* refetch an unparseable hub script cache entry instead of panicking ([#11033](https://github.com/windmill-labs/windmill/issues/11033)) ([88c3ebd](https://github.com/windmill-labs/windmill/commit/88c3ebdfc1325ffbea4521d854e71d231a6409c4)) +* stop a new AI session adopting a legacy sidebar chat ([#11039](https://github.com/windmill-labs/windmill/issues/11039)) ([1076b63](https://github.com/windmill-labs/windmill/commit/1076b638d987ba99c5c27e5478ca580534b9d572)) + +## [1.806.0](https://github.com/windmill-labs/windmill/compare/v1.805.0...v1.806.0) (2026-09-08) + + +### Features + +* bring gitlab repositories to parity for git sync ([#10938](https://github.com/windmill-labs/windmill/issues/10938)) ([9444049](https://github.com/windmill-labs/windmill/commit/9444049d6013c77a5f25f01a736eb5cd741fb3e6)) +* draw a dbt column trace, across projects and the pipeline boundary ([#11014](https://github.com/windmill-labs/windmill/issues/11014)) ([33f9828](https://github.com/windmill-labs/windmill/commit/33f9828c3ed15fe63fccedc1550584f15c0490ab)) +* durable dbt state per environment, and `--defer` onto it ([#10975](https://github.com/windmill-labs/windmill/issues/10975)) ([621fac5](https://github.com/windmill-labs/windmill/commit/621fac55abcd1859e8c8c06e5f4412e61bb85d59)) +* ingest dbt column lineage and real column schemas from the engine's parquet index ([#10977](https://github.com/windmill-labs/windmill/issues/10977)) ([0139467](https://github.com/windmill-labs/windmill/commit/0139467b01b82e4b3d474ca3f205358fa607d19a)) +* let a worker group override the dependency cache object store ([#11019](https://github.com/windmill-labs/windmill/issues/11019)) ([de98adf](https://github.com/windmill-labs/windmill/commit/de98adf055835ab7c4d6305e7d5d3bdab915876b)) +* **nativets:** bound fetch on a peer that never answers ([#11026](https://github.com/windmill-labs/windmill/issues/11026)) ([785277e](https://github.com/windmill-labs/windmill/commit/785277e0bb2ea77b71a89dd4d389d439dfcf9e03)) +* recognize `// volume:` mounts in PHP scripts ([#11018](https://github.com/windmill-labs/windmill/issues/11018)) ([f081fb1](https://github.com/windmill-labs/windmill/commit/f081fb10705cadf99e99dfa786d1cc2ebf0447db)) +* report a WAC task failure the workflow body never awaited ([#11017](https://github.com/windmill-labs/windmill/issues/11017)) ([3e3a41d](https://github.com/windmill-labs/windmill/commit/3e3a41d418d4ee3fe060bd3acf3324f311d89c8a)) +* retry a workflow-as-code task from its task options ([#11013](https://github.com/windmill-labs/windmill/issues/11013)) ([d3f305d](https://github.com/windmill-labs/windmill/commit/d3f305db982b7c5dc49babf9bec8b62adcd2557d)) + + +### Bug Fixes + +* chain redeploys onto a retired path's version history ([#11029](https://github.com/windmill-labs/windmill/issues/11029)) ([0b37226](https://github.com/windmill-labs/windmill/commit/0b372260787edb7e9627ad4fb6637ad5f1024e0a)) +* make the native trigger disable/enable toggle actually save ([#11024](https://github.com/windmill-labs/windmill/issues/11024)) ([448fce9](https://github.com/windmill-labs/windmill/commit/448fce93f743d5b2ef2a2d4496eb2ec238594a0a)) +* offload php signature parsing from async workers ([#11027](https://github.com/windmill-labs/windmill/issues/11027)) ([2cb02e3](https://github.com/windmill-labs/windmill/commit/2cb02e3b3398db49f16377dd79dde2dd6fb5cc02)) +* reduce php parser stack use in debug workers ([#11025](https://github.com/windmill-labs/windmill/issues/11025)) ([2ae8509](https://github.com/windmill-labs/windmill/commit/2ae8509b14f112c9ef5b71321e8fd52596a16c10)) + + +### Performance Improvements + +* reduce shared worker debug polling frames ([#11028](https://github.com/windmill-labs/windmill/issues/11028)) ([946756a](https://github.com/windmill-labs/windmill/commit/946756ae83deb4e7a93111edddbd4d98c596d5a9)) + ## [1.805.0](https://github.com/windmill-labs/windmill/compare/v1.804.0...v1.805.0) (2026-09-07) diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 4501f24810..2e29b9064d 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -132,6 +132,10 @@ export async function runEval( setToolStatus: () => {}, removeToolStatus: () => {}, isPlanModeActive, + // Accepts the run form exactly as the model prefilled it: there is nobody here to + // edit the arguments, so a case can assert what the model proposed but never how + // it reacts to the user changing something. + requestRunArgs: async (_toolId, form) => form.args, onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 338ed8504c..e5b275b86e 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -86,6 +86,7 @@ vi.mock('$lib/gen', async () => { previewBenchmarkSchedule, runBenchmarkDatatableSql, runBenchmarkFlowByPath, + runBenchmarkScriptByPath, runBenchmarkScriptPreview, updateBenchmarkDraft, listBenchmarkMcpTools @@ -279,6 +280,18 @@ vi.mock('$lib/gen', async () => { } return runBenchmarkScriptPreview({ workspace: data.workspace, requestBody }) }, + runScriptByPath: async (data: { + workspace: string + path: string + requestBody?: Record + }) => + hasBenchmarkWorkspace(data.workspace) + ? runBenchmarkScriptByPath({ + workspace: data.workspace, + path: data.path, + args: data.requestBody + }) + : actual.JobService.runScriptByPath(data), runFlowByPath: async (data: { workspace: string path: string diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 94ecb5c029..16d96d10b9 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1974,6 +1974,78 @@ judgeChecklist: - deletes the deployed script via delete_workspace_item rather than a raw API endpoint +- id: global-test33-run-deployed-script-with-form + prompt: |- + Run the deployed script `f/evals/global/format_greeting` for me with the name "ada". + initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json + runtime: + maxTurns: 8 + # A session chat is where the run card has a preview pane beside it; run_script + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_script + # A draft may declare different arguments than the deployed version being run, so + # the names to prefill have to come from the deployed schema. + - read_workspace_item + forbiddenToolsUsed: + - test_run_script + - call_api_endpoint + - write_script + - deploy_workspace_item + # An empty form pushes the work back onto the user, so the prefill is part of + # what the tool is for. + toolCallArgs: + - tool: run_script + field: args.name + stringIncludesAnyOf: + - ada + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - runs the deployed script through run_script rather than a preview test run or a raw API endpoint + - passes the name "ada" so the confirmation form comes up prefilled + +- id: global-test34-run-with-secret-from-variable + prompt: |- + Run the deployed `f/evals/global/billing_sync` for the account `acme` — use the billing + API token we already keep in the workspace. + initial: ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json + runtime: + maxTurns: 10 + # A session chat is where the run card has a preview pane beside it; run_script + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_script + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + # A secret argument is filled by naming the variable that holds it: the value stays in + # the variable and only its path travels. A literal reaches the job as a reference too, + # minted on the way in, but it stays in the tool call the model emitted. + toolCallArgs: + - tool: run_script + field: args.api_token + stringIncludesAnyOf: + - "$var:f/evals/global/stripe_api_token" + - tool: run_script + field: args.account + stringIncludesAnyOf: + - acme + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - fills the secret argument with a reference to the existing workspace variable rather than a literal token + - passes the account "acme" + - does not invent or guess the token's value + - id: global-undo-created-draft prompt: |- Create a draft Postgres resource at `u/admin/scratch_db` for host db.example.com port 5432, database `orders`, user `app`, and tell me what fields it ended up with. diff --git a/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json b/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json new file mode 100644 index 0000000000..ef166b0f9c --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json @@ -0,0 +1,37 @@ +{ + "workspace": { + "variables": [ + { + "path": "f/evals/global/stripe_api_token", + "value": "sk_live_do_not_leak_me", + "is_secret": true, + "description": "Token used by the billing sync job", + "labels": ["billing"] + } + ], + "scripts": [ + { + "path": "f/evals/global/billing_sync", + "summary": "Sync billing records", + "description": "Syncs billing records for one account, authenticating with an API token.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "account": { + "type": "string" + }, + "api_token": { + "type": "string", + "password": true, + "description": "API token to authenticate with" + } + }, + "required": ["account", "api_token"] + }, + "content": "export async function main(account: string, api_token: string) {\n return `synced ${account}`\n}\n" + } + ] + } +} diff --git a/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json b/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json deleted file mode 100644 index 9c1f8d00a7..0000000000 --- a/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value)\n VALUES ($1, to_jsonb((\n SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n FROM v2_job_queue\n WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1\n )))", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f" -} diff --git a/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json b/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json new file mode 100644 index 0000000000..5f3d8300eb --- /dev/null +++ b/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json @@ -0,0 +1,73 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH slots AS (\n SELECT id, slot, min(t) AS first, max(t) AS last, max(v) AS peak,\n (min(ARRAY[t, v]))[2] AS first_value, (max(ARRAY[t, v]))[2] AS last_value,\n (max(ARRAY[t, climbing]))[2] = 1 AS last_climbing,\n COALESCE(bool_and(climbing = 1) AND max(since) - min(since) < $4, false) AS ramp,\n max(ARRAY[t, since]) FILTER (WHERE climbing = 1) AS last_climb\n FROM (\n SELECT id, t,\n CASE jsonb_typeof(value)\n WHEN 'number' THEN value::double precision\n WHEN 'object' THEN t - (value->>'since')::double precision\n END AS v,\n (value->>'since')::double precision AS since,\n (jsonb_typeof(value) = 'object')::int::double precision AS climbing,\n greatest(floor((t - $1::double precision) / $2::double precision), -1)::int\n AS slot\n FROM (\n SELECT id, value, EXTRACT(EPOCH FROM created_at)::double precision AS t\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > to_timestamp($1::double precision - $3::double precision)\n ) m\n ) s\n WHERE v IS NOT NULL\n GROUP BY id, slot\n )\n SELECT id AS \"id!\", slot AS \"slot!\", first AS \"first!\", last AS \"last!\",\n greatest(peak, CASE WHEN last_climb[1] < last THEN (\n SELECT EXTRACT(EPOCH FROM min(n.created_at))::double precision\n FROM metrics n\n WHERE n.id = slots.id AND n.id LIKE 'queue_%'\n AND n.created_at > to_timestamp(last_climb[1] + 0.001)\n AND n.created_at <= to_timestamp(last + 0.001)\n ) - last_climb[2] END) AS \"peak!\",\n first_value AS \"first_value!\", last_value AS \"last_value!\",\n last_climbing AS \"last_climbing!\", ramp AS \"ramp!\"\n FROM slots\n ORDER BY id, slot", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slot!", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "first!", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "last!", + "type_info": "Float8" + }, + { + "ordinal": 4, + "name": "peak!", + "type_info": "Float8" + }, + { + "ordinal": 5, + "name": "first_value!", + "type_info": "Float8" + }, + { + "ordinal": 6, + "name": "last_value!", + "type_info": "Float8" + }, + { + "ordinal": 7, + "name": "last_climbing!", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "ramp!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Float8", + "Float8", + "Float8", + "Float8" + ] + }, + "nullable": [ + false, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863" +} diff --git a/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json b/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json new file mode 100644 index 0000000000..843484dd86 --- /dev/null +++ b/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXTRACT(EPOCH FROM now())::double precision AS \"now!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "now!", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753" +} diff --git a/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json b/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json deleted file mode 100644 index 9f97b97a88..0000000000 --- a/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH queue_metrics as (\n SELECT id, value, created_at\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > now() - interval '14 day'\n )\n SELECT id, array_agg(json_build_object('value', value, 'created_at', created_at) ORDER BY created_at ASC) as \"values!\"\n FROM queue_metrics\n GROUP BY id\n ORDER BY id ASC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "values!", - "type_info": "JsonArray" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - null - ] - }, - "hash": "44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa" -} diff --git a/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json b/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json new file mode 100644 index 0000000000..f0d9aabea3 --- /dev/null +++ b/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(c.id, r.id) AS \"id!\", r.value AS \"value?\",\n EXTRACT(EPOCH FROM r.created_at)::double precision AS \"at?\",\n EXTRACT(EPOCH FROM now() - r.created_at)::double precision AS \"age?\"\n FROM unnest($1::text[]) AS c(id)\n FULL JOIN (\n SELECT DISTINCT ON (id) id, value, created_at\n FROM metrics\n WHERE id LIKE 'queue_%' AND created_at > now() - make_interval(secs => $2)\n ORDER BY id, created_at DESC\n ) r ON r.id = c.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "value?", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "at?", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "age?", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Float8" + ] + }, + "nullable": [ + true, + true, + true, + true + ] + }, + "hash": "58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b" +} diff --git a/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json b/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json new file mode 100644 index 0000000000..8eac288a1b --- /dev/null +++ b/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH running AS (\n SELECT tag, count(*) AS n FROM v2_job_queue WHERE running = true GROUP BY tag\n )\n SELECT t.tag AS \"tag!\", COALESCE(r.n, 0) AS \"running!\",\n (SELECT count(*) FROM worker_ping w\n WHERE w.ping_at > now() - interval '1 minute' AND w.custom_tags @> ARRAY[t.tag]\n ) AS \"workers!\"\n FROM (SELECT tag::text FROM running UNION SELECT unnest($1::text[])) t(tag)\n LEFT JOIN running r ON r.tag = t.tag\n ORDER BY t.tag", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "running!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "workers!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438" +} diff --git a/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json b/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json new file mode 100644 index 0000000000..ae8cc698b9 --- /dev/null +++ b/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag AS \"tag!\", count AS \"count!\",\n EXTRACT(EPOCH FROM now() - head)::double precision AS \"delay!\",\n EXTRACT(EPOCH FROM head)::double precision AS \"head_since!\"\n FROM (\n SELECT tag, sum(n)::bigint AS count,\n (array_agg(head ORDER BY priority DESC NULLS LAST))[1] AS head\n FROM (\n SELECT tag, priority, count(*) AS n, min(scheduled_for) AS head\n FROM v2_job_queue WHERE\n scheduled_for <= now() - ('3 seconds')::interval AND running = false\n GROUP BY tag, priority\n ) g\n GROUP BY tag\n ) t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "delay!", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "head_since!", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null, + null, + null + ] + }, + "hash": "7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5" +} diff --git a/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json b/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json deleted file mode 100644 index f98f3fdeea..0000000000 --- a/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92" -} diff --git a/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json b/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json deleted file mode 100644 index 124a6e36c5..0000000000 --- a/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value) VALUES ($1, $2)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15" -} diff --git a/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json b/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json new file mode 100644 index 0000000000..8c6dd010af --- /dev/null +++ b/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH queue_metrics as (\n SELECT id, created_at,\n CASE WHEN jsonb_typeof(value) = 'object'\n THEN to_jsonb(EXTRACT(EPOCH FROM created_at) - (value->>'since')::numeric)\n ELSE value\n END AS value\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > now() - interval '14 day'\n )\n SELECT id, array_agg(json_build_object('value', value, 'created_at', created_at) ORDER BY created_at ASC) as \"values!\"\n FROM queue_metrics\n GROUP BY id\n ORDER BY id ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "values!", + "type_info": "JsonArray" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null + ] + }, + "hash": "a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16" +} diff --git a/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json b/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json new file mode 100644 index 0000000000..3f73c2171e --- /dev/null +++ b/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO metrics (id, value)\n SELECT id, COALESCE(to_jsonb(EXTRACT(EPOCH FROM now())::double precision - held_head), value)\n FROM unnest($1::text[], $2::jsonb[], $3::double precision[]) AS u(id, value, held_head)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "JsonbArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 4dff91e5d2..26ee0710f3 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1724,7 +1724,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1744,7 +1744,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1787,9 +1787,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" dependencies = [ "serde_core", ] @@ -1958,18 +1958,18 @@ dependencies = [ [[package]] name = "bon" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" +checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9" dependencies = [ "bon-macros", ] [[package]] name = "bon-macros" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" +checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6" dependencies = [ "darling 0.24.1", "ident_case", @@ -5077,7 +5077,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "rustc_version 0.4.1", ] @@ -6194,9 +6194,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "typenum", ] @@ -6463,11 +6463,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" version = "2.3.0" @@ -6517,6 +6538,8 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -6524,6 +6547,28 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_segmenter" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" + [[package]] name = "ident_case" version = "1.0.1" @@ -6644,7 +6689,7 @@ version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -7217,7 +7262,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e668df13f2e97f3eed52d9301f6b1c4c1ccfccc30eab9e6628e4a8c1fc3546" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "lazy_static", "libgssapi-sys", @@ -7266,7 +7311,7 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "plain", "redox_syscall 0.9.4", @@ -7906,7 +7951,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a" dependencies = [ "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "btoi", "byteorder", "bytes", @@ -7965,7 +8010,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -7976,7 +8021,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -7988,7 +8033,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -8000,7 +8045,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -8467,7 +8512,7 @@ version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "once_cell", "onig_sys", @@ -8532,7 +8577,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "foreign-types", "libc", @@ -9485,6 +9530,8 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -9544,7 +9591,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.13+spec-1.1.0", + "toml_edit 0.25.14+spec-1.1.0", ] [[package]] @@ -9633,7 +9680,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "flate2", "hex", @@ -9647,7 +9694,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", ] @@ -9734,7 +9781,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "getopts", "memchr", "unicase", @@ -10028,7 +10075,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10162,7 +10209,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10171,7 +10218,7 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10326,11 +10373,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "encoding_rs", "futures-core", @@ -10378,7 +10425,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.5.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "thiserror 2.0.20", "tower-service", @@ -10396,7 +10443,7 @@ dependencies = [ "getrandom 0.2.17", "http 1.5.0", "hyper 1.11.1", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "retry-policies", "thiserror 2.0.20", @@ -10506,7 +10553,7 @@ dependencies = [ "pastey", "pin-project-lite", "rand 0.10.2", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp-macros", "schemars 1.2.2", "serde", @@ -10524,9 +10571,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf1c49bd4d52014b94db0877410db273c2008f01628b0252a2e9460ad9b7fda" +checksum = "873b730df6f0a9b74b13eb514e0dca4c2db0d8b68b74af98a2e9bf3f9d436585" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -10741,7 +10788,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.4.15", @@ -10754,7 +10801,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.12.1", @@ -11241,7 +11288,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -11254,7 +11301,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -11484,11 +11531,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bs58", "chrono", "hex", @@ -11505,14 +11552,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -12017,7 +12064,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "bytes", "chrono", @@ -12061,7 +12108,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "chrono", "crc", @@ -12335,7 +12382,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c25af97d53cf8aab66a6c68f3418663313fc969ad267fc2a4d19402c329be1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "is-macro", "num-bigint", "once_cell", @@ -12391,7 +12438,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "017d06ea85008234aa9fb34d805c7dc563f2ea6e03869ed5ac5a2dc27d561e4d" dependencies = [ "arrayvec", - "bitflags 2.13.1", + "bitflags 2.13.2", "either", "num-bigint", "phf 0.11.3", @@ -12724,7 +12771,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "enum-as-inner", "libc", @@ -12752,7 +12799,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -13011,11 +13058,11 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +checksum = "b81c0cb5fce14f53e49c1d4da0c508334ff12040221bb8ab01b2dabd91d04b6e" dependencies = [ - "unicode-linebreak", + "icu_segmenter", "unicode-width 0.2.2", ] @@ -13197,6 +13244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -13559,9 +13607,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.14+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "d2195eec204e2764644a4ea619704f9fbe5e0673038eded55ad9956f24fca0cc" dependencies = [ "indexmap 2.14.2", "toml_datetime 1.1.1+spec-1.1.0", @@ -13622,6 +13670,7 @@ dependencies = [ "axum 0.8.9", "base64 0.22.1", "bytes", + "flate2", "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", @@ -13641,6 +13690,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "zstd", ] [[package]] @@ -13706,7 +13756,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-core", "futures-util", @@ -14130,12 +14180,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-linebreak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" - [[package]] name = "unicode-normalization" version = "0.1.25" @@ -14335,9 +14379,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14352,7 +14396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" dependencies = [ "bindgen 0.71.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "fslock", "gzip-header", "home", @@ -14747,7 +14791,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-nats", @@ -14772,7 +14816,7 @@ dependencies = [ "prometheus", "rand 0.9.0", "rdkafka", - "reqwest 0.13.4", + "reqwest 0.13.5", "rumqttc", "rustls 0.23.35", "serde", @@ -14835,7 +14879,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.805.0" +version = "1.809.0" dependencies = [ "async-stream", "async-trait", @@ -14851,7 +14895,7 @@ dependencies = [ "http 1.5.0", "lazy_static", "mime_guess", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -14868,7 +14912,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14881,7 +14925,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "argon2", @@ -14931,7 +14975,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "rsa", "rust-embed", "rustls 0.23.35", @@ -15021,7 +15065,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15044,7 +15088,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15061,7 +15105,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15071,7 +15115,7 @@ dependencies = [ "jsonwebtoken 8.3.0", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -15087,7 +15131,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.805.0" +version = "1.809.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15097,7 +15141,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15114,7 +15158,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15136,7 +15180,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15145,7 +15189,7 @@ dependencies = [ "candle-transformers", "hf-hub", "lazy_static", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -15159,7 +15203,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15175,7 +15219,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15197,7 +15241,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15218,7 +15262,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15232,7 +15276,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-nats", @@ -15246,7 +15290,7 @@ dependencies = [ "hmac", "rand 0.9.0", "rdkafka", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp", "rumqttc", "serde", @@ -15267,7 +15311,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15292,7 +15336,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15301,7 +15345,7 @@ dependencies = [ "hex", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "semver 1.0.28", "serde", "serde_json", @@ -15320,7 +15364,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15342,7 +15386,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15362,7 +15406,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15373,7 +15417,7 @@ dependencies = [ "lazy_static", "prometheus", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sql-builder", @@ -15400,7 +15444,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15428,7 +15472,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.805.0" +version = "1.809.0" dependencies = [ "lazy_static", "serde", @@ -15440,7 +15484,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.805.0" +version = "1.809.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15464,7 +15508,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15478,7 +15522,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.805.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15513,7 +15557,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.805.0" +version = "1.809.0" dependencies = [ "chrono", "lazy_static", @@ -15527,7 +15571,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15546,7 +15590,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.805.0" +version = "1.809.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15563,7 +15607,7 @@ dependencies = [ "axum 0.8.9", "backon", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "chrono", "chrono-tz", @@ -15608,7 +15652,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "reqwest-retry", "rsa", @@ -15652,7 +15696,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.805.0" +version = "1.809.0" dependencies = [ "chrono", "futures", @@ -15672,7 +15716,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.805.0" +version = "1.809.0" dependencies = [ "regex", "serde", @@ -15687,7 +15731,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15714,7 +15758,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "futures", @@ -15731,7 +15775,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.805.0" +version = "1.809.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15747,7 +15791,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -15755,7 +15799,7 @@ dependencies = [ "futures", "http 1.5.0", "oauth2", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp", "serde", "serde_json", @@ -15768,7 +15812,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -15780,7 +15824,7 @@ dependencies = [ "http 1.5.0", "itertools 0.14.0", "lazy_static", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -15799,7 +15843,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "arc-swap", @@ -15824,7 +15868,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-stream", @@ -15842,7 +15886,7 @@ dependencies = [ "lazy_static", "object_store", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "serial_test", @@ -15859,7 +15903,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "futures", @@ -15877,7 +15921,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.805.0" +version = "1.809.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15886,7 +15930,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -15898,7 +15942,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -15910,7 +15954,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "gosyn", @@ -15922,7 +15966,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -15934,7 +15978,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -15946,7 +15990,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "nu-parser", @@ -15957,7 +16001,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15968,7 +16012,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15980,7 +16024,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15991,7 +16035,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-recursion", @@ -16013,7 +16057,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -16025,7 +16069,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -16039,7 +16083,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16056,7 +16100,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -16069,7 +16113,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "serde", @@ -16081,7 +16125,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -16099,7 +16143,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16115,7 +16159,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16131,7 +16175,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -16145,7 +16189,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-recursion", @@ -16166,7 +16210,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "serde_urlencoded", @@ -16184,7 +16228,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "const_format", @@ -16208,7 +16252,7 @@ dependencies = [ "lazy_static", "rcgen", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "rustls 0.23.35", "serde", "serde_json", @@ -16224,7 +16268,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.805.0" +version = "1.809.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16235,7 +16279,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-recursion", @@ -16250,7 +16294,7 @@ dependencies = [ "lazy_static", "magic-crypt", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -16270,7 +16314,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16294,7 +16338,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16305,7 +16349,7 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "rand 0.9.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sql-builder", @@ -16327,7 +16371,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16354,7 +16398,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16369,7 +16413,7 @@ dependencies = [ "lazy_static", "quick_cache", "rand 0.9.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -16387,7 +16431,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16407,7 +16451,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16422,7 +16466,7 @@ dependencies = [ "jsonwebtoken 8.3.0", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -16441,7 +16485,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16477,7 +16521,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16500,7 +16544,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16524,7 +16568,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-nats", @@ -16548,7 +16592,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16583,7 +16627,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16611,7 +16655,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16636,10 +16680,10 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", "itertools 0.14.0", @@ -16655,7 +16699,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-once-cell", @@ -16711,7 +16755,7 @@ dependencies = [ "rand 0.9.0", "rcgen", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "rsa", "rust_decimal", @@ -16773,7 +16817,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.805.0" +version = "1.809.0" dependencies = [ "bytes", "futures", @@ -17024,7 +17068,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "widestring", "windows-sys 0.52.0", ] @@ -17473,18 +17517,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", @@ -17541,6 +17585,7 @@ dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] @@ -17549,6 +17594,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9e8d62316f..be7ab2d192 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.805.0" +version = "1.809.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.805.0" +version = "1.809.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -701,7 +701,7 @@ async-stream = "^0" opentelemetry = "0.30.0" tracing-opentelemetry = "0.31.0" opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio", "testing"] } -opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls", "http-proto"] } +opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls", "http-proto", "gzip-tonic", "zstd-tonic"] } opentelemetry-appender-tracing = "0.30.0" opentelemetry-semantic-conventions = { version = "0.30.0", features = ["semconv_experimental"] } opentelemetry-proto = { version = "0.30.0", features = ["with-serde", "gen-tonic"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 8340c8d90e..f3b1f4732b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -e092518ee60e33160fee9ae91a4d109566f7b0ee +eccad9f68bd7246cc81acb82bdb6c08fc6013f45 diff --git a/backend/migrations/20260909052532_add_missing_primary_keys.down.sql b/backend/migrations/20260909052532_add_missing_primary_keys.down.sql new file mode 100644 index 0000000000..32ae30d2bd --- /dev/null +++ b/backend/migrations/20260909052532_add_missing_primary_keys.down.sql @@ -0,0 +1,9 @@ +-- Dropping the column drops the primary key and the identity sequence with it, and +-- only marks the column dropped in the catalog rather than rewriting the table, so +-- this takes the ACCESS EXCLUSIVE lock but not the time. + +ALTER TABLE workspace_runnable_dependencies DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_node DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_edge DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_column_edge DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_graph_snapshot DROP COLUMN IF EXISTS id; diff --git a/backend/migrations/20260909052532_add_missing_primary_keys.up.sql b/backend/migrations/20260909052532_add_missing_primary_keys.up.sql new file mode 100644 index 0000000000..399fe6cc5f --- /dev/null +++ b/backend/migrations/20260909052532_add_missing_primary_keys.up.sql @@ -0,0 +1,31 @@ +-- These five had neither a PRIMARY KEY nor an explicit REPLICA IDENTITY, which makes +-- PostgreSQL reject UPDATE and DELETE on them under logical replication. +-- `deployment_metadata` and `metrics` are the other two, one migration each after this. +-- +-- The surrogate cannot be swapped for a natural key: every unique index on all five is +-- PARTIAL, split on `script_hash IS NULL`, and a partial index cannot back a primary +-- key. The partial uniques stay; they are what the ON CONFLICT clauses infer. +-- +-- Each ALTER rewrites its table under ACCESS EXCLUSIVE and holds it unavailable for +-- the rewrite. These five share a transaction because each is bounded by what a +-- workspace holds rather than by how long it has run, so none can grow into the one +-- that locks the rest; a transaction holds all its locks until it commits. +-- +-- An instance that cannot afford that lock at startup can set REPLICA IDENTITY FULL +-- on these tables instead, which unblocks replication by itself, and run these +-- idempotent ALTERs in a maintenance window first. + +ALTER TABLE workspace_runnable_dependencies + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_node + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_edge + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_column_edge + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_graph_snapshot + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql new file mode 100644 index 0000000000..3ce4ae52a6 --- /dev/null +++ b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql @@ -0,0 +1 @@ +ALTER TABLE deployment_metadata DROP COLUMN IF EXISTS id; diff --git a/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql new file mode 100644 index 0000000000..951253ce64 --- /dev/null +++ b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql @@ -0,0 +1,13 @@ +-- The sixth of the seven; why any of them need a key is in +-- 20260909052532_add_missing_primary_keys. +-- +-- Kept out of that migration because it is the one table in the set with no retention +-- sweep -- rows accumulate per deployed script hash, flow version and app version for +-- the life of the instance -- so on an instance that upgrades after years of deploys +-- its ACCESS EXCLUSIVE rewrite is the one that could hold the others locked. +-- +-- No natural key: each row is a script, flow OR app deployment, and the three unique +-- indexes are partial on exactly that split, so none of them covers every row. + +ALTER TABLE deployment_metadata + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909085549_add_metrics_primary_key.down.sql b/backend/migrations/20260909085549_add_metrics_primary_key.down.sql new file mode 100644 index 0000000000..1ad428dc02 --- /dev/null +++ b/backend/migrations/20260909085549_add_metrics_primary_key.down.sql @@ -0,0 +1 @@ +ALTER TABLE metrics DROP COLUMN IF EXISTS row_id; diff --git a/backend/migrations/20260909085549_add_metrics_primary_key.up.sql b/backend/migrations/20260909085549_add_metrics_primary_key.up.sql new file mode 100644 index 0000000000..b863035990 --- /dev/null +++ b/backend/migrations/20260909085549_add_metrics_primary_key.up.sql @@ -0,0 +1,12 @@ +-- The last of the seven; why any of them need a key is in +-- 20260909052532_add_missing_primary_keys. +-- +-- Kept out of that migration because it is the largest (~400 MB / 750k rows on the +-- instance this was measured on, a steady state: `queue_%` rows, which are nearly all +-- of them, are swept at 14 days) and its ALTER rewrites it under ACCESS EXCLUSIVE. One +-- migration is one transaction, so alone it holds no lock on the others as it rewrites. +-- +-- The surrogate cannot be called `id` -- `metrics.id` holds the metric NAME. + +ALTER TABLE metrics + ADD COLUMN IF NOT EXISTS row_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql new file mode 100644 index 0000000000..3405d5a681 --- /dev/null +++ b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql @@ -0,0 +1,7 @@ +DROP INDEX IF EXISTS index_app_version_on_app_id; + +DROP INDEX IF EXISTS index_app_script_on_app; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_app_path; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_flow_path; diff --git a/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql new file mode 100644 index 0000000000..6426aec594 --- /dev/null +++ b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql @@ -0,0 +1,32 @@ +-- The FK columns that cascade when a workspace's apps and flows are deleted. Unindexed, +-- Postgres seq-scans the whole child table once per deleted parent row, making a workspace +-- delete cost O(apps and flows deleted x rows in the instance). Fork deletion is where that +-- bites: a fork clones its parent's apps, flows and entire app version history. +-- +-- The workspace_runnable_dependencies pair are partial because the table's check constraint +-- makes app_path and flow_path mutually exclusive, halving each index -- the cascade's +-- equality on the path proves the predicate. The table's existing path indexes are partial on +-- script_hash, which the cascade does not constrain, so they cannot serve it. +-- +-- Dropped before built: a failed concurrent build leaves an invalid index that IF NOT EXISTS +-- would accept forever, unused by the planner yet still maintained on every write. +-- +-- No statement separators outside the statements below, comments included: the CONCURRENTLY +-- rewrite in windmill-api/src/db.rs splits the file on them and would run comment text as SQL. +DROP INDEX IF EXISTS index_app_version_on_app_id; + +CREATE INDEX index_app_version_on_app_id ON app_version (app_id); + +DROP INDEX IF EXISTS index_app_script_on_app; + +CREATE INDEX index_app_script_on_app ON app_script (app); + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_app_path; + +CREATE INDEX index_workspace_runnable_dependencies_on_app_path + ON workspace_runnable_dependencies (app_path, workspace_id) WHERE app_path IS NOT NULL; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_flow_path; + +CREATE INDEX index_workspace_runnable_dependencies_on_flow_path + ON workspace_runnable_dependencies (flow_path, workspace_id) WHERE flow_path IS NOT NULL; diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index b0ed12fb1a..271ad4835b 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -694,6 +694,107 @@ pub fn remove_pinned_imports(code: &str) -> anyhow::Result { Ok(content) } +/// Spans of the string literals naming a loaded module: `import`/`export … from` sources and the +/// argument of a dynamic `import()`. A `require()` call is left out: `require` is an ordinary +/// binding a script can shadow, so its argument is not known to be a module. +struct ImportSpecifierSpans(Vec); + +impl Visit for ImportSpecifierSpans { + noop_visit_type!(); + + fn visit_import_decl(&mut self, n: &swc_ecma_ast::ImportDecl) { + self.0.push(n.src.span); + } + + fn visit_export_all(&mut self, n: &swc_ecma_ast::ExportAll) { + self.0.push(n.src.span); + } + + fn visit_named_export(&mut self, n: &swc_ecma_ast::NamedExport) { + if let Some(src) = &n.src { + self.0.push(src.span); + } + } + + fn visit_call_expr(&mut self, n: &swc_ecma_ast::CallExpr) { + if let (swc_ecma_ast::Callee::Import(_), Some(arg)) = (&n.callee, n.args.first()) { + if let (None, Expr::Lit(Lit::Str(s))) = (arg.spread, &*arg.expr) { + self.0.push(s.span); + } + } + n.visit_children_with(self); + } +} + +/// Drops the `@version` from each pinned module specifier (`pkg@1.2.3/sub` -> `pkg/sub`), +/// rewriting only the specifier literals. Unlike [`remove_pinned_imports`], the same text +/// elsewhere, such as a string the script returns, stays as written. +pub fn remove_pinned_import_specifiers(code: &str) -> anyhow::Result { + let cm: Lrc = Default::default(); + let fm = cm.new_source_file( + FileName::Custom("main.d.ts".into()).into(), + code.to_string(), + ); + let mut tss = TsSyntax::default(); + tss.tsx = true; + tss.no_early_errors = true; + let lexer = Lexer::new( + Syntax::Typescript(tss), + Default::default(), + StringInput::from(&*fm), + None, + ); + let module = Parser::new_from(lexer).parse_module().map_err(|e| { + anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {e:?}") + })?; + let mut specifiers = ImportSpecifierSpans(vec![]); + specifiers.visit_module(&module); + specifiers.0.sort_by_key(|s| s.lo); + + // Spans index the parsed source, which the source map stripped of any UTF-8 BOM. + let bom = if code.starts_with('\u{feff}') { + '\u{feff}'.len_utf8() + } else { + 0 + }; + let offset = + |pos: swc_common::BytePos| pos.0.checked_sub(fm.start_pos.0).map(|o| bom + o as usize); + let mut content = String::with_capacity(code.len()); + let mut copied = 0; + for span in specifiers.0 { + // A span covers the literal's quotes. One that does not land on a matching pair is left + // as written rather than risk rewriting the wrong bytes. + let (Some(open), Some(close)) = ( + offset(span.lo), + offset(span.hi).and_then(|e| e.checked_sub(1)), + ) else { + continue; + }; + let quote = code.as_bytes().get(open); + if open >= close + || open < copied + || !matches!(quote, Some(b'"' | b'\'')) + || code.as_bytes().get(close) != quote + { + continue; + } + let Some(specifier) = code.get(open + 1..close) else { + continue; + }; + let unpinned = IMPORTS_VERSION.captures(specifier).and_then(|x| { + x.get(1) + .map(|y| format!("{}{}", y.as_str(), x.get(2).map_or("", |z| z.as_str()))) + }); + if let Some(unpinned) = unpinned.filter(|u| u != specifier) { + content.push_str(&code[copied..open + 1]); + content.push_str(&unpinned); + copied = close; + } + } + content.push_str(&code[copied..]); + Ok(content) +} + fn resolve_type_ref(type_resolver: &HashMap, typ: &mut Typ) { let mut visited = std::collections::HashSet::new(); resolve_type_ref_with_visited(type_resolver, typ, &mut visited); diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index 4309018fb4..4206dd68f8 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -4,6 +4,7 @@ mod tests { use windmill_parser::{Arg, MainArgSignature, ObjectProperty, ObjectType, Typ}; use windmill_parser_ts::{ parse_deno_signature, parse_expr_for_imports, parse_relative_imports, + remove_pinned_import_specifiers, }; #[test] @@ -33,6 +34,46 @@ mod tests { ); } + #[test] + fn test_remove_pinned_import_specifiers_rewrites_only_specifiers() { + let code = r#"// héllo +import a from "pkg@1.2.3"; +import b from "@scope/pkg@^2/sub"; +export * from "other@3"; +import rel from "./helper"; +const c = await import("dyn@4"); +const require = (v: string) => v; +const d = require("req@5"); +// pkg@1.2.3 +export const label = "pkg@1.2.3"; +"#; + assert_eq!( + remove_pinned_import_specifiers(code).unwrap(), + r#"// héllo +import a from "pkg"; +import b from "@scope/pkg/sub"; +export * from "other"; +import rel from "./helper"; +const c = await import("dyn"); +const require = (v: string) => v; +const d = require("req@5"); +// pkg@1.2.3 +export const label = "pkg@1.2.3"; +"# + ); + assert_eq!( + remove_pinned_import_specifiers("\u{feff}import a from 'pkg@1';").unwrap(), + "\u{feff}import a from 'pkg';" + ); + assert_eq!( + remove_pinned_import_specifiers( + "// a\r\n// b\r\nimport a from \"pkg@1\";\r\nimport b from 'x@2';" + ) + .unwrap(), + "// a\r\n// b\r\nimport a from \"pkg\";\r\nimport b from 'x';" + ); + } + #[test] fn test_parse_empty_main_signature() { let code = r#" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 13597b06c9..5c6ad27103 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.805.0" +version = "1.809.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.805.0" +version = "1.809.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.805.0" +version = "1.809.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.805.0" +version = "1.809.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 7de860c637..f91b32e726 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.805.0" +version = "1.809.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 867d2c8520..9f8605c17a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -109,6 +109,10 @@ use windmill_common::{ APP_WORKSPACED_ROUTE_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, }, + queue_metrics::{ + QueueSample, QUEUE_COUNT_PREFIX, QUEUE_DELAY_PREFIX, QUEUE_DELAY_SAME_HEAD_SECS, + QUEUE_METRIC_HEARTBEAT_SECS, QUEUE_METRIC_STALE_SECS, + }, }; #[cfg(feature = "parquet")] use windmill_object_store::reload_object_store_setting; @@ -5106,155 +5110,303 @@ async fn vacuuming_tables(db: &Pool) -> error::Result<()> { Ok(()) } -pub async fn expose_queue_metrics(db: &Pool) { - let last_check = sqlx::query_scalar!( - "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1" - ) - .fetch_optional(db) - .await - .unwrap_or(Some(chrono::Utc::now())); +/// Shortest spacing between two stored samples of the same queue metric, so a tag whose +/// value moves on every monitor round still writes at most one row per interval. Also how +/// often each server samples the queue when no Prometheus or OTel gauge needs it sooner. +const QUEUE_METRIC_MIN_INTERVAL_SECS: f64 = 25.0; +/// A held delay hovers while the head keeps changing, so an exact-value comparison would rarely +/// dedup it. Only a move the chart would actually render is stored. +const QUEUE_DELAY_TOLERANCE: f64 = 0.1; - let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed); - let save_metrics = last_check - .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25)) - .unwrap_or(true); +/// Append the queue metrics the drawer at `GET /workers/queue_metrics_series` charts, skipping +/// any sample that repeats what is already stored. +/// +/// Only tags with a backlog appear in `queue_stats`, and an arbitrary `?tag=` nobody serves +/// stays backlogged forever, so writing every round would repeat the same pair of rows for +/// the whole 14-day retention. Each metric is written when the value it draws moves, once per +/// heartbeat while it holds, and once more (as a zero) when the tag drains. Gaps therefore +/// mean "unchanged since the last row", which is what the chart interpolates. A delay whose +/// head job stays put is stored as that job's wait start, which the chart draws climbing, so it +/// never moves away from what is stored either. +async fn save_queue_metrics( + db: &Pool, + queue_stats: &std::collections::HashMap, +) { + let sampled_ids = queue_stats + .keys() + .flat_map(|tag| { + [ + format!("{QUEUE_COUNT_PREFIX}{tag}"), + format!("{QUEUE_DELAY_PREFIX}{tag}"), + ] + }) + .collect::>(); - if metrics_enabled || save_metrics || OTEL_METRICS_ENABLED.load(Ordering::Relaxed) { - let queue_counts = windmill_common::queue::get_queue_counts(db).await; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - for q in QUEUE_COUNT_TAGS.read().await.iter() { - if queue_counts.get(q).is_none() { - (*QUEUE_COUNT).with_label_values(&[q]).set(0); - } - } + // Last stored sample of every metric that either has a backlog now or was written + // recently enough to still be believed backlogged. Bounding the lookup by the stale window + // keeps it cheap at any `metrics` size; a per-id `ORDER BY created_at DESC LIMIT 1` does + // not, since the planner may serve it from `metrics_sort_idx` and walk the whole table. + let last_samples = match sqlx::query!( + "SELECT COALESCE(c.id, r.id) AS \"id!\", r.value AS \"value?\", + EXTRACT(EPOCH FROM r.created_at)::double precision AS \"at?\", + EXTRACT(EPOCH FROM now() - r.created_at)::double precision AS \"age?\" + FROM unnest($1::text[]) AS c(id) + FULL JOIN ( + SELECT DISTINCT ON (id) id, value, created_at + FROM metrics + WHERE id LIKE 'queue_%' AND created_at > now() - make_interval(secs => $2) + ORDER BY id, created_at DESC + ) r ON r.id = c.id", + &sampled_ids[..], + QUEUE_METRIC_STALE_SECS, + ) + .fetch_all(db) + .await + { + Ok(rows) => rows, + Err(e) => { + tracing::error!("Failed to read last queue metrics samples: {e:#}"); + return; } + }; - let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed); + let mut ids = vec![]; + let mut values = vec![]; + // The wait start of the head of each held delay, whose value the INSERT computes. + let mut held_heads: Vec> = vec![]; + for row in last_samples { + let Some((prefix, tag)) = [QUEUE_COUNT_PREFIX, QUEUE_DELAY_PREFIX] + .into_iter() + .find_map(|p| row.id.strip_prefix(p).map(|tag| (p, tag))) + else { + continue; + }; + // A stored value that cannot be read cannot be compared, so the next reading is kept. + let last = row + .value + .as_ref() + .and_then(QueueSample::parse) + .zip(row.at) + .zip(row.age) + .map(|((sample, at), age)| (sample, at, age)); + let stat = queue_stats.get(tag); + let current = stat.map(|stat| { + if prefix == QUEUE_COUNT_PREFIX { + stat.count as f64 + } else { + stat.delay + } + }); - if otel_enabled { - for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { - if queue_counts.get(q).is_none() { - otel_set_queue_count(q, 0); + let next_delay = stat + .filter(|_| prefix == QUEUE_DELAY_PREFIX) + .map(|stat| delay_sample(last.map(|(sample, at, _)| sample.head_since(at)), stat)); + let drawn_now = last.map(|(sample, at, age)| (sample.value_at(at + age), age)); + let redraws = last + .zip(next_delay) + .is_some_and(|((sample, ..), next)| redraws(sample, next)); + if should_store(prefix, drawn_now, current, redraws) { + let (value, held_head) = match (stat, next_delay) { + (None, _) => (serde_json::json!(0), None), + (Some(stat), None) => (serde_json::json!(stat.count), None), + (Some(stat), Some(QueueSample::Held(_))) => { + (serde_json::Value::Null, Some(stat.head_since)) } - } - } - - #[allow(unused_mut)] - let mut tags_to_watch = vec![]; - #[allow(unused_mut)] - let mut otel_tags_to_watch = vec![]; - for q in queue_counts { - let count = q.1; - let tag = q.0; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - let metric = (*QUEUE_COUNT).with_label_values(&[&tag]); - metric.set(count as i64); - tags_to_watch.push(tag.to_string()); - } - - if otel_enabled { - otel_tags_to_watch.push(tag.to_string()); - } - otel_set_queue_count(&tag, count as i64); - - // save queue_count and delay metrics per tag - if save_metrics { - sqlx::query!( - "INSERT INTO metrics (id, value) VALUES ($1, $2)", - format!("queue_count_{}", tag), - serde_json::json!(count) - ) - .execute(db) - .await - .ok(); - if count > 0 { - sqlx::query!( - "INSERT INTO metrics (id, value) - VALUES ($1, to_jsonb(( - SELECT EXTRACT(EPOCH FROM now() - scheduled_for) - FROM v2_job_queue - WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval - ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1 - )))", - format!("queue_delay_{}", tag), - tag - ) - .execute(db) - .await - .ok(); - } - } - } - if metrics_enabled { - let mut w = QUEUE_COUNT_TAGS.write().await; - *w = tags_to_watch; - } - if otel_enabled { - let mut w = OTEL_QUEUE_COUNT_TAGS.write().await; - *w = otel_tags_to_watch; - } - - // Single DB query for running counts, shared by Prometheus and OTel - let otel_running = otel_enabled; - #[cfg(feature = "prometheus")] - let need_running_counts = metrics_enabled || otel_running; - #[cfg(not(feature = "prometheus"))] - let need_running_counts = otel_running; - - if need_running_counts { - let queue_running_counts = windmill_common::queue::get_queue_running_counts(db).await; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - for q in QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { - if queue_running_counts.get(q).is_none() { - (*QUEUE_RUNNING_COUNT).with_label_values(&[q]).set(0); - } - } - } - - if otel_running { - for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { - if queue_running_counts.get(q).is_none() { - otel_set_queue_running_count(q, 0); - } - } - } - - #[allow(unused_mut, unused_variables)] - let mut running_tags_to_watch: Vec = vec![]; - #[allow(unused_mut, unused_variables)] - let mut otel_running_tags_to_watch: Vec = vec![]; - for (tag, count) in &queue_running_counts { - #[cfg(feature = "prometheus")] - if metrics_enabled { - let metric = (*QUEUE_RUNNING_COUNT).with_label_values(&[tag]); - metric.set(*count as i64); - running_tags_to_watch.push(tag.to_string()); - } - - if otel_running { - otel_set_queue_running_count(tag, *count as i64); - otel_running_tags_to_watch.push(tag.to_string()); - } - } - - #[cfg(feature = "prometheus")] - if metrics_enabled { - let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await; - *w = running_tags_to_watch; - } - if otel_running { - let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await; - *w = otel_running_tags_to_watch; - } + (Some(_), Some(climbing)) => (climbing.to_json(), None), + }; + ids.push(row.id); + values.push(value); + held_heads.push(held_head); } } + if ids.is_empty() { + return; + } + // A held delay is computed from this statement's `now()`, the row's `created_at` too, so + // `created_at - value` is exactly its head's wait start. That is how the next sample tells + // whether the same job is still at the head, within `QUEUE_DELAY_SAME_HEAD_SECS`, which the + // time between reading the queue and this INSERT could otherwise exceed on a busy database. + if let Err(e) = sqlx::query!( + "INSERT INTO metrics (id, value) + SELECT id, COALESCE(to_jsonb(EXTRACT(EPOCH FROM now())::double precision - held_head), value) + FROM unnest($1::text[], $2::jsonb[], $3::double precision[]) AS u(id, value, held_head)", + &ids[..], + &values[..], + &held_heads[..] as &[Option], + ) + .execute(db) + .await + { + tracing::error!("Failed to save queue metrics: {e:#}"); + } +} + +/// What to store for a delay reading, given when the head job of the last stored sample started +/// waiting. The same job still at the head keeps the delay climbing from its wait start, which +/// the chart draws exactly. A head that changed means a moving queue, whose delay hovers and is +/// held; so is a first sample, which cannot tell yet and must not draw a climb that never was. +fn delay_sample( + last_head_since: Option, + stat: &windmill_common::queue::QueueStat, +) -> QueueSample { + match last_head_since { + Some(since) if (since - stat.head_since).abs() < QUEUE_DELAY_SAME_HEAD_SECS => { + QueueSample::Climbing { since: stat.head_since } + } + _ => QueueSample::Held(stat.delay), + } +} + +/// Whether the next delay sample is drawn differently from the last one even at the same value: +/// a climb whose head left would otherwise go on climbing from the old head, and a held delay +/// whose head stayed would stay flat while the wait grows. +fn redraws(last: QueueSample, next: QueueSample) -> bool { + matches!(last, QueueSample::Climbing { .. }) != matches!(next, QueueSample::Climbing { .. }) +} + +/// Whether a reading deserves a row of its own, given the last one stored for that metric: +/// the value it draws now and how many seconds ago it was written. `current` is `None` once +/// the tag has no backlog left; `redraws` is set when the reading must be drawn differently. +fn should_store( + prefix: &str, + last: Option<(f64, f64)>, + current: Option, + redraws: bool, +) -> bool { + let Some((last_value, age)) = last else { + // Nothing comparable within the lookback window: a tag that just backed up needs a + // first sample, one that was already gone needs nothing. + return current.is_some(); + }; + let Some(current) = current else { + // The tag drained. One zero pins where the line drops; after that the metric matches + // and goes quiet, then falls out of the lookback window entirely. + return last_value != 0.0; + }; + if age >= QUEUE_METRIC_HEARTBEAT_SECS { + return true; + } + age >= QUEUE_METRIC_MIN_INTERVAL_SECS + && (redraws + || if prefix == QUEUE_COUNT_PREFIX { + last_value != current + } else { + (current - last_value).abs() > last_value.abs() * QUEUE_DELAY_TOLERANCE + }) +} + +#[cfg(test)] +mod queue_metric_sampling { + use super::*; + + const RECENT: f64 = QUEUE_METRIC_MIN_INTERVAL_SECS + 1.0; + + #[test] + fn a_holding_backlog_writes_only_on_the_heartbeat() { + let held = Some((3.0, RECENT)); + assert!(!should_store(QUEUE_COUNT_PREFIX, held, Some(3.0), false)); + let due = Some((3.0, QUEUE_METRIC_HEARTBEAT_SECS)); + assert!(should_store(QUEUE_COUNT_PREFIX, due, Some(3.0), false)); + // A held delay hovers, so only a move past the tolerance counts as a change. + let delay = Some((100.0, RECENT)); + assert!(!should_store(QUEUE_DELAY_PREFIX, delay, Some(105.0), false)); + assert!(should_store(QUEUE_DELAY_PREFIX, delay, Some(120.0), false)); + } + + #[test] + fn a_drained_tag_writes_one_zero_then_stops() { + assert!(should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, RECENT)), + None, + false + )); + assert!(!should_store( + QUEUE_COUNT_PREFIX, + Some((0.0, RECENT)), + None, + false + )); + // Including once the heartbeat is due: a tag that is gone stays silent. + let gone = Some((0.0, QUEUE_METRIC_STALE_SECS)); + assert!(!should_store(QUEUE_COUNT_PREFIX, gone, None, false)); + assert!(!should_store(QUEUE_COUNT_PREFIX, None, None, false)); + } + + #[test] + fn a_change_waits_for_the_minimum_interval() { + assert!(!should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, 1.0)), + Some(9.0), + false + )); + assert!(should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, RECENT)), + Some(9.0), + false + )); + // A tag that has just backed up is recorded at once. + assert!(should_store(QUEUE_COUNT_PREFIX, None, Some(9.0), false)); + } + + #[test] + fn a_delay_climbs_while_the_same_job_stays_at_the_head() { + let stat = windmill_common::queue::QueueStat { count: 3, delay: 330.0, head_since: 1000.0 }; + // A held sample written at 1320 saw the same head: it switches to climbing at once, + // although the delay has not moved past the tolerance yet. + let first = QueueSample::Held(320.0); + let climbing = delay_sample(Some(first.head_since(1320.0)), &stat); + assert_eq!(climbing, QueueSample::Climbing { since: 1000.0 }); + assert!(redraws(first, climbing)); + let drawn = Some((first.value_at(1330.0), RECENT)); + assert!(should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(stat.delay), + true + )); + // Stored climbing, it draws the delay exactly: nothing more until the heartbeat. + let drawn = Some((climbing.value_at(1600.0), RECENT)); + assert!(!should_store(QUEUE_DELAY_PREFIX, drawn, Some(600.0), false)); + assert_eq!(delay_sample(None, &stat), QueueSample::Held(330.0)); + } + + #[test] + fn a_climb_whose_head_left_is_held_even_within_the_tolerance() { + // The head waiting since 0 left at 3600 for one queued at 100: 3500s is within 10% of + // the 3600s the climb draws, but kept, the climb would go on from the old head. + let moved = + windmill_common::queue::QueueStat { count: 2, delay: 3500.0, head_since: 100.0 }; + let climbing = QueueSample::Climbing { since: 0.0 }; + let next = delay_sample(Some(climbing.head_since(3000.0)), &moved); + assert_eq!(next, QueueSample::Held(3500.0)); + assert!(redraws(climbing, next)); + let drawn = Some((climbing.value_at(3600.0), RECENT)); + assert!(!should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(moved.delay), + false + )); + assert!(should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(moved.delay), + true + )); + } +} + +/// When this server last sampled the queue into `metrics`, in Unix milliseconds. It only paces +/// how often the queue is scanned for that; whether a sample earns a row is decided from what +/// is already stored. Servers sampling in the same instant can each write it, and the duplicate +/// draws the same. +static LAST_QUEUE_SAMPLE_MS: std::sync::atomic::AtomicI64 = std::sync::atomic::AtomicI64::new(0); + +pub async fn expose_queue_metrics(db: &Pool) { // clean queue metrics older than 14 days sqlx::query!( "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'" @@ -5262,6 +5414,131 @@ pub async fn expose_queue_metrics(db: &Pool) { .execute(db) .await .ok(); + + let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed); + let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed); + let now_ms = chrono::Utc::now().timestamp_millis(); + let save_metrics = now_ms - LAST_QUEUE_SAMPLE_MS.load(Ordering::Relaxed) + >= (QUEUE_METRIC_MIN_INTERVAL_SECS * 1000.0) as i64; + if !(metrics_enabled || otel_enabled || save_metrics) { + return; + } + + // Single DB query for running counts, shared by Prometheus and OTel. It runs ahead of the + // backlog read below, which gives up on the rest of the round when it fails. + let otel_running = otel_enabled; + #[cfg(feature = "prometheus")] + let need_running_counts = metrics_enabled || otel_running; + #[cfg(not(feature = "prometheus"))] + let need_running_counts = otel_running; + + if need_running_counts { + let queue_running_counts = windmill_common::queue::get_queue_running_counts(db).await; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + for q in QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { + if queue_running_counts.get(q).is_none() { + (*QUEUE_RUNNING_COUNT).with_label_values(&[q]).set(0); + } + } + } + + if otel_running { + for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { + if queue_running_counts.get(q).is_none() { + otel_set_queue_running_count(q, 0); + } + } + } + + #[allow(unused_mut, unused_variables)] + let mut running_tags_to_watch: Vec = vec![]; + #[allow(unused_mut, unused_variables)] + let mut otel_running_tags_to_watch: Vec = vec![]; + for (tag, count) in &queue_running_counts { + #[cfg(feature = "prometheus")] + if metrics_enabled { + let metric = (*QUEUE_RUNNING_COUNT).with_label_values(&[tag]); + metric.set(*count as i64); + running_tags_to_watch.push(tag.to_string()); + } + + if otel_running { + otel_set_queue_running_count(tag, *count as i64); + otel_running_tags_to_watch.push(tag.to_string()); + } + } + + #[cfg(feature = "prometheus")] + if metrics_enabled { + let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await; + *w = running_tags_to_watch; + } + if otel_running { + let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await; + *w = otel_running_tags_to_watch; + } + } + + let queue_stats = match windmill_common::queue::get_queue_stats(db).await { + Ok(queue_stats) => queue_stats, + Err(e) => { + tracing::error!("Failed to read queue stats: {e:#}"); + return; + } + }; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + for q in QUEUE_COUNT_TAGS.read().await.iter() { + if queue_stats.get(q).is_none() { + (*QUEUE_COUNT).with_label_values(&[q]).set(0); + } + } + } + + if otel_enabled { + for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { + if queue_stats.get(q).is_none() { + otel_set_queue_count(q, 0); + } + } + } + + #[allow(unused_mut)] + let mut tags_to_watch = vec![]; + #[allow(unused_mut)] + let mut otel_tags_to_watch = vec![]; + for (tag, stat) in queue_stats.iter() { + let count = stat.count; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + let metric = (*QUEUE_COUNT).with_label_values(&[tag]); + metric.set(count as i64); + tags_to_watch.push(tag.to_string()); + } + + if otel_enabled { + otel_tags_to_watch.push(tag.to_string()); + } + otel_set_queue_count(tag, count as i64); + } + + if save_metrics { + LAST_QUEUE_SAMPLE_MS.store(now_ms, Ordering::Relaxed); + save_queue_metrics(db, &queue_stats).await; + } + + if metrics_enabled { + let mut w = QUEUE_COUNT_TAGS.write().await; + *w = tags_to_watch; + } + if otel_enabled { + let mut w = OTEL_QUEUE_COUNT_TAGS.write().await; + *w = otel_tags_to_watch; + } } pub async fn reload_smtp_config(db: &Pool) { diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index a1e0b38518..d0f9d63647 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -70,15 +70,15 @@ ci_test_reference: workspace_id(char), test_script_path(char), test_script_hash( concurrency_settings: hash(bigint), concurrency_key(char), concurrent_limit(int), concurrency_time_window_s(int) config: name(char), config(jsonb) custom_concurrency_key_ended: key(char), ended_at(ts) -dbt_column_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), parent_column(text), child_unique_id(text), child_column(text), lineage_kind(text), ingested_at(ts) +dbt_column_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), parent_column(text), child_unique_id(text), child_column(text), lineage_kind(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts) +dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char) +dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_environment_state: workspace_id(char), script_path(char), environment(text), job_id(uuid), manifest(text), manifest_key(text), run_results(text), run_results_key(text), updated_at(ts) FK: (workspace_id) -> workspace(id) -dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), column_schema(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) +dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), column_schema(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts) FK: (workspace_id) -> workspace(id) @@ -88,7 +88,7 @@ debounce_key: key(char), job_id(uuid), previous_job_id(uuid), first_started_at(t debounce_stale_data: job_id(uuid), to_relock(text[]) debouncing_settings: hash(bigint), debounce_key(char), debounce_delay_s(int), max_total_debouncing_time(int), max_total_debounces_amount(int), debounce_args_to_accumulate(text[]) dependency_map: workspace_id(char), importer_path(char), importer_kind(importer_kind), imported_path(char), importer_node_id(char) -deployment_metadata: workspace_id(char), path(char), script_hash(bigint), app_version(bigint), callback_job_ids(uuid[]), deployment_msg(text), flow_version(bigint), job_id(uuid) +deployment_metadata: workspace_id(char), path(char), script_hash(bigint), app_version(bigint), callback_job_ids(uuid[]), deployment_msg(text), flow_version(bigint), job_id(uuid), id(bigint) FK: (workspace_id) -> workspace(id) draft: workspace_id(char), path(char), typ(draft_type), value(json), created_at(ts) FK: (workspace_id) -> workspace(id) @@ -147,7 +147,7 @@ mcp_oauth_refresh_token: id(bigint), refresh_token(char), access_token_hash(char mcp_oauth_server_client: client_id(char), client_name(char), redirect_uris(text[]), created_at(ts) mcp_oauth_server_code: code(char), client_id(char), user_email(char), workspace_id(char), scopes(text[]), redirect_uri(text), code_challenge(char), code_challenge_method(char), created_at(ts), expires_at(ts) FK: (client_id) -> mcp_oauth_server_client(client_id) -metrics: id(char), value(jsonb), created_at(ts) +metrics: id(char), value(jsonb), created_at(ts), row_id(bigint) mqtt_trigger: mqtt_resource_path(char), subscribe_topics(jsonb[]), client_version(mqtt_client_version), v5_config(jsonb), v3_config(jsonb), client_id(char), path(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) native_trigger: external_id(char), workspace_id(char), service_name(native_trigger_service), script_path(char), is_flow(bool), webhook_token_hash(char), service_config(jsonb), error(text), created_at(ts), updated_at(ts), enabled(bool) FK: (workspace_id) -> workspace(id) @@ -225,7 +225,7 @@ workspace_key: workspace_id(char), kind(workspace_key_kind), key(char) FK: (workspace_id) -> workspace(id) workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_groups(text[]), bypass_users(text[]), created_at(ts) FK: (workspace_id) -> workspace(id) -workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char) +workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text) FK: (workspace_id) -> workspace(id) diff --git a/backend/tests/app_guest_cloud_hosted.rs b/backend/tests/app_guest_cloud_hosted.rs new file mode 100644 index 0000000000..7ab4145a65 --- /dev/null +++ b/backend/tests/app_guest_cloud_hosted.rs @@ -0,0 +1,168 @@ +//! Guests are unavailable on the shared cloud (`CLOUD_HOSTED`). +//! +//! One test in its own binary on purpose: `CLOUD_HOSTED` is read once into a +//! `lazy_static`, so it must be set before anything reads it and cannot be unset for a +//! sibling test in the same process. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const GUEST_TOKEN: &str = "GUEST_SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +#[sqlx::test(fixtures("base"))] +async fn the_cloud_admits_no_guest(db: Pool) -> anyhow::Result<()> { + // Before the server starts, so the flag is what the whole process sees. + unsafe { std::env::set_var("CLOUD_HOSTED", "true") }; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + // The workspace switch cannot be turned on, so no policy can lean on it. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("self-hosted"), + "the refusal must name what guests need" + ); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "guest", "triggerables": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "an app cannot be deployed to guests"); + + // Nor can a key be configured for the JWT way in — the refusal lands before the + // outbound JWKS fetch it would otherwise make. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_jwt_key")), + ADMIN_TOKEN, + ) + .json(&json!({ "jwks_url": "https://issuer.example.com/.well-known/jwks.json" })) + .send() + .await?; + assert_eq!(resp.status(), 400, "a guest JWT key cannot be configured"); + // Clearing one stays allowed: a key nobody can use is still worth removing. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_jwt_key")), + ADMIN_TOKEN, + ) + .json(&json!({})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + // An app already stored in guest mode — pushed by git-sync, or deployed before the + // instance became a cloud one — advertises no entry either. + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "publisher", "triggerables": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + sqlx::query( + "UPDATE app SET policy = jsonb_set(policy, '{execution_mode}', '\"guest\"') + WHERE path = $1 AND workspace_id = 'test-workspace'", + ) + .bind(APP_PATH) + .execute(&db) + .await?; + sqlx::query("UPDATE workspace_settings SET guest_access_enabled = true WHERE workspace_id = 'test-workspace'") + .execute(&db) + .await?; + + // Deploying it again is not refused: only widening an app into guests is, so a + // git-sync push of one already stored that way keeps working (and keeps being inert). + let resp = authed( + client().post(format!("{ws}/apps/update/{APP_PATH}")), + ADMIN_TOKEN, + ) + .json(&json!({ + "policy": { "execution_mode": "guest", "triggerables": {} } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "an app already stored in guest mode must stay deployable: {}", + resp.text().await? + ); + + let resp = authed( + client().get(format!("{ws}/apps/secret_of/{APP_PATH}")), + ADMIN_TOKEN, + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "reading the share secret must succeed"); + let secret: String = resp.text().await?; + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a guest app must not advertise entry where guests are unavailable" + ); + + // And a session issued before the instance became a cloud one stops on its next + // request: the door re-reads the switch, so the credential itself is not enough. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes, workspace_id, expiration) + VALUES (encode(sha256($1::bytea), 'hex'), 'GUEST_SECR', $2, 'guest@example.com', + 'guest_session', $3, 'test-workspace', now() + interval '8 hours')", + ) + .bind(GUEST_TOKEN.as_bytes()) + .bind(GUEST_TOKEN) + .bind(vec![ + "guest".to_string(), + "users:read".to_string(), + format!("apps:read:{APP_PATH}"), + format!("apps:run:{APP_PATH}"), + ]) + .execute(&db) + .await?; + // `whoami` is where an admitted guest resolves as `role: guest`, so a 401 here is + // the door refusing the credential rather than a route saying no. + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a guest session must not authenticate where guests are unavailable" + ); + + Ok(()) +} diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index fc766253bc..95b4a62141 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -939,6 +939,57 @@ export function main() { return midValue(); }"#, Ok(()) } +/// A run with local modules and no lock executes the bundle its lock generation built, which +/// kept the imported script's pin; the run must still load the one copy in node_modules, and +/// leave the script's own data alone even where it matches the pinned specifier. +#[sqlx::test(fixtures("base"))] +async fn test_bun_modules_run_loads_imported_pin_from_node_modules( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + insert_deployed_bun_script( + &db, + "f/pinned_import_modules/module", + 41240002, + r#"import * as isNumber from "is-number@6.0.0"; +export const ns = isNumber; +export const label = "is-number@6.0.0";"#, + ) + .await; + + let job = JobPayload::Code(RawCode { + content: r#"import * as isNumber from "is-number"; +import { ns, label } from "/f/pinned_import_modules/module"; +import { local } from "./helper"; +export function main() { return [ns === isNumber, label, local()]; }"# + .into(), + path: Some("f/pinned_import_modules/main".into()), + language: ScriptLang::Bun, + modules: Some(std::collections::HashMap::from([( + "helper.ts".to_string(), + windmill_common::scripts::ScriptModule { + content: "export const local = () => 'local';".into(), + language: ScriptLang::Bun, + lock: None, + }, + )])), + ..RawCode::default() + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + assert_eq!( + result, + serde_json::json!([true, "is-number@6.0.0", "local"]) + ); + Ok(()) +} + #[sqlx::test(fixtures("base", "bun_edge_cases"))] async fn test_bun_shared_imports_both_styles(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/tests/git_sync_fork_credential.rs b/backend/tests/git_sync_fork_credential.rs index 7e31136727..10cf1e3735 100644 --- a/backend/tests/git_sync_fork_credential.rs +++ b/backend/tests/git_sync_fork_credential.rs @@ -12,8 +12,8 @@ use sqlx::{Pool, Postgres}; use windmill_common::git_sync_ee::{ - git_credential_for_url, repo_provider, repo_supports_managed_git_features, set_git_credential, - GitProvider, + create_repo_webhook, git_credential_for_url, repo_provider, repo_supports_managed_git_features, + set_git_credential, GitProvider, }; use windmill_common::workspaces::GitCredentialProvider; @@ -250,3 +250,31 @@ async fn a_credential_is_not_served_over_a_downgraded_transport( ); Ok(()) } + +/// A GitLab the server cannot reach is the error reported, not the GitHub App +/// lookup that runs after it: for a self-managed GitLab behind a firewall or an +/// untrusted certificate, "no GitHub App installation" names neither the host +/// nor the cause. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn an_unreachable_gitlab_host_is_the_reported_error( + db: Pool, +) -> anyhow::Result<()> { + let err = create_repo_webhook( + &db, + "parent-ws", + "http://glpat-secret@127.0.0.1:1/grp/proj.git", + "https://windmill.example/api/w/parent-ws/git_sync/webhook/gitlab", + "hook-secret", + ) + .await + .expect_err("nothing listens on port 1"); + assert!( + err.to_string().contains("Could not reach the git host"), + "unexpected error: {err}" + ); + assert!( + !err.to_string().contains("glpat-secret"), + "the URL credential leaked into the error: {err}" + ); + Ok(()) +} diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index ddd94ea079..76102656a5 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -1485,3 +1485,44 @@ async fn declarative_sync_rejects_an_unusable_webhook_base_url(db: Pool) { + clear_settings_and_configs(&db).await; + let before = count_global_settings(&db).await; + + let mut desired = BTreeMap::new(); + desired.insert( + "base_url".to_string(), + serde_json::json!("https://wm.example.com"), + ); + desired.insert( + "instance_banner".to_string(), + serde_json::json!({ "enabled": true, "message": "down", "link": "javascript:alert(1)" }), + ); + + let err = windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect_err("a javascript: banner link must fail the sync"); + assert!( + err.to_string().contains("instance_banner"), + "the error should name the offending setting, got: {err}" + ); + + assert_eq!( + count_global_settings(&db).await, + before, + "validation must run before anything is applied" + ); + assert!( + get_global_setting(&db, "base_url").await.is_none(), + "the other settings in the same apply must not have been written either" + ); +} diff --git a/backend/tests/jobs_cross_site_get.rs b/backend/tests/jobs_cross_site_get.rs new file mode 100644 index 0000000000..20aa09a0a5 --- /dev/null +++ b/backend/tests/jobs_cross_site_get.rs @@ -0,0 +1,138 @@ +//! Regression test for cross-site GET CSRF on the job-run endpoints that can run a Hub script. +//! +//! `run_wait_result/p/{path}` and `run_and_stream/p/{path}` answer GET and, for a `hub/` path, +//! run any public Hub script. The session cookie is `SameSite=Lax`, so a browser attaches it +//! to a cross-site top-level GET navigation, and an argument written `$var:` or +//! `$res:` is resolved as the caller: an attacker page could make a logged-in browser +//! run a generic Hub script and hand it the victim's secrets. CORS hides the response but not +//! the side effect. +//! +//! `CrossSiteGetGuard` refuses such a request. Workspace scripts are deliberately not refused, +//! since they only run code the workspace's own members deployed. A request carrying its own +//! credential is allowed; the one case that regresses is a signed-in user clicking a `?token=` +//! Hub-script link from another site, because `extract_token` gives the cookie precedence and +//! exempting the parameter would let `?token=junk` reinstate the vector. +//! +//! This test pins down: +//! - both endpoints refuse a cross-site cookie GET to a Hub script (the core fix), whether +//! `Sec-Fetch-Site` says so or, with no such header (plain http), a cross-host `Referer`, +//! - a junk `token` query parameter does not buy a pass, +//! - the scope: the same request to a workspace script is not refused, +//! - a Hub-script request with its own credential (bearer, or `?token=` and no cookie), or +//! sent as a POST, gets through. +//! +//! No runnable exists and no Hub is contacted. A request that gets past the guard fails as +//! not-found on a workspace path, and on the non-numeric version in `hub/x/...` for a Hub +//! path, which is rejected while resolving the runnable, before any call to the Hub. + +use reqwest::StatusCode; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const HUB_GETS: [&str; 2] = [ + "run_wait_result/p/hub/x/absent", + "run_and_stream/p/hub/x/absent", +]; +const WORKSPACE_GETS: [&str; 2] = [ + "run_wait_result/p/u/test-user/absent", + "run_and_stream/p/u/test-user/absent", +]; + +async fn send(req: reqwest::RequestBuilder) -> anyhow::Result<(StatusCode, String)> { + let resp = req.send().await?; + let status = resp.status(); + Ok((status, resp.text().await?)) +} + +#[sqlx::test(fixtures("base"))] +async fn test_cross_site_get_cannot_run_hub_scripts(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/jobs", + server.addr.port() + ); + let client = reqwest::Client::new(); + let cookie_get = |path: &str| { + client + .get(format!("{base}/{path}")) + .header("Cookie", "token=SECRET_TOKEN") + }; + + // ---- CORE REGRESSION: a cross-site cookie GET cannot run a Hub script. + for path in HUB_GETS { + let refused = [ + ( + "Sec-Fetch-Site: cross-site", + cookie_get(path).header("Sec-Fetch-Site", "cross-site"), + ), + // Plain http gets no `Sec-Fetch-*` at all, so `Referer` is the only signal left. + ( + "cross-host Referer with no Sec-Fetch-Site", + cookie_get(path).header("Referer", "http://attacker.example/page"), + ), + // The cookie outranks a `token` query parameter when authenticating. + ( + "junk ?token= next to the cookie", + client + .get(format!("{base}/{path}?token=junk")) + .header("Cookie", "token=SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site"), + ), + ]; + for (name, req) in refused { + let (status, body) = send(req).await?; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "{path} [{name}] must be refused: {body}" + ); + } + } + + // ---- Scope: the same request to a workspace script is not refused. + for path in WORKSPACE_GETS { + let (status, body) = send(cookie_get(path).header("Sec-Fetch-Site", "cross-site")).await?; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "{path} is a workspace script and must reach the handler: {body}" + ); + } + + // ---- A Hub-script request that carries its own credential, or is a POST, gets through. + let hub = HUB_GETS[1]; + let allowed = [ + ( + "cross-origin bearer token", + client + .get(format!("{base}/{hub}")) + .header("Authorization", "Bearer SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site"), + ), + ( + "cross-origin ?token= with no cookie", + client + .get(format!("{base}/{hub}?token=SECRET_TOKEN")) + .header("Sec-Fetch-Site", "cross-site"), + ), + ( + "POST with the cookie", + client + .post(format!("{base}/{hub}")) + .header("Cookie", "token=SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site") + .json(&serde_json::json!({})), + ), + ]; + for (name, req) in allowed { + let (status, body) = send(req).await?; + assert!( + body.contains("Invalid hub script version"), + "{name} must get past the guard to runnable resolution (got {status}): {body}" + ); + } + + Ok(()) +} diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index 3a81f31021..1a4040d390 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -627,3 +627,121 @@ async fn test_root_job_span_relocated_to_inbound_trace() { expected_uuid_trace ); } + +// ═══════════════════════════════════════════════════════════════════════ +// RESOURCE ATTRIBUTES (OTEL_RESOURCE_ATTRIBUTES) +// ═══════════════════════════════════════════════════════════════════════ + +fn resource_attrs() -> std::collections::HashMap { + otlp_service_resource( + &windmill_common::utils::Mode::Worker, + "fallback-host", + "dev", + ) + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[test] +#[serial_test::serial] +fn test_otlp_resource_merges_env_attributes_without_losing_windmill_identity() { + // These take precedence over the hostname argument and over OTEL_RESOURCE_ATTRIBUTES, + // so clear them or an ambient one fails the assertions below for an unrelated reason. + for var in [ + "OTEL_HOST_NAME", + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + ] { + std::env::remove_var(var); + } + std::env::set_var( + "OTEL_RESOURCE_ATTRIBUTES", + "k8s.pod.uid=abc-123,service.name=injected,host.name=injected", + ); + let attrs = resource_attrs(); + std::env::remove_var("OTEL_RESOURCE_ATTRIBUTES"); + + // Attributes the deployment injects reach the exporters. + assert_eq!( + attrs.get("k8s.pod.uid").map(String::as_str), + Some("abc-123") + ); + // OTEL_RESOURCE_ATTRIBUTES is the secondary resource, so Windmill's own values still win. + assert_eq!( + attrs.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + attrs.get("host.name").map(String::as_str), + Some("fallback-host") + ); +} + +#[test] +#[serial_test::serial] +fn test_otlp_resource_dedicated_overrides_win() { + // A deployment sets these per pod, e.g. from Kubernetes downward-API labels. The + // competing service.name must lose: the spec ranks OTEL_SERVICE_NAME above it. + std::env::set_var("OTEL_RESOURCE_ATTRIBUTES", "service.name=should-lose"); + std::env::set_var("OTEL_SERVICE_NAME", "windmill-workers"); + std::env::set_var("OTEL_SERVICE_VERSION", "1.802.0"); + std::env::set_var("OTEL_HOST_NAME", "pod-7"); + let overridden = resource_attrs(); + + // An empty value means unset, which is what the downward API yields for a missing label. + for var in [ + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + ] { + std::env::set_var(var, ""); + } + let empty = resource_attrs(); + for var in [ + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + "OTEL_RESOURCE_ATTRIBUTES", + ] { + std::env::remove_var(var); + } + let unset = resource_attrs(); + + assert_eq!( + overridden.get("service.name").map(String::as_str), + Some("windmill-workers") + ); + assert_eq!( + overridden.get("service.version").map(String::as_str), + Some("1.802.0") + ); + assert_eq!( + overridden.get("host.name").map(String::as_str), + Some("pod-7") + ); + + assert_eq!( + empty.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + empty.get("host.name").map(String::as_str), + Some("fallback-host") + ); + assert_eq!( + empty.get("service.version").map(String::as_str), + Some(windmill_common::utils::GIT_VERSION) + ); + + // With nothing set at all — the default deployment — SdkProvidedResourceDetector still + // contributes service.name = "unknown_service". Ours has to overwrite it. + assert_eq!( + unset.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + unset.get("service.version").map(String::as_str), + Some(windmill_common::utils::GIT_VERSION) + ); +} diff --git a/backend/tests/script_modules.rs b/backend/tests/script_modules.rs index 8755ebfb57..eec9d3b45a 100644 --- a/backend/tests/script_modules.rs +++ b/backend/tests/script_modules.rs @@ -159,3 +159,58 @@ export function main(name: string) { assert_eq!(result, json!("hello world")); Ok(()) } + +/// A multi-file script run without a lock is bundled by the lockfile build. A pinned import in +/// a workspace script it imports must be installed at that version and still resolve at run time. +#[sqlx::test(fixtures("base"))] +async fn test_bun_module_imports_pinned_workspace_script(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) + VALUES ('test-workspace', 'test-user', $1, '{}', '', '', 'f/system/pinned_module', 12350, 'bun', '')", + ) + .bind( + r#" +import _ from "lodash@4.17.20"; +export function lodashVersion() { return _.VERSION; } +"#, + ) + .execute(&db) + .await?; + + let mut modules = HashMap::new(); + modules.insert( + "helper.ts".to_string(), + ScriptModule { + content: "export function label(v: string) { return v; }\n".to_string(), + language: ScriptLang::Bun, + lock: None, + }, + ); + + let job = JobPayload::Code(RawCode { + content: r#" +import { lodashVersion } from "/f/system/pinned_module"; +import { label } from "./helper.ts"; +export function main() { return label(lodashVersion()); } +"# + .to_owned(), + path: Some("f/system/my_script".to_string()), + language: ScriptLang::Bun, + modules: Some(modules), + tag: None, + ..RawCode::default() + }); + + let result = RunJob::from(job) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!("4.17.20")); + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs index 835c4c7e34..857b2e02e2 100644 --- a/backend/windmill-api-jobs/src/query.rs +++ b/backend/windmill-api-jobs/src/query.rs @@ -585,6 +585,8 @@ pub fn list_completed_jobs_query( let mut sqlb = SqlBuilder::select_from("v2_job_completed") .fields(fields) .order_by( + // The runs page picks its pagination cursor column from this same rule + // (frontend/src/lib/components/runs/useJobsLoader.svelte.ts); change both together. if lq.completed_before.is_some() || lq.completed_after.is_some() || lq.success == Some(false) diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 18a8d52ac0..e7be0d5ede 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -59,11 +59,11 @@ use windmill_common::{ CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, - HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, - RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RUFF_CONFIG_SETTING, - WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, - WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, - WS_BASE_URL_SETTING, + HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_BANNER_SETTING, + MAX_RETENTION_OVERRIDE_WORKSPACES, RETENTION_PERIOD_SECS_OVERRIDES_SETTING, + RUFF_CONFIG_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, + WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, + WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WS_BASE_URL_SETTING, }, instance_config::{self, ApplyMode, InstanceConfig}, server::Smtp, @@ -113,6 +113,9 @@ async fn get_ruff_config_unauthed(Extension(db): Extension) -> error::Result pub fn global_service() -> Router { #[warn(unused_mut)] let r = Router::new() + // `/local` is the path in openapi.yaml, so every generated client (getLocal) calls it; + // `/envs` stays for callers that found the route in the code. + .route("/local", get(get_local_settings)) .route("/envs", get(get_local_settings)) .route( "/global/{key}", @@ -1172,6 +1175,18 @@ async fn run_setting_pre_write_hook( } } } + INSTANCE_BANNER_SETTING => { + match value { + // Clearing (delete row) is handled by the caller; allow it through. + serde_json::Value::Null => {} + serde_json::Value::String(s) if s.trim().is_empty() => {} + v => { + windmill_common::global_settings::validate_instance_banner(v).map_err(|e| { + error::Error::BadRequest(format!("{INSTANCE_BANNER_SETTING}: {e}")) + })?; + } + } + } _ => {} } Ok(()) @@ -1312,6 +1327,7 @@ pub async fn get_global_setting( && key != APP_WORKSPACED_ROUTE_SETTING && key != HTTP_ROUTE_WORKSPACED_ROUTE_SETTING && key != WS_BASE_URL_SETTING + && key != INSTANCE_BANNER_SETTING { require_super_admin(&db, &authed).await?; } diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index 6ee55ed003..07bef364c8 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -19,6 +19,7 @@ use windmill_common::{ db::UserDB, error::JsonResult, jobs::{HIDE_WORKERS_FOR_NON_ADMINS, TAGS_ARE_SENSITIVE}, + queue_metrics::{read_queue_metrics_series, QueueMetricsSeries}, utils::{paginate, Pagination}, worker::{ALL_TAGS, CUSTOM_TAGS_PER_WORKSPACE, DEFAULT_TAGS, DEFAULT_TAGS_PER_WORKSPACE}, workspaces::workspace_with_fork_ancestors, @@ -38,6 +39,8 @@ pub fn global_service() -> Router { ) .route("/get_default_tags", get(get_default_tags)) .route("/queue_metrics", get(get_queue_metrics)) + .route("/queue_metrics_series", get(get_queue_metrics_series)) + .route("/queue_status", get(get_queue_status)) .route("/queue_counts", get(get_queue_counts)) .route("/queue_running_counts", get(get_queue_running_counts)) .route( @@ -270,10 +273,16 @@ async fn get_queue_metrics( ) -> JsonResult> { require_devops_role(&db, &authed).await?; + // The API declares every `value` a number, so a climbing delay, stored as its head's wait + // start, is returned as the delay at the time of its sample. let queue_metrics = sqlx::query_as!( QueueMetric, "WITH queue_metrics as ( - SELECT id, value, created_at + SELECT id, created_at, + CASE WHEN jsonb_typeof(value) = 'object' + THEN to_jsonb(EXTRACT(EPOCH FROM created_at) - (value->>'since')::numeric) + ELSE value + END AS value FROM metrics WHERE id LIKE 'queue_%' AND created_at > now() - interval '14 day' @@ -289,6 +298,88 @@ async fn get_queue_metrics( Ok(Json(queue_metrics)) } +#[derive(Deserialize)] +struct QueueMetricsSeriesQuery { + window_secs: Option, +} + +const QUEUE_METRICS_DEFAULT_WINDOW_SECS: i64 = 24 * 3600; +/// Retention of queue metrics, past which there is nothing left to read. +const QUEUE_METRICS_MAX_WINDOW_SECS: i64 = 14 * 24 * 3600; + +async fn get_queue_metrics_series( + authed: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult { + require_devops_role(&db, &authed).await?; + + let window = query + .window_secs + .unwrap_or(QUEUE_METRICS_DEFAULT_WINDOW_SECS) + .clamp(60, QUEUE_METRICS_MAX_WINDOW_SECS); + Ok(Json(read_queue_metrics_series(&db, window as f64).await?)) +} + +#[derive(Serialize)] +struct QueueTagStatus { + tag: String, + /// Jobs due for more than 3 seconds that no worker has picked up. + waiting: u32, + /// How long the job the next pull would take has been waiting, in seconds. + #[serde(skip_serializing_if = "Option::is_none")] + delay: Option, + running: i64, + /// Workers that pinged in the last minute and pull this tag. + workers: i64, +} + +/// Every tag with jobs waiting or running, read live from the queue. A backlog on a tag no live +/// worker pulls waits for one to start: a worker group scaling up from zero, or none at all for a +/// tag nobody serves. +async fn get_queue_status( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + require_devops_role(&db, &authed).await?; + + let backlog = windmill_common::queue::get_queue_stats(&db).await?; + let backlog_tags = backlog.keys().cloned().collect::>(); + // A job's tag is resolved before it is queued (per-workspace and dedicated worker tags + // included), and the pull matches it exactly against the worker's tags, so containment is + // exact here too. + let rows = sqlx::query!( + "WITH running AS ( + SELECT tag, count(*) AS n FROM v2_job_queue WHERE running = true GROUP BY tag + ) + SELECT t.tag AS \"tag!\", COALESCE(r.n, 0) AS \"running!\", + (SELECT count(*) FROM worker_ping w + WHERE w.ping_at > now() - interval '1 minute' AND w.custom_tags @> ARRAY[t.tag] + ) AS \"workers!\" + FROM (SELECT tag::text FROM running UNION SELECT unnest($1::text[])) t(tag) + LEFT JOIN running r ON r.tag = t.tag + ORDER BY t.tag", + &backlog_tags[..], + ) + .fetch_all(&db) + .await?; + + Ok(Json( + rows.into_iter() + .map(|row| { + let stat = backlog.get(&row.tag); + QueueTagStatus { + waiting: stat.map_or(0, |s| s.count), + delay: stat.map(|s| s.delay), + running: row.running, + workers: row.workers, + tag: row.tag, + } + }) + .collect(), + )) +} + async fn get_queue_counts( authed: ApiAuthed, Extension(db): Extension, diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 85226ddf2c..879e739dc0 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -4691,6 +4691,9 @@ async fn edit_guest_access( Json(EditGuestAccess { guest_access_enabled }): Json, ) -> Result { require_admin(authed.is_admin, &authed.username)?; + if guest_access_enabled { + windmill_common::workspaces::require_guest_support()?; + } let mut tx = db.begin().await?; sqlx::query!( @@ -4747,6 +4750,11 @@ async fn edit_guest_jwt_key( "Set a PEM public key or a JWKS URL, not both".to_string(), )); } + // Clearing stays allowed wherever guests are: a key nobody can use is still worth + // removing. + if public_key.is_some() || jwks_url.is_some() { + windmill_common::workspaces::require_guest_support()?; + } if let Some(pem) = public_key.as_deref() { windmill_common::guest_jwt::decoding_key_from_pem(pem)?; } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ab10b9c19a..a43135d440 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.805.0 + version: 1.809.0 title: Windmill API contact: @@ -5913,7 +5913,8 @@ paths: account; the `guest` app execution mode admits them. Off by default. Re-read where a guest session is minted and at the auth door on every guest request, so turning it off takes effect immediately, for sessions already issued and for - apps whose policy already says `guest`. + apps whose policy already says `guest`. Turning it *on* is refused with a 400 + where guests are unavailable (the shared cloud); turning it off always works. operationId: editGuestAccess tags: - workspace @@ -5948,7 +5949,8 @@ paths: URL, at most one. Both empty clears the workspace key; off cloud, verification then falls back to the instance issuer (`JWT_EXT_JWKS_URL`) if one is set, else no guest JWT is accepted (`guest_access_enabled` is the on/off switch). Workspace-admin gated. - The key is validated before it is stored. + The key is validated before it is stored. Setting a key is refused with a 400 where + guests are unavailable (the shared cloud); clearing one always works. operationId: editGuestJwtKey tags: - workspace @@ -9150,7 +9152,8 @@ paths: description: >- The custom-path counterpart of `getGuestEntry`. Unauthenticated; 404 unless the app's execution mode is `guest` AND its workspace has - `guest_access_enabled` AND the instance has not set `guest_access_disabled`. + `guest_access_enabled` AND the instance has not set `guest_access_disabled`, + and never on a deployment where guests are unavailable (the shared cloud). Returns the workspace too, since a custom URL may not carry it. operationId: getGuestEntryByCustomPath tags: @@ -13332,8 +13335,9 @@ paths: Unauthenticated: what a signed-out visitor reads to learn that signing in would let them in. 404 unless the app's execution mode is `guest` AND the workspace has `guest_access_enabled` AND the instance has not set the - `guest_access_disabled` global setting, so it says nothing about apps that - are not open to guests. Discloses only the app path, to a caller already + `guest_access_disabled` global setting, and never on a deployment where guests + are unavailable (the shared cloud), so it says nothing about apps that are not + open to guests. Discloses only the app path, to a caller already holding the share secret. operationId: getGuestEntry tags: @@ -16847,6 +16851,7 @@ paths: - can_approve - user_auth_required - approvers + - skin properties: flow_id: type: string @@ -16879,6 +16884,16 @@ paths: hide_cancel: type: boolean description: whether to hide the cancel button in the UI + skin: + type: string + enum: [detailed, minimal] + description: how the approval page presents the request + step_summary: + type: string + description: summary of the approval step, for the page title + flow_summary: + type: string + description: summary of the flow or workflow the approval belongs to approvers: type: array items: @@ -21830,6 +21845,98 @@ paths: - id - values + /workers/queue_metrics_series: + get: + summary: get the queue metrics of a time window, as a bounded line per tag + operationId: getQueueMetricsSeries + tags: + - worker + parameters: + - name: window_secs + in: query + required: false + description: how far back to read, in seconds (defaults to one day, capped at the 14-day retention) + schema: + type: integer + responses: + "200": + description: jobs waiting and queue delay per tag, as the vertices of lines joined by straight segments + content: + application/json: + schema: + type: object + properties: + from: + type: integer + description: start of the window, in epoch milliseconds + to: + type: integer + description: end of the window, in epoch milliseconds + tags: + type: array + items: + type: object + properties: + tag: + type: string + count: + type: array + description: "[epoch ms, jobs waiting more than 3 seconds] vertices" + items: + type: array + items: + type: number + delay: + type: array + description: "[epoch ms, seconds the next job has waited] vertices" + items: + type: array + items: + type: number + required: + - tag + - count + - delay + required: + - from + - to + - tags + + /workers/queue_status: + get: + summary: get the live queue status of every tag with jobs waiting or running + operationId: getQueueStatus + tags: + - worker + responses: + "200": + description: queue status per tag + content: + application/json: + schema: + type: array + items: + type: object + properties: + tag: + type: string + waiting: + type: integer + description: jobs due for more than 3 seconds that no worker has picked up + delay: + type: number + description: seconds the job the next pull would take has been waiting, absent when none is + running: + type: integer + workers: + type: integer + description: workers that pinged in the last minute and pull this tag + required: + - tag + - waiting + - running + - workers + /workers/queue_counts: get: summary: get counts of jobs waiting for an executor per tag @@ -25793,6 +25900,30 @@ paths: schema: type: string + /w/{workspace}/hub/projects: + get: + summary: list the hub's published projects + description: | + Forwards to the configured Hub's public project catalogue and returns its + status code and raw response body. Readable by any workspace member: the + listing is not workspace-scoped, and it is proxied only because the Hub's + listing endpoint sends no CORS header. Refused with 400 when the instance + has the Hub disabled, in which case no outbound request is made. + operationId: listHubProjects + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + "400": + description: the Hub is disabled on this instance + /w/{workspace}/hub/project: get: summary: get the hub project linked to a workspace folder @@ -29324,7 +29455,12 @@ components: to one seat: `billable_guests`, `guest_seats`); every other plan and build admits no new email until the count drops. `instance_enabled` is the superadmin switch (`guest_access_disabled` global setting) every workspace switch sits under. + `available` is whether this deployment can have guests at all: false on the shared + cloud, where guest access requires a self-hosted or dedicated deployment, and every + other field and switch is then moot. properties: + available: + type: boolean instance_enabled: type: boolean guest_count: @@ -29344,6 +29480,7 @@ components: type: integer format: int64 required: + - available - instance_enabled - guest_count - window_days diff --git a/backend/windmill-api/src/approvals.rs b/backend/windmill-api/src/approvals.rs index e5b5f019eb..717be4ad0f 100644 --- a/backend/windmill-api/src/approvals.rs +++ b/backend/windmill-api/src/approvals.rs @@ -17,6 +17,7 @@ use std::str::FromStr; use uuid::Uuid; use windmill_common::cache; use windmill_common::error::Error; +use windmill_common::flows::{ApprovalSkin, Suspend}; use windmill_common::jobs::JobKind; use windmill_common::scripts::ScriptHash; @@ -94,6 +95,17 @@ pub struct ApprovalFormDetails { pub message_str: String, pub urls: ResumeUrls, pub schema: Option, + pub skin: ApprovalSkin, +} + +/// The suspended step an approval message is about, and the flow run it belongs to. +struct ApprovalStep { + created_by: String, + created_at: chrono::NaiveDateTime, + script_path: Option, + parent_job_id: Option, + args: Option>>, + suspend: Option, } #[allow(dead_code)] @@ -205,6 +217,90 @@ pub async fn get_approval_form_details( tracing::debug!("Job ID: {:?}", job_id); + let ApprovalStep { created_by, created_at, script_path, parent_job_id, args, suspend } = + fetch_approval_step(&db, w_id, job_id, flow_step_id).await?; + + let schema = suspend.as_ref().map(|suspend| ResumeFormRow { + resume_form: suspend.resume_form.clone(), + hide_cancel: suspend.hide_cancel, + }); + let skin = suspend.and_then(|s| s.skin).unwrap_or_default(); + + let bold_format = match format { + MessageFormat::Slack => "*{}*", + MessageFormat::Teams => "**{}**", + }; + + let message_str = match skin { + ApprovalSkin::Detailed => { + let args_str = args.map_or("None".to_string(), |a| { + serde_json::from_str::(a.get()) + .ok() + .and_then(|v| serde_json::to_string_pretty(&v).ok()) + .unwrap_or_else(|| a.get().to_string()) + }); + let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); + let script_path_str = script_path.as_deref().unwrap_or("None"); + + let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); + + let mut message_str = format!( + "A workflow has been suspended and is waiting for approval:\n\n\ + {}: {created_by}\n\n\ + {}: {created_at_formatted}\n\n\ + {}: {script_path_str}\n\n\ + {}:\n```\n{args_str}\n```\n\n\ + {}: {parent_job_id_str}\n\n", + bold_format.replace("{}", "Created by"), + bold_format.replace("{}", "Created at"), + bold_format.replace("{}", "Script path"), + bold_format.replace("{}", "Args"), + bold_format.replace("{}", "Flow ID") + ); + + // Append custom message if provided + if let Some(msg) = message { + message_str.push_str(msg); + } + message_str + } + ApprovalSkin::Minimal => format!( + "{}\n\n{}: {created_by}", + message.unwrap_or("Your approval is requested."), + bold_format.replace("{}", "Requested by"), + ), + }; + + tracing::debug!("Schema: {:#?}", schema); + + Ok(ApprovalFormDetails { message_str, urls, schema, skin }) +} + +/// The skin of the approval step `flow_step_id` of the flow running `job_id`. Falls back to +/// the detailed skin when the step cannot be resolved, so a message is still sent. +/// Reads through the unrestricted pool without an authorization check of its own: only the +/// skin, which is not sensitive, leaves this function. +pub(crate) async fn get_approval_step_skin( + db: &DB, + w_id: &str, + job_id: Uuid, + flow_step_id: &str, +) -> ApprovalSkin { + match fetch_approval_step(db, w_id, job_id, Some(flow_step_id)).await { + Ok(step) => step.suspend.and_then(|s| s.skin).unwrap_or_default(), + Err(e) => { + tracing::warn!("Could not resolve approval step {flow_step_id} of job {job_id}: {e}"); + ApprovalSkin::default() + } + } +} + +async fn fetch_approval_step( + db: &DB, + w_id: &str, + job_id: Uuid, + flow_step_id: Option<&str>, +) -> Result { // TODO: do we have a helper function for this? let (job_kind, script_hash, raw_flow, parent_job_id, created_at, created_by, script_path, args) = sqlx::query!( "WITH job_info AS ( @@ -240,17 +336,17 @@ pub async fn get_approval_form_details( job_id, &w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await .map_err(|e| Error::BadRequest(e.to_string()))? .ok_or_else(|| Error::BadRequest("This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string())) .map(|r| (r.job_kind, r.script_hash, r.raw_flow, r.parent_job, r.created_at, r.created_by, r.script_path, r.args))?; - let flow_data = match cache::job::fetch_flow(&db, &job_kind, script_hash).await { + let flow_data = match cache::job::fetch_flow(db, &job_kind, script_hash).await { Ok(data) => data, Err(_) => { if let Some(parent_job_id) = parent_job_id.as_ref() { - cache::job::fetch_preview_flow(&db, parent_job_id, raw_flow).await? + cache::job::fetch_preview_flow(db, parent_job_id, raw_flow).await? } else { return Err(Error::BadRequest( "This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string(), @@ -265,49 +361,12 @@ pub async fn get_approval_form_details( tracing::debug!("Module: {:#?}", module); - let schema = module.and_then(|module| { - module.suspend.as_ref().map(|suspend| ResumeFormRow { - resume_form: suspend.resume_form.clone(), - hide_cancel: suspend.hide_cancel, - }) - }); - - let args_str = args.map_or("None".to_string(), |a| { - serde_json::from_str::(a.get()) - .ok() - .and_then(|v| serde_json::to_string_pretty(&v).ok()) - .unwrap_or_else(|| a.get().to_string()) - }); - let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); - let script_path_str = script_path.as_deref().unwrap_or("None"); - - let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); - - let bold_format = match format { - MessageFormat::Slack => "*{}*", - MessageFormat::Teams => "**{}**", - }; - - let mut message_str = format!( - "A workflow has been suspended and is waiting for approval:\n\n\ - {}: {created_by}\n\n\ - {}: {created_at_formatted}\n\n\ - {}: {script_path_str}\n\n\ - {}:\n```\n{args_str}\n```\n\n\ - {}: {parent_job_id_str}\n\n", - bold_format.replace("{}", "Created by"), - bold_format.replace("{}", "Created at"), - bold_format.replace("{}", "Script path"), - bold_format.replace("{}", "Args"), - bold_format.replace("{}", "Flow ID") - ); - - // Append custom message if provided - if let Some(msg) = message { - message_str.push_str(msg); - } - - tracing::debug!("Schema: {:#?}", schema); - - Ok(ApprovalFormDetails { message_str, urls, schema }) + Ok(ApprovalStep { + created_by, + created_at, + script_path, + parent_job_id, + args, + suspend: module.and_then(|m| m.suspend.clone()), + }) } diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 36b6eab0bb..25178ebd80 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -343,6 +343,29 @@ fn refuse_unscopable_guest_app(path: &str, mode: ExecutionMode) -> Result<()> { Ok(()) } +/// Refuse *widening* an app into guests where the deployment has none +/// (`instance_supports_guests`). Only the transition is refused, like the protection +/// rule below it: an app already stored in the mode — deployed before the instance +/// became a cloud one, or pushed by git-sync — keeps deploying, and keeps being inert, +/// since every guest gate refuses it anyway. `deployed_mode` is what the app is stored +/// as, `None` when it is being created. +fn refuse_guest_mode_where_unavailable( + path: &str, + mode: ExecutionMode, + deployed_mode: Option, +) -> Result<()> { + if !matches!(mode, ExecutionMode::Guest) + || deployed_mode == Some(ExecutionMode::Guest) + || windmill_common::workspaces::instance_supports_guests() + { + return Ok(()); + } + Err(Error::BadRequest(format!( + "app {path} cannot be set to Guests: {}", + windmill_common::workspaces::GUESTS_UNAVAILABLE_MESSAGE + ))) +} + /// Gate a viewer on the app's `execution_mode`, as far as can be decided without an /// ACL probe. `Ok(true)` means already authorized — anonymous admits anyone, guest /// admits anyone signed in; `Ok(false)` means the caller is a member and still owes @@ -2518,6 +2541,7 @@ async fn create_app_internal<'a>( // even when the caller did not. app.policy.set_execution_mode(app.policy.execution_mode()); refuse_unscopable_guest_app(&app.path, app.policy.execution_mode())?; + refuse_guest_mode_where_unavailable(&app.path, app.policy.execution_mode(), None)?; if let Some(rule) = deployment_rule_for_mode(app.policy.execution_mode()) { if let RuleCheckResult::Blocked(msg) = check_user_against_rule( w_id, @@ -3565,6 +3589,13 @@ async fn update_app_internal<'a>( ns.path.as_deref().unwrap_or(path), npolicy.execution_mode(), )?; + // An unreadable deployed policy reads as not already-in-mode, the strict + // direction, as for the protection rule below. + refuse_guest_mode_where_unavailable( + ns.path.as_deref().unwrap_or(path), + npolicy.execution_mode(), + deployed_policy.as_ref().map(|d| d.execution_mode()), + )?; if let Some(rule) = deployment_rule_for_mode(npolicy.execution_mode()).filter(|_| !authed.is_admin) { diff --git a/backend/windmill-api/src/csrf.rs b/backend/windmill-api/src/csrf.rs new file mode 100644 index 0000000000..4b2915f791 --- /dev/null +++ b/backend/windmill-api/src/csrf.rs @@ -0,0 +1,251 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::extract::FromRequestParts; +use axum::http::{header, request::Parts, Method}; +use axum::Extension; +use url::Url; +use windmill_common::error::Error; +use windmill_common::users::COOKIE_NAME; + +use crate::triggers::trigger_helpers::RunnableId; + +/// Whether a request is a cross-site GET authenticating on the session cookie alone. A GET +/// handler that runs a script by path resolves it through [`Self::script_runnable`], which +/// refuses a Hub script on such a request. +/// +/// The cookie is `SameSite=Lax`, so browsers attach it to cross-site top-level GET +/// navigations. A `hub/` path runs any public Hub script, and an argument written +/// `$var:` or `$res:` is resolved as the caller before the script sees it: such a +/// GET lets any page pick a generic Hub script and hand it the victim's secrets, which the job +/// can then send anywhere. +/// +/// Workspace scripts and flows are not refused, by choice, so that GET links to them keep +/// working. That is a scope decision, not a safety property: they still take attacker-chosen +/// arguments, `$var:` and `$res:` included, resolved as the victim. What bounds the exposure +/// is that the attacker needs a runnable path and can only run code the workspace deployed. +/// +/// The cookie is the only ambient credential. A bearer header is explicit, and so is the +/// `token` query parameter the webhook URLs carry — a cross-origin `EventSource` has no other +/// way to authenticate, since it cannot set headers. The checks run in `extract_token`'s +/// order, header before cookie, because that is the order it resolves them in: a request +/// carrying both a cookie and `token=` authenticates on the cookie and is therefore still +/// ambient, which is also why a valid `token=` link opened cross-site while signed in is +/// refused. +pub struct CrossSiteGetGuard(Option); + +impl CrossSiteGetGuard { + pub fn script_runnable(&self, script_path: &str) -> windmill_common::error::Result { + let runnable_id = RunnableId::from_script_path(script_path); + let (Some(signal), RunnableId::HubScript(_)) = (&self.0, &runnable_id) else { + return Ok(runnable_id); + }; + // The `Referer` leg is the one that can misfire, on a request that really was + // same-host: it compares against the hosts the backend can see, and a proxy that + // rewrites `Host` without setting `X-Forwarded-Host` leaves none of them matching + // what the browser addressed. Name the comparison so that shows up as a + // misconfiguration rather than as an unexplained 403. + if let CrossSite::RefererMismatch { referer, instance_hosts } = signal { + tracing::warn!( + referer_host = %referer, + ?instance_hosts, + "refusing a cross-site GET Hub script run inferred from Referer; if the request \ + was same-host, set `X-Forwarded-Host` on the proxy or configure `BASE_URL`" + ); + } + Err(Error::PermissionDenied( + "a cross-site GET request cannot run a Hub script with the session cookie, which takes \ + precedence over a `token` query parameter: pass the token in the `Authorization` \ + header, or open the link from the instance itself or from a browser with no Windmill \ + session" + .to_string(), + )) + } +} + +impl FromRequestParts for CrossSiteGetGuard { + type Rejection = std::convert::Infallible; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method != Method::GET { + return Ok(CrossSiteGetGuard(None)); + } + let Some(signal) = cross_site_signal(parts) else { + return Ok(CrossSiteGetGuard(None)); + }; + + let has_bearer = parts + .headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.starts_with("Bearer ")); + if has_bearer { + return Ok(CrossSiteGetGuard(None)); + } + + let has_session_cookie = + Extension::::from_request_parts(parts, state) + .await + .is_ok_and(|Extension(cookies)| cookies.get(COOKIE_NAME).is_some()); + Ok(CrossSiteGetGuard(has_session_cookie.then_some(signal))) + } +} + +enum CrossSite { + Declared, + RefererMismatch { referer: String, instance_hosts: Vec }, +} + +fn cross_site_signal(parts: &Parts) -> Option { + if let Some(site) = parts.headers.get("sec-fetch-site") { + return site + .as_bytes() + .eq_ignore_ascii_case(b"cross-site") + .then_some(CrossSite::Declared); + } + // Fetch Metadata rides only on potentially trustworthy URLs, so an instance served + // over plain http never receives `Sec-Fetch-Site` (nor does Safari before 16.4) while + // the cookie, not being `Secure` there either, still arrives. `Referer` is the only + // other thing a top-level GET navigation carries — `Origin` is not sent on one — so it + // is all that is left there, and it is weak: the default `strict-origin-when-cross- + // origin` policy already drops `Referer` on an https-to-http downgrade, so an https + // attacker page pointing a victim at a plain-http instance sends neither header. This + // leg catches an http-served attacker page and pre-16.4 Safari on https; the guard is + // load-bearing on https and best-effort at best on plain http. An absent `Referer` + // reads as not cross-site, matching how `Sec-Fetch-Site: none` (a bookmark, a typed + // URL) is treated. + let referer = referer_host(parts)?; + let instance_hosts: Vec = instance_hosts(parts).collect(); + (!instance_hosts + .iter() + .any(|host| host.eq_ignore_ascii_case(&referer))) + .then_some(CrossSite::RefererMismatch { referer, instance_hosts }) +} + +/// Every host a legitimate same-host request can name. `Host` alone is not enough: a +/// reverse proxy that forwards without preserving it (nginx `proxy_pass` with no +/// `proxy_set_header Host $host`) hands the backend the upstream's name, which no browser +/// `Referer` will ever match. None of these is browser-settable on a navigation — a +/// navigation carries no custom headers, and `BASE_URL` is instance config — so widening +/// the accepted set costs nothing. +fn instance_hosts(parts: &Parts) -> impl Iterator { + let base_url = windmill_common::BASE_URL.load(); + [ + request_host(parts), + header_host(parts, "x-forwarded-host"), + Url::parse(base_url.as_str()) + .ok() + .and_then(|url| url.host_str().map(str::to_owned)), + ] + .into_iter() + .flatten() +} + +fn referer_host(parts: &Parts) -> Option { + let referer = parts.headers.get(header::REFERER)?.to_str().ok()?; + Url::parse(referer).ok()?.host_str().map(str::to_owned) +} + +fn request_host(parts: &Parts) -> Option { + if let Some(host) = parts.uri.host() { + return Some(host.to_owned()); + } + header_host(parts, header::HOST) +} + +fn header_host(parts: &Parts, name: impl header::AsHeaderName) -> Option { + host_of(parts.headers.get(name)?.to_str().ok()?) +} + +/// The host in a `Host`-shaped header value: `host[:port]`, where `host` may be a bracketed +/// IPv6 literal, and where a chain of proxies appends to `X-Forwarded-Host` so only the +/// first entry is the one the browser addressed. The port is split off by the URL parser +/// rather than by hand-rolling the bracket rules. +fn host_of(value: &str) -> Option { + let host = value.split(',').next()?.trim(); + Url::parse(&format!("http://{host}")) + .ok()? + .host_str() + .map(str::to_owned) +} + +#[cfg(test)] +mod tests { + use super::{cross_site_signal, host_of, CrossSite}; + use axum::http::{request::Parts, Request}; + + fn parts(headers: &[(&str, &str)]) -> Parts { + let mut req = Request::get("/api/w/ws/jobs/run_wait_result/p/hub/1/x"); + for (name, value) in headers { + req = req.header(*name, *value); + } + req.body(()).unwrap().into_parts().0 + } + + #[test] + fn sec_fetch_site_decides_when_present() { + let declared = |site| cross_site_signal(&parts(&[("sec-fetch-site", site)])); + assert!(matches!(declared("cross-site"), Some(CrossSite::Declared))); + for site in ["same-origin", "same-site", "none"] { + assert!(declared(site).is_none(), "{site} is not cross-site"); + } + // The header outranks a `Referer` that disagrees with it. + let with_referer = parts(&[ + ("sec-fetch-site", "same-origin"), + ("host", "windmill.example"), + ("referer", "https://attacker.example/page"), + ]); + assert!(cross_site_signal(&with_referer).is_none()); + } + + #[test] + fn referer_stands_in_when_sec_fetch_site_is_absent() { + let signal = |headers: &[(&str, &str)]| cross_site_signal(&parts(headers)); + assert!(matches!( + signal(&[ + ("host", "windmill.example"), + ("referer", "https://attacker.example/p") + ]), + Some(CrossSite::RefererMismatch { .. }) + )); + // Ports differ between the frontend and the API, and do not make a request cross-site. + assert!(signal(&[ + ("host", "windmill.example:8000"), + ("referer", "http://windmill.example:3000/apps"), + ]) + .is_none()); + // A proxy that rewrote `Host` but forwarded the public name. + assert!(signal(&[ + ("host", "windmill-server.internal"), + ("x-forwarded-host", "windmill.example"), + ("referer", "https://windmill.example/apps"), + ]) + .is_none()); + assert!(signal(&[("host", "windmill.example")]).is_none()); + } + + #[test] + fn host_of_strips_port_brackets_and_proxy_chain() { + assert_eq!(host_of("windmill.example"), Some("windmill.example".into())); + assert_eq!( + host_of("windmill.example:8000"), + Some("windmill.example".into()) + ); + assert_eq!(host_of("[::1]:8000"), Some("[::1]".into())); + assert_eq!(host_of("[::1]"), Some("[::1]".into())); + assert_eq!( + host_of("windmill.example, proxy.internal"), + Some("windmill.example".into()) + ); + assert_eq!(host_of(""), None); + assert_eq!(host_of("not a host"), None); + } +} diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index d781e229d2..e8a6858db7 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -108,6 +108,9 @@ lazy_static::lazy_static! { (20260826214706, include_str!( "../../migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql" ).replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20260909163047, include_str!( + "../../migrations/20260909163047_workspace_delete_cascade_indexes.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), ].into_iter().collect(); } diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs index ccb530ce20..633ad88a34 100644 --- a/backend/windmill-api/src/hub_publish.rs +++ b/backend/windmill-api/src/hub_publish.rs @@ -6,13 +6,14 @@ use axum::{ http::{request::Parts, StatusCode}, response::{IntoResponse, Response}, routing::{get, post}, - Router, + Extension, Router, }; use serde::{Deserialize, Deserializer, Serialize}; use windmill_common::{ error::{to_anyhow, Error}, + global_settings::{load_value_from_global_settings, DISABLE_HUB_SETTING}, utils::require_admin, - HUB_BASE_URL, + DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, }; pub fn workspaced_service() -> Router { @@ -48,6 +49,7 @@ pub fn workspaced_service() -> Router { post(discard_project_update), ) .route("/project", get(get_project_by_source)) + .route("/projects", get(list_projects)) } #[derive(Deserialize)] @@ -548,6 +550,84 @@ async fn get_project_by_source(ctx: HubPublishCtx) -> Result bool { + fn host_of(url: &str) -> Option { + let parsed = url::Url::parse(url.trim()).ok()?; + if !matches!(parsed.scheme(), "http" | "https") { + return None; + } + Some( + parsed + .host_str()? + .trim_end_matches('.') + .to_ascii_lowercase(), + ) + } + match (host_of(hub), host_of(DEFAULT_HUB_BASE_URL)) { + (Some(host), Some(default_host)) => host == default_host, + _ => false, + } +} + +// The hub's project catalogue. Read by any workspace member rather than through +// `HubPublishCtx`, which requires an admin: nothing here is workspace-scoped or +// publishing-related. It exists at all because the hub's listing endpoint sends no +// CORS header, so the browser cannot read it directly the way it reads a single +// project. `accept: application/json` is what makes the hub answer with JSON. +// +// The caller's token is sent only to a hub this instance was pointed at deliberately. +// Every other route here is admin-only; this one is not, so forwarding a member's +// bearer token to `hub.windmill.dev` would put a credential replayable against this +// instance on a host outside it — for a listing that needs no credential at all. +async fn list_projects( + _authed: ApiAuthed, + Extension(db): Extension, + Tokened { token }: Tokened, +) -> Result { + // `disable_hub` turns the hub off for a closed instance, and this handler makes an + // outbound request. The frontend hides its entry points on the same setting, but that + // is presentation: an authenticated member can call this route directly, so the refusal + // has to live here. + let disabled = load_value_from_global_settings(&db, DISABLE_HUB_SETTING) + .await? + .and_then(|v| v.as_bool()) + .unwrap_or(false); + if disabled { + return Err(Error::BadRequest( + "The hub is disabled on this instance".to_string(), + )); + } + + let hub = (**HUB_BASE_URL.load()).clone(); + let url = format!("{}/projects", hub); + let mut req = HTTP_CLIENT.get(&url).header("accept", "application/json"); + if !is_public_hub(&hub) { + req = req.bearer_auth(&token); + } + let res = req + .send() + .await + .map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?; + + let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = res + .text() + .await + .map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?; + + Ok((status, text)) +} + async fn submit_project( ctx: HubPublishCtx, Path((_workspace, slug)): Path<(String, ProjectSlug)>, @@ -645,3 +725,42 @@ async fn forward_to_hub( Ok((status, text)) } + +#[cfg(test)] +mod tests { + use super::is_public_hub; + + #[test] + fn public_hub_recognized_in_every_spelling() { + // The predicate decides whether a workspace member's bearer token leaves the + // instance, so both directions matter: a miss on the public hub sends the token + // to windmill.dev, and a false match withholds it from a private hub that needs it. + // Every spelling here is one `hub_base_url` can hold and `reqwest` will still send. + for hub in [ + "https://hub.windmill.dev", + "http://hub.windmill.dev/", + "HTTPS://hub.windmill.dev", + "https://HUB.WINDMILL.DEV", + "https://hub.windmill.dev:443", + "https://hub.windmill.dev.", + "https://hub.windmill.dev/some/path", + " https://hub.windmill.dev ", + ] { + assert!(is_public_hub(hub), "{hub} should be the public hub"); + } + for hub in [ + "https://hub.internal.example", + "https://hub.windmill.dev.evil.example", + "https://windmill.dev", + // The host is what the request goes to, whatever precedes the `@`. + "https://hub.windmill.dev@hub.internal.example", + // Unparseable, or not a scheme a request can be built from. Grouped with the + // private hubs because the caller then attaches the token, which is harmless here: + // `reqwest` rejects the same value before opening a connection. + "hub.windmill.dev", + "ftp://hub.windmill.dev", + ] { + assert!(!is_public_hub(hub), "{hub} should not be the public hub"); + } + } +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 9a3db7961f..84968ebbcc 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -73,6 +73,7 @@ use crate::{ args::{self, RawWebhookArgs}, auth::{OptTokened, Tokened}, concurrency_groups::join_concurrency_key, + csrf::CrossSiteGetGuard, db::{ApiAuthed, DB}, triggers::trigger_helpers::RunnableId, users::{ @@ -106,7 +107,10 @@ use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error}, flow_status::{Approval, ApprovalConditions, FlowStatus, FlowStatusModule}, - flows::{add_virtual_items_if_necessary, resolve_maybe_value, FlowValue}, + flows::{ + add_virtual_items_if_necessary, resolve_maybe_value, ApprovalSkin, FlowModule, FlowValue, + Suspend, + }, jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode}, oauth2::HmacSha256, query_builders, @@ -4416,6 +4420,15 @@ async fn count_completed_jobs( )) } +lazy_static::lazy_static! { + /// 0 keeps the connection-wide statement_timeout. + static ref LIST_JOBS_STATEMENT_TIMEOUT_SECS: u64 = + std::env::var("LIST_JOBS_STATEMENT_TIMEOUT_SECS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(30); +} + async fn list_jobs( authed: ApiAuthed, Extension(user_db): Extension, @@ -4533,10 +4546,32 @@ async fn list_jobs( // tracing::info!("sql: {}", &sql); let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; + // A client that gives up does not cancel its query, so without this bound every retry of a + // slow filter stacks another scan running until the connection-wide 5min timeout. + let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS; + if timeout_secs > 0 { + sqlx::query(&format!("SET LOCAL statement_timeout = '{timeout_secs}s'")) + .execute(&mut *tx) + .await?; + } + let jobs: Vec = sqlx::query_as(&sql) .fetch_all(&mut *tx) .warn_after_seconds_with_sql(5, format!("list_jobs: {}", sql)) - .await?; + .await + .map_err(|e| match e { + sqlx::Error::Database(ref db_err) + if timeout_secs > 0 && db_err.code().as_deref() == Some("57014") => + { + Error::Generic( + StatusCode::BAD_REQUEST, + format!( + "Listing jobs took more than {timeout_secs}s and was stopped. Set a start date or narrow the filters." + ), + ) + } + e => e.into(), + })?; tx.commit().await?; Ok(Json(jobs.into_iter().map(From::from).collect())) @@ -4826,6 +4861,11 @@ struct ApprovalInfo { user_auth_required: bool, #[serde(skip_serializing_if = "Option::is_none")] hide_cancel: Option, + skin: ApprovalSkin, + #[serde(skip_serializing_if = "Option::is_none")] + step_summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_summary: Option, approvers: Vec, /// Share-read-link token for the flow, minted only for callers allowed to view this /// approval. Lets an authenticated workspace-member approver open the run details of @@ -4879,6 +4919,48 @@ fn can_approve_step( } } +/// The latest approval step the run has passed: a step before the current `step` that ran +/// rather than being skipped. Steps from `step` on don't count, because while an approval is +/// pending the step after it already holds the `WaitingForEvents` status. +fn last_reached_approval_step<'a>( + flow: &'a FlowValue, + status: &FlowStatus, +) -> Option<&'a FlowModule> { + flow.modules + .iter() + .zip(status.modules.iter()) + .take(usize::try_from(status.step).unwrap_or(0)) + .rev() + .filter(|(_, m)| matches!(m, FlowStatusModule::Success { skipped: false, .. })) + .map(|(module, _)| module) + .find(|module| module.suspend.is_some()) +} + +/// The approval conditions a step's own settings give, as the worker records them when the step +/// suspends. The worker drops them from the run once the step is approved, so a run that has +/// moved on is gated by these. Groups computed by an expression can't be re-evaluated outside +/// the run, so such a step falls back to any signed-in user. +fn approval_conditions_from_settings(suspend: &Suspend) -> Option { + let user_auth_required = suspend.user_auth_required.unwrap_or(false); + let self_approval_disabled = suspend.self_approval_disabled.unwrap_or(false); + if !user_auth_required && !self_approval_disabled { + return None; + } + let user_groups_required = match &suspend.user_groups_required { + Some(InputTransform::Static { value }) if user_auth_required => { + serde_json::from_str(value.get()).unwrap_or_default() + } + _ => vec![], + }; + Some(ApprovalConditions { user_auth_required, user_groups_required, self_approval_disabled }) +} + +/// How the approval step presents itself on the approval page. +struct ApprovalStepView { + skin: ApprovalSkin, + summary: Option, +} + async fn get_approval_info( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -4907,13 +4989,33 @@ async fn get_approval_info( script_path: Option, email: String, flow_status: Option, - workflow_as_code_status: Option, + // `v2_job_status` only holds a run that hasn't finished, so the fields below also read + // the completed run's status: a finished run's page keeps its skin and, for workflows as + // code, its description, still gated by the approval conditions the run had. + completed_flow_status: Option, + is_wac: bool, + wac_approval: Option, + approval_conditions: Option, + flow_summary: Option, } let row = sqlx::query_as::<_, ApprovalJobRow>( "SELECT j.id, j.runnable_path as script_path, j.permissioned_as_email as email, - s.flow_status, s.workflow_as_code_status + s.flow_status, + c.flow_status AS completed_flow_status, + COALESCE(s.workflow_as_code_status, c.workflow_as_code_status) IS NOT NULL + AS is_wac, + COALESCE(s.workflow_as_code_status, c.workflow_as_code_status)->'_approval' + AS wac_approval, + COALESCE(s.flow_status, c.flow_status)->'approval_conditions' + AS approval_conditions, + NULLIF(COALESCE(f.summary, sc.summary), '') AS flow_summary FROM v2_job j LEFT JOIN v2_job_status s ON s.id = j.id + LEFT JOIN v2_job_completed c ON c.id = j.id + LEFT JOIN flow f + ON j.kind = 'flow' AND f.workspace_id = j.workspace_id AND f.path = j.runnable_path + LEFT JOIN script sc + ON j.kind = 'script' AND sc.workspace_id = j.workspace_id AND sc.hash = j.runnable_id WHERE j.id = $1 AND j.workspace_id = $2", ) .bind(&job_id) @@ -4922,31 +5024,31 @@ async fn get_approval_info( .await? .ok_or_else(|| Error::NotFound(format!("Job {job_id} not found")))?; - let is_wac = row.workflow_as_code_status.is_some(); + let is_wac = row.is_wac; + let run_ac = row + .approval_conditions + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()); // Extract approval info based on WAC vs classic flow - let (form_schema, description, default_args, enums, approval_conditions, hide_cancel) = + let (form_schema, description, default_args, enums, approval_conditions, hide_cancel, step) = if is_wac { - let approval_meta = row - .workflow_as_code_status - .as_ref() - .and_then(|v| v.get("_approval")); + let approval_meta = row.wac_approval.as_ref(); let form = approval_meta.and_then(|m| m.get("form").cloned()); let default_args = approval_meta.and_then(|m| m.get("default_args").cloned()); let enums = approval_meta.and_then(|m| m.get("enums").cloned()); let description = approval_meta.and_then(|m| m.get("description").cloned()); - let ac = row - .flow_status - .as_ref() - .and_then(|v| v.get("approval_conditions")) - .and_then(|v| serde_json::from_value::(v.clone()).ok()); - (form, description, default_args, enums, ac, None) + let skin = approval_meta + .and_then(|m| m.get("skin")) + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .unwrap_or_default(); + let step = Some(ApprovalStepView { skin, summary: None }); + (form, description, default_args, enums, run_ac, None, step) } else { let fs = row .flow_status .as_ref() .and_then(|v| serde_json::from_value::(v.clone()).ok()); - let ac = fs.as_ref().and_then(|s| s.approval_conditions.clone()); // For classic flows, form/description come from the flow definition and step result let approval_step = fs.as_ref().map(|s| (s.step as usize).saturating_sub(1)); @@ -5006,6 +5108,28 @@ async fn get_approval_info( .and_then(|s| s.resume_form.as_ref()) .map(|rf| serde_json::json!(rf)); let hc = suspend_settings.map(|s| s.hide_cancel.unwrap_or(false)); + let completed_fs = row + .completed_flow_status + .as_ref() + .filter(|_| fs.is_none()) + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + let approval_module = raw_flow + .as_ref() + .zip(fs.as_ref().or(completed_fs.as_ref())) + .and_then(|(flow, status)| last_reached_approval_step(flow, status)); + let ac = run_ac.or_else(|| { + approval_module + .and_then(|module| module.suspend.as_ref()) + .and_then(approval_conditions_from_settings) + }); + let step = approval_module.map(|module| ApprovalStepView { + skin: module + .suspend + .as_ref() + .and_then(|s| s.skin) + .unwrap_or_default(), + summary: module.summary.clone().filter(|s| !s.trim().is_empty()), + }); // Fetch description, default_args, and enums from the step's completed job result let step_job_id = fs @@ -5029,9 +5153,12 @@ async fn get_approval_info( (None, None, None) }; - (form, desc, default_args, enums, ac, hc) + (form, desc, default_args, enums, ac, hc, step) }; + let skin = step.as_ref().map(|s| s.skin).unwrap_or_default(); + let step_summary = step.and_then(|s| s.summary); + let user_auth_required = approval_conditions .as_ref() .map(|ac| ac.user_auth_required) @@ -5061,6 +5188,9 @@ async fn get_approval_info( can_approve: false, user_auth_required, hide_cancel: None, + skin, + step_summary: None, + flow_summary: None, approvers: vec![], view_token: None, })); @@ -5096,6 +5226,9 @@ async fn get_approval_info( can_approve, user_auth_required, hide_cancel, + skin, + step_summary, + flow_summary: row.flow_summary, approvers, view_token, })) @@ -7447,6 +7580,7 @@ async fn log_job_view( } pub async fn run_wait_result_job_by_path_get( + cross_site: CrossSiteGetGuard, method: hyper::http::Method, authed: ApiAuthed, Extension(user_db): Extension, @@ -7459,6 +7593,7 @@ pub async fn run_wait_result_job_by_path_get( check_license_key_valid().await?; let script_path = script_path.to_path(); + let runnable_id = cross_site.script_runnable(script_path)?; check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; if method == http::Method::HEAD { @@ -7471,12 +7606,7 @@ pub async fn run_wait_result_job_by_path_get( args.body = args::Body::HashMap(payload_as_args); let args = args - .to_args_from_runnable( - &db, - &w_id, - RunnableId::from_script_path(script_path), - run_query.skip_preprocessor, - ) + .to_args_from_runnable(&db, &w_id, runnable_id, run_query.skip_preprocessor) .await?; check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; @@ -7895,6 +8025,7 @@ pub async fn stream_flow_by_version( } pub async fn stream_script_by_path( + cross_site: CrossSiteGetGuard, authed: ApiAuthed, Extension(db): Extension, Extension(user_db): Extension, @@ -7903,12 +8034,13 @@ pub async fn stream_script_by_path( method: hyper::http::Method, args: RawWebhookArgs, ) -> error::Result { + let runnable_id = cross_site.script_runnable(script_path.to_path())?; stream_job( authed, db, user_db, w_id, - RunnableId::from_script_path(script_path.to_path()), + runnable_id, args, run_query, method == http::Method::GET, @@ -11877,4 +12009,58 @@ mod approval_view_gate_tests { "trigger@example.com" )); } + + #[test] + fn approval_step_is_the_last_one_passed() { + let flow: FlowValue = serde_json::from_value(serde_json::json!({ "modules": [ + { "id": "a", "value": { "type": "identity" }, "suspend": {} }, + { "id": "b", "value": { "type": "identity" }, "suspend": {} }, + { "id": "c", "value": { "type": "identity" } } + ]})) + .unwrap(); + let step_at = |step: i32, types: [(&str, bool); 3]| { + let mut status = FlowStatus::new(&flow); + status.step = step; + status.modules = ["a", "b", "c"] + .into_iter() + .zip(types) + .map(|(id, (kind, skipped))| { + serde_json::from_value(serde_json::json!({ + "type": kind, "id": id, "job": Uuid::nil(), "count": 1, + "failed_retries": [], "skipped": skipped + })) + .unwrap() + }) + .collect(); + last_reached_approval_step(&flow, &status).map(|module| module.id.clone()) + }; + let waiting = ("WaitingForEvents", false); + let pending = ("WaitingForPriorSteps", false); + let ran = ("Success", false); + let skipped = ("Success", true); + // Awaiting a's approval: b, itself an approval step, already holds `WaitingForEvents`. + assert_eq!(step_at(1, [ran, waiting, pending]).as_deref(), Some("a")); + assert_eq!(step_at(2, [ran, ran, waiting]).as_deref(), Some("b")); + assert_eq!(step_at(3, [ran, skipped, ran]).as_deref(), Some("a")); + assert_eq!(step_at(0, [pending, pending, pending]), None); + } + + #[test] + fn approved_step_stays_gated_by_its_settings() { + let from_settings = |suspend: serde_json::Value| { + approval_conditions_from_settings(&serde_json::from_value(suspend).unwrap()) + }; + let login = from_settings(serde_json::json!({ + "user_auth_required": true, + "user_groups_required": { "type": "static", "value": ["approvers"] } + })); + assert!(!can_view( + &None, + &login, + Some("f/team/flow"), + "trigger@example.com" + )); + assert_eq!(login.unwrap().user_groups_required, ["approvers"]); + assert!(from_settings(serde_json::json!({})).is_none()); + } } diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index f404d7be9b..fc2c773703 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -80,6 +80,7 @@ pub mod azure_proxy_ee; mod azure_proxy_oss; mod capture; mod concurrency_groups; +mod csrf; mod db; mod db_health; mod dbt; diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index aa2415f664..ea673610e6 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -1265,7 +1265,7 @@ is, a different one moves it there and archives the old path"), }, "execution_mode": { "type": "string", - "description": "Who the app's runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Either way `anonymous`, which makes the app publicly executable, is never assumed. Possible values: viewer, publisher, anonymous" + "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { "type": "string" @@ -1380,7 +1380,7 @@ is, a different one moves it there and archives the old path"), }, "execution_mode": { "type": "string", - "description": "Who the app's runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Either way `anonymous`, which makes the app publicly executable, is never assumed. Possible values: viewer, publisher, anonymous" + "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { "type": "string" diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 593ea3b298..204b30010d 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -13,19 +13,25 @@ use sha2::Sha256; use sqlx::types::Uuid; use std::collections::HashMap; use windmill_common::error::{to_anyhow, Error}; +use windmill_common::flows::ApprovalSkin; +use windmill_common::utils::truncate_with_ellipsis; use windmill_common::variables::{get_secret_value_as_admin, get_workspace_key}; use crate::db::{ApiAuthed, DB}; use crate::jobs::{QueryApprover, ResumeUrls}; use crate::{ approvals::{ - extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType, - MessageFormat, QueryButtonText, QueryDefaultArgsJson, QueryDynamicEnumJson, - QueryFlowStepId, QueryMessage, ResumeFormField, ResumeSchema, + extract_w_id_from_resume_url, get_approval_step_skin, handle_resume_action, + ApprovalFormDetails, FieldType, MessageFormat, QueryButtonText, QueryDefaultArgsJson, + QueryDynamicEnumJson, QueryFlowStepId, QueryMessage, ResumeFormField, ResumeSchema, }, auth::OptTokened, }; +// Slack rejects a button value over 2000 characters, and with it the whole post. The button value +// carries the message on to the modal, so the message is shortened to fit. +const SLACK_BUTTON_VALUE_MAX_CHARS: usize = 2000; + #[derive(Deserialize, Debug)] pub struct SlackFormData { payload: String, @@ -127,6 +133,9 @@ struct PrivateMetadata { // HMAC over (w_id, resource_path) keyed on the workspace key; minted when the modal is // built, required by `handle_submission` before the resource_path is decrypted. signature: Option, + // Only selects the wording of the updated channel message, so it is left unsigned. + #[serde(default)] + skin: ApprovalSkin, } // Opportunistic transport-level check: when `SLACK_SIGNING_SECRET` is configured we verify @@ -432,6 +441,7 @@ async fn handle_submission( let container: Container = private_metadata.container; let hide_cancel = private_metadata.hide_cancel; let signature = private_metadata.signature; + let skin = private_metadata.skin; // If hide_cancel is true, we don't need to extract information from the private_metadata if hide_cancel.unwrap_or(false) && action == "cancel" { @@ -463,7 +473,7 @@ async fn handle_submission( tracing::warn!("Failed to resolve slack token for {w_id}/{resource_path}: {e:#}"); Error::BadRequest("Invalid Slack callback request".to_string()) })?; - update_original_slack_message(action, slack_token, container).await?; + update_original_slack_message(action, slack_token, container, skin).await?; Ok(()) } @@ -475,14 +485,19 @@ async fn transform_schemas( required: Option>, default_args_json: Option<&serde_json::Value>, dynamic_enums_json: Option<&serde_json::Value>, + skin: ApprovalSkin, ) -> Result { tracing::debug!("Resume urls: {:#?}", urls); + let link_label = match skin { + ApprovalSkin::Detailed => "Flow suspension details", + ApprovalSkin::Minimal => "View in Windmill", + }; let mut blocks = vec![serde_json::json!({ "type": "section", "text": { "type": "mrkdwn", - "text": format!("{}\n<{}|Flow suspension details>", text, urls.approvalPage), + "text": format!("{}\n<{}|{link_label}>", text, urls.approvalPage), } })]; @@ -918,10 +933,6 @@ async fn send_slack_message( value["approver"] = serde_json::json!(approver); } - if let Some(message) = message { - value["message"] = serde_json::json!(message); - } - if let Some(default_args_json) = default_args_json { value["default_args_json"] = default_args_json.clone(); } @@ -950,33 +961,8 @@ async fn send_slack_message( .map_err(|e| Box::new(e) as Box)?; value["signature"] = serde_json::json!(signature); - let payload = serde_json::json!({ - "channel": channel_id, - "text": "A flow has been suspended. Please approve or reject the flow.", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A flow has been suspended. Please approve or reject the flow." - } - }, - { - "type": "actions", - "elements": [ - { - "type": "button", - "text": { - "type": "plain_text", - "text": "View" - }, - "action_id": "open_modal", - "value": value.to_string() - } - ] - } - ] - }); + let skin = get_approval_step_skin(db, w_id, job_id, flow_step_id).await; + let payload = channel_message_payload(channel_id, skin, message, value); tracing::debug!("Payload: {:?}", payload); @@ -1000,6 +986,88 @@ async fn send_slack_message( Ok(StatusCode::OK) } +/// The channel post announcing the approval. Its button hands `button_value` to the modal, with +/// `message` added, shortened to what Slack's button value limit leaves room for. +fn channel_message_payload( + channel_id: &str, + skin: ApprovalSkin, + message: Option<&str>, + mut button_value: serde_json::Value, +) -> serde_json::Value { + let message = message.map(|m| message_fitting_button_value(&button_value, m)); + if let Some(message) = &message { + button_value["message"] = serde_json::json!(message); + } + let (text, section, button_label) = match skin { + ApprovalSkin::Detailed => { + let text = "A flow has been suspended. Please approve or reject the flow."; + (text, text.to_string(), "View") + } + ApprovalSkin::Minimal => { + let mut section = "*Approval requested*".to_string(); + if let Some(message) = &message { + section.push('\n'); + section.push_str(message); + } + ("Approval requested", section, "Review") + } + }; + + serde_json::json!({ + "channel": channel_id, + "text": text, + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": section + } + }, + { + "type": "actions", + "elements": [ + { + "type": "button", + "text": { + "type": "plain_text", + "text": button_label + }, + "action_id": "open_modal", + "value": button_value.to_string() + } + ] + } + ] + }) +} + +/// The longest prefix of `message` that keeps `button_value` carrying it within Slack's limit. +fn message_fitting_button_value(button_value: &serde_json::Value, message: &str) -> String { + let mut with_message = button_value.clone(); + let mut fits = |max_chars: usize| { + let fitted = truncate_with_ellipsis(message, max_chars); + with_message["message"] = serde_json::json!(fitted); + (with_message.to_string().chars().count() <= SLACK_BUTTON_VALUE_MAX_CHARS).then_some(fitted) + }; + if let Some(whole) = fits(usize::MAX) { + return whole; + } + // Searched on the serialized length, which escaping makes longer than the raw prefix, and + // which grows with every character kept. + let (mut shortest, mut longest) = + (0, message.chars().count().min(SLACK_BUTTON_VALUE_MAX_CHARS)); + while shortest < longest { + let mid = (shortest + longest + 1) / 2; + if fits(mid).is_some() { + shortest = mid; + } else { + longest = mid - 1; + } + } + fits(shortest).unwrap_or_else(|| truncate_with_ellipsis(message, 0)) +} + async fn get_modal_blocks( db: DB, w_id: &str, @@ -1034,7 +1102,7 @@ async fn get_modal_blocks( ) .await?; - let ApprovalFormDetails { message_str, urls, schema } = approval_details; + let ApprovalFormDetails { message_str, urls, schema, skin } = approval_details; // Get the card content let card_content = transform_schemas( @@ -1063,6 +1131,7 @@ async fn get_modal_blocks( }), default_args_json, dynamic_enums_json, + skin, ) .await?; @@ -1077,6 +1146,7 @@ async fn get_modal_blocks( resume_button_text, cancel_button_text, &private_metadata_signature, + skin, ))) } @@ -1090,27 +1160,32 @@ fn construct_payload( resume_button_text: Option<&str>, cancel_button_text: Option<&str>, signature: &str, + skin: ApprovalSkin, ) -> serde_json::Value { + let (title, resume_label, cancel_label) = match skin { + ApprovalSkin::Detailed => ("Workflow Suspended", "Resume Workflow", "Cancel Workflow"), + ApprovalSkin::Minimal => ("Approval request", "Approve", "Reject"), + }; let mut view = serde_json::json!({ "type": "modal", "callback_id": "submit_form", "notify_on_close": true, "title": { "type": "plain_text", - "text": "Workflow Suspended" + "text": title }, "blocks": blocks, "submit": { "type": "plain_text", - "text": resume_button_text.unwrap_or("Resume Workflow") + "text": resume_button_text.unwrap_or(resume_label) }, - "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature }).to_string(), + "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature, "skin": skin }).to_string(), }); if !hide_cancel { view["close"] = serde_json::json!({ "type": "plain_text", - "text": cancel_button_text.unwrap_or("Cancel Workflow") + "text": cancel_button_text.unwrap_or(cancel_label) }); } @@ -1193,11 +1268,13 @@ async fn update_original_slack_message( action: &str, token: String, container: Container, + skin: ApprovalSkin, ) -> Result<(), Error> { - let message = if action == "resume" { - "\n\n*Workflow has been resumed!* :white_check_mark:" - } else { - "\n\n*Workflow has been canceled!* :x:" + let message = match (skin, action == "resume") { + (ApprovalSkin::Detailed, true) => "\n\n*Workflow has been resumed!* :white_check_mark:", + (ApprovalSkin::Detailed, false) => "\n\n*Workflow has been canceled!* :x:", + (ApprovalSkin::Minimal, true) => "*Approved* :white_check_mark:", + (ApprovalSkin::Minimal, false) => "*Rejected* :x:", }; let final_blocks = vec![serde_json::json!({ @@ -1242,3 +1319,66 @@ async fn update_original_slack_message( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn long_message_keeps_the_channel_post_within_slack_limits() { + let button_value = serde_json::json!({ + "w_id": "demo", + "job_id": Uuid::nil(), + "path": "u/admin/slack", + "channel": "C0123456789", + "flow_step_id": "a", + "signature": "f".repeat(64), + }); + let carried = |skin, message: &str| { + let payload = channel_message_payload("C1", skin, Some(message), button_value.clone()); + let button = payload["blocks"][1]["elements"][0]["value"] + .as_str() + .unwrap() + .to_string(); + assert!(button.chars().count() <= SLACK_BUTTON_VALUE_MAX_CHARS); + let section = payload["blocks"][0]["text"]["text"].as_str().unwrap(); + assert!(section.chars().count() <= 3000); + serde_json::from_str::(&button) + .unwrap() + .message + .unwrap() + }; + // Quotes and newlines each cost two characters once escaped into the button value. + let message = "Expense \"offsite\" line\n".repeat(1000); + for skin in [ApprovalSkin::Detailed, ApprovalSkin::Minimal] { + let kept = carried(skin, &message); + let kept = kept.strip_suffix("...").unwrap(); + assert!(message.starts_with(kept)); + assert!(kept.chars().count() > 1_000); + assert_eq!(carried(skin, "Short message"), "Short message"); + } + } + + #[test] + fn minimal_skin_survives_the_modal_round_trip() { + let container = Container { message_ts: "1".to_string(), channel_id: "C1".to_string() }; + let payload = construct_payload( + serde_json::json!([]), + false, + "trigger", + "https://example.com/resume", + "u/admin/slack", + container, + None, + None, + "signature", + ApprovalSkin::Minimal, + ); + let view = &payload["view"]; + assert_eq!(view["submit"]["text"], "Approve"); + assert_eq!(view["close"]["text"], "Reject"); + let metadata: PrivateMetadata = + serde_json::from_str(view["private_metadata"].as_str().unwrap()).unwrap(); + assert_eq!(metadata.skin, ApprovalSkin::Minimal); + } +} diff --git a/backend/windmill-api/src/static_assets.rs b/backend/windmill-api/src/static_assets.rs index 06a79d022e..22645a2ac4 100644 --- a/backend/windmill-api/src/static_assets.rs +++ b/backend/windmill-api/src/static_assets.rs @@ -107,6 +107,14 @@ fn serve_path(path: &str, original_path: &str, query: Option<&str>) -> Response< .header("Cross-Origin-Resource-Policy", "cross-origin"); } + // Login and its siblings carry a different `rd` on every page that links + // to them, so a crawler meets thousands of URLs for one form. The app is + // client-rendered, so a meta tag only exists after a render pass; the + // header is seen on the first fetch. + if original_path.starts_with("/user/") { + res = res.header("X-Robots-Tag", "noindex, nofollow"); + } + // Add Content-Security-Policy header for static assets when policy is set if !CSP_POLICY.is_empty() { if let Ok(header_value) = HeaderValue::try_from(CSP_POLICY.as_str()) { diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 0d5ebdc2c3..da9f5f3df2 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -272,7 +272,10 @@ fn format_db_error(message: &str, detail: Option<&str>, hint: Option<&str>) -> S msg } -fn error_source_chain(e: &dyn std::error::Error) -> String { +/// `e` followed by each of its sources, `: `-separated. The result is meant for +/// users, and a `reqwest::Error` renders its request URL: never pass one built +/// from a URL carrying credentials in its userinfo. +pub fn error_source_chain(e: &dyn std::error::Error) -> String { let mut msg = e.to_string(); let mut source = e.source(); while let Some(cause) = source { diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 8180e820f2..1d590e38bf 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -125,6 +125,112 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; /// `base_url` when unset; set it when the browser-facing URL is not reachable /// from GitHub and a separate ingress fronts the API for inbound webhooks. pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url"; +/// Instance-wide announcement rendered above every page of the app (maintenance +/// windows, incidents). Readable by any authenticated user, unlike most settings: +/// the banner exists to be shown to everyone, so it must never hold anything the +/// whole instance may not see. +pub const INSTANCE_BANNER_SETTING: &str = "instance_banner"; + +/// Ceiling on the banner message. The banner is a one-or-two-line strip above every +/// page, so anything longer is a layout accident rather than an announcement. +pub const INSTANCE_BANNER_MESSAGE_MAX_LEN: usize = 500; + +/// Ceiling on the banner's link label, which renders as a button inside that same strip. +pub const INSTANCE_BANNER_LINK_LABEL_MAX_LEN: usize = 60; + +/// Validate an [`INSTANCE_BANNER_SETTING`] value. +/// +/// The banner is the one setting rendered to every user of the instance, so its +/// shape is checked at the boundary rather than trusted from the writer: a value +/// that reaches the browser malformed breaks the layout for everyone at once. +/// +/// The link is restricted to http(s) so a stored `javascript:`/`data:` URL can +/// never become the href of an anchor every user sees. +/// +/// Only shapes that would *misrender* are rejected. An enabled banner with no message +/// is left alone deliberately: it renders as nothing, and every write path here runs +/// under the bulk settings save, so rejecting it would fail an admin's whole settings +/// edit — retention, SMTP and all — over a half-typed announcement. +pub fn validate_instance_banner(value: &serde_json::Value) -> Result<(), String> { + let obj = value + .as_object() + .ok_or_else(|| "must be a JSON object".to_string())?; + + // Field types are checked before their contents. Every read below is an `as_str`/ + // `as_bool`, which reports a wrong-typed field as absent — so without this a + // `"link": 123` would skip the URL checks entirely and be stored, and the settings + // form would then throw on it (`link.trim()` on a number) instead of rendering. + for (field, expected, ok) in [ + ( + "enabled", + "a boolean", + obj.get("enabled").is_none_or(|v| v.is_boolean()), + ), + ( + "dismissible", + "a boolean", + obj.get("dismissible").is_none_or(|v| v.is_boolean()), + ), + ( + "message", + "a string", + obj.get("message").is_none_or(|v| v.is_string()), + ), + ( + "severity", + "a string", + obj.get("severity").is_none_or(|v| v.is_string()), + ), + ( + "link", + "a string", + obj.get("link").is_none_or(|v| v.is_string()), + ), + ( + "link_label", + "a string", + obj.get("link_label").is_none_or(|v| v.is_string()), + ), + ] { + if !ok { + return Err(format!("{field} must be {expected}")); + } + } + + for (field, max) in [ + ("message", INSTANCE_BANNER_MESSAGE_MAX_LEN), + ("link_label", INSTANCE_BANNER_LINK_LABEL_MAX_LEN), + ] { + let len = obj + .get(field) + .and_then(|v| v.as_str()) + .map_or(0, |s| s.chars().count()); + if len > max { + return Err(format!("{field} must be at most {max} characters")); + } + } + + if let Some(severity) = obj.get("severity").and_then(|v| v.as_str()) { + if !matches!(severity, "info" | "warning" | "error") { + return Err("severity must be one of info, warning, error".to_string()); + } + } + + if let Some(link) = obj.get("link").and_then(|v| v.as_str()) { + if !link.trim().is_empty() { + let url = url::Url::parse(link.trim()) + .map_err(|e| format!("link must be an absolute http(s) URL: {e}"))?; + if !matches!(url.scheme(), "http" | "https") { + return Err("link must use the http or https scheme".to_string()); + } + if !url.has_host() { + return Err("link must include a host".to_string()); + } + } + } + + Ok(()) +} /// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value. /// @@ -327,6 +433,30 @@ pub const ENV_SETTINGS: &[&str] = &[ "OTEL_METRICS", "OTEL_TRACING", "OTEL_LOGS", + // The OTEL_EXPORTER_OTLP_*HEADERS variables are left out: they carry exporter API keys, and + // this list is logged at startup and returned to superadmins by `get_local_settings`. + "OTEL_METRICS_ENABLED", + "OTEL_TRACING_ENABLED", + "OTEL_LOGS_ENABLED", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "OTEL_EXPORTER_OTLP_PROTOCOL", + "OTEL_EXPORTER_OTLP_COMPRESSION", + "OTEL_EXPORTER_OTLP_TIMEOUT", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", + "OTEL_METRIC_EXPORT_INTERVAL", + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + "OTEL_ENVIRONMENT", + "OTEL_RESOURCE_ATTRIBUTES", + "OTEL_JOB_LOGS", + "OTEL_TRACES_RETENTION_SECS", "DISABLE_S3_STORE", "PG_SCHEMA", "PG_LISTENER_REFRESH_PERIOD_SECS", @@ -590,6 +720,61 @@ mod tests { } } + #[test] + fn instance_banner_rejects_unsafe_and_malformed_values() { + // The link becomes the href of an anchor shown to every user of the instance, + // so a non-http(s) scheme must not survive a write. + for link in [ + "javascript:alert(1)", + "data:text/html,", + "vbscript:msgbox(1)", + "not-a-url", + "https://", + ] { + let banner = serde_json::json!({ "enabled": true, "message": "down", "link": link }); + assert!( + validate_instance_banner(&banner).is_err(), + "link '{link}' should be rejected" + ); + } + // A wrong-typed field reads as absent to every accessor here, so without an + // explicit type check it would skip validation and be stored. + for bad in [ + serde_json::json!({ "enabled": true, "message": "down", "link": 123 }), + serde_json::json!({ "enabled": true, "message": "down", "link_label": ["a"] }), + serde_json::json!({ "enabled": true, "message": { "text": "down" } }), + serde_json::json!({ "enabled": true, "message": "down", "severity": 2 }), + serde_json::json!({ "enabled": "yes", "message": "down" }), + serde_json::json!({ "enabled": true, "message": "down", "dismissible": "no" }), + ] { + assert!( + validate_instance_banner(&bad).is_err(), + "{bad} should be rejected" + ); + } + // The strip is one or two lines tall; both of its texts are bounded. + for (field, over) in [ + ("message", INSTANCE_BANNER_MESSAGE_MAX_LEN + 1), + ("link_label", INSTANCE_BANNER_LINK_LABEL_MAX_LEN + 1), + ] { + let mut banner = serde_json::json!({ "enabled": true, "message": "down" }); + banner[field] = serde_json::Value::String("x".repeat(over)); + assert!( + validate_instance_banner(&banner).is_err(), + "an over-long {field} should be rejected" + ); + } + // Enabled with no message renders as nothing and must stay writable: every path + // into this validator is a bulk settings save, so rejecting it would fail an + // admin's unrelated edits over a half-typed announcement. + assert!(validate_instance_banner(&serde_json::json!({ "enabled": true })).is_ok()); + let ok = serde_json::json!({ + "enabled": true, "message": "down", "severity": "warning", + "link": "https://status.example.com", "dismissible": false + }); + assert!(validate_instance_banner(&ok).is_ok()); + } + #[test] fn webhook_base_url_matches_the_ui_validator() { // Kept in lockstep with `isValidWebhookBaseUrl` in diff --git a/backend/windmill-common/src/guest_jwt.rs b/backend/windmill-common/src/guest_jwt.rs index fe05bec6b6..9f3e47d708 100644 --- a/backend/windmill-common/src/guest_jwt.rs +++ b/backend/windmill-common/src/guest_jwt.rs @@ -579,6 +579,15 @@ pub async fn jwks_key_for(url: &str, token: &str) -> Result<(DecodingKey, Vec Result { + // The admit check downstream refuses these anyway; refusing here keeps a deployment + // with no guests from parsing attacker-supplied JWTs at all, and names the reason in + // the log the caller writes. + if !crate::workspaces::instance_supports_guests() { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: {}", + crate::workspaces::GUESTS_UNAVAILABLE_MESSAGE + ))); + } if token.len() > MAX_GUEST_JWT_LEN { return Err(Error::NotAuthorized(format!( "guest JWT refused: token is longer than {MAX_GUEST_JWT_LEN} bytes" diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index c3060e9a89..e28bf139cf 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -1338,6 +1338,16 @@ pub async fn sync_global_settings_declarative( } } + let banner_key = crate::global_settings::INSTANCE_BANNER_SETTING; + match desired.get(banner_key) { + None | Some(serde_json::Value::Null) => {} + Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} + Some(banner) => crate::global_settings::validate_instance_banner(banner) + // The validator's messages name the offending field and its expected type, + // never the submitted value, so they are safe to surface here. + .map_err(|e| anyhow::anyhow!("{banner_key}: {e}"))?, + } + let diff = diff_global_settings(current, desired, ApplyMode::Replace); apply_settings_diff(db, &diff).await?; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 7fb5e85a69..1639b4605d 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -111,6 +111,7 @@ pub use pipeline_advanced_ee as pipeline_advanced; pub use pipeline_advanced_oss as pipeline_advanced; pub mod query_builders; pub mod queue; +pub mod queue_metrics; pub mod result_stream; pub mod runnable_settings; pub mod schedule; diff --git a/backend/windmill-common/src/queue.rs b/backend/windmill-common/src/queue.rs index 2656ce5af5..2008e1ce93 100644 --- a/backend/windmill-common/src/queue.rs +++ b/backend/windmill-common/src/queue.rs @@ -15,6 +15,57 @@ pub async fn get_queue_counts(db: &Pool) -> HashMap { .unwrap_or_else(|| HashMap::new()) } +/// Backlog of a single tag: jobs waiting more than 3 seconds past their `scheduled_for`. +pub struct QueueStat { + pub count: u32, + /// How long the job that would be picked up next has already been waiting, in seconds. + pub delay: f64, + /// When that job started waiting (its `scheduled_for`), in epoch seconds. + pub head_since: f64, +} + +/// Same backlog as [`get_queue_counts`], plus the delay of the job at the head of each +/// tag's queue. The head is picked with the same ordering the worker pull uses, so the +/// delay reported is the one a worker is about to observe. +/// +/// Reads the queue of every workspace: a caller exposing the result MUST restrict it to +/// devops users, as `GET /workers/queue_counts` does. Unlike [`get_queue_counts`], a failed +/// read is an error rather than an empty map, which would read as every backlog draining. +pub async fn get_queue_stats( + db: &Pool, +) -> crate::error::Result> { + // Grouping by (tag, priority) first finds every head in the same single pass as the + // count. A per-tag `ORDER BY ... LIMIT 1` walks `queue_sort_v2`, whose `tag` column comes + // last, through every other tag's backlog queued ahead of it. + let rows = sqlx::query!( + "SELECT tag AS \"tag!\", count AS \"count!\", + EXTRACT(EPOCH FROM now() - head)::double precision AS \"delay!\", + EXTRACT(EPOCH FROM head)::double precision AS \"head_since!\" + FROM ( + SELECT tag, sum(n)::bigint AS count, + (array_agg(head ORDER BY priority DESC NULLS LAST))[1] AS head + FROM ( + SELECT tag, priority, count(*) AS n, min(scheduled_for) AS head + FROM v2_job_queue WHERE + scheduled_for <= now() - ('3 seconds')::interval AND running = false + GROUP BY tag, priority + ) g + GROUP BY tag + ) t", + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|x| { + ( + x.tag, + QueueStat { count: x.count as u32, delay: x.delay, head_since: x.head_since }, + ) + }) + .collect()) +} + pub async fn get_queue_running_counts(db: &Pool) -> HashMap { sqlx::query!( "SELECT tag AS \"tag!\", count(*) AS \"count!\" FROM v2_job_queue WHERE diff --git a/backend/windmill-common/src/queue_metrics.rs b/backend/windmill-common/src/queue_metrics.rs new file mode 100644 index 0000000000..f5affd7346 --- /dev/null +++ b/backend/windmill-common/src/queue_metrics.rs @@ -0,0 +1,482 @@ +//! The queue metrics the monitor samples into `metrics` (`queue_count_{tag}` and +//! `queue_delay_{tag}`), and how a stored series is drawn back. +//! +//! A stored value is a number, held until the next sample, or, for a delay, `{"since": }`: the job at the head of the queue has been waiting since then and was still there +//! when sampled, so the delay climbs one second per second until the next sample. Besides +//! [`QueueSample`], the SQL in [`read_queue_metrics_series`] and in `GET /workers/queue_metrics` +//! decodes both shapes. + +use std::collections::BTreeMap; + +use serde::Serialize; +use sqlx::{Pool, Postgres}; + +pub const QUEUE_COUNT_PREFIX: &str = "queue_count_"; +pub const QUEUE_DELAY_PREFIX: &str = "queue_delay_"; + +/// A backlogged tag whose value has not moved is re-sampled only this often. A longer heartbeat +/// writes fewer rows, but keeps a tag whose drain was never recorded (no server was up when it +/// drained) drawn as backlogged for longer. +pub const QUEUE_METRIC_HEARTBEAT_SECS: f64 = 5.0 * 60.0; + +/// A series silent for longer than this has drained: the sampler stops looking for it, so no +/// closing zero will come, and it is drawn as zero from there. Heartbeats land up to a monitor +/// tick and a sampling slot late, so this must stay well above their real spacing. +pub const QUEUE_METRIC_STALE_SECS: f64 = 3.0 * QUEUE_METRIC_HEARTBEAT_SECS; + +/// Heads that started waiting within this of each other are one wait: jobs queued together +/// leave the head one after another without the delay dropping. +pub const QUEUE_DELAY_SAME_HEAD_SECS: f64 = 1.0; + +/// Slots a series is split into, whatever the window. A slot draws at most four vertices, and a +/// climb one more at each slot boundary it crosses, so a line stays under about 600 points +/// however many rows the window holds. +const QUEUE_METRICS_SERIES_SLOTS: f64 = 120.0; + +/// A stored sample, as it is drawn from the moment it was written until the next one. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum QueueSample { + /// A count, or a delay while the head keeps changing, which hovers. + Held(f64), + /// A delay while the job that started waiting at `since` (epoch seconds) stays at the head. + Climbing { since: f64 }, +} + +impl QueueSample { + pub fn parse(value: &serde_json::Value) -> Option { + match value.get("since") { + Some(since) => since.as_f64().map(|since| Self::Climbing { since }), + None => value.as_f64().map(Self::Held), + } + } + + pub fn to_json(self) -> serde_json::Value { + match self { + Self::Held(value) => serde_json::json!(value), + Self::Climbing { since } => serde_json::json!({ "since": since }), + } + } + + /// Its value at `t`, in epoch seconds. + pub fn value_at(self, t: f64) -> f64 { + match self { + Self::Held(value) => value, + Self::Climbing { since } => t - since, + } + } + + /// When the job at the head of a delay sample written at `at` started waiting. + pub fn head_since(self, at: f64) -> f64 { + match self { + Self::Held(delay) => at - delay, + Self::Climbing { since } => since, + } + } +} + +#[derive(Serialize)] +pub struct QueueMetricsSeries { + /// The window drawn, in epoch milliseconds. + pub from: i64, + pub to: i64, + pub tags: Vec, +} + +#[derive(Serialize)] +pub struct QueueTagSeries { + pub tag: String, + /// Vertices `[epoch ms, value]` of a line joined by straight segments. + pub count: Vec<(i64, f64)>, + pub delay: Vec<(i64, f64)>, +} + +/// The queue metrics of the last `window_secs`, each series aggregated per slot by the database +/// and drawn by [`render_series`], so the size is bounded by the number of tags rather than by +/// how many rows they wrote. +/// +/// Reads the metrics of every workspace's tags: a caller exposing the result MUST restrict it to +/// devops users, as `GET /workers/queue_metrics_series` does. +pub async fn read_queue_metrics_series( + db: &Pool, + window_secs: f64, +) -> crate::error::Result { + let to = sqlx::query_scalar!("SELECT EXTRACT(EPOCH FROM now())::double precision AS \"now!\"") + .fetch_one(db) + .await?; + let from = to - window_secs; + let slot_secs = window_secs / QUEUE_METRICS_SERIES_SLOTS; + + // Slot -1 holds the samples written before the window, of which only the last is used: it + // sets the value in force at the left edge. A series silent for longer than the stale window + // reads as zero, so nothing older can matter. Arrays compare element by element, so + // `max(ARRAY[t, v])` is the slot's latest sample, found without sorting every row. `v` is a + // sample's value when it was written: for a climbing delay, how long its head had waited. + // + // A climb keeps rising until the next sample, so when that sample lands in the same slot + // (the tag drained, or its head moved), the climb's top is higher than any `v`. Looking the + // next sample up for the slot's last climb, rather than ordering every row, keeps the pass a + // plain aggregate; an earlier climb in the same slot still shows up to its last heartbeat. + // `t` round-trips through `to_timestamp` to within a microsecond either way, so both bounds + // carry a millisecond of slack, far less than two distinct samples of a series are apart: + // without it the climbing sample can match itself, or the one at `last` fall outside. + let rows = sqlx::query!( + "WITH slots AS ( + SELECT id, slot, min(t) AS first, max(t) AS last, max(v) AS peak, + (min(ARRAY[t, v]))[2] AS first_value, (max(ARRAY[t, v]))[2] AS last_value, + (max(ARRAY[t, climbing]))[2] = 1 AS last_climbing, + COALESCE(bool_and(climbing = 1) AND max(since) - min(since) < $4, false) AS ramp, + max(ARRAY[t, since]) FILTER (WHERE climbing = 1) AS last_climb + FROM ( + SELECT id, t, + CASE jsonb_typeof(value) + WHEN 'number' THEN value::double precision + WHEN 'object' THEN t - (value->>'since')::double precision + END AS v, + (value->>'since')::double precision AS since, + (jsonb_typeof(value) = 'object')::int::double precision AS climbing, + greatest(floor((t - $1::double precision) / $2::double precision), -1)::int + AS slot + FROM ( + SELECT id, value, EXTRACT(EPOCH FROM created_at)::double precision AS t + FROM metrics + WHERE id LIKE 'queue_%' + AND created_at > to_timestamp($1::double precision - $3::double precision) + ) m + ) s + WHERE v IS NOT NULL + GROUP BY id, slot + ) + SELECT id AS \"id!\", slot AS \"slot!\", first AS \"first!\", last AS \"last!\", + greatest(peak, CASE WHEN last_climb[1] < last THEN ( + SELECT EXTRACT(EPOCH FROM min(n.created_at))::double precision + FROM metrics n + WHERE n.id = slots.id AND n.id LIKE 'queue_%' + AND n.created_at > to_timestamp(last_climb[1] + 0.001) + AND n.created_at <= to_timestamp(last + 0.001) + ) - last_climb[2] END) AS \"peak!\", + first_value AS \"first_value!\", last_value AS \"last_value!\", + last_climbing AS \"last_climbing!\", ramp AS \"ramp!\" + FROM slots + ORDER BY id, slot", + from, + slot_secs, + QUEUE_METRIC_STALE_SECS, + QUEUE_DELAY_SAME_HEAD_SECS, + ) + .fetch_all(db) + .await?; + + #[derive(Default)] + struct Stored { + carried: Option, + slots: Vec, + } + // [count, delay] per tag. + let mut stored: BTreeMap = BTreeMap::new(); + for row in rows { + let (series, tag) = if let Some(tag) = row.id.strip_prefix(QUEUE_COUNT_PREFIX) { + (0, tag) + } else if let Some(tag) = row.id.strip_prefix(QUEUE_DELAY_PREFIX) { + (1, tag) + } else { + continue; + }; + let series = &mut stored.entry(tag.to_string()).or_default()[series]; + let slot = MetricSlot { + first: row.first, + last: row.last, + peak: row.peak, + first_value: row.first_value, + last_value: row.last_value, + last_climbing: row.last_climbing, + ramp: row.ramp, + }; + if row.slot < 0 { + series.carried = Some(slot); + } else { + series.slots.push(slot); + } + } + + let tags = stored + .into_iter() + .map(|(tag, [count, delay])| { + let draw = + |s: &Stored| render_series(s.carried.as_ref(), &s.slots, from, to, slot_secs); + QueueTagSeries { count: draw(&count), delay: draw(&delay), tag } + }) + // A tag that drained before the window has nothing to draw in it. + .filter(|s| s.count.iter().chain(&s.delay).any(|(_, v)| *v != 0.0)) + .collect(); + + Ok(QueueMetricsSeries { + from: (from * 1000.0).round() as i64, + to: (to * 1000.0).round() as i64, + tags, + }) +} + +/// The stored samples of one series that fall in one time slot. +#[derive(Debug, Clone, Copy)] +pub struct MetricSlot { + /// When the first and the last sample of the slot were written, in epoch seconds. + pub first: f64, + pub last: f64, + /// The highest value the series drew over the slot, a climb that ends inside it included. + pub peak: f64, + pub first_value: f64, + /// The value of the last sample, which holds (or climbs, for a climbing delay) until the next. + pub last_value: f64, + pub last_climbing: bool, + /// Every sample of the slot climbs from the same head, so the slot is one exact ramp. + pub ramp: bool, +} + +/// Draw a stored series over `[from, to]` (epoch seconds), split into slots of `slot_secs`, as +/// the vertices of a line joined by straight segments, each `(epoch ms, value)`. +/// +/// A sample holds its value, or a climbing delay keeps climbing, until the next sample or until +/// the series has been silent for [`QUEUE_METRIC_STALE_SECS`]. `carried` is the slot before +/// `from`, whose last sample sets the left edge. A slot draws its peak across the span of its +/// samples, so a spike shorter than a slot still shows at full height, unless it is a single +/// climb, drawn exactly. A climb gets a vertex at every slot boundary it crosses: the delay axis +/// is logarithmic, so one straight segment across many slots would misplace it. +pub fn render_series( + carried: Option<&MetricSlot>, + slots: &[MetricSlot], + from: f64, + to: f64, + slot_secs: f64, +) -> Vec<(i64, f64)> { + let mut line = Line { points: vec![], from, slot_secs }; + let mut held = carried + .map(Held::after) + .filter(|h| from - h.at <= QUEUE_METRIC_STALE_SECS); + if let Some(h) = held { + line.push(from, h.value_at(from)); + } + for slot in slots { + let entering = line.advance(&mut held, slot.first); + line.push(slot.first, entering); + if slot.ramp { + line.push(slot.first, slot.first_value); + } else { + line.push(slot.first, slot.peak); + line.push(slot.last, slot.peak); + } + line.push(slot.last, slot.last_value); + held = Some(Held::after(slot)); + } + if !line.points.is_empty() { + let value = line.advance(&mut held, to); + line.push(to, value); + } + line.points +} + +/// The last sample drawn: when it was written, its value then, and whether it climbs from there. +#[derive(Clone, Copy)] +struct Held { + at: f64, + value: f64, + climbing: bool, +} + +impl Held { + fn after(slot: &MetricSlot) -> Self { + Self { at: slot.last, value: slot.last_value, climbing: slot.last_climbing } + } + + fn value_at(self, t: f64) -> f64 { + if self.climbing { + self.value + (t - self.at) + } else { + self.value + } + } +} + +struct Line { + points: Vec<(i64, f64)>, + from: f64, + slot_secs: f64, +} + +impl Line { + /// The value `held` has at `t`, drawing the climb that leads there and, when the series went + /// silent for too long first, its drop to zero, after which it is forgotten. + fn advance(&mut self, held: &mut Option, t: f64) -> f64 { + let Some(h) = *held else { + return 0.0; + }; + let stale_at = h.at + QUEUE_METRIC_STALE_SECS; + if h.climbing { + let end = t.min(stale_at); + let start = h.at.max(self.from); + let mut boundary = self.from + + ((start - self.from) / self.slot_secs).floor() * self.slot_secs + + self.slot_secs; + while boundary < end { + self.push(boundary, h.value_at(boundary)); + boundary += self.slot_secs; + } + } + if t <= stale_at { + return h.value_at(t); + } + self.push(stale_at, h.value_at(stale_at)); + self.push(stale_at, 0.0); + *held = None; + 0.0 + } + + fn push(&mut self, t: f64, value: f64) { + let point = ((t * 1000.0).round() as i64, value); + match self.points.as_mut_slice() { + [.., last] if *last == point => {} + // A horizontal run only needs its two ends. + [.., a, b] if a.1 == value && b.1 == value => b.0 = point.0, + _ => self.points.push(point), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const FROM: f64 = 1_000_000.0; + const TO: f64 = FROM + 3600.0; + const SLOT: f64 = 30.0; + + fn held(first: f64, last: f64, peak: f64, last_value: f64) -> MetricSlot { + MetricSlot { + first: FROM + first, + last: FROM + last, + peak, + first_value: peak, + last_value, + last_climbing: false, + ramp: false, + } + } + + /// A slot whose samples all climb from a head that started waiting 30s before `FROM`. + fn climbing(first: f64, last: f64) -> MetricSlot { + MetricSlot { + first: FROM + first, + last: FROM + last, + peak: last + 30.0, + first_value: first + 30.0, + last_value: last + 30.0, + last_climbing: true, + ramp: true, + } + } + + fn at(secs: f64, value: f64) -> (i64, f64) { + (((FROM + secs) * 1000.0) as i64, value) + } + + #[test] + fn a_value_holds_until_the_next_sample_and_a_drain_drops_where_it_was_written() { + let line = render_series( + None, + &[ + held(60.0, 60.0, 3.0, 3.0), + held(600.0, 600.0, 2.0, 2.0), + held(900.0, 900.0, 0.0, 0.0), + ], + FROM, + TO, + SLOT, + ); + assert_eq!( + line, + vec![ + at(60.0, 0.0), + at(60.0, 3.0), + at(600.0, 3.0), + at(600.0, 2.0), + at(900.0, 2.0), + at(900.0, 0.0), + at(3600.0, 0.0), + ] + ); + } + + #[test] + fn a_series_silent_past_the_stale_window_drops_to_zero() { + let line = render_series(None, &[held(60.0, 60.0, 3.0, 3.0)], FROM, TO, SLOT); + let dropped = 60.0 + QUEUE_METRIC_STALE_SECS; + assert_eq!( + line, + vec![ + at(60.0, 0.0), + at(60.0, 3.0), + at(dropped, 3.0), + at(dropped, 0.0), + at(3600.0, 0.0) + ] + ); + } + + #[test] + fn a_slot_draws_its_peak_then_continues_from_its_last_sample() { + // Samples at 60 (5), 70 (9), 80 (4) collapsed into one slot. + let line = render_series( + Some(&held(-30.0, -30.0, 2.0, 2.0)), + &[held(60.0, 80.0, 9.0, 4.0)], + FROM, + FROM + 120.0, + SLOT, + ); + assert_eq!( + line, + vec![ + at(0.0, 2.0), + at(60.0, 2.0), + at(60.0, 9.0), + at(80.0, 9.0), + at(80.0, 4.0), + at(120.0, 4.0) + ] + ); + } + + #[test] + fn a_climbing_delay_is_drawn_exactly_up_to_its_drain() { + // 300s slots: one holds two climbing samples, and heartbeats follow until the drain. + let line = render_series( + None, + &[ + climbing(60.0, 120.0), + climbing(360.0, 360.0), + climbing(660.0, 660.0), + held(900.0, 900.0, 0.0, 0.0), + ], + FROM, + TO, + 300.0, + ); + assert_eq!( + line, + vec![ + at(60.0, 0.0), + // The slot is one climb, not its peak held across it. + at(60.0, 90.0), + at(120.0, 150.0), + // A vertex at each slot boundary the climb crosses. + at(300.0, 330.0), + at(360.0, 390.0), + at(600.0, 630.0), + at(660.0, 690.0), + // Still climbing right up to the closing zero. + at(900.0, 930.0), + at(900.0, 0.0), + at(3600.0, 0.0), + ] + ); + } +} diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index d71bc50e60..00aae89b43 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -250,23 +250,37 @@ pub async fn get_full_hub_script_by_path( let version = path_iterator .next() .ok_or_else(|| Error::internal_err(format!("expected hub path to have version number")))?; + // A cache entry that cannot be read or parsed counts as a miss rather than an error: + // a truncated write leaves a file that exists but deserializes to nothing, and refetching + // it is always preferable to failing the job push it was read for. let cache_path = format!("{}/{version}", *HUB_CACHE_DIR); - let script; - if tokio::fs::metadata(&cache_path).await.is_err() { - script = get_full_hub_script_by_path_inner(path, http_client, db).await?; - if let Err(e) = crate::worker::write_file( - &HUB_CACHE_DIR, - &version, - &serde_json::to_string(&script).map_err(to_anyhow)?, - ) { - tracing::error!("failed to write hub script {path} to cache: {e}"); - } else { - tracing::info!("wrote hub script {path} to cache"); + let cached = match tokio::fs::read_to_string(&cache_path).await { + Ok(content) => serde_json::from_str::(&content) + .inspect_err(|e| { + tracing::error!("hub script cache at {cache_path} is unparseable, refetching: {e}") + }) + .ok(), + Err(e) => { + if e.kind() != std::io::ErrorKind::NotFound { + tracing::error!("hub script cache at {cache_path} is unreadable, refetching: {e}"); + } + None } - } else { - let cache_content = tokio::fs::read_to_string(cache_path).await?; - script = serde_json::from_str(&cache_content).unwrap(); + }; + if let Some(script) = cached { tracing::info!("read hub script {path} from cache"); + return Ok(script); + } + + let script = get_full_hub_script_by_path_inner(path, http_client, db).await?; + if let Err(e) = crate::worker::write_file( + &HUB_CACHE_DIR, + &version, + &serde_json::to_string(&script).map_err(to_anyhow)?, + ) { + tracing::error!("failed to write hub script {path} to cache: {e}"); + } else { + tracing::info!("wrote hub script {path} to cache"); } Ok(script) } diff --git a/backend/windmill-common/src/ssrf.rs b/backend/windmill-common/src/ssrf.rs index 713b2e0289..31dee749e6 100644 --- a/backend/windmill-common/src/ssrf.rs +++ b/backend/windmill-common/src/ssrf.rs @@ -8,6 +8,10 @@ pub const ALLOW_PRIVATE_SAML_METADATA_URLS_ENV: &str = "ALLOW_PRIVATE_SAML_METAD pub const ALLOW_PRIVATE_GUEST_JWKS_URLS_ENV: &str = "ALLOW_PRIVATE_GUEST_JWKS_URLS"; +/// Lets every git call reach hosts on a private network, whoever it is made for. +/// Without it, [`private_git_host_allowed`] decides. +pub const ALLOW_LOCAL_GIT_REMOTES_ENV: &str = "ALLOW_LOCAL_GIT_REMOTES"; + /// Why a URL failed SSRF validation. /// /// The distinction matters for callers that gate private endpoints behind a @@ -203,6 +207,54 @@ pub fn allow_private_saml_metadata_urls() -> bool { .is_some_and(|v| v == "true" || v == "1") } +fn allow_local_git_remotes() -> bool { + std::env::var(ALLOW_LOCAL_GIT_REMOTES_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1") +} + +/// Who a git call is made for, which decides whether it may reach a host on a +/// private network. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GitRemoteCaller { + /// A workspace admin's request, or Windmill's own work (polling, webhook and + /// token upkeep, the merge request after a deploy), whose errors only admins read. + AdminOrSystem, + /// A request from anyone who is not a workspace admin. + NonAdmin, +} + +/// Whether a git call made for `caller` may reach a host on a private network. +/// +/// The refusal is for non-admins, who may not be able to run code (operators) +/// and would read git's error output back as a probe of the server's network. +/// An admin can run code, which reaches those hosts from a worker already. On a +/// cloud instance, where a workspace admin is anyone who signed up, every caller +/// is refused. +pub fn private_git_host_allowed(caller: GitRemoteCaller) -> bool { + git_host_policy_allows( + caller, + allow_local_git_remotes(), + *crate::worker::CLOUD_HOSTED, + ) +} + +fn git_host_policy_allows(caller: GitRemoteCaller, opted_in: bool, cloud_hosted: bool) -> bool { + opted_in || (caller == GitRemoteCaller::AdminOrSystem && !cloud_hosted) +} + +/// Appended to a refusal of a private git host, naming what would let `caller` +/// through. `None` where nothing an instance administrator sets would help. +pub fn private_git_host_hint(caller: GitRemoteCaller) -> Option { + (caller == GitRemoteCaller::NonAdmin && !*crate::worker::CLOUD_HOSTED).then(|| { + format!( + "Only workspace admins can reach a git server on a private network. To allow \ + every user, set the {ALLOW_LOCAL_GIT_REMOTES_ENV}=true environment variable on \ + the Windmill servers" + ) + }) +} + pub async fn validate_saml_metadata_url(url: &str) -> Result { let parsed = url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; @@ -632,6 +684,15 @@ mod tests { )); } + #[test] + fn private_git_hosts_are_refused_to_non_admins_and_on_cloud() { + use GitRemoteCaller::{AdminOrSystem, NonAdmin}; + assert!(git_host_policy_allows(AdminOrSystem, false, false)); + assert!(!git_host_policy_allows(NonAdmin, false, false)); + assert!(!git_host_policy_allows(AdminOrSystem, false, true)); + assert!(git_host_policy_allows(NonAdmin, true, true)); + } + #[tokio::test] async fn saml_ssrf_error_message_includes_env_hint_only_for_private_urls() { let private_error = validate_url_for_ssrf("http://127.0.0.1/metadata") diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index ebb0a723ea..ee8a389bb7 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -2512,6 +2512,42 @@ pub fn split_python_requirements>(requirements: T) -> Vec .collect() } +/// Byte offset of the comment marker, per pip's rule: a `#` at line start or preceded by +/// whitespace. A `#` elsewhere belongs to the requirement (`pkg @ https://h/p.whl#sha256=…`). +fn requirement_comment_start(line: &str) -> Option { + line.char_indices() + .find(|(i, c)| *c == '#' && (*i == 0 || line[..*i].ends_with(char::is_whitespace))) + .map(|(i, _)| i) +} + +/// The installable requirement carried by one lockfile line, or `None` for a comment, a +/// `-r`/`-e`/`--flag` directive, or a blank. +/// +/// Windmill installs a lockfile one entry at a time as a `uv pip install` argument, so +/// requirements-file syntax a file-level parser would absorb is an unparseable package name +/// here and has to be stripped first. +pub fn requirement_from_lockfile_line(line: &str) -> Option<&str> { + let requirement = match requirement_comment_start(line) { + Some(i) => &line[..i], + None => line, + } + .trim() + // Continuations are stripped, not joined: right for `--generate-hashes` locks, whose + // continued lines are `--hash=` flags this function drops, but a lock continuing onto a + // marker or extra would lose it. + .trim_end_matches('\\') + .trim_end(); + + (!requirement.is_empty() && !requirement.starts_with('-')).then_some(requirement) +} + +/// Whether a lockfile line continues onto the next one. The continued lines reach the +/// installer as entries of their own rather than being joined, so a caller that cares what +/// they carried — `--hash=` pins, for a `--generate-hashes` lock — has to say so itself. +pub fn lockfile_line_has_continuation(line: &str) -> bool { + line.trim_end().ends_with('\\') +} + #[derive(Eq, PartialEq, Clone, Copy, Default, Debug)] #[repr(u32)] pub enum PyVAlias { @@ -2630,6 +2666,52 @@ mod tests { ids.iter().map(|s| s.to_string()).collect() } + /// Fixtures are verbatim `uv pip compile` output (uv 0.11.28): split and inline + /// annotation styles, and `--generate-hashes`. + #[test] + fn test_requirement_from_lockfile_line() { + assert_eq!(requirement_from_lockfile_line(" # via httpx"), None); + assert_eq!(requirement_from_lockfile_line(" # via"), None); + assert_eq!(requirement_from_lockfile_line(" # anyio"), None); + assert_eq!( + requirement_from_lockfile_line(" # via -r .tmp/requirements.in"), + None + ); + assert_eq!( + requirement_from_lockfile_line("anyio==4.15.1 \\"), + Some("anyio==4.15.1") + ); + assert_eq!( + requirement_from_lockfile_line( + " --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7 \\" + ), + None + ); + assert_eq!(requirement_from_lockfile_line("# py: 3.11"), None); + assert_eq!(requirement_from_lockfile_line("-r other.txt"), None); + assert_eq!( + requirement_from_lockfile_line("--index-url https://x"), + None + ); + assert_eq!(requirement_from_lockfile_line(" "), None); + assert_eq!( + requirement_from_lockfile_line("httpx==0.27.0"), + Some("httpx==0.27.0") + ); + assert_eq!( + requirement_from_lockfile_line("httpx==0.27.0 # via -r requirements.in"), + Some("httpx==0.27.0") + ); + // A `#` not preceded by whitespace is part of the requirement, not a comment. + assert_eq!( + requirement_from_lockfile_line("wmill @ https://h/wmill.whl#sha256=abc"), + Some("wmill @ https://h/wmill.whl#sha256=abc") + ); + + assert!(lockfile_line_has_continuation("anyio==4.15.1 \\")); + assert!(!lockfile_line_has_continuation("anyio==4.15.1")); + } + #[test] fn test_parse_job_oom_score_adj() { assert_eq!(parse_job_oom_score_adj(Some("300")), 300); diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 443c3b6b63..45c8695585 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -183,7 +183,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28949/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28958/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -863,8 +863,12 @@ pub async fn guest_count_in_window<'c, E: sqlx::Executor<'c, Database = sqlx::Po /// The instance's standing against the guest allowance, as every surface reports it. #[derive(Clone, Debug, Serialize)] pub struct GuestUsage { + /// Whether this deployment can admit guests at all ([`instance_supports_guests`]). + /// Off, every other field is moot and no switch below can turn guests on. + pub available: bool, /// The superadmin switch (`GUEST_ACCESS_DISABLED_SETTING`), which every workspace - /// switch sits under. + /// switch sits under. Reported as stored, so a superadmin sees what they set even + /// where `available` overrules it. pub instance_enabled: bool, /// Distinct guest emails over the trailing `window_days`. pub guest_count: i64, @@ -877,9 +881,33 @@ pub struct GuestUsage { pub guest_seats: i64, } -/// SQL for "the instance admits guests": the superadmin switch, absent meaning on. The -/// setting is read as text before the cast so `true` and `"true"` both count. -fn instance_admits_guests_sql() -> String { +/// What a caller is told when it asks for guests on a deployment that cannot have them. +pub const GUESTS_UNAVAILABLE_MESSAGE: &str = + "Guest access is not available on Windmill Cloud. It requires a self-hosted instance \ + or a dedicated Windmill Cloud deployment."; + +/// Whether guests can exist on this deployment at all. They cannot on the shared cloud: +/// a guest is an identity Windmill itself never vouched for, admitted on the say-so of +/// whoever runs the instance, which is not a call a multi-tenant deployment can make for +/// its tenants. Folded into every guest gate below, so a workspace switch or an app +/// policy left saying `guest` is inert rather than honored. +pub fn instance_supports_guests() -> bool { + !*crate::worker::CLOUD_HOSTED +} + +/// [`instance_supports_guests`] as an error, for the writes that would otherwise store a +/// setting that can never take effect. +pub fn require_guest_support() -> Result<()> { + if instance_supports_guests() { + Ok(()) + } else { + Err(Error::BadRequest(GUESTS_UNAVAILABLE_MESSAGE.to_string())) + } +} + +/// SQL for the superadmin switch alone, absent meaning on. The setting is read as text +/// before the cast so `true` and `"true"` both count. +fn instance_switch_sql() -> String { format!( "NOT COALESCE((SELECT (value #>> '{{}}')::boolean FROM global_settings \ WHERE name = '{}'), false)", @@ -887,9 +915,18 @@ fn instance_admits_guests_sql() -> String { ) } +/// SQL for "the instance admits guests": the superadmin switch, under +/// [`instance_supports_guests`]. +fn instance_admits_guests_sql() -> String { + if !instance_supports_guests() { + return "false".to_string(); + } + instance_switch_sql() +} + pub async fn guest_usage(db: &crate::DB) -> Result { - let instance_admits = instance_admits_guests_sql(); - let instance_enabled: bool = sqlx::query_scalar(&format!("SELECT {instance_admits}")) + let instance_switch = instance_switch_sql(); + let instance_enabled: bool = sqlx::query_scalar(&format!("SELECT {instance_switch}")) .fetch_one(db) .await .map_err(|e| Error::internal_err(format!("reading the instance guest switch: {e:#}")))?; @@ -901,6 +938,7 @@ pub async fn guest_usage(db: &crate::DB) -> Result { 0 }; Ok(GuestUsage { + available: instance_supports_guests(), instance_enabled, guest_count, window_days: GUEST_WINDOW_DAYS, diff --git a/backend/windmill-common/tests/queue_metrics_series.rs b/backend/windmill-common/tests/queue_metrics_series.rs new file mode 100644 index 0000000000..f6bc7b4056 --- /dev/null +++ b/backend/windmill-common/tests/queue_metrics_series.rs @@ -0,0 +1,136 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::queue_metrics::{read_queue_metrics_series, QUEUE_METRIC_STALE_SECS}; + +const WINDOW: f64 = 3600.0; + +/// Store a sample written `at` seconds after the start of a `WINDOW` ending now. +async fn sample(db: &Pool, id: &str, value: serde_json::Value, at: f64) { + sqlx::query( + "INSERT INTO metrics (id, value, created_at) VALUES ($1, $2, now() - make_interval(secs => $3))", + ) + .bind(id) + .bind(value) + .bind(WINDOW - at) + .execute(db) + .await + .expect("failed to store a metric sample"); +} + +/// The database hands the renderer the last sample before the window, which sets the left edge, +/// and for each slot its peak and its latest value, which the line continues from. +#[sqlx::test(migrations = "../migrations")] +async fn a_series_starts_from_the_sample_before_the_window_and_keeps_each_slot_peak( + db: Pool, +) { + // Before the window: 1, then 2, which is what is in force at the left edge. + sample(&db, "queue_count_t", json!(1), -120.0).await; + sample(&db, "queue_count_t", json!(2), -60.0).await; + // Three samples inside one 30s slot: the line rises to their peak, then drops to the last. + sample(&db, "queue_count_t", json!(5), 605.0).await; + sample(&db, "queue_count_t", json!(9), 612.0).await; + sample(&db, "queue_count_t", json!(4), 620.0).await; + // Drained before the window: nothing left to draw. + sample(&db, "queue_count_gone", json!(3), -300.0).await; + sample(&db, "queue_count_gone", json!(0), -200.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + + assert_eq!( + series.tags.len(), + 1, + "a tag drained before the window is left out" + ); + let tag = &series.tags[0]; + assert_eq!(tag.tag, "t"); + assert!(tag.delay.is_empty()); + + let stale = 620.0 + QUEUE_METRIC_STALE_SECS; + let expected = [ + (0.0, 2.0), + (605.0, 2.0), + (605.0, 9.0), + (620.0, 9.0), + (620.0, 4.0), + (stale, 4.0), + (stale, 0.0), + (WINDOW, 0.0), + ]; + assert_eq!(tag.count.len(), expected.len(), "vertices: {:?}", tag.count); + for ((ms, value), (at, expected_value)) in tag.count.iter().zip(expected) { + let secs = (*ms - series.from) as f64 / 1000.0; + // Samples are stored a few milliseconds before the window is read. + assert!( + (secs - at).abs() < 2.0 && *value == expected_value, + "expected ({at}, {expected_value}), got ({secs}, {value}) in {:?}", + tag.count + ); + } +} + +/// A delay stored as its head's wait start is drawn as that wait, growing a second per second, +/// right up to the zero that closes it. +#[sqlx::test(migrations = "../migrations")] +async fn a_climbing_delay_is_drawn_as_the_wait_of_its_head(db: Pool) { + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + // The head started waiting 30s before the window; heartbeats restate it until the drain. + let head = json!({ "since": now - WINDOW - 30.0 }); + for at in [60.0, 360.0, 660.0] { + sample(&db, "queue_delay_t", head.clone(), at).await; + } + sample(&db, "queue_delay_t", json!(0), 900.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + let points = series.tags[0] + .delay + .iter() + .map(|(ms, value)| ((*ms - series.from) as f64 / 1000.0, *value)) + .collect::>(); + + let climb = points + .iter() + .filter(|(_, value)| *value > 0.0) + .collect::>(); + assert!( + climb.len() > 4, + "the climb has vertices along the way: {points:?}" + ); + for (at, value) in &climb { + assert!( + (value - (at + 30.0)).abs() < 2.0, + "off the climb at {at}: {points:?}" + ); + } + let (first, _) = climb[0]; + let (top, _) = climb[climb.len() - 1]; + assert!( + (first - 60.0).abs() < 2.0 && (top - 900.0).abs() < 2.0, + "{points:?}" + ); +} + +/// A climb that drains inside its slot keeps its top, which no stored value holds: it is reached +/// at the next sample. +#[sqlx::test(migrations = "../migrations")] +async fn a_climb_that_drains_inside_its_slot_keeps_its_top(db: Pool) { + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + // All in the 30s slot starting at 600: held at 5s, then climbing from a head queued at 597, + // which is still there when the tag drains at 627, 30s into its wait. + sample(&db, "queue_delay_t", json!(5), 602.0).await; + sample(&db, "queue_delay_t", json!({ "since": now - WINDOW + 597.0 }), 610.0).await; + sample(&db, "queue_delay_t", json!(0), 627.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + let top = series.tags[0] + .delay + .iter() + .map(|(_, value)| *value) + .fold(0.0, f64::max); + assert!((top - 30.0).abs() < 2.0, "{:?}", series.tags[0].delay); +} diff --git a/backend/windmill-common/tests/queue_stats.rs b/backend/windmill-common/tests/queue_stats.rs new file mode 100644 index 0000000000..419ad98d56 --- /dev/null +++ b/backend/windmill-common/tests/queue_stats.rs @@ -0,0 +1,69 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::queue::get_queue_stats; + +const WORKSPACE: &str = "test-workspace"; + +async fn queue_job( + db: &Pool, + tag: &str, + priority: Option, + waited_secs: f64, + running: bool, +) { + sqlx::query( + "WITH job AS ( + INSERT INTO v2_job (id, workspace_id, tag) VALUES (gen_random_uuid(), $1, $2) + RETURNING id + ) + INSERT INTO v2_job_queue (id, workspace_id, tag, priority, running, scheduled_for) + SELECT id, $1, $2, $3, $4, now() - make_interval(secs => $5) FROM job", + ) + .bind(WORKSPACE) + .bind(tag) + .bind(priority) + .bind(running) + .bind(waited_secs) + .execute(db) + .await + .expect("failed to queue job"); +} + +/// The delay reported for a tag is that of the job the worker pull takes first, ordered +/// `priority DESC NULLS LAST, scheduled_for`, not simply the oldest one waiting. Running jobs +/// and jobs less than 3 seconds past due are not part of the backlog at all. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn queue_stats_report_the_delay_of_the_job_pulled_next(db: Pool) { + // The oldest job has no priority, so every prioritized job runs before it. + queue_job(&db, "mixed", None, 900.0, false).await; + queue_job(&db, "mixed", Some(1), 600.0, false).await; + queue_job(&db, "mixed", Some(5), 300.0, false).await; + queue_job(&db, "mixed", Some(5), 100.0, false).await; + // Highest priority, but not backlog: already running, or not yet 3 seconds past due. + queue_job(&db, "mixed", Some(9), 1200.0, true).await; + queue_job(&db, "mixed", Some(9), 1.0, false).await; + queue_job(&db, "unprioritized", None, 500.0, false).await; + queue_job(&db, "unprioritized", None, 50.0, false).await; + + let stats = get_queue_stats(&db).await.unwrap(); + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + + let mixed = &stats["mixed"]; + assert_eq!(mixed.count, 4); + assert!( + (mixed.delay - 300.0).abs() < 5.0, + "expected the oldest job of the highest priority, got a delay of {}", + mixed.delay + ); + // The same job's wait start, which the delay is measured from. + assert!((mixed.head_since + mixed.delay - now).abs() < 5.0); + let unprioritized = &stats["unprioritized"]; + assert_eq!(unprioritized.count, 2); + assert!( + (unprioritized.delay - 500.0).abs() < 5.0, + "expected the oldest job, got a delay of {}", + unprioritized.delay + ); +} diff --git a/backend/windmill-common/tests/schema_replica_identity.rs b/backend/windmill-common/tests/schema_replica_identity.rs new file mode 100644 index 0000000000..fb14a5c78e --- /dev/null +++ b/backend/windmill-common/tests/schema_replica_identity.rs @@ -0,0 +1,46 @@ +//! Every table must be replicable. +//! +//! PostgreSQL refuses UPDATE and DELETE on a table that has neither a PRIMARY KEY +//! nor an explicit REPLICA IDENTITY once the database is published to a logical +//! replication slot. That is what a low-downtime major-version upgrade runs on +//! (RDS and Aurora Blue/Green, pglogical) and what every CDC pipeline reads, so a +//! single keyless table blocks the upgrade outright. This runs against a freshly +//! migrated database and fails on the migration that introduces one. + +use sqlx::{Pool, Postgres}; + +/// Partitioned parents are checked alongside ordinary tables: a parent without a +/// key hands the same defect to every partition created under it later. +#[sqlx::test(migrations = "../migrations")] +async fn every_table_is_replicable(db: Pool) -> anyhow::Result<()> { + let offenders: Vec = sqlx::query_scalar( + "SELECT n.nspname || '.' || c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND NOT ( + -- FULL and USING INDEX replicate on their own. + c.relreplident IN ('f', 'i') + -- DEFAULT resolves to the primary key, so it needs one to exist. + -- NOTHING never replicates, primary key or not. + OR (c.relreplident = 'd' AND EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indrelid = c.oid AND i.indisprimary + )) + ) + ORDER BY 1", + ) + .fetch_all(&db) + .await?; + + assert!( + offenders.is_empty(), + "logical replication will reject UPDATE and DELETE on these tables, because \ + they carry no replica identity it can use: {}. \ + Give each one a primary key -- a natural composite key where every column \ + is NOT NULL, otherwise a surrogate `BIGINT GENERATED ALWAYS AS IDENTITY`.", + offenders.join(", ") + ); + + Ok(()) +} diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 1ea5545b68..2df5dbab5f 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -17,6 +17,7 @@ use windmill_api_auth::{ }; use windmill_common::db::DB; use windmill_common::per_minute_counter::PerMinuteCounter; +use windmill_common::ssrf::{private_git_host_allowed, private_git_host_hint, GitRemoteCaller}; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use crate::secret_backend_ext::rename_vault_secret; @@ -3429,8 +3430,10 @@ fn git_url_userinfo(url: &str) -> Option<&str> { git_url_userinfo_range(url).map(|r| &url[r]) } -/// Validates a git URL to prevent option injection, SSRF, and local file read. -async fn validate_git_url(url: &str) -> Result<()> { +/// Validates a git URL to prevent option injection, SSRF, and local file read. The +/// syntax and scheme checks apply to every caller; the private-host refusal only +/// where [`private_git_host_allowed`] refuses `caller`. +async fn validate_git_url(url: &str, caller: GitRemoteCaller) -> Result<()> { let url = url.trim(); if url.is_empty() { return Err(Error::BadRequest("Git URL cannot be empty".to_string())); @@ -3484,25 +3487,26 @@ async fn validate_git_url(url: &str) -> Result<()> { let host = extract_host_from_git_url(url) .ok_or_else(|| Error::BadRequest("Could not parse hostname from git URL".to_string()))?; - // CI/dev escape hatch: integration tests run their git remote (a Gitea - // container) on localhost, which the network-target checks below reject. - // Scheme and option-injection validation above still applies. - if std::env::var("ALLOW_LOCAL_GIT_REMOTES").is_ok_and(|v| v == "true" || v == "1") { + // Scheme and option-injection validation above applies to every caller. + if private_git_host_allowed(caller) { return Ok(()); } + let hint = private_git_host_hint(caller) + .map(|h| format!(" {h}")) + .unwrap_or_default(); if host == "localhost" || host.ends_with(".local") || host == "[::1]" { - return Err(Error::BadRequest( - "Git URLs targeting localhost or local network are not allowed".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URLs targeting localhost or local network are not allowed.{hint}" + ))); } // Check literal IP addresses if let Ok(ip) = host.parse::() { if is_private_or_reserved_ip(&ip) { - return Err(Error::BadRequest( - "Git URLs targeting private or reserved IP addresses are not allowed".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URLs targeting private or reserved IP addresses are not allowed.{hint}" + ))); } } else { // Hostname — resolve via DNS and reject if any address is private. Fail @@ -3523,9 +3527,9 @@ async fn validate_git_url(url: &str) -> Result<()> { } for addr in addrs { if is_private_or_reserved_ip(&addr.ip()) { - return Err(Error::BadRequest( - "Git URL hostname resolves to a private or reserved IP address".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URL hostname resolves to a private or reserved IP address.{hint}" + ))); } } } @@ -3627,8 +3631,14 @@ async fn get_git_commit_hash( .map_err(|e| { Error::BadRequest(format!("Invalid git repository resource format: {}", e)) })?; + let caller = if authed.is_admin { + GitRemoteCaller::AdminOrSystem + } else { + GitRemoteCaller::NonAdmin + }; git_resource.url = - resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false).await?; + resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false, caller) + .await?; // A credential is stored under the repository it was issued for, so a // resource repointed elsewhere finds none. Which credential can be attached // is bounded by that; who may use it is bounded here, on the same terms as @@ -3658,7 +3668,7 @@ async fn get_git_commit_hash( let (git_ssh_cmd, filenames) = get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; - let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; + let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd, caller).await; delete_paths(&filenames).await; @@ -3762,12 +3772,17 @@ async fn get_git_ssh_cmd( const GIT_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); /// `git` command for a remote probe, with HTTP redirects disabled. `validate_git_url` -/// only vets the host in the URL; git's default (`http.followRedirects=initial`) -/// would let a validated public remote 302 the probe onto a private or link-local -/// address that no check ever sees. Build every probe through this. +/// checks the host in the URL, never one a redirect names; git's default +/// (`http.followRedirects=initial`) would let a public remote 302 the probe of a +/// caller refused private hosts onto one. Build every probe through this. +/// +/// The transports are pinned too: an SCP-shaped remote-helper string such as +/// `ext::@host:path` passes the URL check for a caller allowed private +/// hosts, and only git's own config would stop it from running the command. fn git_probe_command() -> Command { let mut git_cmd = Command::new("git"); git_cmd.args(["-c", "http.followRedirects=false"]); + git_cmd.env("GIT_ALLOW_PROTOCOL", "http:https:ssh:git"); git_cmd } @@ -3827,8 +3842,8 @@ fn dot_git_url(url: &str) -> Option { } /// Run a remote probe, retrying against [`dot_git_url`] if the remote answered the -/// URL as given with a redirect. Extending the path keeps the retry on the host -/// `validate_git_url` already cleared, which is exactly what following the redirect +/// URL as given with a redirect. Extending the path keeps the retry on the host of +/// the URL `validate_git_url` checked, which is exactly what following the redirect /// would not guarantee. `build` must produce the probe for the URL it is handed. /// /// A retry that also fails reports the *original* failure, so the caller's message @@ -3968,6 +3983,7 @@ async fn resolve_azure_devops_url( w_id: &str, url: &str, allow_cache: bool, + caller: GitRemoteCaller, ) -> Result { // Trim first: the http(s) gates the callers apply trim too, so a stored URL with // leading whitespace must not reach the scheme check here as a non-http one. @@ -3980,7 +3996,7 @@ async fn resolve_azure_devops_url( // cost a live credential (nor cache one), and whoever can edit the URL would // otherwise drive a token mint per poll tick. let probe_url = url.replace(placeholder, "windmill"); - validate_git_url(&probe_url).await?; + validate_git_url(&probe_url, caller).await?; // The background poller reads the referenced resource under the system identity, // which bypasses RLS. Confining the destination is what keeps that from becoming an @@ -4180,9 +4196,10 @@ fn git_sync_system_dba(db: &DB) -> DbWithOptAuthed<'static, ApiAuthed> { async fn get_repo_latest_commit_hash( git_resource: &GitRepositoryResource, git_ssh_command: Option, + caller: GitRemoteCaller, ) -> Result { // Validate URL and branch to prevent option injection and SSRF attacks - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, caller).await?; let ref_spec = git_resource .branch @@ -4315,8 +4332,14 @@ pub async fn get_git_repo_head_for_autopull( "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), )); } - git_resource.url = - resolve_azure_devops_url(&git_sync_system_dba(db), w_id, &git_resource.url, true).await?; + git_resource.url = resolve_azure_devops_url( + &git_sync_system_dba(db), + w_id, + &git_resource.url, + true, + GitRemoteCaller::AdminOrSystem, + ) + .await?; // A repo whose credential Windmill holds carries none in its URL, so the // poller has to attach it here or every probe would be unauthenticated. git_resource.url = @@ -4324,14 +4347,15 @@ pub async fn get_git_repo_head_for_autopull( if let Some(branch) = git_resource.branch.as_deref().filter(|s| !s.is_empty()) { let branch = branch.to_string(); - let sha = get_repo_latest_commit_hash(&git_resource, None).await?; + let sha = get_repo_latest_commit_hash(&git_resource, None, GitRemoteCaller::AdminOrSystem) + .await?; return Ok(Some((branch, sha))); } // No explicit branch: resolve the remote's default-branch NAME along with // its head in one call. Fork sync needs the concrete name to scope // `wm-fork//*`, so a bare "HEAD" ref would silently disable it. - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, GitRemoteCaller::AdminOrSystem).await?; let output = run_git_probe_for_url(&git_resource.url, "ls-remote --symref HEAD", |url| { let mut git_cmd = git_probe_command(); git_cmd.args(["ls-remote", "--symref", url, "HEAD"]); @@ -4426,13 +4450,20 @@ pub async fn get_git_repo_fork_heads_for_autopull( "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), )); } - git_resource.url = resolve_azure_devops_url(&dba, w_id, &git_resource.url, true).await?; + git_resource.url = resolve_azure_devops_url( + &dba, + w_id, + &git_resource.url, + true, + GitRemoteCaller::AdminOrSystem, + ) + .await?; // Same reason as the head probe above: a repository whose credential Windmill // holds carries none in its URL, and listing the fork branches is the half of // polling that would otherwise go out unauthenticated. git_resource.url = windmill_common::git_sync_oss::with_stored_credential(db, w_id, git_resource.url).await?; - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, GitRemoteCaller::AdminOrSystem).await?; validate_git_ref(base_branch)?; for r in extra_refs { @@ -4818,46 +4849,52 @@ mod tests { )); } + // A caller let through to private hosts must still hit the scheme check. #[tokio::test] async fn test_validate_git_url_blocks_file_scheme() { - let result = validate_git_url("file:///etc/passwd").await; + let result = validate_git_url("file:///etc/passwd", GitRemoteCaller::AdminOrSystem).await; assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("https://")); } #[tokio::test] async fn test_validate_git_url_blocks_private_ips() { - assert!(validate_git_url("http://127.0.0.1/repo.git").await.is_err()); - assert!(validate_git_url("http://169.254.169.254/latest/meta-data/") - .await - .is_err()); - assert!(validate_git_url("http://10.0.0.1/repo.git").await.is_err()); - assert!(validate_git_url("http://172.16.0.1/repo.git") - .await - .is_err()); - assert!(validate_git_url("http://192.168.1.1/repo.git") - .await - .is_err()); - assert!(validate_git_url("git://0.0.0.0/repo.git").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://127.0.0.1/repo.git").await.is_err()); + assert!(v("http://169.254.169.254/latest/meta-data/").await.is_err()); + let err = v("http://10.0.0.1/repo.git").await.unwrap_err(); + assert!(err.to_string().contains("ALLOW_LOCAL_GIT_REMOTES"), "{err}"); + assert!(v("http://172.16.0.1/repo.git").await.is_err()); + assert!(v("http://192.168.1.1/repo.git").await.is_err()); + assert!(v("git://0.0.0.0/repo.git").await.is_err()); // IPv6 loopback, unique-local, and link-local literals - assert!(validate_git_url("git://[::1]/repo.git").await.is_err()); - assert!(validate_git_url("git://[fd00::1]/repo.git").await.is_err()); - assert!(validate_git_url("git://[fe80::1]/repo.git").await.is_err()); + assert!(v("git://[::1]/repo.git").await.is_err()); + assert!(v("git://[fd00::1]/repo.git").await.is_err()); + assert!(v("git://[fe80::1]/repo.git").await.is_err()); + } + + #[tokio::test] + async fn test_validate_git_url_lets_admins_reach_private_hosts() { + assert!( + validate_git_url("http://10.0.0.1/repo.git", GitRemoteCaller::AdminOrSystem) + .await + .is_ok() + ); } #[tokio::test] async fn test_validate_git_url_blocks_localhost() { - assert!(validate_git_url("http://localhost/repo.git").await.is_err()); - assert!(validate_git_url("http://myhost.local/repo.git") - .await - .is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://localhost/repo.git").await.is_err()); + assert!(v("http://myhost.local/repo.git").await.is_err()); } #[tokio::test] async fn test_validate_git_url_blocks_local_paths() { - assert!(validate_git_url("/etc/passwd").await.is_err()); - assert!(validate_git_url("../relative/path").await.is_err()); - assert!(validate_git_url("./local/repo").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::AdminOrSystem); + assert!(v("/etc/passwd").await.is_err()); + assert!(v("../relative/path").await.is_err()); + assert!(v("./local/repo").await.is_err()); } /// Minimal loopback HTTP server: replies to every request with `response` and @@ -4987,7 +5024,11 @@ mod tests { async fn test_validate_git_url_fails_closed_on_unresolvable_host() { // `.invalid` never resolves (RFC 6761). The private-IP check is only // meaningful if a failed lookup rejects instead of falling through. - let result = validate_git_url("https://this-host-does-not-exist.invalid/repo.git").await; + let result = validate_git_url( + "https://this-host-does-not-exist.invalid/repo.git", + GitRemoteCaller::NonAdmin, + ) + .await; assert!( result.is_err(), "an unresolvable host was allowed — does this resolver synthesize records for NXDOMAIN?" @@ -4998,21 +5039,33 @@ mod tests { #[tokio::test] async fn test_validate_git_url_allows_valid_urls() { // Needs DNS: validation fails closed on a host it cannot resolve. - assert!(validate_git_url("https://github.com/user/repo.git") + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("https://github.com/user/repo.git").await.is_ok()); + assert!(v("git@github.com:user/repo.git").await.is_ok()); + assert!(v("ssh://git@github.com/user/repo.git").await.is_ok()); + } + + #[tokio::test] + async fn test_git_probe_refuses_remote_helpers() { + // A caller allowed private hosts skips the DNS step that would reject this + // SCP-shaped string, so the transport pin is what keeps git from running it. + let output = git_probe_command() + .args(["ls-remote", "testhelper::x@127.0.0.1:repo"]) + .output() .await - .is_ok()); - assert!(validate_git_url("git@github.com:user/repo.git") - .await - .is_ok()); - assert!(validate_git_url("ssh://git@github.com/user/repo.git") - .await - .is_ok()); + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("transport 'testhelper' not allowed"), + "{stderr}" + ); } #[tokio::test] async fn test_validate_git_url_blocks_option_injection() { - assert!(validate_git_url("-evil").await.is_err()); - assert!(validate_git_url("--upload-pack=evil").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::AdminOrSystem); + assert!(v("-evil").await.is_err()); + assert!(v("--upload-pack=evil").await.is_err()); } #[test] @@ -5108,24 +5161,21 @@ mod tests { // GHSA-p5cj-8cfh-mjv6: a loopback authority must stay blocked, and the // fragment/query `@public-host` bypasses of #8600 must be rejected so the // host git dials can never diverge from the validated host. - assert!(validate_git_url("http://127.0.0.1:40173/repo.git") - .await - .is_err()); - assert!(validate_git_url( - "http://127.0.0.1:40173/repo.git#@github.com/windmill-labs/windmill.git" - ) - .await - .is_err()); - assert!(validate_git_url( - "http://127.0.0.1:40173/repo.git?@github.com/windmill-labs/windmill.git" - ) - .await - .is_err()); - // A legitimate public repo URL still validates. + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://127.0.0.1:40173/repo.git").await.is_err()); assert!( - validate_git_url("https://github.com/windmill-labs/windmill.git") + v("http://127.0.0.1:40173/repo.git#@github.com/windmill-labs/windmill.git") .await - .is_ok() + .is_err() ); + assert!( + v("http://127.0.0.1:40173/repo.git?@github.com/windmill-labs/windmill.git") + .await + .is_err() + ); + // A legitimate public repo URL still validates. + assert!(v("https://github.com/windmill-labs/windmill.git") + .await + .is_ok()); } } diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs index 02feeb421a..cb1c53a579 100644 --- a/backend/windmill-types/src/flows.rs +++ b/backend/windmill-types/src/flows.rs @@ -538,6 +538,20 @@ pub struct Suspend { pub hide_cancel: Option, #[serde(skip_serializing_if = "false_or_empty")] pub continue_on_disapprove_timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skin: Option, +} + +/// How an approval request is presented, on the approval page and in Slack/Teams messages. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum ApprovalSkin { + Minimal, + /// A skin this server does not know renders as the detailed one rather than failing to + /// deserialize the whole flow, so a flow authored against a newer version still runs. + #[default] + #[serde(other)] + Detailed, } fn false_or_empty(v: &Option) -> bool { @@ -1365,6 +1379,23 @@ mod tests { assert_eq!(val.modules.len(), 1); } + #[test] + fn suspend_skin_unknown_value_falls_back_to_detailed() { + let skin_of = |skin: &str| { + let val: FlowValue = serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": {"type": "identity"}, + "suspend": {"required_events": 1, "skin": skin} + }] + })) + .unwrap(); + val.modules[0].suspend.as_ref().unwrap().skin + }; + assert_eq!(skin_of("minimal"), Some(ApprovalSkin::Minimal)); + assert_eq!(skin_of("not_a_skin_yet"), Some(ApprovalSkin::Detailed)); + } + #[test] fn agent_tool_keeps_description_through_locking() { // #10244: the dependency job rebuilds each tool from its locked FlowModule; the diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index d03f19b6ba..7a38369dd7 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -105,9 +105,11 @@ const p = { const normalized = (isRelative ? join(dirname(file_path), pathNoExt) : pathNoExt.slice(1)).replace(/\\/g, "/"); const hash = TEMP_SCRIPT_REFS?.[normalized]; + // Lock generation substitutes `raw`: the dependency scan reads versions from the + // `pkg@version` specifiers in imported scripts, which `raw_unpinned` strips. const url = (isRelative - ? `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${file_path}/../${args.path}${endExt}` - : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}` + ? `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${file_path}/../${args.path}${endExt}` + : `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${args.path}${endExt}` ) + (hash ? `?temp_script_hash=${hash}` : ""); const file = isRelative ? resolve("./" + file_path + "/../" + args.path + ".url") diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index a9844e5cfb..16f480815e 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -13,7 +13,7 @@ use itertools::Itertools; use serde_json::value::RawValue; use uuid::Uuid; -use windmill_parser_ts::remove_pinned_imports; +use windmill_parser_ts::{remove_pinned_import_specifiers, remove_pinned_imports}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PrecomputedAgentInfo}; @@ -1784,8 +1784,19 @@ pub async fn handle_bun_job( if modules.as_ref().is_some_and(|m| !m.is_empty()) { let bundle_path = std::path::Path::new(job_dir).join("out").join("main.js"); if bundle_path.exists() { + // The lock-generation build kept every `pkg@version` specifier, and bun resolves + // a pinned specifier outside node_modules, loading a second copy of the package. + // The bundle holds the user's code too, so only the specifiers are rewritten, and + // a bundle the parser rejects still runs as built, pins and all. let bundled = std::fs::read_to_string(&bundle_path)?; - write_file(job_dir, "main.ts", &bundled)?; + let unpinned = remove_pinned_import_specifiers(&bundled).unwrap_or_else(|e| { + tracing::warn!( + job_id = %job.id, + "could not unpin the modules bundle, running it as built: {e:#}" + ); + bundled + }); + write_file(job_dir, "main.ts", &unpinned)?; } } "\n\n--- BUN CODE EXECUTION ---\n".to_string() @@ -1975,7 +1986,7 @@ async function run() {{ return {{ type: "inline_checkpoint", key: dispatch.key, result: dispatch.result ?? null, started_at: dispatch.started_at, duration_ms: dispatch.duration_ms }}; }} if (dispatch.mode === "approval") {{ - return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled }}; + return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled, skin: dispatch.skin, description: dispatch.description }}; }} if (dispatch.mode === "sleep") {{ return {{ type: "sleep", key: dispatch.key, seconds: dispatch.seconds }}; @@ -3206,7 +3217,7 @@ pub async fn handle_wac_v2_output( job.id, num_steps ))) } - WacOutput::Approval { key, timeout, form, self_approval_disabled } => { + WacOutput::Approval { key, timeout, form, self_approval_disabled, skin, description } => { let db = match conn { Connection::Sql(db) => db, _ => { @@ -3322,15 +3333,19 @@ pub async fn handle_wac_v2_output( }; // Store approval form metadata for the approval page endpoint - let approval_meta = serde_json::json!({ + let mut approval_meta = serde_json::json!({ "key": key, "form": form, "timeout": timeout_secs as u32, "self_approval_disabled": sad, + "skin": skin.unwrap_or_default(), "resume": resume_url, "cancel": cancel_url, "approvalPage": approval_page_url, }); + if let Some(description) = description.filter(|d| !d.is_null()) { + approval_meta["description"] = description; + } sqlx::query( "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( COALESCE(workflow_as_code_status, '{}'::jsonb), diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index bb449f1810..413448b644 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -2271,7 +2271,11 @@ pub(crate) fn dbt_command(p: &PreparedProject, args: &[&str]) -> Command { .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - .env("GIT_PATH", GIT_PATH.as_str()); + .env("GIT_PATH", GIT_PATH.as_str()) + // dbt reports anonymous usage to dbt Labs from every invocation unless told + // not to, and a project's `flags:` block cannot override the variable. Set + // before the project's environment so a descriptor can still opt back in. + .env("DBT_SEND_ANONYMOUS_USAGE_STATS", "false"); // Both environments belong to the child. Under a sandbox they reach it through // the jail profile instead: set here, they would reach the dynamic loader that // execs nsjail itself, so an `LD_PRELOAD` from the project would run as the diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 9f363a7f90..175ed8295d 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -37,8 +37,9 @@ use windmill_common::{ scripts::ScriptLang, utils::calculate_hash, worker::{ - copy_dir_recursively, is_allowed_file_location, pad_string, split_python_requirements, - write_file, Connection, PyVAlias, PythonAnnotations, WORKER_CONFIG, + copy_dir_recursively, is_allowed_file_location, lockfile_line_has_continuation, pad_string, + requirement_from_lockfile_line, split_python_requirements, write_file, Connection, + PyVAlias, PythonAnnotations, WORKER_CONFIG, }, }; @@ -227,9 +228,9 @@ fn filter_pip_local_dependencies(lines: Vec) -> (Vec, Vec, compiled_deps: &[Regex]) -> (Vec, Vec) { - let (ignored, kept): (Vec, Vec) = lines - .into_iter() - .partition(|s| !s.starts_with('#') && compiled_deps.iter().any(|dep| dep.is_match(s))); + let (ignored, kept): (Vec, Vec) = lines.into_iter().partition(|s| { + !s.trim_start().starts_with('#') && compiled_deps.iter().any(|dep| dep.is_match(s)) + }); (kept, ignored) } @@ -323,6 +324,11 @@ pub async fn uv_pip_compile( "compile", "-q", "--no-header", + // The `#`-line filter applied to the output below only catches whole-line + // annotations, and uv's annotation style is configurable: `[pip] + // annotation-style = "line"` in the worker HOME's uv.toml emits them inline + // ("anyio==4.15.1 # via httpx"), which that filter keeps. + "--no-annotate", file, "--strip-extras", "-o", @@ -2520,11 +2526,23 @@ pub async fn handle_python_reqs( // Find out if there is already cached dependencies // If so, skip them let mut in_cache = vec![]; + if requirements + .iter() + .any(|r| lockfile_line_has_continuation(r)) + { + tracing::warn!(workspace_id = %w_id, job_id = %job_id, "lockfile continues entries across lines; the continued lines are dropped"); + append_logs( + job_id, + w_id, + "\n[!] lockfile continues entries across lines and the continued lines are dropped: `--hash=` pins, extras and markers written that way do not apply\n".to_string(), + conn, + ) + .await; + } for req in &requirements { - // Ignore python version annotation backed into lockfile - if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { + let Some(req) = requirement_from_lockfile_line(req) else { continue; - } + }; let py_prefix = &py_version.to_cache_dir(false); let venv_p = format!( diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index cecf0e316c..798cd347b1 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -46,6 +46,10 @@ pub enum WacOutput { form: Option, #[serde(default)] self_approval_disabled: Option, + #[serde(default)] + skin: Option, + #[serde(default)] + description: Option, }, /// Server-side sleep — suspend the workflow for a duration without holding a worker. #[serde(rename = "sleep")] diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 2899ff694c..d5efec8ff0 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.805.0"; +export const VERSION = "v1.809.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/TESTING.md b/cli/TESTING.md index b7a1e701b8..df0311677d 100644 --- a/cli/TESTING.md +++ b/cli/TESTING.md @@ -35,6 +35,28 @@ binary and starts a shared backend instance. Examples: `sync_pull_push`, `dev_server`, `standalone_commands` +## Module mocks + +`mock.module` replaces a module for the **whole process**, and it does reach modules that +were already imported — a stub one file installs lands on a consumer an earlier file +loaded. + +Handing the module back in `afterAll` is not a reliable undo. Files do run one at a time +(a root-level `afterAll` completes before the next file's body evaluates), so it looks +like it should be — but stubbing `bundle.ts` and restoring it that way still left +`raw_app_svelte_plugin_unit.test.ts` asserting against an empty bundle, green on Linux +and red on Windows, where the `readdir` file order differs. Treat a stub as permanent for +the run. + +So the rule is about what you stub, not how you clean up: **stub only a module no other +in-process suite imports.** Check with `grep -rl "" test/` before reaching +for one. A suite that drives the CLI through a spawned process is out of reach of a +module mock and doesn't count. + +`raw_app_push_policy_unit.test.ts` is the worked example: it stubs `gen/services.gen.ts`, +which passes the rule because nothing else in `test/` imports the three API functions it +replaces, and deliberately does not stub `bundle.ts`, which failed it. + ## AI Benchmark Caveats The repo-level benchmark CLI lives under `ai_evals/`, but it currently depends on diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index b35efc02a3..0d4ad634c8 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -119,9 +119,10 @@ export function isExecutionModeAnonymous(app: any) { export function isExecutionModeGuest(app: any) { return app?.["policy"]?.["execution_mode"] == "guest"; } -export type AppExecutionMode = "anonymous" | "guest" | "publisher"; +export type AppExecutionMode = "anonymous" | "guest" | "publisher" | "viewer"; /** The access mode is the one policy field a tracked app keeps, as `public` (anonymous) - * or `guests` (guest); the rest of the policy is regenerated on push. */ + * or `guests` (guest); the rest of the policy is preserved from the deployed app on + * push (see `generatingPolicy`). */ export function markAccessFromPolicy(app: any) { if (isExecutionModeAnonymous(app)) { app.public = true; @@ -129,14 +130,35 @@ export function markAccessFromPolicy(app: any) { app.guests = true; } } -export function executionModeFromAppFile(app: any): AppExecutionMode { +/** The mode the tracked file states, or `undefined` when it states none — the + * normal case, since a pull writes only the two open-access markers. `viewer` + * and `publisher` have no marker of their own, so a file can only name them + * through a policy block it was hand-written with. */ +function statedExecutionMode(app: any): AppExecutionMode | undefined { if (app?.["public"] ?? isExecutionModeAnonymous(app)) { return "anonymous"; } if (app?.["guests"] ?? isExecutionModeGuest(app)) { return "guest"; } - return "publisher"; + const mode = app?.["policy"]?.["execution_mode"]; + return mode === "viewer" || mode === "publisher" ? mode : undefined; +} + +/** The mode this push deploys under. A file that states one is authoritative, in + * both directions. Otherwise the two open-access markers are all it says, so + * their absence closes a deployed `anonymous`/`guest` app back down to + * `publisher` — while a deployed `viewer` is not a grant those markers revoke, + * so it carries over rather than widening to `publisher`. */ +export function executionModeForPush( + localApp: any, + deployedPolicy: Policy | undefined, +): AppExecutionMode { + const stated = statedExecutionMode(localApp); + if (stated) { + return stated; + } + return deployedPolicy?.execution_mode === "viewer" ? "viewer" : "publisher"; } export async function pushApp( workspace: string, @@ -161,12 +183,9 @@ export async function pushApp( //ignore } - let remoteOnBehalfOf: string | undefined; - let remoteOnBehalfOfEmail: string | undefined; - if (app?.policy) { - remoteOnBehalfOf = app.policy.on_behalf_of; - remoteOnBehalfOfEmail = app.policy.on_behalf_of_email; - } + // `app.policy` is cleared a few lines down, so capture it first: it is the + // base the regenerated policy is built on. + const deployedPolicy: Policy | undefined = app?.policy; markAccessFromPolicy(app); // console.log(app); @@ -181,20 +200,18 @@ export async function pushApp( const localApp = (await yamlParseFile(path)) as AppFile; replaceInlineScripts(localApp.value, localPath, true); - await generatingPolicy(localApp, remotePath, executionModeFromAppFile(localApp)); - - const preserveFields: { preserve_on_behalf_of?: boolean } = {}; - if (permissionedAsContext?.userIsAdminOrDeployer) { - if (app) { - if (localApp.policy && remoteOnBehalfOf) { - (localApp.policy as any).on_behalf_of = remoteOnBehalfOf; - (localApp.policy as any).on_behalf_of_email = remoteOnBehalfOfEmail; - preserveFields.preserve_on_behalf_of = true; - log.info(`Preserving ${remoteOnBehalfOfEmail ?? remoteOnBehalfOf} as permissioned_as for app ${remotePath}`); - } - } - // On create: backend applies folder defaults - } + // On create the backend applies folder defaults, so there is nothing to preserve. + const preserveFields = preserveOnBehalfOfFields( + remotePath, + deployedPolicy, + permissionedAsContext + ); + await generatingPolicy( + localApp, + remotePath, + executionModeForPush(localApp, deployedPolicy), + basePolicy(localApp, deployedPolicy, !!preserveFields.preserve_on_behalf_of) + ); // extra_perms goes through /acls/* — strip from the body so a perms-only // edit never bumps the app version (see applyExtraPermsDiff for details). @@ -251,18 +268,76 @@ export async function pushApp( export async function generatingPolicy( app: any, path: string, - executionMode: AppExecutionMode + executionMode: AppExecutionMode, + base: Policy | undefined ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { - app.policy = await windmillUtils.updatePolicy(app.value, undefined); - app.policy.execution_mode = executionMode; + app.policy = await windmillUtils.updatePolicy(app.value, base); + finalizeDerivedPolicy(app.policy, executionMode); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; } } +/** What the regenerated policy starts from: the deployed one, so a push keeps + * settings the tracked file doesn't record; on a first push, whatever the file + * states. The run identity rides along only when `claimsOnBehalfOf` — never + * from the file, never from a pusher who may not preserve one, since `wmill` + * is regularly pointed at servers older than the rewrite that would fix it. */ +export function basePolicy( + localApp: any, + deployedPolicy: Policy | undefined, + claimsOnBehalfOf: boolean +): Policy | undefined { + const stated = deployedPolicy ?? (localApp?.policy as Policy | undefined); + if (!stated || claimsOnBehalfOf) { + return stated; + } + const base: Policy = { ...stated }; + delete base.on_behalf_of; + delete base.on_behalf_of_email; + return base; +} + +/** Claim the run-as identity the regenerated policy carries over from the + * deployed app. Only a deployed identity may be claimed, never one the tracked + * file states — a repo doesn't get to pick who an app runs as. Without the flag + * the backend rewrites `on_behalf_of` to whoever is pushing, and it only honors + * the flag for an admin or a `wm_deployers` member, so a caller who is neither + * doesn't get to claim it here either. */ +export function preserveOnBehalfOfFields( + remotePath: string, + deployedPolicy: Policy | undefined, + permissionedAsContext: PermissionedAsContext | undefined +): { preserve_on_behalf_of?: boolean } { + const onBehalfOf = deployedPolicy?.on_behalf_of; + if (!permissionedAsContext?.userIsAdminOrDeployer || !onBehalfOf) { + return {}; + } + log.info( + `Preserving ${deployedPolicy?.on_behalf_of_email ?? onBehalfOf} as permissioned_as for app ${remotePath}` + ); + return { preserve_on_behalf_of: true }; +} + +/** The policy is written wholesale by the deploy, so the fields it does not + * derive from the tracked sources have to survive the trip. The policy builder + * has already recomputed what it can — the triggerables on both paths, plus the + * S3 rules on the low-code one, which `updateRawAppPolicy` has no equivalent of + * and so carries over. This sets the two left: the access mode, and the legacy + * `triggerables`, which still grant execution (the backend folds them into + * `triggerables_v2` at run time) and so are dropped rather than carried, or a + * deployed app would keep being able to run runnables this push removed. */ +export function finalizeDerivedPolicy( + policy: Policy, + executionMode: AppExecutionMode +) { + policy.triggerables = undefined; + policy.execution_mode = executionMode; +} + async function list(opts: GlobalOptions & { includeDraftOnly?: boolean; json?: boolean }) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -425,14 +500,16 @@ async function push( if (isRawAppByName || hasRawAppYaml) { const { pushRawApp } = await import("./raw_apps.ts"); const merged = await mergeConfigWithConfigFile(opts); - // Raw-app ownership preservation is not implemented on either push - // path: sync push hands pushRawApp no context either. await pushRawApp( workspace.workspaceId, remotePath, absoluteFilePath, undefined, merged.defaultTs, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), ); log.info(colors.bold.underline.green("Raw app pushed")); } else { diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 901a47593f..8d82edb9df 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -1,6 +1,9 @@ import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; -import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; @@ -17,13 +20,22 @@ import { deepEqual, readTextFile } from "../../utils/utils.ts"; import { type AppExecutionMode, - executionModeFromAppFile, + basePolicy, + executionModeForPush, + finalizeDerivedPolicy, markAccessFromPolicy, + preserveOnBehalfOfFields, replaceInlineScripts, repopulateFields, } from "./app.ts"; +import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER, RECORDINGS_FOLDER } from "./app_metadata.ts"; +import { + NEVER_DEPLOYED_DIRS, + NEVER_DEPLOYED_FILES, +} from "../../utils/app_files.ts"; import { writeIfChanged } from "../../utils/utils.ts"; import { yamlOptions } from "../sync/sync.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; @@ -316,13 +328,11 @@ async function collectAppFiles( const relativePath = basePath + entry.name; if (entry.isDirectory()) { - // Skip the runnables, node_modules, and sql_to_apply subfolders + // The backend folder deploys as `value.runnables`, not as a bundled + // file; the rest reach the server through no channel at all. if ( entry.name === APP_BACKEND_FOLDER || - entry.name === "node_modules" || - entry.name === "dist" || - entry.name === ".claude" || - entry.name === "sql_to_apply" + NEVER_DEPLOYED_DIRS.has(entry.name) ) { continue; } @@ -334,13 +344,11 @@ async function collectAppFiles( } await readDirRecursive(fullPath + SEP, relativePath + "/"); } else if (entry.isFile()) { - // Skip generated/metadata files that shouldn't be part of the app + // `raw_app.yaml` deploys as the request's metadata rather than as a + // bundled file; the rest reach the server through no channel at all. if ( entry.name === "raw_app.yaml" || - entry.name === "package-lock.json" || - entry.name === "DATATABLES.md" || - entry.name === "AGENTS.md" || - entry.name === "wmill.d.ts" + NEVER_DEPLOYED_FILES.has(entry.name) ) { continue; } @@ -360,6 +368,7 @@ export async function pushRawApp( localPath: string, message?: string, defaultTs: "bun" | "deno" = "bun", + permissionedAsContext?: PermissionedAsContext, ): Promise { if (alreadySynced.includes(localPath)) { return; @@ -376,6 +385,10 @@ export async function pushRawApp( } catch { //ignore } + // `app.policy` is cleared a few lines down, so capture it first. `raw_app.yaml` + // records none of the policy, so anything the deploy drawer set is only here. + const deployedPolicy: Policy | undefined = app?.policy; + markAccessFromPolicy(app); // console.log(app); if (app) { @@ -424,10 +437,21 @@ export async function pushRawApp( // Create a temporary app object for policy generation const appForPolicy = { ...localApp, runnables }; + // On create the backend applies folder defaults, so there is nothing to preserve. + const preserveFields = preserveOnBehalfOfFields( + remotePath, + deployedPolicy, + permissionedAsContext, + ); await generatingPolicy( appForPolicy, remotePath, - executionModeFromAppFile(localApp), + executionModeForPush(localApp, deployedPolicy), + basePolicy( + localApp, + deployedPolicy, + !!preserveFields.preserve_on_behalf_of, + ), ); const files = await collectAppFiles(localPath); @@ -482,6 +506,7 @@ export async function pushRawApp( path: remotePath, summary: localApp.summary, policy: appForPolicy.policy, + ...preserveFields, deployment_message: message, // Preserve any user draft at this path (see backend skip_draft_deletion). skip_draft_deletion: true, @@ -532,14 +557,12 @@ export async function generatingPolicy( app: any, path: string, executionMode: AppExecutionMode, + base: Policy | undefined, ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { - app.policy = await windmillUtils.updateRawAppPolicy( - app.runnables, - app.policy, - ); - app.policy.execution_mode = executionMode; + app.policy = await windmillUtils.updateRawAppPolicy(app.runnables, base); + finalizeDerivedPolicy(app.policy, executionMode); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; @@ -564,6 +587,10 @@ async function pushRawAppCommand( filePath, undefined, merged.defaultTs, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), ); log.info(colors.bold.underline.green("Raw app pushed")); } diff --git a/cli/src/commands/lint/lint.ts b/cli/src/commands/lint/lint.ts index f6a35429ef..d031c16b3f 100644 --- a/cli/src/commands/lint/lint.ts +++ b/cli/src/commands/lint/lint.ts @@ -31,10 +31,18 @@ import { import { isFlowInlineScriptPath, isAppInlineScriptPath, - isRawAppPath, + isFolderResourcePathAnyFormat, getFolderSuffix, + getScriptBasePathFromModulePath, } from "../../utils/resource_folders.ts"; -import { exts } from "../script/script.ts"; +import { isFilesetResource } from "../../utils/utils.ts"; +import { + exts, + findContentFile, + hasScriptExt, + isModuleEntryMetadata, + UnresolvableScriptContentFileError, +} from "../script/script.ts"; interface LintOptions extends GlobalOptions { json?: boolean; @@ -67,6 +75,9 @@ export interface LintReport { const YAML_FILE_REGEX = /\.ya?ml$/i; const NATIVE_TRIGGER_REGEX = /\.[^.]+_native_trigger\.ya?ml$/i; +// The metadata suffixes `findContentFile` resolves a flat script from. `.yml` is +// deliberately absent, since the push does not accept it there either. +const FLAT_SCRIPT_METADATA_REGEX = /\.script\.(yaml|json|lock)$/; function normalizePath(p: string): string { return p.replaceAll(SEP, "/"); @@ -643,6 +654,83 @@ export async function checkMissingLocks( return issues; } +/** + * Whether a path is a script's own metadata, as opposed to metadata the push + * deploys through some parent: a folder resource's inline scripts, a fileset's + * children (arbitrarily named, so one may be spelled exactly like a script's + * metadata) and the files of a module or dbt bundle all belong to that parent. + * + * Takes the path as the SYNC ROOT spells it, like the push. Relative to the + * lint target the enclosing folder is gone whenever the target IS that folder; + * absolute, the classifiers match their suffixes ANYWHERE in the string, so a + * checkout under `acme.app` reads as one app and nothing is ever reported. + */ +function isStandaloneScriptMetadata(rootedPath: string): boolean { + // Both suffix formats, because the dotted/non-dotted setting is read from the + // invocation directory and an explicit lint target may not share it. + if ( + isFolderResourcePathAnyFormat(rootedPath) || + isFilesetResource(rootedPath) + ) { + return false; + } + // A module folder keeps its metadata inside itself (`__mod/script.yaml`), + // which is standalone even though every other path under `__mod/` is not. + if (isModuleEntryMetadata(rootedPath)) return true; + if (getScriptBasePathFromModulePath(rootedPath) !== undefined) return false; + return FLAT_SCRIPT_METADATA_REGEX.test(rootedPath); +} + +/** + * `findContentFile` quotes the paths it was given back in its errors, so the + * lint target's own prefix comes off them again. Anchored at a path start: a + * plain substring replace of `f/` also eats the one inside `conf/`, mangling + * the very filename the message is telling the reader to delete. + */ +function relativizeMessage(message: string, prefix: string): string { + if (!prefix) return message; + const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return message.replaceAll(new RegExp(`(^|[\\s(])${escaped}/`, "g"), "$1"); +} + +/** + * Script metadata files that cannot be paired with exactly one content file: + * the push refuses those, and no metadata format makes them deployable, so the + * inactive twin of a format switch (`foo.script.json` in a yaml repo) is dead + * weight worth reporting even though the push skips it rather than refusing it. + * + * Resolved through `findContentFile` rather than by probing `exts` directly, so + * lint and push agree on what counts as paired: a dbt project's descriptor is + * optional and its absence is not an orphan, while two content files beside one + * metadata file is just as undeployable as none. It classifies what it is given + * and looks under `syncRoot`, so where the command was invoked from is not part + * of the answer. + */ +async function checkOrphanScriptMetadata( + syncRoot: string, + prefix: string, + metadataPaths: string[], +): Promise { + const issues: FileIssue[] = []; + for (const metadataPath of metadataPaths) { + const rootedPath = prefix ? `${prefix}/${metadataPath}` : metadataPath; + try { + await findContentFile(rootedPath, syncRoot); + } catch (e) { + if (!(e instanceof UnresolvableScriptContentFileError)) { + log.debug(`Failed to resolve content file for ${rootedPath}: ${e}`); + continue; + } + issues.push({ + path: metadataPath, + target: "script", + errors: [relativizeMessage(e.message, prefix)], + }); + } + } + return issues; +} + export async function runLint( opts: LintOptions, directory?: string, @@ -674,8 +762,16 @@ export async function runLint( const root = await FSFSElement(targetDirectory, [], false); const validator = new WindmillYamlValidator(); + // Walked paths are relative to the lint target; this puts them back the way + // the sync root spells them, which is what the two below are written against. + const syncRoot = await findSyncRoot(targetDirectory); + const metadataPrefix = normalizePath( + path.relative(syncRoot, targetDirectory), + ); + const warnings: LintWarning[] = []; const issues: FileIssue[] = []; + const scriptMetadataPaths: string[] = []; let scannedFiles = 0; let validatedFiles = 0; let validFiles = 0; @@ -689,6 +785,17 @@ export async function runLint( const normalizedPath = normalizePath(entry.path); scannedFiles += 1; + + // Collected before the YAML filter below: `.script.lock` and `.script.json` + // are metadata too, and both fail the push when nothing pairs with them. + if ( + isStandaloneScriptMetadata( + metadataPrefix ? `${metadataPrefix}/${normalizedPath}` : normalizedPath, + ) + ) { + scriptMetadataPaths.push(normalizedPath); + } + if (!YAML_FILE_REGEX.test(normalizedPath)) { continue; } @@ -727,6 +834,16 @@ export async function runLint( } } + // Unconditional: unlike a missing lock, metadata with no content file fails + // every push, so there is no mode in which it is acceptable. + issues.push( + ...(await checkOrphanScriptMetadata( + syncRoot, + metadataPrefix, + scriptMetadataPaths, + )), + ); + // Check for missing locks if --locks-required is set if (opts.locksRequired) { const lockIssues = await checkMissingLocks(opts, explicitTargetDirectory); @@ -820,6 +937,15 @@ async function lint(opts: LintOptions & { watch?: boolean }, directory?: string) } } +/** + * Whether a changed file can change what a lint run reports: metadata in any of + * its formats, and the content files whose presence is what keeps that metadata + * from being an orphan. + */ +function affectsLint(filename: string): boolean { + return /\.(ya?ml|json|lock)$/i.test(filename) || hasScriptExt(filename); +} + async function lintWatch(opts: LintOptions, directory?: string) { const { watch } = await import("node:fs"); const targetDir = directory ? path.resolve(process.cwd(), directory) : process.cwd(); @@ -842,7 +968,7 @@ async function lintWatch(opts: LintOptions, directory?: string) { let debounce: ReturnType | null = null; watch(targetDir, { recursive: true }, (_event, filename) => { - if (!filename || !filename.toString().endsWith(".yaml") && !filename.toString().endsWith(".yml")) return; + if (!filename || !affectsLint(filename.toString())) return; if (debounce) clearTimeout(debounce); debounce = setTimeout(runAndReport, 300); }); @@ -853,7 +979,7 @@ async function lintWatch(opts: LintOptions, directory?: string) { const command = new Command() .description( - "Validate Windmill flow, schedule, and trigger YAML files in a directory", + "Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file", ) .arguments("[directory:string]") .option("--json", "Output results in JSON format") diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index c44d9928f6..fe193e5d85 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -1075,10 +1075,12 @@ export class DbtPathCollisionError extends UnresolvableScriptContentFileError {} * guard on one of them leaves the other silently overwriting. */ export async function collidingDbtProject( - basePath: string + basePath: string, + baseDir?: string ): Promise { const project = basePath + "__dbt/dbt_project.yml"; - return (await stat(project).then(() => true).catch(() => false)) + const onDisk = baseDir ? path.join(baseDir, project) : project; + return (await stat(onDisk).then(() => true).catch(() => false)) ? project : undefined; } @@ -1139,7 +1141,14 @@ async function readScriptContent(filePath: string): Promise { } } -export async function findContentFile(filePath: string) { +/** + * The script file `filePath`'s metadata belongs to. `baseDir`, when given, is + * where the disk lookups happen, leaving `filePath` classified as written: the + * layout helpers below match their suffixes ANYWHERE in a path, so a caller + * that prefixed a checkout named `repo__mod` would have it read as the module. + */ +export async function findContentFile(filePath: string, baseDir?: string) { + const onDisk = (p: string) => (baseDir ? path.join(baseDir, p) : p); // Folder layout: __mod/script.yaml -> __mod/script.ts const isModuleFolderMeta = isModuleEntryMetadata(filePath); const toCandidate = (ext: string) => @@ -1163,7 +1172,7 @@ export async function findContentFile(filePath: string) { const validCandidates = ( await Promise.all( candidates.map((x) => { - return stat(x) + return stat(onDisk(x)) .catch(() => undefined) .then((x) => x?.isFile()) .then((e) => { @@ -1183,6 +1192,7 @@ export async function findContentFile(filePath: string) { const dbtCandidate = toCandidate("__dbt/" + DBT_DESCRIPTOR_NAME); const dbtProject = await collidingDbtProject( dbtCandidate.slice(0, -("__dbt/" + DBT_DESCRIPTOR_NAME).length), + baseDir, ); const nonDbtCandidates = validCandidates.filter((c) => c !== dbtCandidate); if (dbtProject && nonDbtCandidates.length > 0) { diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 99a098f848..2159ea2539 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -152,6 +152,7 @@ import { generateAppLocksInternal, RECORDINGS_FOLDER, } from "../app/app_metadata.ts"; +import { deploysWithRawApp } from "../../utils/app_files.ts"; import { isFlowPath, isAppPath, @@ -2018,20 +2019,18 @@ export async function elementsToMap( } if (isRawAppFile(path)) { - // FSFSElement builds paths with the platform separator, while the checks - // below are written with "/": without normalizing, none of them match on - // Windows and the push collector's own exclusions become perpetual diffs. + // FSFSElement builds paths with the platform separator, while + // `deploysWithRawApp` is written with "/": without normalizing it matches + // nothing on Windows and the push collector's own exclusions become + // perpetual diffs. const suffix = path .split(getFolderSuffix("raw_app") + SEP) .pop() ?.replaceAll(SEP, "/"); - if ( - suffix?.startsWith("dist/") || - suffix?.startsWith(RECORDINGS_FOLDER + "/") || - suffix == "wmill.d.ts" || - suffix == "package-lock.json" || - suffix == "DATATABLES.md" - ) { + // A file no push sends is not a change to track. Listing it leaves it + // pending forever — nothing ever uploads it — and pushing it redeploys + // the whole app, reassigning its run-as user, to ship nothing. + if (suffix && !deploysWithRawApp(suffix)) { continue; } } @@ -6142,7 +6141,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message }, + { message: opts.message, permissionedAsContext }, ); } else { // Flow folder doesn't exist locally — delete on server @@ -6187,7 +6186,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message }, + { message: opts.message, permissionedAsContext }, ); } else { // App folder doesn't exist locally — delete on server @@ -6233,7 +6232,11 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message, defaultTs: opts.defaultTs }, + { + message: opts.message, + defaultTs: opts.defaultTs, + permissionedAsContext, + }, ); } else { // The entire raw app folder was deleted locally, diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index b5494b112c..228d0444a5 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.805.0"; +export const VERSION = "1.809.0"; diff --git a/cli/src/core/permissioned_as.ts b/cli/src/core/permissioned_as.ts index 57570b3549..d0301373da 100644 --- a/cli/src/core/permissioned_as.ts +++ b/cli/src/core/permissioned_as.ts @@ -3,6 +3,12 @@ import * as log from "./log.ts"; import { colors } from "@cliffy/ansi/colors"; import { Confirm } from "@cliffy/prompt/confirm"; import { getTypeStrFromPath } from "../types.ts"; +import { + extractFolderPath, + isAppFolderMetadataFile, + isRawAppFolderMetadataFile, +} from "../utils/resource_folders.ts"; +import { deploysWithRawApp } from "../utils/app_files.ts"; import { parseSyncBehavior } from "./conf.ts"; export interface PermissionedAsContext { @@ -89,6 +95,45 @@ function contentHasOnBehalfOf(content: string, typeStr: string): boolean { return false; } +type AppTypeStr = "app" | "raw_app"; + +function isAppTypeStr(typeStr: string): typeStr is AppTypeStr { + return typeStr === "app" || typeStr === "raw_app"; +} + +/** The app folder a file belongs to. `isAppFolderMetadataFile` and its raw twin + * match a literal `/`, unlike `extractFolderPath` — so normalize before either, + * or a Windows path takes a different branch from the same file on Linux. */ +function appFolderOf(path: string, typeStr: AppTypeStr): string { + return extractFolderPath(path, typeStr) ?? path; +} + +function toPosix(path: string): string { + return path.replaceAll("\\", "/"); +} + +/** App folders whose own metadata file is being added or deleted, which is how a + * whole app arrives or goes rather than being redeployed. Neither takes an owner + * over: a create has none yet, and a delete leaves none behind. */ +function appsArrivingOrLeaving(changes: Change[]): Set { + const folders = new Set(); + for (const change of changes) { + if (change.name === "edited") continue; + const path = toPosix(change.path); + if (!isAppFolderMetadataFile(path) && !isRawAppFolderMetadataFile(path)) { + continue; + } + let typeStr: string; + try { + typeStr = getTypeStrFromPath(path); + } catch { + continue; + } + if (isAppTypeStr(typeStr)) folders.add(appFolderOf(path, typeStr)); + } + return folders; +} + export async function preCheckPermissionedAs( changes: Change[], userEmail: string, @@ -101,6 +146,12 @@ export async function preCheckPermissionedAs( if (userIsAdminOrDeployer) return; const wouldChangeItems: { path: string; currentOwner: string }[] = []; + const addItem = (item: { path: string; currentOwner: string }) => { + if (!wouldChangeItems.some((i) => i.path === item.path)) { + wouldChangeItems.push(item); + } + }; + const arrivingOrLeaving = appsArrivingOrLeaving(changes); for (const change of changes) { let typeStr: string; @@ -110,6 +161,22 @@ export async function preCheckPermissionedAs( continue; } + // An app is redeployed whole by any change to any of the files it actually + // sends — added, edited or deleted alike — so its policy is rewritten + // regardless of what the file holds. Settled here, before the content the + // other kinds parse to find their owner, which an app has none of to parse. + if (isAppTypeStr(typeStr)) { + const path = toPosix(change.path); + const folder = appFolderOf(path, typeStr); + if ( + !arrivingOrLeaving.has(folder) && + (typeStr === "app" || deploysWithRawApp(path.slice(folder.length))) + ) { + addItem({ path: folder, currentOwner: "(app policy owner)" }); + } + continue; + } + if (change.name === "added") { const content = change.content; if (!content) continue; @@ -130,11 +197,6 @@ export async function preCheckPermissionedAs( const label = typeStr === "script" ? "(script owner)" : "(flow owner)"; wouldChangeItems.push({ path: change.path, currentOwner: label }); - } else if (typeStr === "app") { - wouldChangeItems.push({ - path: change.path, - currentOwner: "(app policy owner)", - }); } continue; } @@ -177,12 +239,6 @@ export async function preCheckPermissionedAs( } } continue; - } else if (typeStr === "app") { - wouldChangeItems.push({ - path: change.path, - currentOwner: "(app policy owner)", - }); - continue; } else if (typeStr === "schedule") { const match = beforeContent.match( /email:\s*["']?([^\s"']+)["']?/ diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index a8790f35a7..0c7b91887d 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -1037,12 +1037,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -1824,12 +1828,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -2705,12 +2713,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -4673,13 +4685,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Process items in parallel with optional concurrency control. # @@ -5558,7 +5574,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -6794,12 +6810,16 @@ export async function sleep(seconds: number): Promise * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -6975,13 +6995,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Get the resume/cancel/approval-page URLs bound to one \`\`wait_for_approval\`\` step. # @@ -7403,7 +7427,7 @@ Manage jobs (import/export) ### lint -Validate Windmill flow, schedule, and trigger YAML files in a directory +Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file **Arguments:** \`[directory:string]\` diff --git a/cli/src/types.ts b/cli/src/types.ts index 9c517ff3c7..5713d06390 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -234,7 +234,7 @@ export async function pushObj( if (!rawAppName) { throw new Error(`Could not extract raw app name from path: ${p}`); } - await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message, defaultTs); + await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message, defaultTs, permissionedAsContext); } else if (typeEnding === "folder") { await pushFolder(workspace, p, befObj, newObj); } else if (typeEnding === "variable") { diff --git a/cli/src/utils/app_files.ts b/cli/src/utils/app_files.ts new file mode 100644 index 0000000000..ca1be1de0e --- /dev/null +++ b/cli/src/utils/app_files.ts @@ -0,0 +1,48 @@ +import { + APP_BACKEND_FOLDER, + RECORDINGS_FOLDER, +} from "../commands/app/app_metadata.ts"; + +/** Directories under a raw app that no push sends. */ +const NEVER_DEPLOYED_DIRS = new Set([ + "node_modules", + "dist", + ".claude", + "sql_to_apply", +]); + +/** Files under a raw app that no push sends. */ +const NEVER_DEPLOYED_FILES = new Set([ + "package-lock.json", + "DATATABLES.md", + "AGENTS.md", + "wmill.d.ts", +]); + +/** + * Whether an app-root-relative path (`/` separators, leading slash optional) + * reaches the server through any of a push's three channels: `raw_app.yaml` as + * metadata, the backend folder as runnables, the rest bundled by + * `collectAppFiles`. A path this rejects deploys nothing, so changing it is not + * a change to the app however much the sync diff lists it. `collectAppFiles` + * must not drift from this — it reads the same two sets. + */ +export function deploysWithRawApp(relativePath: string): boolean { + const segments = relativePath.split("/").filter(Boolean); + if (segments.length === 0) return false; + const name = segments[segments.length - 1]; + const dirs = segments.slice(0, -1); + // The sets below describe the bundle, which never walks into the backend + // folder — applying them there would strip a runnable whose file shares a + // name (`backend/wmill.d.ts` is the runnable `wmill.d`). Depth 1 because + // `loadRunnablesFromBackend` reads that folder's top level only. + if (dirs[0] === APP_BACKEND_FOLDER) return dirs.length === 1; + if (NEVER_DEPLOYED_FILES.has(name)) return false; + if (dirs.some((d) => NEVER_DEPLOYED_DIRS.has(d))) return false; + // Session recordings are written at the app root only, so an app with a + // `recordings/` component folder of its own still ships it. + if (dirs[0] === RECORDINGS_FOLDER) return false; + return true; +} + +export { NEVER_DEPLOYED_DIRS, NEVER_DEPLOYED_FILES }; diff --git a/cli/test/app_access_mode_unit.test.ts b/cli/test/app_access_mode_unit.test.ts index 12185d6d99..42a3fd082a 100644 --- a/cli/test/app_access_mode_unit.test.ts +++ b/cli/test/app_access_mode_unit.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test"; import { - executionModeFromAppFile, + executionModeForPush, generatingPolicy, markAccessFromPolicy, } from "../src/commands/app/app.ts"; @@ -13,16 +13,39 @@ test("the access mode survives the app.yaml round trip", async () => { guest.policy = undefined; expect(guest.guests).toBe(true); expect(guest.public).toBeUndefined(); - expect(executionModeFromAppFile(guest)).toBe("guest"); - await generatingPolicy(guest, "u/test/app", executionModeFromAppFile(guest)); + expect(executionModeForPush(guest, undefined)).toBe("guest"); + await generatingPolicy( + guest, + "u/test/app", + executionModeForPush(guest, undefined), + undefined, + ); expect(guest.policy.execution_mode).toBe("guest"); const anonymous: any = { policy: { execution_mode: "anonymous" }, value: {} }; markAccessFromPolicy(anonymous); anonymous.policy = undefined; expect(anonymous.public).toBe(true); - expect(executionModeFromAppFile(anonymous)).toBe("anonymous"); + expect(executionModeForPush(anonymous, undefined)).toBe("anonymous"); - expect(executionModeFromAppFile({ policy: { execution_mode: "publisher" } })).toBe("publisher"); - expect(executionModeFromAppFile({})).toBe("publisher"); + expect(executionModeForPush({ policy: { execution_mode: "publisher" } }, undefined)).toBe("publisher"); + expect(executionModeForPush({}, undefined)).toBe("publisher"); +}); + +// `viewer` is the narrowest mode — each runnable runs as the viewer, not as the +// app's identity — and the only one with no marker in the file, so both ways it +// can reach a push must survive rather than widen to `publisher`. +test("viewer is never widened to publisher by a push", () => { + // Carried over from the deployed app: a pull writes no marker for it. + expect(executionModeForPush({}, { execution_mode: "viewer" })).toBe("viewer"); + // Stated by the file, which is all a first push has to go on. + expect(executionModeForPush({ policy: { execution_mode: "viewer" } }, undefined)).toBe("viewer"); + // The open-access markers still win, in either direction. + expect(executionModeForPush({ public: true }, { execution_mode: "viewer" })).toBe("anonymous"); + expect(executionModeForPush({}, { execution_mode: "anonymous" })).toBe("publisher"); + // A stated mode is authoritative both ways: the carry-over is for a file that + // says nothing, so it must not pin a deployed app to `viewer` forever. + expect( + executionModeForPush({ policy: { execution_mode: "publisher" } }, { execution_mode: "viewer" }) + ).toBe("publisher"); }); diff --git a/cli/test/lint_orphan_metadata_unit.test.ts b/cli/test/lint_orphan_metadata_unit.test.ts new file mode 100644 index 0000000000..7066d10dca --- /dev/null +++ b/cli/test/lint_orphan_metadata_unit.test.ts @@ -0,0 +1,166 @@ +import { expect, test, describe } from "bun:test"; +import { mkdtemp, rm, mkdir, writeFile } from "node:fs/promises"; +import os from "node:os"; +import * as path from "node:path"; +import { runLint } from "../src/commands/lint/lint.ts"; + +const WMILL_YAML = "defaultTs: bun\nincludes:\n - f/**\nexcludes: []\n"; +const METADATA = "summary: test\nlock: ''\nschema:\n properties: {}\n"; + +async function write(dir: string, rel: string, content: string) { + const full = path.join(dir, rel); + await mkdir(path.dirname(full), { recursive: true }); + await writeFile(full, content, "utf-8"); +} + +/** + * Runs `fn` with a sync root at `/`, from which lint resolves + * every walked path. The name is a parameter because it is load-bearing: the + * folder suffixes lint classifies by (`.app`, `__mod`, …) are matched anywhere + * in a path, so a root carrying one must not change what lint reports. + */ +async function withSyncRoot( + rootName: string, + fn: (syncRoot: string) => Promise, + opts: { runFromParent?: boolean } = {}, +): Promise { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "wmill_lint_orphan_")); + const syncRoot = path.join(tempDir, rootName); + const originalCwd = process.cwd(); + try { + await write(syncRoot, "wmill.yaml", WMILL_YAML); + process.chdir(opts.runFromParent ? tempDir : syncRoot); + await fn(syncRoot); + } finally { + process.chdir(originalCwd); + await rm(tempDir, { recursive: true }); + } +} + +describe("orphan script metadata", () => { + test("reports metadata with no content file, with locks not required", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/paired.py", "def main():\n pass\n"); + await write(syncRoot, "f/paired.script.yaml", METADATA); + await write(syncRoot, "f/orphan.script.yaml", METADATA); + await write(syncRoot, "f/orphan_json.script.json", "{}\n"); + await write(syncRoot, "f/orphan_lock.script.lock", "some-dep==1.0.0\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.exitCode).toBe(1); + expect(report.issues.map((i) => i.path).sort()).toEqual([ + "f/orphan.script.yaml", + "f/orphan_json.script.json", + "f/orphan_lock.script.lock", + ]); + expect(report.issues[0].target).toBe("script"); + expect(report.issues[0].errors[0]).toContain("No script file found next to"); + }); + }); + + test("reports a module folder's own metadata with no content file", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/orphan__mod/script.yaml", METADATA); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues.map((i) => i.path)).toEqual([ + "f/orphan__mod/script.yaml", + ]); + + // Linting the module folder itself: the walked paths no longer carry the + // `__mod/` boundary that says this is a module's metadata. + const inFolder = await runLint( + {} as any, + path.join(syncRoot, "f/orphan__mod"), + ); + + expect(inFolder.issues.map((i) => i.path)).toEqual(["script.yaml"]); + }); + }); + + test("reports orphans under a sync root named like a resource folder", async () => { + await withSyncRoot("acme.app", async (syncRoot) => { + await write(syncRoot, "f/orphan.script.yaml", METADATA); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues.map((i) => i.path)).toEqual(["f/orphan.script.yaml"]); + }); + }); + + test("does not report a paired module under a sync root named like a module folder", async () => { + // Run from OUTSIDE the checkout, the one invocation whose paths carry the + // root's own name: nothing above the sync root may be classified. + await withSyncRoot( + "repo__mod", + async (syncRoot) => { + await write(syncRoot, "f/example__mod/script.yaml", METADATA); + await write( + syncRoot, + "f/example__mod/script.ts", + "export function main() {}\n", + ); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }, + { runFromParent: true }, + ); + }); + + test("does not report a non-dotted folder resource's child", async () => { + // The dotted/non-dotted setting is read from the invocation directory, so + // an explicit target configured the other way must still be recognized. + await withSyncRoot( + "repo", + async (syncRoot) => { + await write(syncRoot, "f/a__raw_app/raw_app.yaml", "value: {}\n"); + await write(syncRoot, "f/a__raw_app/backend/config.script.lock", "x\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }, + { runFromParent: true }, + ); + }); + + test("keeps the reported path whole when the lint target is a path segment", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/conf/orphan.script.yaml", METADATA); + + const report = await runLint({} as any, path.join(syncRoot, "f")); + + expect(report.issues.map((i) => i.path)).toEqual([ + "conf/orphan.script.yaml", + ]); + expect(report.issues[0].errors[0]).toContain("conf/orphan.script.yaml"); + }); + }); + + test("does not report a fileset child spelled like script metadata", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/data.resource.yaml", "value: {}\n"); + await write(syncRoot, "f/data.fileset/config.script.yaml", "a: 1\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }); + }); + + test("does not report a dbt project whose optional descriptor is absent", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/proj.script.yaml", METADATA); + await write(syncRoot, "f/proj__dbt/dbt_project.yml", "name: proj\n"); + await write(syncRoot, "f/proj__dbt/models/a.sql", "select 1\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }); + }); +}); diff --git a/cli/test/precheck_permissioned_as_apps_unit.test.ts b/cli/test/precheck_permissioned_as_apps_unit.test.ts new file mode 100644 index 0000000000..4a224a43c0 --- /dev/null +++ b/cli/test/precheck_permissioned_as_apps_unit.test.ts @@ -0,0 +1,147 @@ +/** + * The pre-check is what stops a push from silently reassigning an item's run-as + * user. Raw apps were missing from it, so the one kind whose file records no + * policy at all was also the one that changed owner without a word. + */ + +import { expect, test } from "bun:test"; +import { preCheckPermissionedAs } from "../src/core/permissioned_as.ts"; + +/** Non-interactive and without the override flag, the pre-check exits rather + * than reassigning silently — so a thrown exit is the signal it fired. */ +type Shape = "edited" | "added" | "deleted"; + +function change(path: string, name: Shape = "edited") { + return { name, path, before: "summary: x\n", content: "summary: x\n" }; +} + +async function precheck( + changes: ReturnType[], +): Promise { + const exit = process.exit; + let code: number | undefined; + (process as any).exit = (c?: number) => { + code = c; + throw new Error(`exit:${c}`); + }; + const logged: string[] = []; + const err = console.error; + console.error = (...a: unknown[]) => void logged.push(a.join(" ")); + try { + await preCheckPermissionedAs(changes, "pusher@corp", false, false, false); + } catch (e) { + if (!String(e).startsWith("Error: exit:")) throw e; + } finally { + (process as any).exit = exit; + console.error = err; + } + return code === undefined ? undefined : logged.join("\n"); +} + +test("a raw-app push warns the non-deployer it will take over the run-as user", async () => { + const message = await precheck([change("f/test/myapp.raw_app/index.tsx")]); + + expect(message).toBeDefined(); + expect(message).toContain("f/test/myapp.raw_app"); + expect(message).toContain("pusher@corp"); +}); + +// Deleting one file re-pushes the whole app rather than deleting it, so the +// takeover happens there too. +test("deleting one of an app's files warns like editing one", async () => { + const message = await precheck([ + change("f/test/myapp.raw_app/gone.tsx", "deleted"), + ]); + + expect(message).toContain("f/test/myapp.raw_app"); +}); + +// The metadata file going with it means the app itself is created or removed — +// neither takes an owner over. +test("an app arriving or leaving whole is not a takeover", async () => { + const created = await precheck([ + change("f/test/new.raw_app/raw_app.yaml", "added"), + change("f/test/new.raw_app/index.tsx", "added"), + ]); + const removed = await precheck([ + change("f/test/old.raw_app/raw_app.yaml", "deleted"), + change("f/test/old.raw_app/index.tsx", "deleted"), + ]); + + expect(created).toBeUndefined(); + expect(removed).toBeUndefined(); +}); + +// An app carries no owner in its files, so nothing about it depends on their +// content — an empty one redeploys it exactly like any other. +test("an empty file still counts as a change to the app", async () => { + const added = await precheck([ + { name: "added", path: "f/test/myapp.raw_app/blank.ts", content: "" }, + ]); + const edited = await precheck([ + { name: "edited", path: "f/test/myapp.raw_app/blank.ts", before: "" }, + ]); + + expect(added).toContain("f/test/myapp.raw_app"); + expect(edited).toContain("f/test/myapp.raw_app"); +}); + +// `extractFolderPath` normalizes separators but the metadata predicates match a +// literal `/`, so a Windows path must not take a different branch. +test("a Windows path classifies the same as its posix twin", async () => { + const created = await precheck([ + change("f\\test\\new.raw_app\\raw_app.yaml", "added"), + change("f\\test\\new.raw_app\\index.tsx", "added"), + ]); + const edited = await precheck([ + change("f\\test\\myapp.raw_app\\index.tsx"), + ]); + + expect(created).toBeUndefined(); + expect(edited).toContain("f/test/myapp.raw_app"); +}); + +// `collectAppFiles` never sends these, and the sync diff never stops listing +// them (nothing uploads them, so they stay "added" forever) — so warning on one +// would gate every push of a scaffolded app on the override flag. +test("a file the push never sends is not a change to the app", async () => { + const artifacts = await precheck([ + change("f/test/myapp.raw_app/AGENTS.md", "added"), + change("f/test/myapp.raw_app/sql_to_apply/a.sql", "added"), + change("f/test/myapp.raw_app/node_modules/dep/index.js", "added"), + change("f/test/myapp.raw_app/recordings/r.json", "added"), + change("f/test/myapp.raw_app/package-lock.json"), + change("f/test/myapp.raw_app/wmill.d.ts"), + // Only the backend folder's *top level* is a runnable; nothing reads deeper, + // so the depth limit is what keeps a `backend/node_modules/` from becoming + // the perpetual diff this predicate exists to remove. + change("f/test/myapp.raw_app/backend/node_modules/dep/index.js", "added"), + ]); + // The three channels a push does send through: bundled file, metadata, runnable. + const sent = await precheck([change("f/test/myapp.raw_app/index.tsx")]); + const meta = await precheck([change("f/test/myapp.raw_app/raw_app.yaml")]); + const runnable = await precheck([change("f/test/myapp.raw_app/backend/a.ts")]); + // The runnable channel is not the bundle: the bundle's name exclusions don't + // reach into it, so a runnable file sharing one of those names still deploys. + const namesake = await precheck([ + change("f/test/myapp.raw_app/backend/wmill.d.ts"), + ]); + + expect(artifacts).toBeUndefined(); + expect(sent).toContain("f/test/myapp.raw_app"); + expect(meta).toContain("f/test/myapp.raw_app"); + expect(runnable).toContain("f/test/myapp.raw_app"); + expect(namesake).toContain("f/test/myapp.raw_app"); +}); + +test("an app is listed once however many of its files changed", async () => { + const message = await precheck([ + change("f/test/myapp.raw_app/index.tsx"), + change("f/test/myapp.raw_app/raw_app.yaml"), + change("f/test/myapp.raw_app/backend/a.ts"), + change("f/test/low.app/app.yaml"), + change("f/test/low.app/inline.ts"), + ]); + + expect(message).toContain("2 item(s)"); +}); diff --git a/cli/test/raw_app_push_policy_unit.test.ts b/cli/test/raw_app_push_policy_unit.test.ts new file mode 100644 index 0000000000..fbbc7b55c0 --- /dev/null +++ b/cli/test/raw_app_push_policy_unit.test.ts @@ -0,0 +1,142 @@ +/** + * `raw_app.yaml` records none of the policy but the access-mode markers, so a + * push that regenerated the whole policy reset the deploy drawer's settings — + * run-as identity, sandbox isolation — to the pushing user's. Pin that the + * deployed policy is carried over, that a first push still starts from what the + * file states, and that the markers still close a deployed open app back down. + */ + +import { afterAll, beforeEach, expect, mock, test } from "bun:test"; +import { mkdtemp, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let calls: any[] = []; +let deployedPolicy: any; +/** No app deployed at the path: `getAppByPath` 404s and the push creates one. */ +let deployed = true; + +// Stub only what no other in-process suite imports, and treat a stub as +// permanent for the run (see "Module mocks" in cli/TESTING.md). These three API +// functions qualify — nothing else in `test/` imports them. `bundle.ts` did not: +// stubbing it left `raw_app_svelte_plugin_unit.test.ts` asserting against an +// empty bundle, which an `afterAll` hand-back did not prevent. So the real +// bundler runs instead, on the app each push writes below. +const realServices = await import("../gen/services.gen.ts"); + +mock.module("../gen/services.gen.ts", () => ({ + ...realServices, + getAppByPath: async () => { + if (!deployed) throw new Error("not found"); + return { + path: "f/test/raw", + summary: "raw", + value: { files: {}, runnables: {} }, + policy: deployedPolicy, + }; + }, + updateAppRaw: async (a: unknown) => { + calls.push(a); + }, + createAppRaw: async (a: unknown) => { + calls.push(a); + }, +})); + +// Belt and braces: nothing else in-process calls these, and a hand-back is not +// what makes that safe. +afterAll(() => { + mock.module("../gen/services.gen.ts", () => realServices); +}); + +const { pushRawApp } = await import("../src/commands/app/raw_apps.ts"); + +const ADMIN = { + userCache: new Map(), + userIsAdminOrDeployer: true, + userEmail: "deployer@windmill.dev", +}; + +async function push(yamlTail: string, admin = true): Promise { + calls = []; + const dir = await mkdtemp(join(tmpdir(), "windmill_raw_push_")); + await writeFile( + join(dir, "raw_app.yaml"), + `summary: raw\nrunnables: {}\n${yamlTail}`, + "utf-8", + ); + // Any file the remote doesn't have, so the push isn't short-circuited as + // up to date. It is also the bundler's entry point, so it has to compile. + await writeFile(join(dir, "index.tsx"), "export default 1\n", "utf-8"); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ name: "app", private: true }), + "utf-8", + ); + // `ensureNodeModules` only checks the directory is there; borrowing the CLI's + // own skips an npm install per push. + await symlink(join(process.cwd(), "node_modules"), join(dir, "node_modules")); + await pushRawApp("w", "f/test/raw", dir, undefined, "bun", admin ? ADMIN : undefined); + expect(calls).toHaveLength(1); + return calls[0].formData.app; +} + +beforeEach(() => { + deployed = true; + deployedPolicy = { + on_behalf_of: "u/svc", + on_behalf_of_email: "svc@corp", + sandbox: true, + frontend_sdk_scopes: ["jobs:run"], + execution_mode: "anonymous", + // Legacy v1 grants: the backend folds them into v2 at run time, so keeping + // them would keep granting runnables a push has removed. + triggerables: { "script/f/test/gone": {} }, + triggerables_v2: { "a:script/f/test/gone": {} }, + }; +}); + +test("a raw-app push keeps the deployed run-as and sandbox settings", async () => { + const body = await push("public: true\n"); + + expect(body.policy.on_behalf_of).toBe("u/svc"); + expect(body.policy.on_behalf_of_email).toBe("svc@corp"); + expect(body.preserve_on_behalf_of).toBe(true); + expect(body.policy.sandbox).toBe(true); + expect(body.policy.frontend_sdk_scopes).toEqual(["jobs:run"]); + expect(body.policy.execution_mode).toBe("anonymous"); + expect(body.policy.triggerables).toBeUndefined(); + expect(body.policy.triggerables_v2).toEqual({}); +}); + +test("a raw-app push without the marker closes an anonymous app back down", async () => { + const body = await push(""); + + expect(body.policy.execution_mode).toBe("publisher"); +}); + +test("a push that may not claim the deployed identity doesn't send it", async () => { + const body = await push("", false); + + expect(body.preserve_on_behalf_of).toBeUndefined(); + // Not just the flag: the identity itself stays off the wire, so no server can + // deploy this push under it. + expect(body.policy.on_behalf_of).toBeUndefined(); + expect(body.policy.on_behalf_of_email).toBeUndefined(); + // Everything the pusher is entitled to carry over still comes along. + expect(body.policy.sandbox).toBe(true); +}); + +test("a first raw-app push deploys the policy its file states", async () => { + deployed = false; + const body = await push( + "policy:\n sandbox: true\n on_behalf_of: u/impostor\n on_behalf_of_email: impostor@corp\n", + ); + + expect(body.policy.sandbox).toBe(true); + // A repo doesn't get to pick who an app runs as: the identity never reaches + // the wire, so no server can be talked into deploying under it. + expect(body.policy.on_behalf_of).toBeUndefined(); + expect(body.policy.on_behalf_of_email).toBeUndefined(); + expect(body.preserve_on_behalf_of).toBeUndefined(); +}); diff --git a/cli/test/sync_push_auto_metadata_repro.test.ts b/cli/test/sync_push_auto_metadata_repro.test.ts index e9ea167ffc..4854198d64 100644 --- a/cli/test/sync_push_auto_metadata_repro.test.ts +++ b/cli/test/sync_push_auto_metadata_repro.test.ts @@ -168,7 +168,7 @@ test( // Customer scenario: a barrel file (f/lib/errors/index.ts) re-exports from // siblings (./types.ts, ./WorkflowError.ts, ...). An importer in a different // folder imports from the barrel. On a fresh DB, the dep job for the importer -// fetches index.ts via raw_unpinned + temp_script_hash, but bun's resolver +// fetches index.ts via raw + temp_script_hash, but bun's resolver // then has to resolve the barrel's *sibling* imports — and those need to be // in TEMP_SCRIPT_REFS too. test( diff --git a/docs/clone_repo_and_upload_to_instance_storage.bun.ts b/docs/clone_repo_and_upload_to_instance_storage.bun.ts deleted file mode 100644 index 485ac0c766..0000000000 --- a/docs/clone_repo_and_upload_to_instance_storage.bun.ts +++ /dev/null @@ -1,390 +0,0 @@ -import * as wmillclient from "windmill-client"; -import { basename, join } from "node:path"; -import { existsSync, rmSync } from "fs"; -import process from "process"; -import { spawn } from 'child_process'; -import * as fs_async from 'fs/promises'; -import * as fs from 'node:fs'; - -const UPLOAD_CONCURRENCY = 16; -const CLONE_MARKER_FILE = ".windmill_clone_complete"; - -type GitRepository = { - url: string; - branch: string; - folder: string; - gpg_key: any; - is_github_app: boolean; -}; - -export async function main( - resource_path: string, - workspace: string, - git_ssh_identity?: string[], - commit?: string -) { - let clonedRepoPath: string | undefined; - - try { - console.log("Starting git clone and Blob storage upload process"); - - // Get the git repository resource - const repo_resource: GitRepository = await wmillclient.getResource(resource_path); - - const cwd = process.cwd(); - - if (git_ssh_identity) { - process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity) - } - - // Handle GitHub App authentication if needed - if (repo_resource.is_github_app) { - const token = await get_gh_app_token(); - repo_resource.url = prependTokenToGitHubUrl(repo_resource.url, token); - } - - process.env["HOME"] = "."; - process.env.GIT_TERMINAL_PROMPT = "0"; - - // Clone the repository - const { repo_name, commitHash } = await git_clone(cwd, repo_resource, commit); - clonedRepoPath = join(cwd, repo_name); - - // Remove .git directory to avoid uploading git history - const gitDir = join(clonedRepoPath, ".git"); - if (existsSync(gitDir)) { - rmSync(gitDir, { recursive: true, force: true }); - console.log("Removed .git directory"); - } - - // Upload to S3 - const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`; - const fileCount = await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace); - - return { - success: true, - message: "Repository cloned and uploaded to S3 successfully", - s3_path: s3Path, - commit_hash: commitHash, - file_count: fileCount, - }; - - } catch (error) { - console.error("Error in git clone and upload:", error); - throw error; - } finally { - // Clean up cloned repository - if (clonedRepoPath && existsSync(clonedRepoPath)) { - rmSync(clonedRepoPath, { recursive: true, force: true }); - console.log("Cleaned up cloned repository"); - } - } -} - -async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise { - const sshIdFiles = await Promise.all( - git_ssh_identity.map(async (varPath, i) => { - const filePath = join(cwd, `./ssh_id_priv_${i}`); - - try { - // Get variable value using windmill - let content = await wmillclient.getVariable(varPath); - content += '\n'; - - // Write file with content - await fs_async.writeFile(filePath, content, { encoding: 'utf8' }); - - // Set file permissions to 0o600 (read/write for owner only) - await fs_async.chmod(filePath, 0o600); - - // Escape single quotes for shell command - const escapedPath = filePath.replace(/'/g, "'\\''"); - return ` -i '${escapedPath}'`; - } catch (error) { - console.error( - `Variable ${varPath} not found for git ssh identity: ${error}` - ); - return ''; - } - }) - ); - - const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`; - return gitSshCmd; -} - -async function git_clone( - cwd: string, - repo_resource: GitRepository, - commit?: string, -): Promise<{ repo_name: string; commitHash: string }> { - if (commit) { - return git_clone_at_commit(cwd, repo_resource, commit); - } else { - return git_clone_at_latest(cwd, repo_resource); - } -} - -async function git_clone_at_commit( - cwd: string, - repo_resource: GitRepository, - commit: string, -): Promise<{ repo_name: string; commitHash: string }> { - let repo_url = repo_resource.url; - const subfolder = repo_resource.folder ?? ""; - let branch = repo_resource.branch ?? ""; - const repo_name = basename(repo_url, ".git"); - - const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?.+)\)/); - if (azureMatch) { - console.log("Fetching Azure DevOps access token..."); - const azureResource = await wmillclient.getResource(azureMatch.groups.url); - const response = await fetch( - `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, - { - method: "POST", - body: new URLSearchParams({ - client_id: azureResource.azureClientId, - client_secret: azureResource.azureClientSecret, - grant_type: "client_credentials", - resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", - }), - } - ); - const { access_token } = await response.json(); - repo_url = repo_url.replace(azureMatch[0], access_token); - } - - const repoPath = join(cwd, repo_name); - await fs_async.mkdir(repoPath, { recursive: true }); - - process.chdir(repoPath); - - let args = ['init', '--quiet'] - if (branch) { - args.push(`--initial-branch=${branch}`) - } - await runCommand(undefined, 'git', ...args); - - await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url); - - await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit); - - await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD'); - - const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim(); - - // Return to original directory - process.chdir(cwd); - - return { repo_name, commitHash }; -} - -async function git_clone_at_latest( - cwd: string, - repo_resource: GitRepository -): Promise<{ repo_name: string; commitHash: string }> { - let repo_url = repo_resource.url; - const subfolder = repo_resource.folder ?? ""; - let branch = repo_resource.branch ?? ""; - const repo_name = basename(repo_url, ".git"); - - // Handle Azure DevOps token if needed - const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?.+)\)/); - if (azureMatch) { - console.log("Fetching Azure DevOps access token..."); - const azureResource = await wmillclient.getResource(azureMatch.groups.url); - const response = await fetch( - `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, - { - method: "POST", - body: new URLSearchParams({ - client_id: azureResource.azureClientId, - client_secret: azureResource.azureClientSecret, - grant_type: "client_credentials", - resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", - }), - } - ); - const { access_token } = await response.json(); - repo_url = repo_url.replace(azureMatch[0], access_token); - } - - const args = ["clone", "--quiet", "--depth", "1"]; - if (subfolder !== "") args.push("--sparse"); - if (branch !== "") args.push("--branch", branch); - args.push(repo_url, repo_name); - - await runCommand(-1, "git", ...args); - - const fullPath = join(cwd, repo_name); - process.chdir(fullPath); - - if (subfolder !== "") { - await runCommand(undefined, "git", "sparse-checkout", "add", subfolder); - const subfolderPath = join(fullPath, subfolder); - - if (!existsSync(subfolderPath)) { - throw new Error(`Subfolder ${subfolder} does not exist.`); - } - - process.chdir(subfolderPath); - } - - // Get the commit hash - const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim(); - - // Return to original directory - process.chdir(cwd); - - return { repo_name, commitHash }; -} - -async function uploadDirectoryToS3( - directoryPath: string, - s3BasePath: string, - workspace: string, -): Promise { - console.log(`Uploading ${directoryPath} -> ${s3BasePath}`); - - // Walk once into a flat task list so we can drive a bounded-concurrency pool. - const tasks: { localPath: string; s3Key: string }[] = []; - function walk(dir: string, s3Path: string) { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name); - const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - walk(fullPath, s3Key); - } else if (entry.isFile()) { - tasks.push({ localPath: fullPath, s3Key }); - } - } - } - walk(directoryPath, s3BasePath); - console.log(`Discovered ${tasks.length} files to upload`); - - let nextIndex = 0; - let uploaded = 0; - let lastReport = 0; - async function worker() { - while (true) { - const idx = nextIndex++; - if (idx >= tasks.length) return; - const { localPath, s3Key } = tasks[idx]; - const fileContent = fs.readFileSync(localPath); - const blob = new Blob([fileContent], { type: 'application/octet-stream' }); - await wmillclient.HelpersService.gitRepoViewerFileUpload({ - workspace, - fileKey: s3Key, - requestBody: blob, - }); - uploaded++; - if (uploaded - lastReport >= 25 || uploaded === tasks.length) { - lastReport = uploaded; - console.log(`Uploaded ${uploaded} / ${tasks.length} files`); - } - } - } - await Promise.all( - Array.from({ length: Math.min(UPLOAD_CONCURRENCY, tasks.length) }, () => worker()) - ); - - // Marker is the LAST write — its presence is what the viewer checks for. - const markerKey = `${s3BasePath}/${CLONE_MARKER_FILE}`; - const markerBody = JSON.stringify({ - completed_at: new Date().toISOString(), - file_count: tasks.length, - }); - await wmillclient.HelpersService.gitRepoViewerFileUpload({ - workspace, - fileKey: markerKey, - requestBody: new Blob([markerBody], { type: 'application/json' }), - }); - console.log(`Wrote completion marker: ${markerKey}`); - - return tasks.length; -} - -function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise { - const nargs = secret_position != undefined ? args.slice() : args; - if (secret_position && secret_position < 0) - secret_position = nargs.length - 1 + secret_position; - - let secret: string | undefined = undefined; - if (secret_position != undefined) { - nargs[secret_position] = "***"; - secret = args[secret_position]; - } - console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`); - - return new Promise((resolve, reject) => { - const process = spawn(cmd, args); - - let stdout = ''; - let stderr = ''; - - process.stdout.on('data', (data) => { - stdout += data.toString(); - }); - - process.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - process.on('error', (error) => { - let errorString = error.toString(); - if (secret) errorString = errorString.replace(secret, "***"); - console.log(`Shell command FAILED: ${cmd}`, errorString); - const e = new Error( - `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}` - ); - reject(e); - }); - - process.on('close', (code) => { - if (stdout.length > 0) { - console.log("Shell stdout:", stdout); - } - if (stderr.length > 0) { - console.log("Shell stderr:", stderr); - } - if (code === 0) { - console.log(`Shell command completed successfully: ${cmd}`); - resolve(stdout); - } else { - reject(new Error(`Command failed with code ${code}: ${stderr}`)); - } - }); - }); -} - -async function get_gh_app_token() { - const workspace = process.env["WM_WORKSPACE"]; - const jobToken = process.env["WM_TOKEN"]; - const baseUrl = - process.env["BASE_INTERNAL_URL"] ?? - process.env["BASE_URL"] ?? - "http://localhost:8000"; - const url = `${baseUrl}/api/w/${workspace}/github_app/token`; - - const response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${jobToken}`, - }, - body: JSON.stringify({ job_token: jobToken }), - }); - - if (!response.ok) { - const errorBody = await response.text().catch(() => ""); - throw new Error(`GitHub App token error (${response.status}): ${errorBody || response.statusText}`); - } - const data = await response.json(); - return data.token; -} - -function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) { - const url = new URL(gitHubUrl); - return `https://x-access-token:${installationToken}@${url.hostname}${url.pathname}`; -} diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index f5ce2357ca..ac64800c36 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,10 +4,10 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 42 registered actions across seventeen features (`ai_session`, `ai_chat`, +It currently carries 49 registered actions across eighteen features (`ai_session`, `ai_chat`, `ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `flow_editor`, `flow_run`, -`flow_step`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`, -`sso_groups_claim`). Nearly all of the +`flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, +`usage_meter`, `sso_groups_claim`). Nearly all of the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/docs/git-repo-viewer-hub-script.md b/docs/git-repo-viewer-hub-script.md index 7bfbe8160f..a1b5ea1482 100644 --- a/docs/git-repo-viewer-hub-script.md +++ b/docs/git-repo-viewer-hub-script.md @@ -1,7 +1,10 @@ # Git repo viewer — hub script -The hub script `clone_repo_and_upload_to_instance_storage` is published from -`windmill-integrations` and pinned in `frontend/src/lib/hubPaths.json` as +The hub script `clone_repo_and_upload_to_instance_storage` +([hub page](https://hub.windmill.dev/scripts/windmill/13968)) is published from +`windmill-integrations` +(`hub/windmill/scripts/action/13968_clone_repo_and_upload_to_instance_storage/script.ts`) +and pinned in `frontend/src/lib/hubPaths.json` as `cloneRepoToS3forGitRepoViewer`. Hub paths are exact version pins, so editing the script means publishing a new version and repointing that entry. @@ -23,6 +26,11 @@ The repo viewer in the Windmill app expects the hub script to: 3. **Write a completion marker** as the very last action of a successful run, so the API and frontend can distinguish a fully-populated S3 directory from a partial / interrupted upload. +4. **Follow symlinks that stay inside the checkout.** Both the git clone and + the archive extraction keep a repository's symlinks as links, and + `Dirent.isFile()` / `isDirectory()` are both false for a link, so a walk + that only checks those drops every linked file and directory from the + viewer. See [Symlinks](#symlinks). The marker file the frontend looks for is `.windmill_clone_complete` at the root of the per-commit directory: @@ -43,28 +51,74 @@ after the walk completes: ```ts const UPLOAD_CONCURRENCY = 16 const CLONE_MARKER_FILE = ".windmill_clone_complete" +const MAX_SYMLINKED_ENTRIES = 20_000 +const MAX_SYMLINKED_BYTES = 512 * 1024 * 1024 async function uploadDirectoryToS3( directoryPath: string, s3BasePath: string, workspace: string, -) { +): Promise { console.log(`Uploading ${directoryPath} -> ${s3BasePath}`) // Walk the directory once, producing a flat list of (localPath, s3Key) pairs. const tasks: { localPath: string; s3Key: string }[] = [] - function walk(dir: string, s3Path: string) { + const root = fs.realpathSync(directoryPath) + // Real paths of the directories being descended through. + const ancestors = new Set() + // What entries reached through a link have cost so far; see Symlinks below. + let symlinkedEntries = 0 + let symlinkedBytes = 0 + let symlinkBudgetSpent = false + function chargeSymlinkBudget(relPath: string, entries: number, bytes: number): boolean { + if (symlinkBudgetSpent) return false + symlinkedEntries += entries + symlinkedBytes += bytes + if (symlinkedEntries <= MAX_SYMLINKED_ENTRIES && symlinkedBytes <= MAX_SYMLINKED_BYTES) { + return true + } + symlinkBudgetSpent = true + console.log( + `Skipping ${relPath} and every symlinked entry after it: symlinks reach more than ` + + `${MAX_SYMLINKED_ENTRIES} entries or ${MAX_SYMLINKED_BYTES / 2 ** 20} MiB` + ) + return false + } + function walk(dir: string, relDir: string, viaLink: boolean) { + ancestors.add(dir) for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name) - const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name - if (entry.isDirectory()) { - walk(fullPath, s3Key) - } else if (entry.isFile()) { - tasks.push({ localPath: fullPath, s3Key }) + const relPath = relDir ? `${relDir}/${entry.name}` : entry.name + const linked = viaLink || entry.isSymbolicLink() + if (linked && !chargeSymlinkBudget(relPath, 1, 0)) continue + let localPath = join(dir, entry.name) + if (entry.isSymbolicLink()) { + const link = fs.readlinkSync(localPath) + try { + localPath = fs.realpathSync(localPath) + } catch (e: any) { + console.log(`Skipping symlink ${relPath} -> ${link}: cannot resolve target (${e.code})`) + continue + } + if (localPath !== root && !localPath.startsWith(root + sep)) { + console.log(`Skipping symlink ${relPath} -> ${link}: target is outside the repository`) + continue + } + } + const stat = fs.statSync(localPath) + if (stat.isDirectory() && ancestors.has(localPath)) { + console.log(`Skipping ${relPath}: links back to a directory it is inside`) + continue + } + if (linked && stat.isFile() && !chargeSymlinkBudget(relPath, 0, stat.size)) continue + if (stat.isDirectory()) { + walk(localPath, relPath, linked) + } else if (stat.isFile()) { + tasks.push({ localPath, s3Key: `${s3BasePath}/${relPath}` }) } } + ancestors.delete(dir) } - walk(directoryPath, s3BasePath) + walk(root, "", false) console.log(`Discovered ${tasks.length} files to upload`) @@ -114,9 +168,40 @@ async function uploadDirectoryToS3( requestBody: new Blob([markerBody], { type: "application/json" }), }) console.log(`Wrote completion marker: ${markerKey}`) + + return tasks.length } ``` +## Symlinks + +A link is resolved with `realpathSync` and followed only when its target lies +inside the checkout's real path. A file target is uploaded under the link's own +path; a directory target is walked as if it sat there, so +`inventories/prod/group_vars -> ../../shared/group_vars` shows up in the viewer +with its files. Everything else is skipped and logged: + +- **A target outside the checkout.** The repository chooses the target, and the + checkout sits in the job's working directory next to the ssh key + `get_git_ssh_cmd` writes (`../ssh_id_priv_0`) and the job's `args.json`. A + link to one of those, or to `/proc/self/environ` with the caller's + `WM_TOKEN`, would put it in storage for every reader of the resource. This + is why the walk does not follow links the way `aws s3 sync` does. +- **A target that cannot be resolved**: a dangling link, or a link loop + (`ELOOP`). +- **A directory that is already being walked higher up** (`loop -> .`, + `up -> ..`). The guard holds the real paths of the current descent only, as + `find -L` does, not every directory seen so far: a directory reachable + through two links is uploaded under both paths, as the checkout presents it. +- **Anything reached through a link once the budget is spent.** Because a + directory can be reached along many paths, two links to the next directory + at each level double the tree, and a repository a few dozen links deep would + expand past what the job can hold in memory. Every entry reached through a + link counts against a budget of 20,000 entries and 512 MiB. It is charged + before the link is resolved, so links that end up skipped count too, and + neither their work nor their log lines can multiply. Past the budget, the rest + are skipped with one log line. The checkout's own files are always uploaded. + ## Notes for review - **Concurrency level**: 16 is a starting point; tune based on instance @@ -127,6 +212,12 @@ async function uploadDirectoryToS3( paths on retry, so a partial upload + retry naturally heals. Old commit directories from before this patch are unreachable through the UI but still consume storage; an instance admin can prune them manually if desired. +- **A new pin doesn't refresh commits already uploaded**: the viewer keys + storage on the commit hash (`gitrepos/{workspace}/{resource_path}/{commit_hash}/`) + and only checks that the marker exists. So a commit uploaded by an earlier + script version keeps that version's tree (hub/28905's had no symlinks) until + the repository's head moves to a new commit, or an admin deletes that + commit's directory. - **Error propagation**: keep the existing `try/catch` in `main` so an upload failure surfaces in the job result and is shown in the new viewer error banner. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a200c878e3..4c9b9350ef 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.805.0", + "version": "1.809.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.805.0", + "version": "1.809.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -5579,9 +5579,9 @@ } }, "node_modules/driver.js": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.3.6.tgz", - "integrity": "sha512-g2nNuu+tWmPpuoyk3ffpT9vKhjPz4NrJzq6mkRDZIwXCrFhrKdDJ9TX5tJOBpvCTBrBYjgRQ17XlcQB15q4gMg==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.8.0.tgz", + "integrity": "sha512-+8/IO7h1v14IzWh2GP60N7T3PFZweXwdn5e5POuxRSBoCYUojsBxzqawPeXh3YZIibRy7EehYNEyxe7slwwtdg==", "license": "MIT" }, "node_modules/dts-bundle-generator": { diff --git a/frontend/package.json b/frontend/package.json index d96e512c00..9b7df8228a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.805.0", + "version": "1.809.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/components/AppTutorials.svelte b/frontend/src/lib/components/AppTutorials.svelte deleted file mode 100644 index 25d06e0167..0000000000 --- a/frontend/src/lib/components/AppTutorials.svelte +++ /dev/null @@ -1,29 +0,0 @@ - - - diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 7f50794a52..b7001059c2 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -1078,6 +1078,7 @@ {otherArgs} {helperScript} {workspace} + {disabled} bind:value format={format ?? ''} /> diff --git a/frontend/src/lib/components/CenteredModal.svelte b/frontend/src/lib/components/CenteredModal.svelte index 3250b3a16a..08234c25b1 100644 --- a/frontend/src/lib/components/CenteredModal.svelte +++ b/frontend/src/lib/components/CenteredModal.svelte @@ -14,6 +14,8 @@ centerVertically?: boolean loading?: boolean containOverflow?: boolean + /** The Windmill version and update notice in the header. */ + showVersion?: boolean children?: import('svelte').Snippet } @@ -25,6 +27,7 @@ centerVertically = true, loading = false, containOverflow = false, + showVersion = true, children }: Props = $props() @@ -82,5 +85,5 @@ {/if} - + diff --git a/frontend/src/lib/components/DdlMigrationGuard.svelte b/frontend/src/lib/components/DdlMigrationGuard.svelte index ae5182f560..3bda6aa216 100644 --- a/frontend/src/lib/components/DdlMigrationGuard.svelte +++ b/frontend/src/lib/components/DdlMigrationGuard.svelte @@ -3,7 +3,7 @@ import Modal2 from './common/modal/Modal2.svelte' import NewDataTableMigrationModal from './workspaceSettings/NewDataTableMigrationModal.svelte' import DataTableMigrationsButton from './workspaceSettings/DataTableMigrationsButton.svelte' - import { splitSqlStatements, isDdlStatement } from './sqlDdl' + import { joinSqlStatements, splitSqlRuns } from './sqlDdl' import { logDdlGuardChoice } from './workspaceSettings/datatableTelemetry' import { CornerDownLeft } from 'lucide-svelte' @@ -11,7 +11,7 @@ type Choice = 'run' | 'migrate' | 'cancel' - let promptStatement = $state(undefined) + let promptStatements = $state([]) let promptOpen = $state(false) let resolvePrompt: ((choice: Choice) => void) | undefined = undefined let resolveMigrationClosed: ((created: boolean) => void) | undefined = undefined @@ -23,11 +23,15 @@ // toast action after a migration is created here. let migrationsModal = $state(undefined) + // The block is shown as-is in the prompt and becomes the migration body, where + // every statement inside the BEGIN; ... END; frame must be `;`-terminated. + let promptSql = $derived(joinSqlStatements(promptStatements)) + function finishPrompt(choice: Choice) { const r = resolvePrompt resolvePrompt = undefined promptOpen = false - promptStatement = undefined + promptStatements = [] r?.(choice) } @@ -49,10 +53,10 @@ } } - function promptDdl(statement: string): Promise { + function promptDdl(statements: string[]): Promise { return new Promise((resolve) => { resolvePrompt = resolve - promptStatement = statement + promptStatements = statements promptOpen = true }) } @@ -66,49 +70,51 @@ // Open the prefilled new-migration modal. Resolves with whether a migration // was actually created (false if the user cancelled / closed it). - function openMigrationModal(statement: string): Promise { + function openMigrationModal(sql: string): Promise { return new Promise((resolve) => { resolveMigrationClosed = (created: boolean) => resolve(created) - newMigrationModal?.open({ codeUp: statement }) + newMigrationModal?.open({ codeUp: sql }) }) } /** - * Inspect `code` for DDL statements. For each one, prompt the user to run it - * anyway or turn it into a migration (prompts shown one at a time). Returns - * whether to proceed and the code to run (with migrated statements stripped). + * Inspect `code` for DDL statements. Each run of adjacent DDL statements is + * prompted for once (runs shown one at a time) and becomes a single migration, + * so a chain of schema changes applies in one transaction instead of asking + * once per statement. Returns whether to proceed and the code to run (with + * migrated statements stripped). */ export async function guard( code: string ): Promise<{ proceed: boolean; code: string; ranMigration: boolean }> { migrationRan = false - const statements = splitSqlStatements(code) - if (!statements.some((s) => isDdlStatement(s))) { + const runs = splitSqlRuns(code) + if (!runs.some((r) => r.isDdl)) { return { proceed: true, code, ranMigration: false } } const kept: string[] = [] - for (const statement of statements) { - if (!isDdlStatement(statement)) { - kept.push(statement) + for (const run of runs) { + if (!run.isDdl) { + kept.push(...run.statements) continue } - // Re-prompt for this statement until the user makes a terminal choice; + // Re-prompt for this run until the user makes a terminal choice; // cancelling the migration modal returns to the prompt with the DDL intact. for (;;) { - const choice = await promptDdl(statement) + const choice = await promptDdl(run.statements) if (choice === 'cancel') { logDdlGuardChoice('cancelled') return { proceed: false, code, ranMigration: migrationRan } } if (choice === 'run') { logDdlGuardChoice('run_anyway') - kept.push(statement) + kept.push(...run.statements) break } - // migrate: only strip the statement once a migration is actually + // migrate: only strip the statements once a migration is actually // created; if the modal was cancelled, loop back to the prompt. - const created = await openMigrationModal(statement) + const created = await openMigrationModal(joinSqlStatements(run.statements)) if (created) { logDdlGuardChoice('migrated') break @@ -116,14 +122,14 @@ } } - return { proceed: true, code: kept.join(';\n'), ranMigration: migrationRan } + return { proceed: true, code: joinSqlStatements(kept), ranMigration: migrationRan } } 1 ? 'Schema changes detected' : 'Schema change detected'} fixedWidth="md" fixedHeight="adaptive" bind:isOpen={promptOpen} @@ -131,12 +137,17 @@ >

- This looks like a schema-changing (DDL) statement. Schema changes are best tracked as - migrations rather than run ad-hoc. Create a migration for it instead? + {#if promptStatements.length > 1} + These {promptStatements.length} consecutive statements are schema-changing (DDL). Schema changes + are best tracked as migrations rather than run ad-hoc. Create a single migration for them instead? + {:else} + This looks like a schema-changing (DDL) statement. Schema changes are best tracked as + migrations rather than run ad-hoc. Create a migration for it instead? + {/if} 

{promptStatement ?? ''}
{promptSql}
diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index c4674d7313..1bf9e7fde3 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -1245,7 +1245,6 @@ {:else if inputType === 'dynselect'}
+ + Service name, environment, resource attributes, metrics temporality and other + options are set with environment variables. + {/if}
{:else if setting.fieldType == 'otel_tracing_proxy'} @@ -861,6 +870,8 @@ {:else if setting.fieldType == 'ws_connectivity'} + {:else if setting.fieldType == 'instance_banner'} + {/if} {#if hasError} diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index d124569929..f965362659 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -14,6 +14,7 @@ import { sleep } from '$lib/utils' import { enterpriseLicense } from '$lib/stores' + import { isCloudHosted } from '$lib/cloud' import { createEventDispatcher } from 'svelte' import { setLicense } from '$lib/enterpriseUtils' @@ -100,7 +101,8 @@ otel: {}, indexer_settings: {}, critical_error_channels: [], - github_enterprise_app: {} + github_enterprise_app: {}, + instance_banner: {} } function applyFormDefaults(vals: Record): void { @@ -524,6 +526,10 @@ for (const category of settingsKeys) { const categorySettings = getSettingsForCategory(category) result[category] = categorySettings.some((s) => { + // A field the build never renders must not be able to block Save: off-cloud its + // value is unreachable, so an invalid one (from config sync, say) would leave the + // category permanently unsaveable with nothing on screen to fix. + if (s.cloudonly && !isCloudHosted()) return false if (s.isValid && !s.isValid(currentValues?.[s.key])) return true if (s.validate) { const errors = s.validate(currentValues?.[s.key]) @@ -1061,8 +1067,7 @@
  • worker usage (worker, worker instance, vCPUs, memory)
  • user usage (author count, operator count, the distinct guests of the last 30 days, - the seats they add past the free allowance, and the workspaces that allow - guests)
  • superadmin email addresses
  • development instance status
    • @@ -1078,8 +1083,11 @@ loaded, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a membership, the plan tier and quota shown when the execution meter is opened, whether app sandbox isolation is turned on, whether a step's workspace script is edited from - the flow editor, and how data tables and their migrations are set up and used, last 30 - days)
  • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1129,8 +1137,7 @@
  • worker usage (worker, worker instance, vCPUs, memory)
  • user usage (author count, operator count, the distinct guests of the last 30 days, - the seats they add past the free allowance, and the workspaces that allow - guests)
  • development instance status
  • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/JobArgs.svelte b/frontend/src/lib/components/JobArgs.svelte index eb6cc9bc97..64ca912608 100644 --- a/frontend/src/lib/components/JobArgs.svelte +++ b/frontend/src/lib/components/JobArgs.svelte @@ -21,9 +21,18 @@ args: any argLabel?: string | undefined workspace?: string | undefined + /** Drop the header's expand-into-a-drawer button, for a caller that already offers a + * way to open the run in full. */ + disableExpand?: boolean } - let { id = undefined, args, argLabel = undefined, workspace = undefined }: Props = $props() + let { + id = undefined, + args, + argLabel = undefined, + workspace = undefined, + disableExpand = false + }: Props = $props() // Internal flag injected by "test this step" runs to suppress the asset // dispatcher. Not a real input: shown as a badge instead of a table row, @@ -125,18 +134,20 @@ ${Object.entries(displayArgs) Value {#snippet headerAction()} -
    - -
    + {#if !disableExpand} +
    + +
    + {/if} {/snippet} diff --git a/frontend/src/lib/components/LocalDraftBanner.svelte b/frontend/src/lib/components/LocalDraftBanner.svelte index dff131b281..af2787222e 100644 --- a/frontend/src/lib/components/LocalDraftBanner.svelte +++ b/frontend/src/lib/components/LocalDraftBanner.svelte @@ -2,12 +2,8 @@ import { Button } from '$lib/components/common' import DiffDrawer from '$lib/components/DiffDrawer.svelte' import { classes } from '$lib/components/common/alert/model' - import { - cleanValueProperties, - orderedYamlStringify, - replaceFalseWithUndefined, - type Value - } from '$lib/utils' + import { cleanValueProperties, replaceFalseWithUndefined, type Value } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { AlertCircle, Diff } from 'lucide-svelte' import { twMerge } from 'tailwind-merge' import { fade } from 'svelte/transition' diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index 0307643a08..e4b73478aa 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -16,15 +16,13 @@
    @@ -31,9 +32,11 @@
    -
    - - -
    + {#if showVersion} +
    + + +
    + {/if}
    diff --git a/frontend/src/lib/components/PasswordArgInput.svelte b/frontend/src/lib/components/PasswordArgInput.svelte index a45ad6579d..ee1f67e9c3 100644 --- a/frontend/src/lib/components/PasswordArgInput.svelte +++ b/frontend/src/lib/components/PasswordArgInput.svelte @@ -1,7 +1,7 @@
    - {#if loading} + {#snippet action()} + + {#snippet children({ item })} + {#each Object.keys(WINDOWS) as key (key)} + + {/each} + {/snippet} + + {/snippet} + + {#if metrics.error} + {metrics.error.message} + {:else if metrics.current === undefined} - {:else if noMetrics} -

    No jobs delayed by more than 3 seconds in the last 14 days

    + {:else if metrics.current.tags.length === 0} +

    + No jobs delayed by more than 3 seconds in the last {WINDOWS[windowKey].label} +

    {:else}
    - {#if countData} - 3s)' - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - y: { - title: { - display: true, - text: 'count' - } - } - } - }} - /> - {/if} - {#if delayData} - 3s)' - }, - tooltip: { - callbacks: { - label: function (context) { - // @ts-ignore - if (context.raw.y === 1) { - return context.dataset.label + ': 0' - } else { - // @ts-ignore - return context.dataset.label + ': ' + context.raw.y - } - } - } - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - - y: { - type: 'logarithmic', - title: { - display: true, - text: 'delay (s)' - }, - ticks: { - callback: (value, _) => (value === 1 ? '0' : value) - } - } - } - }} - /> - {/if} + + - Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are - included in the graph. + Only tags with jobs delayed by more than 3 seconds in this window are included. At wide + windows a line shows the highest value of each time slot, so short spikes stay visible.
    {/if} diff --git a/frontend/src/lib/components/QueueStatusTable.svelte b/frontend/src/lib/components/QueueStatusTable.svelte new file mode 100644 index 0000000000..5f34e80c60 --- /dev/null +++ b/frontend/src/lib/components/QueueStatusTable.svelte @@ -0,0 +1,115 @@ + + +
    + {#snippet action()} +
    diff --git a/frontend/src/lib/components/RunForm.svelte b/frontend/src/lib/components/RunForm.svelte index a371f3eea9..510f774587 100644 --- a/frontend/src/lib/components/RunForm.svelte +++ b/frontend/src/lib/components/RunForm.svelte @@ -24,6 +24,7 @@ import InputSelectedBadge from './schema/InputSelectedBadge.svelte' import { untrack } from 'svelte' import { processSecretArgs } from './secretArgUtils' + import { enforceDisabledDefaults, resetKeysToast } from './job_args' import PowerShellCommonParams from './PowerShellCommonParams.svelte' let reloadArgs = $state(0) @@ -60,11 +61,12 @@ export async function run(overrideScheduledForStr?: string | undefined | null) { let processedArgs: Record + const { args: withDefaults, resetKeys } = enforceDisabledDefaults(args ?? {}, runnable?.schema) + if (resetKeys.length > 0) { + sendUserToast(resetKeysToast(resetKeys)) + } try { - processedArgs = await processSecretArgs( - enforceDisabledDefaults(args ?? {}, true), - runnable?.schema - ) + processedArgs = await processSecretArgs(withDefaults, runnable?.schema) } catch (e) { sendUserToast('Failed to process sensitive args: ' + e, true) return @@ -178,30 +180,6 @@ } } - function enforceDisabledDefaults( - args: Record, - notify: boolean = false - ): Record { - const schema = runnable?.schema - if (!schema?.properties) return args - const result = { ...args } - const resetKeys: string[] = [] - for (const [key, prop] of Object.entries(schema.properties) as [string, any][]) { - if (prop?.disabled && 'default' in prop) { - if (notify && result[key] !== prop.default) { - resetKeys.push(key) - } - result[key] = prop.default - } - } - if (resetKeys.length > 0) { - sendUserToast( - `Disabled field${resetKeys.length > 1 ? 's' : ''} ${resetKeys.map((k) => `'${k}'`).join(', ')} reset to default value${resetKeys.length > 1 ? 's' : ''}` - ) - } - return result - } - /** Rewrite the open JSON editor from the current args. Only for args replaced from outside * the editor: entering the JSON view already starts from whatever `args` holds. */ export function syncJsonEditor() { @@ -322,7 +300,7 @@ bind:this={jsonEditor} on:select={(e) => { if (e.detail) { - args = enforceDisabledDefaults(e.detail) + args = enforceDisabledDefaults(e.detail, runnable?.schema).args } }} initialCode={argsToJsonPayload(runnable.schema, args)} diff --git a/frontend/src/lib/components/RunPageTutorials.svelte b/frontend/src/lib/components/RunPageTutorials.svelte deleted file mode 100644 index e46b1c9019..0000000000 --- a/frontend/src/lib/components/RunPageTutorials.svelte +++ /dev/null @@ -1,25 +0,0 @@ - - - diff --git a/frontend/src/lib/components/WorkspaceItemDiffViewer.svelte b/frontend/src/lib/components/WorkspaceItemDiffViewer.svelte index 77aae182fa..cd0622959c 100644 --- a/frontend/src/lib/components/WorkspaceItemDiffViewer.svelte +++ b/frontend/src/lib/components/WorkspaceItemDiffViewer.svelte @@ -24,7 +24,8 @@ doesn't reflow the parent. import RawAppFileDiff from './raw_apps/RawAppFileDiff.svelte' import type { RawAppFileItem } from './raw_apps/rawAppDiffUtils' import { Loader2 } from 'lucide-svelte' - import { cleanValueProperties, orderedYamlStringify, replaceFalseWithUndefined } from '$lib/utils' + import { cleanValueProperties, replaceFalseWithUndefined } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { scriptLangToEditorLang } from '$lib/scripts' interface Props { diff --git a/frontend/src/lib/components/WorkspaceTutorials.svelte b/frontend/src/lib/components/WorkspaceTutorials.svelte deleted file mode 100644 index f2f1bb63b6..0000000000 --- a/frontend/src/lib/components/WorkspaceTutorials.svelte +++ /dev/null @@ -1,25 +0,0 @@ - - - diff --git a/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte b/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte new file mode 100644 index 0000000000..a1a58a4ce5 --- /dev/null +++ b/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte @@ -0,0 +1,219 @@ + + +
    +
    +
    + {#if context} + + {context} + + {/if} + {#if job} +

    + Requested by {job.created_by} · +

    + {/if} +
    + {STATUS_BADGE[status].label} +
    + + {#if typeof approvalInfo.description === 'string'} +

    {approvalInfo.description}

    + {:else if approvalInfo.description != undefined} + + {/if} + + {#if status === 'pending'} + {#if hasForm} + {#if emptyString($enterpriseLicense)} + + {:else} + + {/if} + {/if} + + {#if approvalInfo.can_approve} +
    + {#if approvalInfo.hide_cancel !== true} + + {:else} +
    + {/if} + +
    + {#if isSelfApprovalBypass} + + As an administrator, by approving or rejecting this request, you bypass the self-approval + interdiction. + + {/if} + {:else if approvalInfo.user_auth_required && !$userStore} +

    Sign in to review this request.

    + + {:else} +
    +

    You are not authorized to approve this request.

    + {#if isSelfApprovalRefused} +

    Self-approval is disabled for this step.

    + {/if} + {#if groupsRequired.length > 0} +

    + Only members of the following groups can approve: + {groupsRequired.join(', ')} +

    + {/if} +
    + {/if} + {:else} +
    + {#if status === 'approved'} + + {:else if status === 'rejected'} + + {:else} + + {/if} +
    + + {status === 'closed' ? 'This request is closed' : STATUS_BADGE[status].label} + + + {#if status === 'approved'} + Your approval was recorded. You can close this page. + {:else if status === 'rejected'} + Your rejection was recorded. You can close this page. + {:else} + The flow is no longer waiting for approval. + {/if} + +
    +
    + {/if} + + {#if !isLocked && ((status === 'pending' && approvalInfo.approvers.length > 0) || isWorkspaceMember)} +
    + + {#if status === 'pending' && approvalInfo.approvers.length > 0} + Already approved by {approvalInfo.approvers.map((a) => a.approver).join(', ')} + {/if} + + {#if isWorkspaceMember} + + {/if} +
    + {/if} +
    diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index 8ef4ffbf95..565ac5d5d3 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -473,15 +473,6 @@ let appEditorHeader: AppEditorHeader | undefined = $state(undefined) - export function triggerTutorial() { - const urlParams = new URLSearchParams(window.location.search) - const tutorial = urlParams.get('tutorial') - - if (tutorial) { - appEditorHeader?.runTutorialById(tutorial) - } - } - let box: HTMLElement | undefined = $state(undefined) function parseScroll() { $yTop = box?.scrollTop ?? 0 diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index a57dc87622..ab043f81a1 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -7,25 +7,13 @@ import { redo, undo } from '$lib/history.svelte' import { discardDraftAfterDeploy } from '$lib/userDraftToast' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' - import { - enterpriseLicense, - tutorialsToDo, - userStore, - userWorkspaces, - workspaceStore - } from '$lib/stores' + import { enterpriseLicense, userStore, userWorkspaces, workspaceStore } from '$lib/stores' import { isMac, type Item, userPathPrefix } from '$lib/utils' - import { resetAllTodos, skipAllTodos } from '$lib/tutorialUtils' - import { getTutorialIndex } from '$lib/tutorials/config' import { random_adj } from '$lib/components/random_positive_adjetive' import { AlignHorizontalSpaceAround, BellOff, - BookOpen, Bug, - CheckCheck, - CheckCircle, - Circle, DiffIcon, Expand, FileJson, @@ -33,7 +21,6 @@ FormInput, History, Laptop2, - RefreshCw, Save, Smartphone, FileClock, @@ -61,7 +48,6 @@ import Awareness from '$lib/components/Awareness.svelte' import { secondaryMenuLeftStore, secondaryMenuRightStore } from './settingsPanel/secondaryMenu' import Dropdown from '$lib/components/DropdownV2.svelte' - import AppEditorTutorial from './AppEditorTutorial.svelte' import AppReportsDrawer from './AppReportsDrawer.svelte' import DebugPanel from './contextPanel/DebugPanel.svelte' @@ -679,49 +665,9 @@ action: () => { appExport?.open(toStatic($app, $staticExporter, $summary).app) } - }, - { - displayName: 'Tutorials', - icon: BookOpen, - separatorTop: true, - submenuItems: [ - { - displayName: 'Background runnables', - action: () => appEditorTutorial?.runTutorialById('backgroundrunnables'), - icon: $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) - ? Circle - : CheckCircle, - iconColor: $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) - ? undefined - : 'green' - }, - { - displayName: 'Connection', - action: () => appEditorTutorial?.runTutorialById('connection'), - icon: $tutorialsToDo.includes(getTutorialIndex('connection')) ? Circle : CheckCircle, - iconColor: $tutorialsToDo.includes(getTutorialIndex('connection')) ? undefined : 'green' - }, - { - displayName: 'Reset tutorials', - action: () => resetAllTodos(), - icon: RefreshCw, - separatorTop: true - }, - { - displayName: 'Skip tutorials', - action: () => skipAllTodos(), - icon: CheckCheck - } - ] } ]) as Item[] - let appEditorTutorial: AppEditorTutorial | undefined = $state(undefined) - - export function runTutorialById(id: string, options?: { skipStepsCount?: number }) { - appEditorTutorial?.runTutorialById(id, options) - } - let appReportingDrawerOpen = $state(false) export function openTroubleshootPanel() { @@ -1090,15 +1036,7 @@ {/if}
    -
    - - {#if $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) || $tutorialsToDo.includes(getTutorialIndex('connection'))} - - {/if} -
    - +
    {#if hasErrors} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte index 7b68389d82..836af68db6 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte @@ -115,6 +115,10 @@ // letting the publisher believe the app is open. let guestAccessEnabled: boolean | undefined = $state(undefined) let guestUsage: GuestUsage | undefined = $state(undefined) + // Whether the deployment can have guests at all; off, the mode is not on offer. The + // backend decides; the hostname stands in until it has answered, the shared cloud + // being the only deployment where guests are unavailable. + let guestsAvailable = $derived.by(() => guestUsage?.available ?? !isCloudHosted()) $effect(() => { const ws = opWs @@ -458,7 +462,7 @@
    {/if} - {#if rulesetsLoaded && !canSetGuest && policy.execution_mode != 'guest'} + {#if rulesetsLoaded && !canSetGuest && policy.execution_mode != 'guest' && guestsAvailable} Opening this app to guests is restricted to workspace admins and bypass users by a workspace protection rule @@ -481,8 +485,10 @@ - {#if embedMode && policy.execution_mode == 'guest' && guestAccessEnabled && guestJwtBase} + {#if embedMode && policy.execution_mode == 'guest' && guestAccessEnabled && guestJwtBase && guestsAvailable}
    Embed for your own authenticated users (guest JWT) diff --git a/frontend/src/lib/components/apps/editor/AppEditorTutorial.svelte b/frontend/src/lib/components/apps/editor/AppEditorTutorial.svelte deleted file mode 100644 index c268058717..0000000000 --- a/frontend/src/lib/components/apps/editor/AppEditorTutorial.svelte +++ /dev/null @@ -1,35 +0,0 @@ - - -) => { - targetTutorial = event.detail.detail - }} -/> - - { - targetTutorial = undefined - }} - on:confirmed={async () => { - window.open(`/apps/add?tutorial=${targetTutorial}`, '_blank') - }} -> -
    - This tutorial can only be run on a new app. -
    -     diff --git a/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte b/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte index 4ae67dbb0e..614d1f4fa4 100644 --- a/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte +++ b/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte @@ -9,10 +9,10 @@ cleanValueProperties, displayDate, emptyString, - orderedYamlStringify, replaceFalseWithUndefined, type Value } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { AppService, type AppWithLastVersion, type AppHistory } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { Skeleton } from '$lib/components/common' diff --git a/frontend/src/lib/components/apps/editor/PublicApp.svelte b/frontend/src/lib/components/apps/editor/PublicApp.svelte index 6523f13ffa..6de8f9df0c 100644 --- a/frontend/src/lib/components/apps/editor/PublicApp.svelte +++ b/frontend/src/lib/components/apps/editor/PublicApp.svelte @@ -1,14 +1,14 @@ @@ -111,18 +111,6 @@ >Powered by   Windmill
    - - {#snippet userInfo(child)} -
    {child}
    - {/snippet} - -
    {#if $userStore} - {@render userInfo($userStore.username)} - {:else if globalUser} - {@render userInfo(globalUser.email)} - {:else}{/if} -
    {/if} {#if notExists} @@ -188,27 +176,35 @@ )} style={app?.value?.['css']?.['app']?.['viewer']?.style} > - goto(path)} - gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} - /> + {#await loadAppPreview()} + + {:then Module} + goto(path)} + gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} + /> + {:catch} +
    + Reload the page to try again. +
    + {/await}
    {/if} {/key} diff --git a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte index dbe8f8d503..e994767488 100644 --- a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte +++ b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte @@ -26,12 +26,14 @@ import { OpenAPI, UserService } from '$lib/gen' import { page } from '$app/state' import { onDestroy, onMount, setContext, type Snippet } from 'svelte' - import { Alert, Skeleton } from '$lib/components/common' + import Alert from '$lib/components/common/alert/Alert.svelte' + import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import { base } from '$app/paths' import { goto } from '$app/navigation' import Login from '$lib/components/Login.svelte' import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils' import { EMBED_NAV_CONTEXT_KEY, type EmbedNav } from '../types' + import { loadAppPreview } from './loadAppPreview' import RawAppSdkConsent from '$lib/components/raw_apps/RawAppSdkConsent.svelte' import { hasStoredSdkConsent, storeSdkConsent } from '$lib/components/raw_apps/sdkScopes' @@ -420,7 +422,8 @@ if (unsandboxed || isRaw) { // Render the app directly on this origin: same-origin when unsandboxed // (the default), or a single opaque bundle iframe when it's a sandboxed - // raw app. + // raw app. A low-code app's runtime downloads alongside the app payload. + if (!isRaw) loadAppPreview().catch(() => {}) onViewerReady?.(undefined, requestTokenRefresh) } else { // Sandboxed low-code: hand the scoped token to the opaque viewer iframe. @@ -529,6 +532,8 @@ onMount(() => { if (isViewer) { + // Only a sandboxed low-code app is ever framed as a viewer. + loadAppPreview().catch(() => {}) window.addEventListener('message', handleViewerMessage) installHashRelay() // Announce readiness so the embedder sends us the token. diff --git a/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte b/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte index 2ac045feaa..aa81b7f5cc 100644 --- a/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte +++ b/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte @@ -11,7 +11,6 @@ left } from './componentCallbacks.svelte' import type { AppEditorContext, AppViewerContext } from '../../types' - import { isCurrentlyInTutorial } from '$lib/stores' const { history, movingcomponents, jobsDrawerOpen, runnableJobEditorPanel } = getContext('AppEditorContext') as AppEditorContext @@ -34,8 +33,7 @@ if ( (typeof classes === 'string' && classes.includes('inputarea')) || ['INPUT', 'TEXTAREA'].includes(document.activeElement?.tagName!) || - $runnableJobEditorPanel.focused || - isCurrentlyInTutorial.val + $runnableJobEditorPanel.focused ) { return } diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte index 214ab3e31d..a4bed4affb 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte @@ -14,10 +14,9 @@ import { defaultCode } from '../component' import WorkspaceScriptList from '../settingsPanel/mainInput/WorkspaceScriptList.svelte' import RunnableSelector from '../settingsPanel/mainInput/RunnableSelector.svelte' - import { defaultScripts, isCurrentlyInTutorial } from '$lib/stores' + import { defaultScripts } from '$lib/stores' import DefaultScripts from '$lib/components/DefaultScripts.svelte' import type { Preview } from '$lib/gen' - import { twMerge } from 'tailwind-merge' import type { InlineScript } from '../../sharedTypes' interface Props { @@ -122,13 +121,7 @@ -
    +
    Choose a language
    diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte index aaffb1feaf..56304ca3e3 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte @@ -7,10 +7,6 @@ import { getAllScriptNames } from '../../utils' import PanelSection from '../settingsPanel/common/PanelSection.svelte' import { getAppScripts } from './utils' - import AppTutorials from '$lib/components/AppTutorials.svelte' - import { tutorialsToDo } from '$lib/stores' - import { ignoredTutorials } from '$lib/components/tutorials/ignoredTutorials' - import { tutorialInProgress } from '$lib/tutorialUtils' import DocLink from '../settingsPanel/DocLink.svelte' import HideButton from '../settingsPanel/HideButton.svelte' import { BG_PREFIX } from '../appUtilsCore' @@ -37,10 +33,6 @@ } function createBackgroundScript() { - if ($tutorialsToDo.includes(5) && !$ignoredTutorials?.includes(5) && !tutorialInProgress()) { - appTutorials?.runTutorialById('backgroundrunnables', { skipStepsCount: 2 }) - } - for (const [index, script] of $app.hiddenInlineScripts.entries()) { if (script.hidden) { delete script.hidden @@ -75,7 +67,6 @@ selectScript(`${BG_PREFIX}${$app.hiddenInlineScripts.length - 1}`) } - let appTutorials: AppTutorials | undefined = $state(undefined) const dispatch = createEventDispatcher() let runnables = $derived(getAppScripts($app.grid, $app.subgrids)) // When selected component changes, update selectedScriptComponentId @@ -248,5 +239,3 @@
    - - diff --git a/frontend/src/lib/components/apps/editor/loadAppPreview.ts b/frontend/src/lib/components/apps/editor/loadAppPreview.ts new file mode 100644 index 0000000000..57e82e09f8 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/loadAppPreview.ts @@ -0,0 +1,12 @@ +// AppPreview must stay a dynamic import: it statically reaches every low-code component +// (and through them monaco, ag-grid, chart.js...), none of which a raw app renders. +// assertLeanPublicAppRoutes in vite.config.js fails the build if that regresses. +let pending: Promise | undefined + +export function loadAppPreview() { + pending ??= import('./AppPreview.svelte').catch((e) => { + console.error('Could not load the low-code app viewer', e) + throw e + }) + return pending +} diff --git a/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts b/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts index 4da6f18f32..c7325698e3 100644 --- a/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts +++ b/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts @@ -35,9 +35,9 @@ export function useListHighlight(opts: { untrack(() => (index = resting)) }) - function move(delta: number) { + function land(next: number) { const count = opts.count() - if (count === 0) return + if (count === 0 || next < 0 || next >= count) return pointerOwns = false // Rows are tabbable, so focus can sit on one. Enter then activates whatever is // focused, which has to stay the highlighted row — so any row counts, not just @@ -47,16 +47,29 @@ export function useListHighlight(opts: { const focusedId = document.activeElement?.id const rowWasFocused = !!focusedId && Array.from({ length: count }, (_, i) => opts.rowId(i)).includes(focusedId) - index = index < 0 ? (delta > 0 ? 0 : count - 1) : (index + delta + count) % count + index = next const row = document.getElementById(opts.rowId(index)) row?.scrollIntoView({ block: 'nearest' }) if (rowWasFocused) row?.focus() } + function move(delta: number) { + const count = opts.count() + if (count === 0) return + land(index < 0 ? (delta > 0 ? 0 : count - 1) : (index + delta + count) % count) + } + return { get index() { return index }, + /** Step the highlight, for a list whose own keys move it beyond Up and Down — + * a tree stepping into the children a folder just revealed. */ + move, + /** Put the highlight on a row named outright, rather than a step from wherever + * it is — the row a caller's own key landed on, or the one that has focus. A + * step cannot say this: from nothing lit it can only reach an end of the list. */ + moveTo: land, /** Wire to each row's `onMouseEnter`. */ hovered(i: number) { if (pointerOwns) index = i diff --git a/frontend/src/lib/components/common/modal/PagedContent.svelte b/frontend/src/lib/components/common/modal/PagedContent.svelte index 674c147c52..bfae0473f1 100644 --- a/frontend/src/lib/components/common/modal/PagedContent.svelte +++ b/frontend/src/lib/components/common/modal/PagedContent.svelte @@ -4,7 +4,18 @@ /** One level of a paginated dialog. Order is the order given: the page on screen sits at rest * and every other waits off the side it is listed on, so a deeper page arrives from the right * and the way back arrives from the left without anyone naming a direction. */ - export type ModalPage = { key: string; content: Snippet } + export type ModalPage = { + key: string + content: Snippet + /** + * Drawn in place of `content` for a page that has not been opened yet, so the first + * navigation to it has something to slide in — without one, the box arrives empty and + * fills a frame later, which reads as the animation being broken rather than as + * loading. A skeleton is enough: it is on screen for the length of the transition. + * Unnecessary under `warm`, which builds every page up front. + */ + placeholder?: Snippet + } + + + +
    + +
    + +
    +
    + {#if hasArgs} + + + {:else} +

    This {runnableKind} takes no arguments.

    + {/if} +
    +
    + + {#if fades.bottom} +
    + {/if} +
    + + +
    + {#if runForm.clearedKeys?.length} +

    + Sent in a shape this field has no reading of, so it opened empty: + {runForm.clearedKeys.join(', ')} +

    + {/if} + {#if runForm.resetKeys?.length} +

    + Disabled by this {runnableKind}, so it will run with its default: + {runForm.resetKeys.join(', ')} +

    + {/if} + {#if runForm.strippedKeys?.length} +

    + A file, so it opened empty for you to attach: + {runForm.strippedKeys.join(', ')} +

    + {/if} + {#if planMode} +

    {PLAN_MODE_MESSAGES.runFormRefused}

    + {/if} + + +
    + + +
    +
    +
    diff --git a/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte new file mode 100644 index 0000000000..edd1704bb2 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte @@ -0,0 +1,530 @@ + + + +{#snippet status()} + {#if !pending} + + {statusTime} + + {/if} +{/snippet} + + +{#snippet previewChip()} + +{/snippet} + + + + (toggled = { id: message.tool_call_id, open: !expanded })} + headerLeft={status} + headerRight={previewTarget ? previewChip : undefined} + class="scroll-mb-8" + labelClass="font-main font-medium text-primary" + contentClass="p-0 overflow-hidden" +> + {#if formInPreview} +
    + These inputs are open in the preview panel. +
    + {:else if pending} + + {:else} + +
    + + (userTab = { id: message.tool_call_id, value: e.detail })} + class="h-8 px-3 font-main" + wrapperClass="shrink-0" + slidingIndicator + > + {#if !rawView} + {#each tabs as tab (tab.value)} + + + + {#snippet extra()} + {#if tab.value === 'logs' && logLineCount > 0} + {logLineCount} + {/if} + {/snippet} + + + {/each} + {/if} +
    + (jsonView = { id: message.tool_call_id, on: e.detail })} + size="2xs" + options={{ right: 'JSON', rightTooltip: 'Show this call as raw JSON' }} + lightMode + /> +
    +     + + +
    + +
    + {#if rawView} +
    + + + + +
    + {:else} + + {#key activeTab} +
    + {#if activeTab === 'input'} + + + {:else if activeTab === 'logs'} + {#if logs.trim()} + {#if logs.length >= MAX_LOG_LENGTH} +

    + Tail of the logs, the last {MAX_LOG_LENGTH} characters. +

    + {/if} + 
    {logs}
    + {:else} +

    No logs yet.

    + {/if} + {#if running} +
    + + streaming +
    + {/if} + {:else if failed} + 
    {message.error}
    + {:else if streaming} + + + {:else if resultValue !== undefined} + + + {:else if canceled} + +
    + +

    {cancelReason}

    + {#if !ran} +

    + The inputs it would have run with are on the Inputs tab. +

    + {/if} +
    + {:else} +

    This run returned no result.

    + {/if} +
    + {/key} + {/if} +
    +
    + + + {#if fades.bottom && fadeBody} +
    + {#if activeTab === 'logs'} +
    + {/if} + {/if} +
    + + {#if running && chatJob} + +
    + +
    + {/if} + {/if} +     diff --git a/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte index c4b2d0cc21..9d1bf698f7 100644 --- a/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte @@ -1,6 +1,7 @@ @@ -136,17 +127,18 @@ {:else if hasContent}
    - 
    {formatJson($state.snapshot(content))}
    - {#if showFade && canScrollDown} + {#if showFade && fades.bottom}
    {/if}
    diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 34b1adcf68..f5ae22a332 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -37,6 +37,7 @@ import ToolMessageActions from './ToolMessageActions.svelte' import ToolPreviewCard from './ToolPreviewCard.svelte' import AskUserQuestionDisplay from './AskUserQuestionDisplay.svelte' + import RunScriptCard from './RunScriptCard.svelte' import WebSearchSourcesDisplay from './WebSearchSourcesDisplay.svelte' import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' @@ -117,6 +118,10 @@ isActiveUserQuestion(message) ? message.userQuestion : undefined ) + // The run card owns this call from the form to whatever settled it, cancelling included: + // the card is the call, and a run the user stopped is not a different kind of thing. + const isRunCard = $derived(Boolean(message.runForm)) + // The preview chip sits on the header row (to the right of the tool-call text); // shown once the tool settled, never while loading/erroring/awaiting confirmation. const showPreviewChip = $derived( @@ -140,6 +145,8 @@ {message.toolName} {/if}
    +{:else if isRunCard} + {:else if planState} diff --git a/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte b/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte index 81d00fd8e4..b5f4fa8bae 100644 --- a/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte @@ -8,15 +8,26 @@ interface Props { card: { kind: PreviewCardKind; path: string } + /** Opens something other than the item's own preview — the run card opens the call + * it owns, which is a form before it is a run. */ + onOpen?: () => void + title?: string + /** The kind icon says what the chip opens. A card that already names its own runnable + * in the row above has said it, and repeating it there reads as a second subject. */ + kindIcon?: boolean } - let { card }: Props = $props() + let { card, onOpen, title, kindIcon = true }: Props = $props() const kindLabel = $derived(card.kind === 'raw_app' ? 'app' : card.kind) let opening = $state(false) async function open() { if (opening) return + if (onOpen) { + onOpen() + return + } opening = true try { await runToolDisplayAction(openItemPreviewAction(card.kind, card.path)) @@ -30,9 +41,11 @@ variant="default" unifiedSize="2xs" disabled={opening} - title="Open {kindLabel} preview: {card.path}" + title={title ?? `Open ${kindLabel} preview: ${card.path}`} onClick={open} - startIcon={{ icon: RowIcon as unknown as IconType, props: { kind: card.kind, size: 12 } }} + startIcon={kindIcon + ? { icon: RowIcon as unknown as IconType, props: { kind: card.kind, size: 12 } } + : undefined} endIcon={{ icon: PanelRight }} wrapperClasses="shrink-0" > diff --git a/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts b/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts index 5290528dba..e257e40975 100644 --- a/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts +++ b/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts @@ -2,36 +2,61 @@ import { get } from 'svelte/store' import { userStore } from '$lib/stores' /** - * A set of workspace-object paths the chat may act through, remembered per - * workspace and per account. + * Which workspace-object paths the chat may act through, remembered per workspace + * and per account. * - * Being able to read a resource is not the same as wanting the chat to use it: a - * resource in a shared folder is readable by a whole team, and each enabled entry - * costs something on every turn — an MCP server puts its tool descriptions in the - * model's context and reaches an external system, a skill puts its description - * there. So an entry is off until it is turned on. + * Only the paths someone actually decided about are stored; everything else is the + * kind's default. That is what lets a default flip — skills went from off-until-on + * to on-until-off — without rewriting anyone's storage: the stored entries keep + * meaning what the person chose, and only the paths they never touched move. * * Stored per browser, like the chat's other per-user preferences, but keyed by - * email as well as workspace: browser storage outlives a logout, and inheriting - * the previous account's selection would hand the next person capabilities they - * never turned on. Workspace ids cannot contain `:`, so the composite key is - * unambiguous. + * email as well as workspace: browser storage outlives a logout, and one person's + * decisions must not be read as the next person's — whichever way they went. + * Workspace ids cannot contain `:`, so the composite key is unambiguous. */ -export type EnabledPathsPreference = { - enabledPaths: (workspace: string) => string[] +export type PathsPreference = { + /** Paths stored as on, which is the enabled set only for a kind that defaults to + * off. For one that defaults to on this is whatever happens to be written down — + * nothing for a decision made since the default flipped, and the whole selection + * for a browser still carrying storage from before it. Neither is the answer to + * "what is enabled": ask `isEnabled` per path for that. */ + explicitlyEnabledPaths: (workspace: string) => string[] isEnabled: (workspace: string, path: string) => boolean /** Returns false when there is no account to record the preference against, so * a caller that just created the object can say it did not stay on. */ setEnabled: (workspace: string, path: string, enabled: boolean) => boolean + /** Drop any decision about `path`, leaving it at the default. What a caller wants + * when the thing at that path is gone — a deleted or renamed skill — rather than + * writing the default as a decision, which reads as one and would mean the + * opposite the day the default moves. */ + forget: (workspace: string, path: string) => void } -export function createEnabledPathsPreference(storageKey: string): EnabledPathsPreference { +/** One decision per path. The older shape was an array of the paths that were on, + * which says exactly that and so needs no conversion to be read. */ +type Choices = Record + +function toChoices(raw: unknown): Choices { + if (Array.isArray(raw)) return Object.fromEntries(raw.map((path) => [String(path), true])) + if (raw && typeof raw === 'object') { + return Object.fromEntries( + Object.entries(raw as Record).map(([path, on]) => [path, on === true]) + ) + } + return {} +} + +export function createPathsPreference( + storageKey: string, + defaultEnabled: boolean +): PathsPreference { function scope(workspace: string): string | undefined { const email = get(userStore)?.email return email ? `${workspace}:${email}` : undefined } - function read(): Record { + function read(): Record { if (typeof localStorage === 'undefined') return {} try { return JSON.parse(localStorage.getItem(storageKey) ?? '{}') @@ -40,35 +65,42 @@ export function createEnabledPathsPreference(storageKey: string): EnabledPathsPr } } - function write(all: Record) { + function choices(workspace: string): Choices { + const key = scope(workspace) + return key ? toChoices(read()[key]) : {} + } + + /** `decision` of undefined drops the entry. */ + function write(workspace: string, path: string, decision: boolean | undefined): boolean { + const key = scope(workspace) + if (!key) return false + const all = read() + const current = toChoices(all[key]) + if (decision === undefined) { + delete current[path] + } else { + current[path] = decision + } + all[key] = current try { localStorage.setItem(storageKey, JSON.stringify(all)) } catch (e) { console.error(`Failed to persist ${storageKey}`, e) } - } - - function enabledPaths(workspace: string): string[] { - const key = scope(workspace) - return key ? (read()[key] ?? []) : [] + return true } return { - enabledPaths, - isEnabled: (workspace, path) => enabledPaths(workspace).includes(path), - setEnabled: (workspace, path, enabled) => { - const key = scope(workspace) - if (!key) return false - const all = read() - const current = new Set(all[key] ?? []) - if (enabled) { - current.add(path) - } else { - current.delete(path) - } - all[key] = [...current] - write(all) - return true - } + explicitlyEnabledPaths: (workspace) => + Object.entries(choices(workspace)) + .filter(([, on]) => on) + .map(([path]) => path), + isEnabled: (workspace, path) => choices(workspace)[path] ?? defaultEnabled, + // A path put back to the default is dropped rather than stored as one: the + // entries are the decisions, so a re-enabled skill leaves nothing behind and a + // later skill at that path starts from the default like any other. + setEnabled: (workspace, path, enabled) => + write(workspace, path, enabled === defaultEnabled ? undefined : enabled), + forget: (workspace, path) => void write(workspace, path, undefined) } } diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index 47a6b0b169..c6bed6988c 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -14,6 +14,9 @@ import type { ScriptLintResult } from '../shared' import { applyFlowJsonUpdate, updateRawScriptModuleContent } from './helperUtils' import { findModuleInFlow } from '$lib/components/flows/flowTree' + import { getEditorStoragePath } from '$lib/components/editorStoragePathContext' + + const editorStoragePath = getEditorStoragePath() let { flowModuleSchemaMap, @@ -162,6 +165,8 @@ selectionManager.selectId(id, { openPanel: true }) }, + getStoragePath: () => editorStoragePath?.(), + testFlow: async (args, conversationId) => { // Set preview args if provided if (args) { diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 63c4ffe94f..51fea34d58 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -136,6 +136,10 @@ export interface FlowAIChatHelpers { /** Run a test of the current flow using the UI's preview mechanism */ testFlow: (args?: Record, conversationId?: string) => Promise + /** The path this editor's draft is stored under. Tells a caller which of several mounted + * editors is the one an active-editor context names. */ + getStoragePath: () => string | undefined + /** Get lint errors from a specific module (focuses it first, waits for Monaco to analyze) */ getLintErrors: (moduleId: string) => Promise } diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlow.json b/frontend/src/lib/components/copilot/chat/flow/openFlow.json index cab2839d48..7226329062 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlow.json +++ b/frontend/src/lib/components/copilot/chat/flow/openFlow.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"version":"1.791.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file +{"openapi":"3.0.3","info":{"version":"1.808.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts index 5bd36b85ce..f821413351 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts @@ -1,6 +1,6 @@ import { z } from "zod" -export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { +export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { const schemas = [z.intersection(z.object({ "tool_type": z.literal("flowmodule") }), z.lazy(() => flowModuleValueSchema)).describe("A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module"), z.object({ "tool_type": z.literal("mcp"), "resource_path": z.string().describe("Path to the MCP resource/server configuration"), "include_tools": z.array(z.string()).describe("Whitelist of specific tools to include from this MCP server").optional(), "exclude_tools": z.array(z.string()).describe("Blacklist of tools to exclude from this MCP server").optional() }).describe("Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers"), z.object({ "tool_type": z.literal("websearch") }).describe("A tool implemented as a websearch tool. The AI can call this like any other websearch tool")]; const errors = schemas.reduce( (errors, schema) => @@ -37,7 +37,7 @@ export const flowModuleSchema = z.object({ "id": z.string().describe("Unique ide message: "Invalid input: Should pass single schema", }); } - }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type"), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch") + }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type"), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch") export const flowModulesSchema = z.array(flowModuleSchema) diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts index 8158fe62b8..deb3694e99 100644 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts @@ -88,6 +88,19 @@ const CATALOG = [ required: ['workspace', 'hash'] } }, + { + name: 'runScriptByPath', + description: 'Run script by path', + instructions: 'Trigger a run of a deployed script', + path: '/w/{workspace}/jobs/run/p/{path}', + method: 'POST', + path_params_schema: { + type: 'object', + properties: { workspace: { type: 'string' }, path: { type: 'string' } }, + required: ['workspace', 'path'] + }, + body_schema: { type: 'object', additionalProperties: true } + }, { name: 'runFlowByPath', description: 'Run flow by path', @@ -178,6 +191,19 @@ describe('call_api_get', () => { expect(search.matches.map((m: any) => m.name)).not.toContain('deleteScriptByHash') }) + // Left reachable, this endpoint is the way around the argument form: it runs the + // deployed script on the model's arguments, unstripped and unshown. + it('refuses a deployed script run, pointing at run_script', async () => { + const called = await run('call_api_endpoint', { name: 'runScriptByPath' }) + expect(called.error).toContain('run_script') + expect(called.success).toBe(false) + + // And it is gone from search, so the model is redirected before it ever calls. + const search = await run('search_api_endpoints', { query: 'run deployed script' }) + expect(search.matches.map((m: any) => m.name)).not.toContain('runScriptByPath') + expect(search.covered_by_dedicated_tools?.join(' ')).toContain('run_script') + }) + it('refuses draft-blind item reads and lists, pointing at the draft-aware tools', async () => { for (const name of ['getScriptByPath', 'getResource', 'getSchedule']) { const result = await run('call_api_get', { name }) diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts index 5d1324ebe3..2271ff91e0 100644 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts +++ b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts @@ -36,6 +36,7 @@ const COVERED_ENDPOINTS: Record = { listFlows: 'list_workspace_items (it includes your drafts)', listResource: 'list_workspace_items (it includes your drafts)', listSchedules: 'list_workspace_items (it includes your drafts)', + runScriptByPath: 'run_script (it shows the user an argument form to confirm)', deleteScriptByPath: 'delete_workspace_item', deleteScriptByHash: 'delete_workspace_item', deleteFlowByPath: 'delete_workspace_item', diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 19050bb7da..d8abb38cdf 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -82,12 +82,21 @@ vi.mock('$lib/gen', async () => { runScriptPreview: vi.fn(async () => 'job-script-preview'), runFlowPreview: vi.fn(async () => 'job-flow-preview'), runFlowByPath: vi.fn(async () => 'job-flow-by-path'), + runScriptByPath: vi.fn(async () => 'job-script-by-path'), getJob: vi.fn(async () => ({ type: 'CompletedJob', success: true, result: { ok: true }, logs: 'test logs' })), + // What every job wait polls first; unmocked it reaches the real client and the + // wait never returns. Answers completed, so one tick settles the job. + getJobUpdates: vi.fn(async () => ({ + completed: true, + running: false, + new_logs: 'test logs', + log_offset: 'test logs'.length + })), getJobLogs: vi.fn(async () => 'job log line 1\njob log line 2'), listJobs: vi.fn(async () => [ { @@ -211,6 +220,9 @@ vi.mock('$lib/gen', async () => { createResource: vi.fn(async () => 'created'), updateResource: vi.fn(async () => 'updated'), deleteResource: vi.fn(async () => 'deleted'), + // A workspace with no skills, which is what makes `read_skill` refuse a path + // the model composed: the gate is membership in the listing. + listResource: vi.fn(async () => []), getResourceValue: vi.fn(async () => ({ content: 'skill body' })) }), VariableService: wrapService(actual.VariableService, { @@ -294,6 +306,12 @@ vi.mock('$lib/gen', async () => { } }) +// Minting reaches the API and is covered in secretArgUtils.test.ts; what matters here is that a +// run the posture answers goes through it and starts on what came back. +vi.mock('$lib/components/secretArgUtils', () => ({ + processSecretArgs: vi.fn(async (args: Record) => args) +})) + vi.mock('./rawAppBundlerBridge', () => ({ bundleRawAppDraft: vi.fn(async () => ({ js: 'bundled js', @@ -349,6 +367,7 @@ import { VariableService } from '$lib/gen' import { superadmin, userStore, usersWorkspaceStore } from '$lib/stores' +import { processSecretArgs } from '$lib/components/secretArgUtils' import { clearWorkspaceRoleCache } from '$lib/user' import { get } from 'svelte/store' import type { Tool, ToolCallbacks } from '../shared' @@ -368,7 +387,10 @@ function getBackendDraft(kind: string, path: string, _opts?: unknown): const toolCallbacks: ToolCallbacks = { setToolStatus: vi.fn(), - removeToolStatus: vi.fn() + removeToolStatus: vi.fn(), + // Every host that can run a script mounts the form, so the default answers it with what it + // opened with. A test meaning to exercise a host without one overrides this with undefined. + requestRunArgs: async (_toolId, form) => form.args } function getGlobalTool(name: string): Tool<{}> { @@ -4318,8 +4340,7 @@ describe('global AI tools', () => { const result = await withCompletedTestJob(() => callGlobalTool('test_run_script', { - path: 'f/scripts/draft-test', - args: { name: 'Ada' } + path: 'f/scripts/draft-test' }) ) @@ -4328,7 +4349,7 @@ describe('global AI tools', () => { requestBody: { path: 'f/scripts/draft-test', content, - args: { name: 'Ada' }, + args: {}, language: 'bun' } }) @@ -4347,8 +4368,7 @@ describe('global AI tools', () => { await withCompletedTestJob(() => callGlobalTool('test_run_script', { - path: 'f/scripts/deployed-test', - args: { name: 'Grace' } + path: 'f/scripts/deployed-test' }) ) @@ -4361,18 +4381,412 @@ describe('global AI tools', () => { requestBody: { path: 'f/scripts/deployed-test', content: 'def main(name):\n return name', - args: { name: 'Grace' }, + args: {}, language: 'python3' } }) }) + // A test run meets the same card as a deployed one, so what previews is what the form + // submitted — not what the model proposed. + it('test_run_script previews the arguments the form submitted', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/formed-test', + summary: 'Formed test script', + content: 'export async function main(name: string) {}', + language: 'bun', + schema: { properties: { name: { type: 'string' } } } + } as any) + + let opened: Record | undefined + let kind: string | undefined + let helperSource: { code?: string; lang?: string } | undefined + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_script', + { path: 'f/scripts/formed-test', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, form) => { + opened = form.args + kind = form.kind + helperSource = { code: form.code, lang: form.lang } + return { name: 'Grace' } + } + } + ) + ) + + expect(opened).toEqual({ name: 'Ada' }) + // Drives the card's tense: a test says it tested, not that it ran. + expect(kind).toBe('test') + // The draft itself, so a dynselect field offers the options this code returns rather + // than the deployed version's — which is stale, or absent for a draft never deployed. + expect(helperSource).toEqual({ + code: 'export async function main(name: string) {}', + lang: 'bun' + }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/scripts/formed-test', + content: 'export async function main(name: string) {}', + args: { name: 'Grace' }, + language: 'bun' + } + }) + }) + + // The bypass posture answers a run form as it answers any other confirmation, for a + // deployed run as much as a test. The card must never render one first: a form nobody + // will fill in is attached already settled, so no field is ever mounted, and the schema + // it would have built them from never reaches the transcript. + it('mounts no field on either run form under yolo', async () => { + const script = { + path: 'f/scripts/yolo', + content: 'export async function main(name: string) {}', + language: 'bun', + schema: { properties: { name: { type: 'string' } } } + } as any + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce(script) + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce(script) + + const statuses: any[] = [] + const requestRunArgs = vi.fn(async (_toolId: string, form: any) => form.args) + const yolo = { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs + } + + await withCompletedTestJob(() => + callGlobalTool('test_run_script', { path: 'f/scripts/yolo', args: { name: 'Ada' } }, yolo) + ) + + const testForm = statuses.find((s) => s.runForm)?.runForm + expect(testForm.submitted).toBe(true) + // Nothing is left to render it, and a card carrying one persists it forever. + expect(testForm.schema).toBeUndefined() + // Told the form is already answered, or the loop parks on a card with no fields. + expect(requestRunArgs.mock.calls[0][2]).toEqual({ autoAccepted: true }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: expect.objectContaining({ args: { name: 'Ada' } }) }) + ) + + statuses.length = 0 + await withCompletedTestJob(() => + callGlobalTool('run_script', { path: 'f/scripts/yolo', args: { name: 'Ada' } }, yolo) + ) + + const deployedForm = statuses.find((s) => s.runForm)?.runForm + expect(deployedForm.submitted).toBe(true) + expect(deployedForm.schema).toBeUndefined() + // skipPreprocessor: the form fills the main input schema, so a preprocessor would take + // these arguments for a webhook body and run main on its output instead. + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada' }, skipPreprocessor: true }) + ) + }) + + // The posture is the user's standing answer to whether to ask, so a run it answers starts on + // the model's arguments as sent — no default filled in, no required field second-guessed. + // Predicting what a mounted field would hold starts runs the form itself would refuse; the + // schema's own defaults are the worker's job, from the code's signature. + it('sends the model arguments as proposed when the posture answers', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/defaulted', + schema: { + properties: { name: { type: 'string' }, retries: { type: 'number', default: 3 } }, + required: ['name', 'retries'] + } + } as any) + const statuses: any[] = [] + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/defaulted', args: { name: 'Ada' } }, + { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(statuses.find((x) => x.runForm)?.runForm.submitted).toBe(true) + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada' } }) + ) + }) + + // A disabled field is declared as not the caller's to set, and the posture answering the + // form does not make the model one of the callers it is kept from. + it('holds a disabled default against the model when the posture answers', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/locked', + schema: { + properties: { + name: { type: 'string' }, + mode: { type: 'string', disabled: true, default: 'safe' } + } + } + } as any) + + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/locked', args: { name: 'Ada', mode: 'destructive' } }, + { + ...toolCallbacks, + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada', mode: 'safe' } }) + ) + // Or the next call proposes the same override again. + expect(result).toContain('disables mode') + }) + + // With no form there is no PasswordArgInput to turn a proposed secret into a reference, and + // a job's arguments are readable by everyone who can see its run. What starts the job must + // be what came back from the minting, never the proposal. + it('mints a proposed secret into a reference before starting a run the posture answers', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/secret', + schema: { properties: { token: { type: 'string', password: true } }, required: ['token'] } + } as any) + vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ + token: '$var:u/ada/secret_arg/minted' + })) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/secret', args: { token: 'hunter2' } }, + { + ...toolCallbacks, + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(processSecretArgs).toHaveBeenCalledWith( + { token: 'hunter2' }, + expect.anything(), + expect.anything() + ) + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) + ) + }) + + // The bypass is the user's standing answer, not a licence for the host to skip asking: + // a chat with nowhere to put a form still refuses the run under any other posture. + it('run_script refuses a host with no form unless the posture answers for it', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/noform', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const refused = await callGlobalTool( + 'run_script', + { path: 'f/scripts/noform', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: undefined } + ) + + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(refused).toContain('cannot show a run form') + }) + + // The posture answers wherever it is set, form or no form: what it answers is consent, and a + // host without one has nothing left to ask. A secret still becomes a reference first, which + // is the only thing the missing form would have done. + it('run_script runs on a formless host under yolo', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/noform-secret', + schema: { + properties: { token: { type: 'string', password: true } }, + required: ['token'] + } + } as any) + vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ + token: '$var:u/ada/secret_arg/minted' + })) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/noform-secret', args: { token: 'hunter2' } }, + { + ...toolCallbacks, + requestRunArgs: undefined, + shouldAutoAcceptToolConfirmations: () => true + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) + ) + }) + + // The transcript is re-cloned into IndexedDB on every save, and a form takes as much text + // as the user pastes. What the card stores is bounded; what the job runs is not. + it('run_script stores a marker for oversized arguments but runs them in full', async () => { + const huge = 'x'.repeat(120_000) + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/big', + content: 'export async function main(blob: string) {}', + language: 'bun', + schema: { properties: { blob: { type: 'string' } } } + } as any) + + const statuses: any[] = [] + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/big', args: { blob: 'small' } }, + { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + // The user pastes into the field the model left small: the model's own + // proposal is bounded by what it can emit, this is not. + requestRunArgs: async () => ({ blob: huge }) + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { blob: huge } }) + ) + const persisted = statuses.filter((s) => s.parameters !== undefined).at(-1)?.parameters + expect(persisted).toEqual({ reason: 'WINDMILL_TOO_BIG' }) + expect(JSON.stringify(statuses)).not.toContain(huge) + }) + + // A dropped pass-through fails silently: the argument is accepted and ignored, and the + // run just waits out the default budget. + it('run_script detaches on background and on wait_seconds', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/slow', + schema: { properties: { name: { type: 'string' } } } + } as any) + + // wait_seconds 0 detaches the same way, so one run of each pins both lines. + for (const detachArg of [{ background: true }, { wait_seconds: 0 }]) { + const onJobDetached = vi.fn() + await callGlobalTool( + 'run_script', + { path: 'f/scripts/slow', args: { name: 'Ada' }, ...detachArg }, + { ...toolCallbacks, onJobStarted: vi.fn(), onJobDetached } + ) + + expect(onJobDetached).toHaveBeenCalledWith('job-script-by-path') + } + }) + + // A schema with no fields still opens a form: an empty one is still the Run button, and + // that button is the whole confirmation this tool has. Skipping it because there is + // nothing to fill in starts the script with no confirmation at all. + it('test_run_script opens a form and starts no job when the schema declares no field', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/noargs-test', + content: 'export async function main() {}', + language: 'bun', + schema: { properties: {} } + } as any) + + const cancelled = await callGlobalTool( + 'test_run_script', + { path: 'f/scripts/noargs-test', args: { force_delete: true } }, + { ...toolCallbacks, requestRunArgs: async () => undefined } + ) + + expect(JobService.runScriptPreview).not.toHaveBeenCalled() + expect(cancelled).toContain('The user cancelled the run form') + + // A schema declaring nothing is a form with no field to hold this, and the card says + // as much — so answering it must not send an argument that was never on screen. + const answered = await withCompletedTestJob(() => + callGlobalTool( + 'test_run_script', + { path: 'f/scripts/noargs-test', args: { force_delete: true } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, form) => form.args } + ) + ) + + expect(JobService.runScriptPreview).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: expect.objectContaining({ args: {} }) }) + ) + expect(answered).toContain('does not declare force_delete') + }) + + // A host that answers the form without minting, as the eval harness does by returning the + // proposal verbatim: the job's arguments are readable by everyone who can see its run. + it('mints a secret the host handed back as a literal', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/host-literal', + schema: { properties: { token: { type: 'string', password: true } } } + } as any) + vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ + token: '$var:u/ada/secret_arg/minted' + })) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/host-literal', args: { token: 'hunter2' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, form) => form.args } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) + ) + }) + + // The bypass has no form to have shown them either, so the rule holds there too. + it('drops an undeclared argument when the posture answers too', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/noargs-yolo', + schema: { properties: { name: { type: 'string' } } } + } as any) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/noargs-yolo', args: { name: 'Ada', force_delete: true } }, + { + ...toolCallbacks, + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada' } }) + ) + }) + + // The form offers the arguments the flow declares and no others, so a fixture flow that + // takes one has to say so — as a real flow does, since nothing else could render a field. + const FLOW_NAME_SCHEMA = { type: 'object', properties: { name: { type: 'string' } } } + it('test_run_flow previews draft flow content by path', async () => { const modules = [{ id: 'start', value: { type: 'identity' } }] await callGlobalTool('write_flow', { path: 'f/flows/draft-test', summary: 'Draft test flow', - modules: JSON.stringify(modules) + modules: JSON.stringify(modules), + schema: JSON.stringify(FLOW_NAME_SCHEMA) }) await withCompletedTestJob(() => @@ -4399,7 +4813,7 @@ describe('global AI tools', () => { path: 'f/flows/deployed-test', summary: 'Deployed test flow', value: { modules }, - schema: {} + schema: FLOW_NAME_SCHEMA } as any) await withCompletedTestJob(() => @@ -4423,15 +4837,17 @@ describe('global AI tools', () => { }) }) - it('test_run_flow uses the live flow editor test hook when the active editor matches the path', async () => { + // The editor is driven by the key its draft is stored under, which reads and edits of the + // path resolve through too: a staged rename leaves that key where it was. + it('test_run_flow drives the live flow editor by its storage path', async () => { seedBackendDraft( 'flow', - '', + 'u/admin/live_flow_storage', { path: 'u/admin/live_flow', summary: 'Live flow', value: { modules: [{ id: 'live_step', value: { type: 'identity' } }] }, - schema: {}, + schema: FLOW_NAME_SCHEMA, edited_by: '', edited_at: '', archived: false, @@ -4442,7 +4858,7 @@ describe('global AI tools', () => { UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', - storagePath: '', + storagePath: 'u/admin/live_flow_storage', effectivePath: 'u/admin/live_flow' }) const testActiveFlow = vi.fn(async () => 'job-live-flow') @@ -4454,12 +4870,14 @@ describe('global AI tools', () => { path: 'u/admin/live_flow', args: { name: 'Ada' } }, - toolCallbacks, + { ...toolCallbacks, requestRunArgs: async () => ({ name: 'Grace' }) }, { testActiveFlow } ) ) - expect(testActiveFlow).toHaveBeenCalledWith({ name: 'Ada' }) + // What the form submitted, not what the model proposed: the editor runs the flow, but + // the arguments are the user's. + expect(testActiveFlow).toHaveBeenCalledWith('u/admin/live_flow_storage', { name: 'Grace' }) expect(FlowService.getFlowByPath).not.toHaveBeenCalled() expect(JobService.runFlowPreview).not.toHaveBeenCalled() expect(result).toContain('Result (SUCCESS)') @@ -4468,12 +4886,12 @@ describe('global AI tools', () => { it('test_run_flow falls back to preview when the live flow editor test hook returns undefined', async () => { seedBackendDraft( 'flow', - '', + 'u/admin/live_flow_fallback', { path: 'u/admin/live_flow_fallback', summary: 'Live flow fallback', value: { modules: [{ id: 'fallback_step', value: { type: 'identity' } }] }, - schema: {}, + schema: FLOW_NAME_SCHEMA, edited_by: '', edited_at: '', archived: false, @@ -4484,7 +4902,7 @@ describe('global AI tools', () => { UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', - storagePath: '', + storagePath: 'u/admin/live_flow_fallback', effectivePath: 'u/admin/live_flow_fallback' }) const testActiveFlow = vi.fn(async () => undefined) @@ -4501,7 +4919,7 @@ describe('global AI tools', () => { ) ) - expect(testActiveFlow).toHaveBeenCalledWith({ name: 'Ada' }) + expect(testActiveFlow).toHaveBeenCalledWith('u/admin/live_flow_fallback', { name: 'Ada' }) expect(FlowService.getFlowByPath).not.toHaveBeenCalled() expect(JobService.runFlowPreview).toHaveBeenCalledWith({ workspace: WORKSPACE, @@ -4513,6 +4931,156 @@ describe('global AI tools', () => { }) }) + // A flow reaches its run form the way a script does: opened on the flow's own input schema, + // with the dynamic-option picker the schema carries, and the run takes what came back. + it('test_run_flow opens the form on the flow schema and runs what it submitted', async () => { + const modules = [{ id: 'start', value: { type: 'identity' } }] + await callGlobalTool('write_flow', { + path: 'f/flows/formed-flow', + summary: 'Formed flow', + modules: JSON.stringify(modules), + schema: JSON.stringify({ + ...FLOW_NAME_SCHEMA, + 'x-windmill-dyn-select-code': 'export function names() { return ["Ada"] }', + 'x-windmill-dyn-select-lang': 'bun' + }) + }) + + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'f/flows/formed-flow', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, f) => { + form = f + return { name: 'Grace' } + } + } + ) + ) + + expect(form.args).toEqual({ name: 'Ada' }) + expect(form.runnableKind).toBe('flow') + expect(form.schema?.properties).toEqual(FLOW_NAME_SCHEMA.properties) + // The flow's own dynselect script, which the schema carries rather than a step. + expect({ code: form.code, lang: form.lang }).toEqual({ + code: 'export function names() { return ["Ada"] }', + lang: 'bun' + }) + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/flows/formed-flow', + value: { modules }, + args: { name: 'Grace' } + } + }) + }) + + // The form waits as long as the user does, so which editor is on screen is only known when + // they press Run — checking it when the card appeared would drive an editor they have since + // moved away from. + it('test_run_flow re-checks the editor on screen when the form is submitted', async () => { + const modules = [{ id: 'moved_step', value: { type: 'identity' } }] + seedBackendDraft( + 'flow', + 'u/admin/moved_flow', + { + path: 'u/admin/moved_flow', + summary: 'Moved flow', + value: { modules }, + schema: FLOW_NAME_SCHEMA, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/moved_flow', + effectivePath: 'u/admin/moved_flow' + }) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'u/admin/moved_flow', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, form) => { + // The preview panel moves to another flow while the form sits open. + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/other_flow', + effectivePath: 'u/admin/other_flow' + }) + return form.args + } + }, + { testActiveFlow } + ) + ) + + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { path: 'u/admin/moved_flow', value: { modules }, args: { name: 'Ada' } } + }) + }) + + // The flow may be open in a session tab that isn't the one on screen: driving its editor + // would paint the run into a tab the user is not looking at. + it('test_run_flow previews rather than driving an editor the user is not looking at', async () => { + seedBackendDraft( + 'flow', + 'u/admin/background_flow', + { + path: 'u/admin/background_flow', + summary: 'Background flow', + value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, + schema: FLOW_NAME_SCHEMA, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/flow_on_screen', + effectivePath: 'u/admin/flow_on_screen' + }) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'u/admin/background_flow', args: { name: 'Ada' } }, + toolCallbacks, + { testActiveFlow } + ) + ) + + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'u/admin/background_flow', + value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, + args: { name: 'Ada' } + } + }) + }) + it('test_run_step previews rawscript steps from the draft flow', async () => { const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' await callGlobalTool('write_flow', { @@ -4633,6 +5201,220 @@ describe('global AI tools', () => { }) }) + // The form IS the consent, so a dismissed one must leave the script unrun. + it('run_script starts no job when the user cancels the form', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/deployed', + summary: 'Deployed', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const result = await callGlobalTool( + 'run_script', + { path: 'f/scripts/deployed', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async () => undefined } + ) + + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(result).toContain('The user cancelled the run form') + expect(result).toContain('Do not call run_script again') + }) + + // job_args.test.ts owns what each step of the argument pipeline does; this owns that + // run_script still runs them. Delete a call from runThroughForm and every one of those + // unit tests still passes, so one call has to cross all of them here. + it('run_script puts the proposed arguments through the whole pipeline', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/everything', + schema: { + properties: { + count: { type: 'number' }, + ratio: { type: 'number' }, + size: { type: 'number' }, + token: { type: 'string', password: true }, + locked: { type: 'string', default: 'fixed', disabled: true }, + doc: { type: 'string', contentEncoding: 'base64' } + } + } + } as any) + + const bytes = 'QUJD'.repeat(1024) + const statuses: any[] = [] + let shown: Record | undefined + let cleared: string[] | undefined + let reset: string[] | undefined + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { + path: 'f/scripts/everything', + args: { + count: '7', + ratio: 'abc', + size: '$var:u/admin/batch_size', + token: 'hunter2', + locked: 'tampered', + doc: bytes, + force_delete: true + } + }, + { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + requestRunArgs: async (_toolId, form) => { + shown = form.args + cleared = form.clearedKeys + reset = form.resetKeys + // What the user does with the form: attaches the file no model can produce, + // and names a variable for the secret it was not allowed to fill. + return { ...form.args, doc: bytes, token: '$var:u/ada/prod_api_key' } + } + } + ) + ) + + // Coerced, cleared, left alone, reset, dropped and emptied of bytes — every rule reached + // through the tool rather than called directly. The proposed secret is not emptied: it is + // already in the model's own tool call in the same stored record, and PasswordArgInput + // mints whatever the field opens with before the job sees it. + expect(shown).toEqual({ + count: 7, + size: '$var:u/admin/batch_size', + token: 'hunter2', + locked: 'fixed' + }) + expect(cleared).toEqual(['ratio']) + expect(reset).toEqual(['locked']) + + // The bytes belong in the job request and nowhere else: the card is persisted, and a + // file small enough to survive truncation would otherwise reach the model whole. + expect(JobService.runScriptByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/scripts/everything', + requestBody: { + count: 7, + size: '$var:u/admin/batch_size', + locked: 'fixed', + doc: bytes, + token: '$var:u/ada/prod_api_key' + }, + skipPreprocessor: true + }) + expect(result).toContain('does not declare force_delete') + expect(result).toContain('') + // The bytes are the value; the reference is not, and the run page shows it for this + // same job. + expect(result).not.toContain(bytes) + expect(JSON.stringify(statuses)).not.toContain(bytes) + expect(result).toContain('$var:u/ada/prod_api_key') + expect(JSON.stringify(statuses)).toContain('$var:u/ada/prod_api_key') + // Named, or an emptied field reads as the user having deleted the value and the next + // call proposes the same bytes again. + for (const named of ['ratio', 'doc', 'locked']) expect(result).toContain(named) + }) + + // The form is its own confirmation, so it never reaches processToolCall's second gate. + // Plan mode can be switched on while it sits open, and the job must not start. + it('run_script starts no job when plan mode is entered while the form is open', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/noargs', + schema: { properties: {} } + } as any) + + let planning = false + const result = await callGlobalTool( + 'run_script', + { path: 'f/scripts/noargs', args: {} }, + { + ...toolCallbacks, + isPlanModeActive: () => planning, + requestRunArgs: async (_toolId, form) => { + planning = true + return form.args + } + } + ) + + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(result).toContain('plan mode is active') + }) + + // A form that reaches the screen mints a proposed secret on mount, which the gate after + // the user answers is too late to unmake — so plan mode arriving during the fetch counts. + it('run_script opens no form when plan mode is entered during the schema fetch', async () => { + let planning = false + vi.mocked(ScriptService.getScriptByPath).mockImplementationOnce(async () => { + planning = true + return { + path: 'f/scripts/pw', + schema: { properties: { token: { type: 'string', password: true } } } + } as any + }) + + let formOpened = false + const result = await callGlobalTool( + 'run_script', + { path: 'f/scripts/pw', args: { token: 'hunter2' } }, + { + ...toolCallbacks, + isPlanModeActive: () => planning, + requestRunArgs: async (_toolId, form) => { + formOpened = true + return form.args + } + } + ) + + expect(formOpened).toBe(false) + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(result).toContain('plan mode is active') + }) + + it('run_script runs the arguments the user submitted, not the ones proposed', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/greet', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/greet', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async () => ({ name: 'Grace' }) } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/scripts/greet', + requestBody: { name: 'Grace' }, + skipPreprocessor: true + }) + // The model must not assume its proposal is what ran. + expect(result).toContain('Ran with arguments: {"name":"Grace"}') + }) + + // The arguments are already in the call this result answers, and every way the form's + // own differ from the proposed ones has its own clause — so an untouched form has + // nothing to name, and naming it anyway pays for the copy on every later iteration. + it('run_script names the arguments only when the user changed them', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/greet', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/greet', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, form) => form.args } + ) + ) + + expect(result).not.toContain('Ran with arguments') + expect(result).toContain('unedited') + }) + it('test_run_step lists nested step ids when a step is not found', async () => { await callGlobalTool('write_flow', { path: 'f/flows/nested-step-error', @@ -5323,6 +6105,9 @@ describe('session-only preview tools gating', () => { expect(names).not.toContain('list_app_runs') expect(names).not.toContain('search_dom') expect(names).not.toContain('read_dom') + // Not withheld: without it the side panel's only route to a deployed run is the raw + // endpoint, which confirms an opaque request body instead of the arguments. + expect(names).toContain('run_script') // other tools are still present expect(names).toContain('write_script') }) @@ -5335,6 +6120,7 @@ describe('session-only preview tools gating', () => { expect(names).toContain('list_app_runs') expect(names).toContain('search_dom') expect(names).toContain('read_dom') + expect(names).toContain('run_script') // The session set is the full globalTools minus capability-gated tools: // this environment is not Chromium, so take_screenshot is withheld (DOM // capture is only faithful on Blink). search_dom / read_dom are not gated. @@ -5437,13 +6223,16 @@ describe('session-only preview tools gating', () => { }) describe('read_skill', () => { - it('refuses a path the user has not selected, without reading it', async () => { + // Every path is enabled by default now, so the listing is what keeps the tool to + // skills: without it the model could name any resource holding a string `content` + // and have it read back. + it('refuses a path that is not a skill in the workspace, without reading it', async () => { localStorage.clear() userStore.set({ username: 'bob', email: 'bob@windmill.dev', workspace_id: WORKSPACE } as any) const res = await callGlobalTool('read_skill', { path: 'u/someone/private-notes' }) - expect(res).toContain('not one of the skills selected') + expect(res).toContain('not one of the skills available') expect(vi.mocked(ResourceService.getResourceValue)).not.toHaveBeenCalled() }) }) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index e919769e7a..adb4f4e6e6 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -20,6 +20,7 @@ import { WebsocketTriggerService } from '$lib/gen' import { createTwoFilesPatch } from 'diff' +import { deepEqual } from 'fast-equals' import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' import { $ScriptLang } from '$lib/gen/schemas.gen' import type { @@ -47,6 +48,16 @@ import { STARTER_RUNNABLE_KEY, type FrameworkKey } from '$lib/components/raw_apps/templates' +import { + coerceArgsToSchema, + dropUndeclaredArgs, + enforceDisabledDefaults, + redactFileArgs, + redactSecretArgs, + stripFileArgs +} from '$lib/components/job_args' +import { processSecretArgs } from '$lib/components/secretArgUtils' +import { PLAN_MODE_MESSAGES } from '../planModeMessages' import { DEFAULT_DATA as DEFAULT_RAW_APP_DATA } from '$lib/components/raw_apps/dataTableRefUtils' import { appSourceToDraftValue } from '$lib/components/raw_apps/rawAppDraftValue' import type { RawAppDomQuery } from '$lib/components/raw_apps/rawAppDom' @@ -83,7 +94,7 @@ import { import { searchNpmPackagesTool } from '../script/core' import type { McpServer } from './mcpTools' import { logFeatureUsage } from '$lib/utils/featureUsage' -import { enabledSkillPaths } from '../skills/enabledSkills' +import { isSkillEnabled } from '../skills/enabledSkills' import { listSkillResources, readSkillBody, @@ -120,6 +131,7 @@ import { isHubPath, type CreatedResourceTriggerKind, type PreviewCardKind, + type RunFormDisplay, type Tool, type ToolCallbacks, type ToolDisplayAction @@ -276,6 +288,8 @@ const ACTIVE_GLOBAL_EDITOR_DRAFTS: readonly { export type GlobalActiveEditorContext = { type: ActiveGlobalEditorType path: string + /** The key the draft is stored under, which `path` leaves behind on a rename. */ + storagePath: string isLiveDraft: true } @@ -889,7 +903,21 @@ const testRunScriptSchema = z.object({ const testRunScriptToolDef = createToolDef( testRunScriptSchema, 'test_run_script', - 'Execute a preview-style test run of a script by path, preferring draft content when it exists.', + 'Execute a preview-style test run of a script by path, preferring draft content when it exists. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.', + { strict: false } +) + +const runScriptSchema = z.object({ + path: z.string().describe('Workspace path of the deployed script to run.'), + args: testRunArgsSchema, + background: backgroundArgSchema, + wait_seconds: waitSecondsArgSchema +}) + +const runScriptToolDef = createToolDef( + runScriptSchema, + 'run_script', + 'Run a DEPLOYED script for real, under the user\'s own permissions. Fill in every argument you can infer: the user gets an argument form prefilled with `args` and decides what runs. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call. A required file is the user\'s to attach, so call this even when you cannot supply one rather than asking in chat. Use only when the user names the deployed version ("the deployed X", "in production", "for real"); otherwise use test_run_script.', { strict: false } ) @@ -903,7 +931,7 @@ const testRunFlowSchema = z.object({ const testRunFlowToolDef = createToolDef( testRunFlowSchema, 'test_run_flow', - 'Execute a preview-style test run of a flow by path, preferring draft content when it exists.', + 'Execute a preview-style test run of a flow by path, preferring draft content when it exists. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.', { strict: false } ) @@ -1322,7 +1350,7 @@ ${pipelineBullet} : ' Pass items (":" entries naming the items you changed) so the review is scoped to them — omitting items preselects every pending change in the workspace' }, or mode ("draft" or "fork") to force which comparison is shown. Prefer offering this review page over calling deploy_workspace_item directly when several items changed. - For a Windmill operation no other tool covers (workers, queue state, a run's args, ...), use search_api_endpoints to find a REST endpoint, then call_api_get for reads or call_api_endpoint for mutations (the user is asked to confirm those). Always prefer a dedicated tool when one exists; endpoints for authoring or deleting scripts, flows, apps, schedules, resources, or variables are not available through the API catalog tools — use the draft tools and delete_workspace_item instead. -- runScriptByPath / runFlowByPath from the API catalog run the DEPLOYED version of an item. Use them only when the user explicitly asks to run the deployed version, and read the item with read_workspace_item version: "deployed" first so the arguments match the deployed input schema (a draft may have different inputs). To test something you are editing or just wrote, always use test_run_script, test_run_flow, or test_run_step — they run the draft. +- Default to test_run_script, test_run_flow, or test_run_step for any run request, an existing script included; they prefer drafts and need no deployment. Use run_script only when the user names the deployed version ("the deployed X", "in production", "for real") — a bare "run X" is not that. For run_script, read the item with read_workspace_item version: "deployed" first so the arguments match the deployed schema, and fill in every one you can infer. test_run_script, run_script and test_run_flow all show the user an argument form prefilled with what you sent, so fill in every argument you can infer rather than asking for it in chat. runFlowByPath from the API catalog is the exception — it runs a deployed flow with no form at all: only for a flow the user asked to run deployed. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. Set multiSelect: true only when the answers can genuinely co-apply and the user may pick several (not mutually exclusive). - When the user asks you to remember a lasting preference, always/never do something, or change/stop a behavior going forward, call update_user_instructions to persist it. It edits only the USER INSTRUCTIONS block (not WORKSPACE INSTRUCTIONS). Keep each instruction concise; do not use it for one-off requests scoped to the current task. - Keep context targeted.${ @@ -1373,7 +1401,7 @@ Data Tables: ? ` Skills: -- Skills are reusable instruction sets the user selected for this chat, each covering a specific kind of task. The available skills are listed below by resource path and description. +- Skills are reusable instruction sets available in this workspace, each covering a specific kind of task. The available skills are listed below by resource path and description. - When a user's request matches a skill's description, call read_skill with its exact path to load the full instructions BEFORE acting, then follow them. ${skills.map((s) => `- ${s.path}: ${s.description}`).join('\n')}` : '' @@ -2340,23 +2368,17 @@ export type ChatCommandItem = { } /** - * The skills this user turned on in this workspace, for the global system prompt. - * A readable `ai_skill` resource is only a candidate — enabling one is a personal - * choice, since each enabled skill spends context on every turn. + * The skills in play in this workspace, for the global system prompt: every + * readable `ai_skill` resource except the ones this user turned off. */ export async function loadWorkspaceSkills(workspace: string): Promise { if (!workspace) return [] try { - const enabled = new Set(enabledSkillPaths(workspace)) - if (enabled.size === 0) return [] - // Filtered against what is actually readable now, so a skill that was - // deleted or whose folder access was revoked drops out instead of being - // advertised to the model as something read_skill can load. // A truncated listing still carries most of the workspace, and the drawer is // where that is surfaced; dropping everything here would silently empty the // Skills section instead. return (await listSkillResources(workspace)).skills - .filter((s) => enabled.has(s.path)) + .filter((s) => isSkillEnabled(workspace, s.path)) .map(({ path, name, description }) => ({ path, name, @@ -2382,24 +2404,28 @@ export const readSkillTool: Tool<{}> = { def: createToolDef( readSkillSchema, 'read_skill', - 'Load the full instructions for a selected AI skill by resource path. Skills are listed in the system prompt under "Skills"; call this before acting on a task a skill covers, then follow its instructions.' + 'Load the full instructions for an AI skill by resource path. Skills are listed in the system prompt under "Skills"; call this before acting on a task a skill covers, then follow its instructions.' ), planModeSafe: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = readSkillSchema.parse(args) const name = skillNameFromPath(parsed.path) - // The prompt lists only selected skills, but the tool takes a path the model - // composed, so the selection is enforced here too rather than assumed. Without - // it the tool reads any resource holding a string `content` — the user's own - // access, but not what "load a selected skill" says it does. - if (!enabledSkillPaths(workspace).includes(parsed.path)) { - toolCallbacks.setToolStatus(toolId, { content: `Skill "${name}" is not selected` }) - return `"${parsed.path}" is not one of the skills selected for this chat. Only the paths listed under "Skills" in the system prompt can be read.` - } toolCallbacks.setToolStatus(toolId, { content: `Reading skill "${name}"...` }) try { - // Bounded here rather than in the reader: any `ai_skill` resource can be - // selected, including ones written through git sync or the resource editor + // The prompt lists the skills in play, but the tool takes a path the model + // composed, so what may be read is checked here rather than assumed. Against + // the listing, not just the off-switch: any other path is now enabled too, + // and without this the tool reads any resource holding a string `content` — + // the user's own access, but not what "load a skill" says it does. + const isSkill = (await listSkillResources(workspace)).skills.some( + (s) => s.path === parsed.path + ) + if (!isSkill || !isSkillEnabled(workspace, parsed.path)) { + toolCallbacks.setToolStatus(toolId, { content: `Skill "${name}" is not available` }) + return `"${parsed.path}" is not one of the skills available to this chat. Only the paths listed under "Skills" in the system prompt can be read.` + } + // Bounded here rather than in the reader: any `ai_skill` resource is in play, + // including ones written through git sync or the resource editor // that never passed the authoring form's limits, and an unbounded body // would exhaust the context on one tool call. The editor reads the same // resource untruncated, so opening a long skill cannot rewrite it short. @@ -2408,7 +2434,7 @@ export const readSkillTool: Tool<{}> = { MAX_SKILL_INSTRUCTIONS_LENGTH ) toolCallbacks.setToolStatus(toolId, { content: `Read skill "${name}"` }) - // Whether a selected skill is actually reached for. No key: the path is + // Whether a skill is actually reached for. No key: the path is // workspace-authored text. logFeatureUsage('ai_session', 'skill_read', { workspace }) return `Skill: ${parsed.path}\n\nInstructions:\n${instructions}` @@ -3639,20 +3665,41 @@ export const globalTools: Tool<{}>[] = [ const parsed = testRunScriptSchema.parse(ctx.args) return testRunScriptByPath(parsed, ctx) }, - requiresConfirmation: true, - confirmationMessage: (args) => `Run a test of ${pathLeaf(args?.path, 'the script')}`, + // No requiresConfirmation: the argument form is the confirmation, and the bypass posture + // answers it with what the form opened with — a decision made for the user, so the + // posture's own list has to name it. One thing does run before Run: see the note on + // the form's SchemaForm. + bypassedByAutoAccept: true, + confirmationMessage: 'Run a test of a script', + streamingLabel: 'Preparing the test form...', queuedLabel: (args) => `Test ${args?.path ?? 'the script'}`, showDetails: true, autoCollapseDetails: false }, + { + def: runScriptToolDef, + fn: async (ctx) => { + const parsed = runScriptSchema.parse(ctx.args) + return runDeployedScript(parsed, ctx) + }, + // No requiresConfirmation, for the reason test_run_script carries. + bypassedByAutoAccept: true, + confirmationMessage: 'Run a deployed script', + streamingLabel: 'Preparing the run form...', + queuedLabel: (args) => `Run ${args?.path ?? 'a script'}`, + showDetails: true, + autoCollapseDetails: false + }, { def: testRunFlowToolDef, fn: async (ctx) => { const parsed = testRunFlowSchema.parse(ctx.args) return testRunFlowByPath(parsed, ctx) }, - requiresConfirmation: true, - confirmationMessage: (args) => `Run a test of ${pathLeaf(args?.path, 'the flow')}`, + // No requiresConfirmation, for the reason test_run_script carries. + bypassedByAutoAccept: true, + streamingLabel: 'Preparing the test form...', + confirmationMessage: 'Run a test of a flow', queuedLabel: (args) => `Test ${args?.path ?? 'the flow'}`, showDetails: true, autoCollapseDetails: false @@ -4282,8 +4329,8 @@ export const SESSION_PREVIEW_TOOL_NAMES = new Set([ /** * The global tool set for a given chat: the full `globalTools` for a session - * chat, or `globalTools` minus the session-only preview tools for the regular - * global side-panel chat. + * chat, or `globalTools` minus the preview tools for the regular global + * side-panel chat. */ export function globalToolsFor({ sessionPreview }: { sessionPreview: boolean }): Tool<{}>[] { const tools = sessionPreview @@ -4324,7 +4371,8 @@ type WriteDraftCtx = { export type SessionToolHelpers = { sessionId?: string } export type GlobalToolHelpers = SessionToolHelpers & { - testActiveFlow?: (args?: Record) => Promise + /** Runs the flow editor mounted on `storagePath`, if one is. */ + testActiveFlow?: (storagePath: string, args?: Record) => Promise attachedFiles?: AttachedFilesStore // Read/write the user-level Global instructions. `setUserInstructions` persists the // value and rebuilds the system message so the change applies on the next chat-loop @@ -4353,15 +4401,21 @@ function operatingWorkspaceFromHelpers(helpers: unknown): string | undefined { return (helpers as GlobalToolHelpers | undefined)?.operatingWorkspace } -function activeFlowTestFromCtx( +// Drive a live editor only for the flow on screen: several can be open at once (session tabs), +// and a run painted into a background tab is a side effect the user never sees. Undefined +// sends the caller to a preview run, which reports into the chat alone. The hook is bound to +// that editor's storage path — the same key reads and edits of `path` resolve through, so a +// staged rename cannot send the run to a different editor than the one being edited. +function liveFlowTestHookFromCtx( ctx: { workspace: string; helpers?: unknown }, path: string -): GlobalToolHelpers['testActiveFlow'] | undefined { +): ((args?: Record) => Promise) | undefined { const activeEditor = getActiveGlobalEditorContext(ctx.workspace) if (activeEditor?.type !== 'flow' || activeEditor.path !== path) { return undefined } - return (ctx.helpers as GlobalToolHelpers | undefined)?.testActiveFlow + const testActiveFlow = (ctx.helpers as GlobalToolHelpers | undefined)?.testActiveFlow + return testActiveFlow && ((args) => testActiveFlow(activeEditor.storagePath, args)) } export type OpenPreviewHandler = (req: { @@ -5116,16 +5170,52 @@ function writeVariableDraft(args: WriteVariableArgs, ctx: WriteDraftCtx): Promis async function loadScriptForEdit( path: string, workspace: string -): Promise<{ content: string; language: ScriptLang; summary?: string }> { +): Promise<{ + content: string + language: ScriptLang + summary?: string + schema?: Record +}> { const draft = await getGlobalDraft(workspace, 'script', path) if (draft) { if (typeof draft.value !== 'string' || !draft.language) { throw new Error(`Draft script "${path}" is missing content or language.`) } - return { content: draft.value, language: draft.language, summary: draft.summary } + return { + content: draft.value, + language: draft.language, + summary: draft.summary, + schema: draft.schema as Record | undefined + } } const script = await ScriptService.getScriptByPath({ workspace, path }) - return { content: script.content, language: script.language, summary: script.summary } + return { + content: script.content, + language: script.language, + summary: script.summary, + schema: script.schema as Record | undefined + } +} + +/** The fields a test form offers, for code that may never have been deployed. A draft the + * chat wrote carries the schema it inferred at write time; anything else — a draft written + * elsewhere, a deployed script whose schema predates an edit — is inferred here from the + * content that is about to run, so the form cannot offer a field the code no longer takes. */ +async function schemaForTestRun(script: { + content: string + language: ScriptLang + schema?: Record +}): Promise> { + // Emptily declared is not declared: a stored `properties: {}` means the schema predates + // the arguments the code now takes, so infer rather than offer a form with no fields. + if (Object.keys(script.schema?.properties ?? {}).length > 0) return script.schema! + const schema = emptySchema() + try { + await inferArgs(script.language, script.content, schema) + } catch (e) { + console.error('Failed to infer script schema for the test run form', e) + } + return schema as unknown as Record } async function editScript( @@ -5370,90 +5460,389 @@ async function testRunScriptByPath( args: z.infer, ctx: WriteDraftCtx ): Promise { - const { workspace, toolId, toolCallbacks } = ctx + const { workspace } = ctx const script = await loadScriptForEdit(args.path, workspace) - const testArgs = normalizeTestRunArgs(args.args) + const schema = await schemaForTestRun(script) - return executeTestRun({ - jobStarter: () => - JobService.runScriptPreview({ - workspace, - requestBody: { - path: args.path, - content: script.content, - args: testArgs, - language: script.language - } - }), + return runThroughForm( + { + path: args.path, + schema, + summary: script.summary, + kind: 'test', + code: script.content, + lang: script.language, + // Never "deployed" here: the code about to run is the draft the model is still + // writing, and a line telling it to re-read the deployed schema would send it + // to the wrong version. + schemaNoun: 'script', + toolName: 'test_run_script', + proposed: args.args, + startMessage: `Running test for script "${args.path}"...`, + contextName: 'script', + // Its own loop: the model is told to test and iterate, so the posture answers the + // form with what it opened with rather than parking the loop on a card. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: (submitted) => + JobService.runScriptPreview({ + workspace, + requestBody: { + path: args.path, + content: script.content, + args: submitted, + language: script.language + } + }) + }, + ctx + ) +} + +/** The "do not call again" half is load-bearing: without it the model re-proposes the + * call, which re-opens the form the user just dismissed, and Stop becomes their only + * way out. */ +const runFormCancelled = (toolName: string, noun: string) => + `The user cancelled the run form. The ${noun} did NOT run. Do not call ${toolName} again unless the user asks for it.` + +/** The model only needs to see what the user changed, and nothing bounds an object or + * array argument the form let them paste into. */ +const MAX_SUBMITTED_ARGS_LENGTH = 4000 + +/** The card's own copy is bounded separately, and far higher: it is what the details pane + * renders, and JobArgs stops rendering the JSON in full at this size regardless. */ +const MAX_PERSISTED_ARGS_LENGTH = 100_000 + +/** One run through an argument form: conform what the model proposed to the schema of the + * version about to run, open the form on it, then run whatever came back. Both tools that + * run a script are this, differing only in where the schema comes from and how the job + * starts — so the user meets one card whichever they asked for. */ +type FormRunSpec = { + path: string + schema: Record + summary?: string + kind: 'run' | 'test' + /** The code a test run is about to preview, so its form can offer the same dynamic-option + * pickers the script editor's test panel does. Omitted for a deployed run, which names a + * path instead. */ + code?: string + lang?: ScriptLang + /** How the lines the model reads back name the version this ran: telling it to re-read + * the "deployed schema" of a draft would send it to the wrong code. */ + schemaNoun: string + toolName: string + proposed: Record | null | undefined + startMessage: string + contextName: 'script' | 'flow' + /** Whether the bypass posture may answer this form with what it opened with. */ + autoAcceptable?: boolean + background?: boolean + detachAfterMs?: number + startJob: (submitted: Record) => Promise +} + +async function runThroughForm(spec: FormRunSpec, ctx: WriteDraftCtx): Promise { + const { workspace, toolId, toolCallbacks } = ctx + // Asked of the posture, not of the tool: every run tool is auto-acceptable, so a + // host with no form would otherwise run one on the model's arguments alone, in any + // posture. What a bypass answers is a decision the user already made; without it there + // is no consent to be had here and nothing to fall back on. + const postureAnswers = Boolean( + spec.autoAcceptable && toolCallbacks.shouldAutoAcceptToolConfirmations?.(spec.toolName) + ) + if (!toolCallbacks.requestRunArgs && !postureAnswers) { + return `This chat cannot show a run form, so a ${spec.contextName} cannot be run from here.` + } + + // processToolCall gates plan mode once, before the schema fetch, and this form is its own + // confirmation so it never reaches that gate again. Repeated wherever a write follows: a + // mounted field mints on its own, which no later gate can unmake. + const blockedByPlanMode = (): string | undefined => { + if (!toolCallbacks.isPlanModeActive?.()) return undefined + toolCallbacks.onToolBlockedByPlanMode?.() + toolCallbacks.setToolStatus(toolId, { + content: PLAN_MODE_MESSAGES.blockedLabel, + isLoading: false, + isStreamingArguments: false, + error: PLAN_MODE_MESSAGES.blockedResult, + blockedByPlanMode: true + }) + return PLAN_MODE_MESSAGES.blockedResult + } + const blockedBeforeForm = blockedByPlanMode() + if (blockedBeforeForm) return blockedBeforeForm + + const schema = spec.schema + // Whether to ask is the only question decided here. What a mounted field would hold — a + // default, a synthesised empty, whether Run lights up — is the form's own business: any + // second derivation of it here can start a run the form itself would refuse. + const autoAccepted = postureAnswers + const strippedKeys: string[] = [] + const coerced = autoAccepted + ? undefined + : coerceArgsToSchema(normalizeTestRunArgs(spec.proposed), schema) + let proposed: Record + let resetKeys: string[] + let undeclaredKeys: string[] + if (coerced) { + resetKeys = coerced.resetKeys + undeclaredKeys = coerced.undeclaredKeys + // Left as the model proposed it, minted by the widget the field mounts: a reference put + // here instead would be normalised away by the nested form an object secret renders as. + proposed = stripFileArgs(coerced.args, schema as any, strippedKeys) + } else { + // Both rules hold against every caller, not only the ones a form stands in front of: + // an undeclared argument has no field anywhere, and a disabled one is nobody's to set. + // Without the rest of the coercion, which answers what a mounted widget would show. + const declared = dropUndeclaredArgs(normalizeTestRunArgs(spec.proposed), schema) + undeclaredKeys = declared.undeclaredKeys + const enforced = enforceDisabledDefaults(declared.args, schema) + resetKeys = enforced.resetKeys + // In the widget's stead: with no form there is no PasswordArgInput to turn a proposed + // secret into a reference, and the job's arguments outlive the run. + try { + proposed = await processSecretArgs(enforced.args, schema as any, workspace) + } catch (e) { + const message = `Failed to store the sensitive arguments of "${spec.path}": ${e}` + toolCallbacks.setToolStatus(toolId, { + content: message, + isLoading: false, + isStreamingArguments: false, + error: message + }) + return message + } + } + const form: RunFormDisplay = { + path: spec.path, + summary: spec.summary || undefined, + kind: spec.kind, + runnableKind: spec.contextName, + schema: autoAccepted ? undefined : schema, + code: autoAccepted ? undefined : spec.code, + lang: autoAccepted ? undefined : spec.lang, + submitted: autoAccepted || undefined, + args: proposed, + clearedKeys: coerced?.clearedKeys.length ? coerced.clearedKeys : undefined, + resetKeys: resetKeys.length ? resetKeys : undefined, + strippedKeys: strippedKeys.length ? strippedKeys : undefined + } + + // Files only: nothing rewrites `runForm.args` after this, so bytes left in it outlive the + // size guard that covers `parameters`. + const persisted = { ...form, args: redactFileArgs(proposed, schema as any) } + + toolCallbacks.setToolStatus(toolId, { + content: autoAccepted + ? spec.startMessage + : `Waiting for you to confirm the arguments of "${spec.path}"`, + runForm: persisted, + // Not the raw tool-call arguments: the card settles on what the form opened with. + // Only settles it — the raw proposal still renders while the call streams in. + parameters: persisted.args, + isLoading: true + }) + + // `form`, not `persisted`: a password field mints from what it opens with. + const submitted = toolCallbacks.requestRunArgs + ? await toolCallbacks.requestRunArgs(toolId, form, { autoAccepted }) + : proposed + if (!submitted) { + toolCallbacks.setToolStatus(toolId, { + content: `Run of "${spec.path}" cancelled by user`, + isLoading: false, + isStreamingArguments: false, + error: 'Cancelled by user', + declinedByUser: true + }) + return runFormCancelled(spec.toolName, spec.contextName) + } + + const blockedBeforeRun = blockedByPlanMode() + if (blockedBeforeRun) return blockedBeforeRun + + // Every job leaves through here, so this is where a sensitive argument becomes a reference: + // the form mints as the user types and the bypass mints in its stead, but a host answering + // the form its own way — the eval harness does — would hand over a literal. Idempotent, so + // the two that already minted pay a walk and no round trip. + let toRun: Record + try { + toRun = await processSecretArgs(submitted, schema as any, workspace) + } catch (e) { + const message = `Failed to store the sensitive arguments of "${spec.path}": ${e}` + toolCallbacks.setToolStatus(toolId, { + content: message, + isLoading: false, + isStreamingArguments: false, + error: message + }) + return message + } + + // The card's details pane must show what ran, not what was proposed. Bytes are marked by + // size because the card is persisted; everything else stands as the run page shows it for + // the same job. + const forCard = redactFileArgs(toRun, schema as any) + // The transcript is re-cloned into IndexedDB on every save and a form carries whatever was + // pasted into it, so past what the pane would render the card reads the arguments off the + // job instead. Only once there is a job to read them from: substituting the marker any + // earlier would leave a run that never started showing nothing but the marker. + const oversized = JSON.stringify(forCard).length > MAX_PERSISTED_ARGS_LENGTH + if (!oversized) { + toolCallbacks.setToolStatus(toolId, { parameters: forCard }) + } + + const outcome = await executeTestRun({ + jobStarter: async () => { + const jobId = await spec.startJob(toRun) + // The form's own submitted flag flips a round trip earlier, when the user presses + // Run; only from here is there a job for a stopped turn to say it left running. + toolCallbacks.markRunFormStarted?.(toolId) + if (oversized) { + toolCallbacks.setToolStatus(toolId, { parameters: { reason: 'WINDMILL_TOO_BIG' } }) + } + return jobId + }, workspace, toolCallbacks, toolId, - startMessage: `Running test for script "${args.path}"...`, - contextName: 'script', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path + startMessage: spec.startMessage, + contextName: spec.contextName, + actionNoun: spec.kind === 'test' ? 'test' : 'run', + background: spec.background, + detachAfterMs: spec.detachAfterMs, + label: spec.path }) + + const schemaNoun = `${spec.schemaNoun} schema` + // Only what the form could make no reading of: a wrong-typed value it can read is + // converted silently, since the field then shows what the run carries and there is + // nothing to report. + const clearedKeys = coerced?.clearedKeys ?? [] + const cleared = clearedKeys.length + ? `\nThe ${schemaNoun} declares ${clearedKeys.join(', ')}, but you sent ${clearedKeys.length > 1 ? 'them in shapes' : 'it in a shape'} with no reading in the declared ${clearedKeys.length > 1 ? 'types' : 'type'}, so the ${clearedKeys.length > 1 ? 'fields opened' : 'field opened'} empty and the run did not carry ${clearedKeys.length > 1 ? 'them' : 'it'}. Re-read the input schema and match ${clearedKeys.length > 1 ? 'their declared types' : 'its declared type'}.` + : '' + const reset = resetKeys.length + ? `\nThe ${schemaNoun} disables ${resetKeys.join(', ')}, so the run used ${resetKeys.length > 1 ? 'their defaults' : 'its default'} rather than the proposed ${resetKeys.length > 1 ? 'values' : 'value'}. Do not propose ${resetKeys.length > 1 ? 'them' : 'it'} again.` + : '' + // Nothing renders these, so the model is the only one who can be told they went nowhere. + const undeclared = undeclaredKeys.length + ? `\nThe ${schemaNoun} does not declare ${undeclaredKeys.join(', ')}, so ${undeclaredKeys.length > 1 ? 'they were' : 'it was'} not sent — no run form in Windmill offers a field the schema does not name. Re-read the input schema and use the arguments it declares.` + : '' + // Otherwise an emptied field reads as the user having deleted it, and the next call + // proposes the same bytes again. + const stripped = strippedKeys.length + ? `\n${strippedKeys.join(', ')} ${strippedKeys.length > 1 ? 'are file arguments' : 'is a file argument'}, so the form opened ${strippedKeys.length > 1 ? 'them' : 'it'} empty for the user to attach. ${strippedKeys.length > 1 ? 'They are' : 'It is'} theirs to provide, not yours: do not propose ${strippedKeys.length > 1 ? 'them' : 'it'} again.` + : '' + // Redacted for the model alone: what it proposed is already in its own tool call, but a + // secret the user typed into the form would be entering its context here. + const redacted = redactFileArgs(redactSecretArgs(toRun, schema as any), schema as any) + const submittedJson = JSON.stringify(redacted) + const shown = + submittedJson.length > MAX_SUBMITTED_ARGS_LENGTH + ? submittedJson.slice(0, MAX_SUBMITTED_ARGS_LENGTH) + '... (truncated)' + : submittedJson + // Naming them costs a copy of arguments already in the call above, and cleared/reset/ + // stripped name every way the form's own differ from the proposed ones — so only what + // the user changed is news. + const ran = deepEqual(redacted, proposed) + ? 'Ran with the arguments the form opened with, unedited.' + : `Ran with arguments: ${shown}` + return `${ran}${cleared}${reset}${stripped}${undeclared}\n${outcome}` +} + +async function runDeployedScript( + args: z.infer, + ctx: WriteDraftCtx +): Promise { + const { workspace } = ctx + // No getDraft: this runs the script as it is live, so the form has to offer the + // inputs the live version accepts and not a draft's. + const script = await ScriptService.getScriptByPath({ workspace, path: args.path }) + return runThroughForm( + { + path: args.path, + schema: (script.schema as Record) ?? {}, + summary: script.summary, + kind: 'run', + schemaNoun: 'deployed', + toolName: 'run_script', + proposed: args.args, + startMessage: `Running "${args.path}"...`, + contextName: 'script', + // Bypassable like a test run: the posture is the user's standing answer, and a form + // it parks on is a card nobody is watching. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: (submitted) => + JobService.runScriptByPath({ + workspace, + path: args.path, + requestBody: submitted, + // As the script's own run page does: the form fills the main input schema, and a + // preprocessor would take these arguments for a webhook body and hand the script + // its own output instead. + skipPreprocessor: true + }) + }, + ctx + ) } async function testRunFlowByPath( args: z.infer, ctx: WriteDraftCtx ): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const testArgs = normalizeTestRunArgs(args.args) - const testActiveFlow = activeFlowTestFromCtx(ctx, args.path) + const { workspace } = ctx + // The schema must be in hand before the form is built, and the value rides along from the + // same read so the fields and the previewed flow are one version. With an editor open on + // this path this reads its in-memory cell rather than the network. + const flow = await loadFlowDraftValue(args.path, workspace) + const schema = (flow.flow.schema as Record | null | undefined) ?? {} - if (testActiveFlow) { - return executeTestRun({ - jobStarter: async () => { - const jobId = await testActiveFlow(testArgs) + return runThroughForm( + { + path: args.path, + schema, + summary: flow.summary, + kind: 'test', + // A flow's dynamic-option pickers are one script stored on the schema itself, which is + // where the editor's own test form reads them from (FlowPreviewContent). + code: schema['x-windmill-dyn-select-code'], + lang: schema['x-windmill-dyn-select-lang'], + // Never "deployed": a test run previews the draft, so a line sending the model to the + // deployed schema would name the wrong version. + schemaNoun: 'flow', + toolName: 'test_run_flow', + proposed: args.args, + startMessage: `Starting flow test run for "${args.path}"...`, + contextName: 'flow', + // The model is told to test and iterate, so the bypass posture answers the form. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: async (submitted) => { + // An open editor runs its own in-memory flow and paints the run in its graph. + // Resolved here rather than before the form: the form waits as long as the user + // does, and the editor on screen when they press Run is the one it belongs in. + const jobId = await liveFlowTestHookFromCtx(ctx, args.path)?.(submitted) if (jobId) { return jobId } - - const flow = await loadFlowDraftValue(args.path, workspace) return JobService.runFlowPreview({ workspace, requestBody: { path: args.path, value: flowDraftValueForPreview(flow.flow), - args: testArgs + args: submitted } }) - }, - workspace, - toolCallbacks, - toolId, - startMessage: `Starting flow test run for "${args.path}"...`, - contextName: 'flow', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path - }) - } - - const flow = await loadFlowDraftValue(args.path, workspace) - - return executeTestRun({ - jobStarter: () => - JobService.runFlowPreview({ - workspace, - requestBody: { - path: args.path, - value: flowDraftValueForPreview(flow.flow), - args: testArgs - } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting flow test run for "${args.path}"...`, - contextName: 'flow', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path - }) + } + }, + ctx + ) } async function testRunFlowStepByPath( @@ -7798,9 +8187,10 @@ export function getActiveGlobalEditorContext( ): GlobalActiveEditorContext | undefined { for (const { itemKind, type } of ACTIVE_GLOBAL_EDITOR_DRAFTS) { const liveDraft = UserDraft.getLiveEditorDraft(itemKind, { workspace }) - const path = liveDraft?.effectivePath || liveDraft?.storagePath + if (!liveDraft) continue + const path = liveDraft.effectivePath || liveDraft.storagePath if (!path) continue - return { type, path, isLiveDraft: true } + return { type, path, storagePath: liveDraft.storagePath, isLiveDraft: true } } } diff --git a/frontend/src/lib/components/copilot/chat/planModeMessages.ts b/frontend/src/lib/components/copilot/chat/planModeMessages.ts index 0b502ba839..85710d0c54 100644 --- a/frontend/src/lib/components/copilot/chat/planModeMessages.ts +++ b/frontend/src/lib/components/copilot/chat/planModeMessages.ts @@ -12,6 +12,9 @@ export const PLAN_MODE_MESSAGES = { /** Sits beside the autonomy picker while plan mode holds. The picker's tooltip carries * the rest, so this states only the constraint. */ modeNote: 'Read-only', + /** Refuses a pending run form. Its own string because nothing is settled: the form stays + * live, so this names the way out rather than telling the user their run was blocked. */ + runFormRefused: 'Plan mode is read-only — switch it off to run this script.', // One pair for both artifact tools: the fact and the way forward are the same whether the // model tried to mint the plan or to rewrite it, and the generic refusal above ("put this // change in your plan") reads as nonsense for a call that writes a document. diff --git a/frontend/src/lib/components/copilot/chat/scrollFades.svelte.ts b/frontend/src/lib/components/copilot/chat/scrollFades.svelte.ts new file mode 100644 index 0000000000..0deeae535d --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/scrollFades.svelte.ts @@ -0,0 +1,56 @@ +/** + * Live "is there more past this edge" for one scroll region. Measured, not assumed: these + * boxes change height under a still scroll offset — a tool result streams in, a dynamic + * field fills its options. Put `container` on the scrolling element with + * `onscroll={measure}`, and `content` on the element inside it whose height moves. + */ +export function scrollFades() { + let node: HTMLElement | undefined = undefined + let top = $state(false) + let bottom = $state(false) + // Built on first attach, never at call time: this runs during component init, where + // ResizeObserver does not exist on the server. + let observer: ResizeObserver | undefined = undefined + + function measure() { + if (!node) { + top = false + bottom = false + return + } + top = node.scrollTop > 1 + bottom = node.scrollHeight - node.scrollTop - node.clientHeight > 1 + } + + function observe(el: HTMLElement) { + observer ??= new ResizeObserver(measure) + observer.observe(el) + return { + destroy() { + observer?.unobserve(el) + } + } + } + + return { + get top() { + return top + }, + get bottom() { + return bottom + }, + measure, + container(el: HTMLElement) { + node = el + measure() + const handle = observe(el) + return { + destroy() { + handle.destroy() + node = undefined + } + } + }, + content: observe + } +} diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index c734691e29..768b6d1b65 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -38,7 +38,7 @@ vi.mock('$lib/components/flows/flowTree', () => ({ vi.mock('$lib/gen', () => ({ ScriptService: {}, FlowService: {}, - JobService: { getJob: vi.fn() }, + JobService: { getJob: vi.fn(), getJobUpdates: vi.fn() }, ScheduleService: { previewSchedule: vi.fn(), createSchedule: vi.fn() @@ -945,6 +945,16 @@ describe('processToolCall', () => { { requestConfirmation: vi.fn().mockResolvedValue(false) } ) ).toEqual([['ai_chat', 'tool', 'run_script:declined']]) + // A tool that runs its own consent surface (the run form) declines by settling + // the card, then returns normally — that must not read as a successful run. + expect( + await outcomeKeys({ + fn: vi.fn(async ({ toolCallbacks, toolId }: any) => { + toolCallbacks.setToolStatus(toolId, { declinedByUser: true }) + return 'cancelled' + }) + }) + ).toEqual([['ai_chat', 'tool', 'run_script:declined']]) expect(await outcomeKeys({}, { isPlanModeActive: () => true })).toEqual([ ['ai_chat', 'tool', 'run_script:blocked_plan_mode'] ]) @@ -1272,6 +1282,28 @@ describe('isActiveUserQuestion', () => { }) }) +describe('isActiveRunForm', () => { + const runForm = { path: 'f/a/b', schema: {}, args: { name: 'ada' } } + function toolMessage(overrides: Partial = {}): ToolDisplayMessage { + return { + role: 'tool', + tool_call_id: 'call_r', + content: 'waiting for arguments', + isLoading: true, + runForm, + ...overrides + } + } + + // Either flag unmounts the card, so the loop must stop waiting on it. + it('is false once submitted or cancelled', async () => { + const { isActiveRunForm } = await import('./shared') + expect(isActiveRunForm(toolMessage())).toBe(true) + expect(isActiveRunForm(toolMessage({ runForm: { ...runForm, submitted: true } }))).toBe(false) + expect(isActiveRunForm(toolMessage({ runForm: { ...runForm, canceled: true } }))).toBe(false) + }) +}) + describe('pendingUserAction', () => { const toolMessage = (overrides: Partial = {}): ToolDisplayMessage => ({ role: 'tool', @@ -1289,6 +1321,15 @@ describe('pendingUserAction', () => { expect(pendingUserAction([toolMessage({ needsConfirmation: true })])).toBe('confirmation') }) + // A run form asks for arguments rather than a yes/no, but it blocks the loop the + // same way, so the chat reports it as a confirmation. + it('reports an unsubmitted run form as a confirmation', async () => { + const { pendingUserAction } = await import('./shared') + expect( + pendingUserAction([toolMessage({ runForm: { path: 'f/a/b', schema: {}, args: {} } })]) + ).toBe('confirmation') + }) + it('is undefined for a tool the AI is running on its own', async () => { const { pendingUserAction } = await import('./shared') expect(pendingUserAction([toolMessage()])).toBe(undefined) @@ -1365,6 +1406,9 @@ describe('pollJobCompletion detach', () => { const getJob = vi.mocked(JobService.getJob) getJob.mockReset() getJob.mockResolvedValue({ type: 'QueuedJob', running: true } as any) + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockResolvedValue({ running: true, completed: false } as any) const cbs = makeCallbacks() // detachAfterMs 2000 → 2 polls at 1s each, then detach. @@ -1390,14 +1434,50 @@ describe('pollJobCompletion detach', () => { const { JobService } = await import('$lib/gen') const getJob = vi.mocked(JobService.getJob) getJob.mockReset() - const completed = { type: 'CompletedJob', success: true, result: 42 } + const completed = { type: 'CompletedJob', success: true, result: 42, logs: 'ran' } getJob.mockResolvedValue(completed as any) + // Still landing on a tick the updates endpoint calls unfinished: the job can + // complete between the two calls, and `getJob` is what says so. + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockResolvedValue({ completed: false, running: true } as any) const cbs = makeCallbacks() const promise = pollJobCompletion('job1', 'w', 'tool1', cbs as any, { detachAfterMs: 15000 }) await vi.advanceTimersByTimeAsync(1000) - expect(await promise).toBe(completed) + const landed = await promise + expect(landed).toBe(completed) + // Fetched again with its logs rather than settled on the logless tick fetch, + // which would reach the model as "No logs available". + expect((landed as any).logs).toBe('ran') + } finally { + vi.useRealTimers() + } + }) + + // Streaming rides on a second endpoint; landing the job must not. A failing updates + // endpoint costs live logs, never the run. + it('returns the completed job with its logs when the updates endpoint fails', async () => { + vi.useFakeTimers() + try { + const { pollJobCompletion } = await import('./shared') + const { JobService } = await import('$lib/gen') + const getJob = vi.mocked(JobService.getJob) + getJob.mockReset() + const completed = { type: 'CompletedJob', success: true, result: 42, logs: 'ran' } + getJob.mockResolvedValue(completed as any) + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockRejectedValue(new Error('updates unavailable')) + const cbs = makeCallbacks() + + const promise = pollJobCompletion('job1', 'w', 'tool1', cbs as any, { detachAfterMs: 15000 }) + await vi.advanceTimersByTimeAsync(1000) + + const landed = await promise + expect(landed).toBe(completed) + expect((landed as any).logs).toBe('ran') } finally { vi.useRealTimers() } @@ -1411,6 +1491,9 @@ describe('pollJobCompletion detach', () => { const getJob = vi.mocked(JobService.getJob) getJob.mockReset() getJob.mockResolvedValue({ type: 'QueuedJob', running: true } as any) + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockResolvedValue({ running: true, completed: false } as any) const cbs = makeCallbacks() const promise = pollJobCompletion('job1', 'w', 'tool1', cbs as any) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 6d6794b56c..adb0f63be4 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -553,6 +553,57 @@ export function answeredChoices(q: UserQuestionDisplay): string[] | undefined { return q.selectedChoices ?? (q.selectedChoice ? [q.selectedChoice] : undefined) } +/** Argument form for a deployed-script run, persisted with the transcript — every + * field has to stay plain JSON. */ +export type RunFormDisplay = { + path: string + summary?: string + /** What the run is, in the card's own words: a deployed script run, or a preview of the + * draft being written. Only the tense of the row's label turns on it. */ + kind?: 'run' | 'test' + /** What is being run, for the noun the card says it in. + * Absent on cards recorded before flows had a form, which were all scripts. */ + runnableKind?: 'script' | 'flow' + /** Of whatever version is about to run: the deployed script, or the draft a test + * previews. Only the rendered form reads it, so it is dropped once one of the flags + * below unmounts that form: kept, every settled card would carry a copy of the schema + * — password and file defaults included — in history forever. */ + schema?: Record + /** The script the `dynselect-` helper runs: the draft a script test run previews, since a + * deployed helper would answer for the wrong version, or the one a flow schema carries on + * itself. Dropped with the schema once the form unmounts, so no settled card keeps a copy + * of the code. */ + code?: string + lang?: ScriptLang + /** Prefill only: the card's `parameters` records what the job started with. */ + args: Record + /** Proposed arguments emptied because their declared type had no reading of them. + * Named on the card: an empty field is otherwise the caller having sent nothing. */ + clearedKeys?: string[] + /** Proposed arguments a disabled field overrode with its default. Named for the same + * reason: the field renders locked, so the value it holds is not the proposed one. */ + resetKeys?: string[] + /** File arguments emptied out of the proposal. Named so an empty field reads as the + * caller's value having been removed, not as the field having none. */ + strippedKeys?: string[] + /** Either one unmounts the form, so set exactly one, and only once the loop has + * stopped waiting on this card. */ + submitted?: boolean + canceled?: boolean + /** The job exists. Distinct from `submitted`, which flips a round trip earlier — in + * between, whether the server queued a job is unknown, so a turn stopped there is + * recorded as neither started nor canceled. */ + started?: boolean +} + +/** What a run form is being filled with while it waits. Held by the chat manager, not by + * the form, so the chat card and the preview pane edit one draft rather than two copies. + * The schema rides along because SchemaForm binds and reorders it. */ +export type RunFormDraft = { + args: Record + schema: Record +} + /** One page hit from a provider-side web search (OpenAI sources carry no title). */ export type WebSearchSource = { url: string @@ -565,6 +616,9 @@ export type ToolDisplayMessage = { content: string parameters?: any result?: any + /** What the job has streamed of its result so far, while it is still running. + * Cleared when the job lands: `result` is then the whole of it. */ + resultStream?: string logs?: string isLoading?: boolean /** Arguments fully streamed but execution not started (see queuedToolStatus). */ @@ -578,6 +632,7 @@ export type ToolDisplayMessage = { showFade?: boolean actions?: ToolDisplayAction[] userQuestion?: UserQuestionDisplay + runForm?: RunFormDisplay webSearchSources?: WebSearchSource[] /** Data URL of an image the tool produced (e.g. take_screenshot), shown on the card. */ imageUrl?: string @@ -653,6 +708,18 @@ export function isActiveUserQuestion(message: DisplayMessage | undefined): boole ) } +export function isActiveRunForm(message: DisplayMessage | undefined): boolean { + return Boolean( + message && + message.role === 'tool' && + message.runForm && + message.isLoading && + !message.error && + !message.runForm.submitted && + !message.runForm.canceled + ) +} + // The loop is parked on the user: an unanswered askUserQuestion, or a tool call // staged for confirmation. The manager stays `loading` through both, so anything // rendering progress must ask here first or it reports "the AI is working". @@ -676,6 +743,12 @@ export function pendingUserActionDetail( if (isActiveUserQuestion(message)) { return { action: 'question', toolCallId: message.tool_call_id } } + // A run form is a confirmation carrying arguments, not a question: it parks the + // turn the same way, but Run or Cancel resolves it and typing never does — so it + // must not claim the answer affordance a pending question offers. + if (isActiveRunForm(message)) { + return { action: 'confirmation', toolCallId: message.tool_call_id } + } if (message.needsConfirmation && message.isLoading) { return { action: 'confirmation', toolCallId: message.tool_call_id } } @@ -958,6 +1031,9 @@ export async function processToolCall({ } let result = '' + // A tool that asks for consent itself settles the call as declined or blocked and + // returns normally, so without this both telemeter as successful runs. + let settledInsideTool: 'declined' | 'blocked_plan_mode' | undefined = undefined try { result = await callTool({ tools, @@ -965,10 +1041,17 @@ export async function processToolCall({ args, workspace: workspaceId, helpers, - toolCallbacks, + toolCallbacks: { + ...toolCallbacks, + setToolStatus: (toolId, status) => { + if (status?.declinedByUser) settledInsideTool = 'declined' + else if (status?.blockedByPlanMode) settledInsideTool = 'blocked_plan_mode' + toolCallbacks.setToolStatus(toolId, status) + } + }, toolId: toolCall.id }) - logToolOutcome('ok') + logToolOutcome(settledInsideTool ?? 'ok') toolCallbacks.setToolStatus(toolCall.id, { isLoading: false, isStreamingArguments: false @@ -1069,6 +1152,10 @@ export interface Tool { * is true. */ refuseInPlanMode?: (p: { args: any; helpers: T }) => ToolRejection | undefined requiresConfirmation?: boolean + /** The tool's own argument form is its confirmation, and the bypass posture answers that + * form — so no card is waited on, yet a decision is still being made for the user. The + * list of what the posture bypasses is built from both this and `requiresConfirmation`. */ + bypassedByAutoAccept?: boolean /** Header shown on the confirmation card before the tool runs. Pass a function * to derive it from the parsed arguments (e.g. name the script being tested). */ confirmationMessage?: string | ((args: any) => string) @@ -1229,6 +1316,16 @@ export interface ToolCallbacks { toolId: string, question: UserQuestionDisplay ) => Promise + /** Park the loop on an argument form and resolve with the args the user submitted, or + * undefined if they cancelled. Wired only where the form can be rendered. `autoAccepted` + * says YOLO already answered it with what it opened with, so there is no card to wait on. */ + requestRunArgs?: ( + toolId: string, + form: RunFormDisplay, + opts?: { autoAccepted?: boolean } + ) => Promise | undefined> + /** The submitted form's job is queued. Wired alongside requestRunArgs. */ + markRunFormStarted?: (toolId: string) => void /** Records a workspace item the tool call created/edited/deleted, by its * canonical (itemKind, storagePath). Session chats wire this to accumulate the * chat's modified-items mask; the global side-panel chat omits it (no-op). */ @@ -1488,7 +1585,7 @@ export async function buildSchemaForTool( // Constants for result formatting const MAX_RESULT_LENGTH = 12000 -const MAX_LOG_LENGTH = 4000 +export const MAX_LOG_LENGTH = 4000 export const MAX_RUNNABLE_CONTENT_LENGTH = 20000 /** How long a test run is awaited inline before it detaches into the background @@ -1517,13 +1614,16 @@ export interface TestRunConfig { detachAfterMs?: number /** Human label for the jobs tray row (path / step id). Defaults to the job id. */ label?: string - /** Overrides the default "…test started, waiting for completion" status while the + /** Overrides the default "…started, waiting for completion" status while the * job runs inline (e.g. an SQL tool shows "SQL running…"). */ runningMessage?: string - /** Noun for the human-facing status strings (" test completed successfully"). - * Defaults to `contextName`, which also carries the jobs-tray kind and so cannot - * always name what ran: an app's path runnable queues a flow job. */ + /** The item noun in the human-facing status strings ("Flow test completed + * successfully"). Defaults to `contextName`, which also carries the jobs-tray kind + * and so cannot always name what ran: an app's path runnable queues a flow job. */ completionName?: string + /** The action noun in those same strings ("Script run completed successfully"). + * Defaults to "test", so a tool running the deployed item for real passes "run". */ + actionNoun?: string /** Custom terminal formatting for the INLINE completion path (callers whose * result isn't a plain test-run summary, e.g. exec_datatable_sql shaping rows). * Returns the string handed to the model plus the tool-card patch. When omitted, @@ -1544,6 +1644,46 @@ export type BackgroundJobFormatter = (job: CompletedJob) => { card: Partial } +/** Reads a running job's output incrementally through `getJobUpdates`, the only endpoint + * carrying `new_result_stream` — `getJob` returns logs but never the partial result. Each + * reader keeps its own offsets, so one starting over refetches from zero. Best-effort: a + * failed poll answers `undefined` and mutates nothing, so the next resumes from the same + * offsets and a run always lands on `getJob` alone. */ +export function createJobUpdateReader(jobId: string, workspace: string) { + let logs = '' + let resultStream = '' + let logOffset = 0 + let streamOffset = 0 + let started = false + return { + async poll(): Promise<{ completed: boolean; logs: string; resultStream: string } | undefined> { + let update: Awaited> + try { + update = await JobService.getJobUpdates({ + workspace, + id: jobId, + running: started, + logOffset, + streamOffset + }) + } catch { + return undefined + } + started ||= update.running ?? false + // Both kept as a tail: the offsets come from the server, so dropping the head + // costs nothing here, and neither is the record of the run — the logs are on the + // job, and a streamed partial is replaced by the result the moment it lands. + if (update.new_logs) logs = (logs + update.new_logs).slice(-MAX_LOG_LENGTH) + if (update.new_result_stream) { + resultStream = (resultStream + update.new_result_stream).slice(-MAX_LOG_LENGTH) + } + if (update.log_offset) logOffset = update.log_offset + if (update.stream_offset) streamOffset = update.stream_offset + return { completed: update.completed ?? false, logs, resultStream } + } + } +} + // Common job polling function. // // Two modes, selected by whether `detachAfterMs` is provided: @@ -1563,24 +1703,54 @@ export async function pollJobCompletion( const maxAttempts = detachEnabled ? Math.ceil((options?.detachAfterMs ?? 0) / 1000) : 60 let attempts = 0 let job: CompletedJob | null = null + const reader = createJobUpdateReader(jobId, workspace) while (attempts < maxAttempts) { await new Promise((resolve) => setTimeout(resolve, 1000)) attempts++ try { + const update = await reader.poll() + // The tray's snapshot is trimmed of logs (it is persisted), so the card is the + // only place a running job's output can land. Cards that hide their logs while + // loading are unaffected; the run card follows them line by line. + if (update) { + toolCallbacks.setToolStatus(toolId, { + logs: formatLogs(update.logs), + resultStream: update.resultStream || undefined + }) + } + + // Ask for the logs when the run may be over — the tail written between the last + // poll and the end is only on the job itself — or when there is no reader output + // to have collected them. + const wantLogs = !update || update.completed const fetchedJob = await JobService.getJob({ workspace: workspace, id: jobId, - noLogs: false, + noLogs: !wantLogs, noCode: true }) - if (fetchedJob.type === 'CompletedJob') { - job = fetchedJob + // The updates can still call a landed job unfinished, so a completion seen on + // a logless fetch is fetched again rather than settled without them: the model + // reads these logs, and their absence is indistinguishable from a silent run. + job = wantLogs + ? fetchedJob + : ((await JobService.getJob({ + workspace: workspace, + id: jobId, + noLogs: false, + noCode: true + })) as CompletedJob) break } - // Keep the tray's status + Job snapshot fresh during the inline wait. + // With no reader, this is the only place the card's logs can come from. + if (!update) { + toolCallbacks.setToolStatus(toolId, { logs: formatLogs(fetchedJob.logs) }) + } + // The badge needs the real Job to tell running from suspended or scheduled, which + // the updates do not say. toolCallbacks.onJobStatus?.(jobId, { status: deriveChatJobStatus(fetchedJob), job: trimJob(fetchedJob) @@ -1694,14 +1864,22 @@ function backgroundedSummary(jobId: string, label: string): string { // fills its card the same way one that finished inline does. export function completedJobToolStatus(job: CompletedJob): Partial { // A canceled job isn't a `success`, but it isn't a failure either — the user - // stopped it — so don't dress the card as an error. + // stopped it — so don't dress the card as an error. It still has the result the run + // page shows for a canceled run, which names who stopped it, so keep that. if (job.canceled) { - return { content: 'Background job canceled', logs: formatLogs(job.logs) } + return { + content: 'Background job canceled', + result: formatResult(job.result), + logs: formatLogs(job.logs), + resultStream: undefined + } } return { content: `Background job ${job.success ? 'completed successfully' : 'failed'}`, result: formatResult(job.result), logs: formatLogs(job.logs), + // The partial is the result now, so nothing streamed is kept beside it. + resultStream: undefined, ...(job.success ? {} : { error: getErrorMessage(job.result) }) } } @@ -1731,12 +1909,15 @@ export function backgroundJobCompletionNote( ) } -// Main execution function for test runs export async function executeTestRun(config: TestRunConfig): Promise { // Detach-into-background is enabled only when the host wired the job hooks // (global/sessions chat). Otherwise this stays a blocking call. const detachEnabled = !!config.toolCallbacks.onJobStarted const label = config.label ?? config.contextName + const actionNoun = config.actionNoun ?? 'test' + // Stands on its own where the status strings are prefixed by the item, so its + // default carries the noun. + const failureNoun = config.actionNoun ?? 'test run' try { config.toolCallbacks.setToolStatus(config.toolId, { content: config.startMessage || `Starting ${config.contextName} test...` @@ -1761,7 +1942,8 @@ export async function executeTestRun(config: TestRunConfig): Promise { }) config.toolCallbacks.setToolStatus(config.toolId, { - content: config.runningMessage ?? `${contextName} test started, waiting for completion...` + content: + config.runningMessage ?? `${contextName} ${actionNoun} started, waiting for completion...` }) const outcome = await pollJobCompletion( @@ -1781,7 +1963,7 @@ export async function executeTestRun(config: TestRunConfig): Promise { if (outcome === 'detached') { config.toolCallbacks.onJobDetached?.(jobId) config.toolCallbacks.setToolStatus(config.toolId, { - content: `${contextName} test running in background (job ${jobId})` + content: `${contextName} ${actionNoun} running in background (job ${jobId})` }) return backgroundedSummary(jobId, label) } @@ -1800,9 +1982,12 @@ export async function executeTestRun(config: TestRunConfig): Promise { } config.toolCallbacks.setToolStatus(config.toolId, { - content: `${contextName} test ${job.success ? 'completed successfully' : 'failed'}`, + content: `${contextName} ${actionNoun} ${job.success ? 'completed successfully' : 'failed'}`, result: formatResult(job.result), logs: formatLogs(job.logs), + // The partial is the result now, so the card reads it off `result` alone and the + // transcript stops carrying a second copy of a streamed answer. + resultStream: undefined, ...(job.success ? {} : { error: getErrorMessage(job.result) }) }) @@ -1823,10 +2008,10 @@ export async function executeTestRun(config: TestRunConfig): Promise { // the flow it could not find — losing the one diagnostic the run exists for. const errorMessage = formatToolError(error) config.toolCallbacks.setToolStatus(config.toolId, { - content: `Test execution failed`, + content: `Execution failed`, error: errorMessage }) - throw new Error(`Failed to execute test run: ${errorMessage}`) + throw new Error(`Failed to execute ${failureNoun}: ${errorMessage}`) } } diff --git a/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts b/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts index e91d40bff0..1ac4439a9b 100644 --- a/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts +++ b/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts @@ -1,10 +1,11 @@ -import { createEnabledPathsPreference } from '../enabledPathsPreference' +import { createPathsPreference } from '../enabledPathsPreference' /** Which `ai_skill` resources the chat may follow, per workspace and per account. - * Every enabled skill spends context on every turn, so selecting one is a personal - * choice rather than a consequence of being able to read it. */ -const preference = createEnabledPathsPreference('wm_skills_enabled') + * Every skill readable in the workspace is on unless someone turned it off: a skill + * is instructions the workspace wrote for the assistant to use, so what carrying one + * costs is context, not access. */ +const preference = createPathsPreference('wm_skills_enabled', true) -export const enabledSkillPaths = preference.enabledPaths export const isSkillEnabled = preference.isEnabled export const setSkillEnabled = preference.setEnabled +export const forgetSkill = preference.forget diff --git a/frontend/src/lib/components/copilot/chat/skills/skillResources.ts b/frontend/src/lib/components/copilot/chat/skills/skillResources.ts index 20737caffa..7a80705d4e 100644 --- a/frontend/src/lib/components/copilot/chat/skills/skillResources.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skillResources.ts @@ -38,14 +38,14 @@ export function ambiguousSkillNames(skills: readonly { name: string }[]): Set/…`, a folder the user owns), which is why this goes through * the shared `canWrite` rather than reading `extra_perms` alone. */ -export async function listSkillResources( - workspace: string, - user?: UserExt -): Promise { +export async function listSkillResources(workspace: string, user?: UserExt): Promise { if (!workspace) return { skills: [], truncated: false } const rows: SkillResource[] = [] for (let page = 1; page <= MAX_SKILLS_PAGES; page++) { diff --git a/frontend/src/lib/components/copilot/chat/skills/skillTree.ts b/frontend/src/lib/components/copilot/chat/skills/skillTree.ts new file mode 100644 index 0000000000..d2de4eb655 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/skills/skillTree.ts @@ -0,0 +1,119 @@ +/** Grouping for the Skills list: skills sorted into the folders that hold them. + * + * Only worth showing when the skills are spread over more than one folder — see + * `skillFolderPaths` — since a tree over a single folder is a list with a header + * on top of it. */ + +/** A folder in the skills tree. `path` is the whole prefix the node covers, so it + * is unique across the tree and can key collapsed state. */ +export type SkillTreeNode = { + path: string + /** Header text: the whole scope (`u/admin`, `f/skills`) at the root, one + * segment (`deploy`) deeper, where the ancestors are already on screen. */ + label: string + /** A `u/`, `f/` or `g/` root rather than a subfolder. */ + scope: boolean + children: SkillTreeNode[] + skills: T[] +} + +type Skill = { path: string; name: string } + +/** The folders holding these skills — one entry per distinct parent path. */ +export function skillFolderPaths(skills: readonly Skill[]): Set { + return new Set(skills.map((s) => s.path.split('/').slice(0, -1).join('/'))) +} + +/** Every skill under `node`, its subfolders included. */ +export function countSkills(node: SkillTreeNode): number { + return node.skills.length + node.children.reduce((n, c) => n + countSkills(c), 0) +} + +/** The skills a folder switch acts on: everything under it, however deep. */ +export function nodeSkills(node: SkillTreeNode): T[] { + return [...node.skills, ...node.children.flatMap((c) => nodeSkills(c))] +} + +/** One line of the rendered list, folders included. */ +export type SkillTreeEntry = { key: string; parentKey?: string } & ( + | { kind: 'folder'; node: SkillTreeNode } + | { kind: 'skill'; skill: T } +) + +/** A folder and a skill can hold the same path — `f/skills/deploy` names both when + * `f/skills/deploy/rollback` exists beside it — so the kind is part of the key. */ +export function folderKey(path: string): string { + return `folder:${path}` +} +export function skillKey(path: string): string { + return `skill:${path}` +} + +/** The list as it stands on screen, top to bottom, with collapsed folders holding + * their contents back. This is what the keyboard walks; the markup renders the same + * forest recursively under the same `isCollapsed`, so the two agree line for line. */ +export function visibleEntries( + forest: readonly SkillTreeNode[], + isCollapsed: (path: string) => boolean +): SkillTreeEntry[] { + const out: SkillTreeEntry[] = [] + const walk = (node: SkillTreeNode, parentKey?: string) => { + const key = folderKey(node.path) + out.push({ kind: 'folder', key, parentKey, node }) + if (isCollapsed(node.path)) return + for (const child of node.children) walk(child, key) + for (const skill of node.skills) { + out.push({ kind: 'skill', key: skillKey(skill.path), parentKey: key, skill }) + } + } + for (const root of forest) walk(root) + return out +} + +/** Sort a skill's path into the forest, creating the folders it names. + * + * Paths are `[ufg]//` and may nest further (`f/skills/deploy/rollback` + * passes the resource path CHECK), so the first two segments are one node and each + * segment after that is a node of its own. */ +export function buildSkillTree(skills: readonly T[]): SkillTreeNode[] { + const roots = new Map>() + for (const skill of skills) { + const parts = skill.path.split('/') + // Two segments cannot happen through the API, but a hand-written path in a + // test or a future path shape should land somewhere rather than vanish. + const rootPath = parts.slice(0, 2).join('/') + let node: SkillTreeNode = roots.get(rootPath) ?? { + path: rootPath, + label: rootPath, + scope: true, + children: [], + skills: [] + } + roots.set(rootPath, node) + for (const segment of parts.slice(2, -1)) { + const path = `${node.path}/${segment}` + let child = node.children.find((c) => c.path === path) + if (child === undefined) { + child = { path, label: segment, scope: false, children: [], skills: [] } + node.children.push(child) + } + node = child + } + node.skills.push(skill) + } + + const sort = (node: SkillTreeNode) => { + node.children.sort((a, b) => a.label.localeCompare(b.label)) + node.skills.sort((a, b) => a.name.localeCompare(b.name)) + node.children.forEach(sort) + } + const forest = [...roots.values()] + forest.forEach(sort) + // Personal scopes before shared folders, alphabetical within each: `u/` is + // where this modal's New skill and folder import write, so it is the half of the + // list someone is most often here to change. + return forest.sort((a, b) => { + const scopeRank = (p: string) => (p.startsWith('u/') ? 0 : 1) + return scopeRank(a.path) - scopeRank(b.path) || a.path.localeCompare(b.path) + }) +} diff --git a/frontend/src/lib/components/copilot/chat/skills/skills.test.ts b/frontend/src/lib/components/copilot/chat/skills/skills.test.ts index 5b0d2196ca..ae2e737af2 100644 --- a/frontend/src/lib/components/copilot/chat/skills/skills.test.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skills.test.ts @@ -9,8 +9,10 @@ vi.mock('$lib/stores', () => ({ userStore: { subscribe: (run: (v: unknown) => void) => (run({ ...session }), () => {}) } })) -import { enabledSkillPaths, isSkillEnabled, setSkillEnabled } from './enabledSkills' +import { isMcpEnabled } from '$lib/components/mcp/enabledServers' +import { isSkillEnabled, setSkillEnabled } from './enabledSkills' import { ambiguousSkillNames, truncateChars, truncateForPrompt } from './skillResources' +import { buildSkillTree, countSkills, skillFolderPaths, visibleEntries } from './skillTree' describe('enabledSkills', () => { beforeEach(() => { @@ -18,22 +20,103 @@ describe('enabledSkills', () => { session.email = 'first@windmill.dev' }) - it('keeps the selection separate per workspace', () => { - setSkillEnabled('ws_a', 'u/me/deploy', true) - expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(true) - expect(isSkillEnabled('ws_b', 'u/me/deploy')).toBe(false) + it('keeps a skill turned off separate per workspace', () => { + setSkillEnabled('ws_a', 'u/me/deploy', false) + expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(false) + expect(isSkillEnabled('ws_b', 'u/me/deploy')).toBe(true) }) - it('does not hand the next account the previous one’s selection', () => { - setSkillEnabled('ws_a', 'u/me/deploy', true) + it('does not hand the next account the previous one’s choice', () => { + setSkillEnabled('ws_a', 'u/me/deploy', false) session.email = 'second@windmill.dev' - expect(enabledSkillPaths('ws_a')).toEqual([]) + expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(true) }) it('reports failure when there is no account to record the choice against', () => { session.email = undefined - expect(setSkillEnabled('ws_a', 'u/me/deploy', true)).toBe(false) - expect(enabledSkillPaths('ws_a')).toEqual([]) + expect(setSkillEnabled('ws_a', 'u/me/deploy', false)).toBe(false) + }) +}) + +// Storage written before skills defaulted to on holds an array of the paths that +// were on. Nothing rewrites it, so reading it wrong is what would silently move +// somebody's choices — in either direction, for either default. +describe('choices stored under the older shape', () => { + const scope = 'ws_a:first@windmill.dev' + + beforeEach(() => { + localStorage.clear() + session.email = 'first@windmill.dev' + }) + + it('keeps a skill that was turned on, and defaults the rest to on', () => { + localStorage.setItem('wm_skills_enabled', JSON.stringify({ [scope]: ['u/me/deploy'] })) + expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(true) + expect(isSkillEnabled('ws_a', 'u/me/never-picked')).toBe(true) + }) + + it('leaves an MCP server that was never turned on off', () => { + localStorage.setItem('wm_mcp_enabled', JSON.stringify({ [scope]: ['u/me/github'] })) + expect(isMcpEnabled('ws_a', 'u/me/github')).toBe(true) + expect(isMcpEnabled('ws_a', 'u/me/other')).toBe(false) + }) + + it('records an off decision beside the entries already there', () => { + localStorage.setItem('wm_skills_enabled', JSON.stringify({ [scope]: ['u/me/deploy'] })) + setSkillEnabled('ws_a', 'u/me/review', false) + expect(isSkillEnabled('ws_a', 'u/me/review')).toBe(false) + expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(true) + }) + + it('drops a path put back to the default instead of storing it', () => { + setSkillEnabled('ws_a', 'u/me/review', false) + setSkillEnabled('ws_a', 'u/me/review', true) + expect(JSON.parse(localStorage.getItem('wm_skills_enabled') ?? '{}')[scope]).toEqual({}) + }) +}) + +describe('skill tree', () => { + const skills = [ + { path: 'f/skills/deploy/rollback', name: 'rollback' }, + { path: 'f/skills/onboarding', name: 'onboarding' }, + { path: 'u/admin/release-notes', name: 'release-notes' } + ] + + it('nests each path segment under its owner, own folder first', () => { + const tree = buildSkillTree(skills) + expect(tree.map((n) => n.path)).toEqual(['u/admin', 'f/skills']) + const shared = tree[1] + expect(shared.skills.map((s) => s.name)).toEqual(['onboarding']) + // A skill two levels down gets its folder, rather than being flattened into + // the owner's own rows where the path it came from is lost. + expect(shared.children.map((c) => c.path)).toEqual(['f/skills/deploy']) + expect(shared.children[0].skills.map((s) => s.name)).toEqual(['rollback']) + expect(countSkills(shared)).toBe(2) + }) + + // The keyboard walks this list while the markup renders the forest recursively. If + // the two disagree, Down lands on a row that is not the one lit. + it('lists what is on screen, and holds back what a collapsed folder hides', () => { + const tree = buildSkillTree(skills) + expect(visibleEntries(tree, () => false).map((e) => e.key)).toEqual([ + 'folder:u/admin', + 'skill:u/admin/release-notes', + 'folder:f/skills', + 'folder:f/skills/deploy', + 'skill:f/skills/deploy/rollback', + 'skill:f/skills/onboarding' + ]) + expect(visibleEntries(tree, (path) => path === 'f/skills').map((e) => e.key)).toEqual([ + 'folder:u/admin', + 'skill:u/admin/release-notes', + 'folder:f/skills' + ]) + }) + + // What the list uses to choose between the tree and a flat list. + it('counts the folders holding the skills, not the skills', () => { + expect(skillFolderPaths(skills).size).toBe(3) + expect(skillFolderPaths([skills[1], { path: 'f/skills/deploy', name: 'deploy' }]).size).toBe(1) }) }) @@ -55,7 +138,9 @@ describe('prompt truncation', () => { const body = '漢'.repeat(100) // 300 bytes expect(truncateForPrompt(body, 3000)).toBe(body) const cut = truncateForPrompt(body, 30) - expect(new TextEncoder().encode(cut.replace('… [truncated]', '')).byteLength).toBeLessThanOrEqual(30) + expect( + new TextEncoder().encode(cut.replace('… [truncated]', '')).byteLength + ).toBeLessThanOrEqual(30) expect(cut).toContain('[truncated]') // A byte-aligned cut must not leave a broken code point behind. expect(cut).not.toContain('\ufffd') diff --git a/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts b/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts index 15f6f2839d..dd480c5e97 100644 --- a/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts @@ -65,26 +65,26 @@ export class SkillsMenu { async #toggle(ws: string, path: string, enabled: boolean) { // A session whose fork is still staged has no workspace of its own yet, so `ws` - // is the PARENT: the selection would be stored under it and quietly stop + // is the PARENT: the choice would be stored under it and quietly stop // applying the moment the first send commits the fork. const pendingForkOf = this.#manager.sessionContextResolver?.()?.pendingForkOf if (pendingForkOf !== undefined) { sendUserToast( - `This session has not created its workspace yet, so the selection would be stored under "${pendingForkOf}". Send a message first.`, + `This session has not created its workspace yet, so the choice would be stored under "${pendingForkOf}". Send a message first.`, true ) return } if (!setSkillEnabled(ws, path, enabled)) { - sendUserToast('Could not save the selection for this account.', true) + sendUserToast('Could not save this choice for this account.', true) return } const row = this.#row(path) if (row) row.enabled = enabled - // Whether people select skills at all. Never the skill itself: a path is + // Whether people turn skills off at all. Never the skill itself: a path is // workspace-authored text. logFeatureUsage('ai_session', 'skill_toggle', { key: enabled ? 'on' : 'off', workspace: ws }) - // The prompt lists exactly the enabled skills, so it has to be rebuilt + // The prompt lists exactly the skills that are on, so it has to be rebuilt // before the next message rather than on the next mode change. await this.#manager.refreshGlobalSkills(ws) } @@ -103,10 +103,10 @@ export class SkillsMenu { void this.#load(ws) } const ambiguous = ambiguousSkillNames(this.#rows) - // Enabled first: those are the ones a quick visit is most likely about. - const ordered = [...this.#rows].sort( - (a, b) => Number(b.enabled) - Number(a.enabled) || a.path.localeCompare(b.path) - ) + // By path. Ordering the ones that are on first would put every row in the same + // bucket now that skills start that way, and drop the one row it did move — a + // skill just turned off here — out of the shortcut that turns it back on. + const ordered = [...this.#rows].sort((a, b) => a.path.localeCompare(b.path)) const shown = ordered.slice(0, MAX_MENU_SKILLS) const manage = () => { closeMenu?.() diff --git a/frontend/src/lib/components/custom_ui.ts b/frontend/src/lib/components/custom_ui.ts index 9771521bf4..2a3e6a2176 100644 --- a/frontend/src/lib/components/custom_ui.ts +++ b/frontend/src/lib/components/custom_ui.ts @@ -9,7 +9,6 @@ export type FlowBuilderWhitelabelCustomUi = { export?: boolean history?: boolean aiBuilder?: boolean - tutorials?: boolean diff?: boolean extraDeployOptions?: boolean editableSummary?: boolean diff --git a/frontend/src/lib/components/editorStoragePathContext.ts b/frontend/src/lib/components/editorStoragePathContext.ts new file mode 100644 index 0000000000..e3685995c4 --- /dev/null +++ b/frontend/src/lib/components/editorStoragePathContext.ts @@ -0,0 +1,19 @@ +import { getContext, setContext } from 'svelte' + +// The path an editor's draft is stored under, the same key the live-editor draft +// registers. Published by whichever ancestor owns it (the session tab, or the +// full-page editor) because the parts that must say which item an editor is open +// on sit below both, and the path they can see themselves is the renamed one. + +const KEY = 'EditorStoragePath' + +/** `undefined` where the editor has no stored draft to speak of (a drawer mount). */ +export type EditorStoragePath = () => string | undefined + +export function setEditorStoragePath(storagePath: EditorStoragePath): void { + setContext(KEY, storagePath) +} + +export function getEditorStoragePath(): EditorStoragePath | undefined { + return getContext(KEY) +} diff --git a/frontend/src/lib/components/flows/FlowEditor.svelte b/frontend/src/lib/components/flows/FlowEditor.svelte index 812e971da9..73482817ec 100644 --- a/frontend/src/lib/components/flows/FlowEditor.svelte +++ b/frontend/src/lib/components/flows/FlowEditor.svelte @@ -55,7 +55,6 @@ interface Props { loading: boolean disableStaticInputs?: boolean - disableTutorials?: boolean disableAi?: boolean disableSettings?: boolean disabledFlowInputs?: boolean @@ -99,7 +98,6 @@ let { loading, disableStaticInputs = false, - disableTutorials = false, disableAi = false, disableSettings = false, disabledFlowInputs = false, @@ -391,7 +389,6 @@ bind:this={flowModuleSchemaMap} controlsPosition={compactGraphOverlay ? 'bottom' : 'top'} {disableStaticInputs} - {disableTutorials} {disableAi} {disableSettings} {smallErrorHandler} diff --git a/frontend/src/lib/components/flows/FlowEditorTutorial.svelte b/frontend/src/lib/components/flows/FlowEditorTutorial.svelte deleted file mode 100644 index ca1972914e..0000000000 --- a/frontend/src/lib/components/flows/FlowEditorTutorial.svelte +++ /dev/null @@ -1,62 +0,0 @@ - - -{#key $tutorialsToDo} - - {#snippet buttonReplacement()} -
    - -{#if !disableTutorials} - -{/if} diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index f83b67267a..4c004a4f56 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -734,11 +734,9 @@ return false } - // Clear SvelteFlow's internal selection by creating new nodes array function clearFlowSelection() { - // xyflow owns `selected` on the objects it was handed, and drops it only when it sees a - // node it does not recognise. Serving the cached mapping back would hand it the very - // object it marked selected, so the clear has to go through fresh objects. + // Resetting the cache and reassigning `nodes` hands xyflow objects it has not seen, the + // only lever on its selection available from our own array. offsetNodeCache = new WeakMap() nodes = nodes.map((node) => { if (node.selected) { @@ -855,7 +853,8 @@ ...aiToolNodesResult.toolNodes ] - // Collect module IDs hidden inside collapsed groups so note cleanup preserves them + // Module IDs hidden inside collapsed groups: a note whose members are all in here has + // nothing on screen to wrap, so it is skipped. const collapsedModuleIds = new Set() for (const n of finalNodes) { if (n.type === 'collapsedGroup') { @@ -990,6 +989,9 @@ // FlowRunStatus instead. flowJob suspendStatus + // Dataflow edges hang off the selected step. Selection is otherwise xyflow's own state, + // so it only rebuilds the graph while those edges are shown. + if ($useDataflow) selectedId const collapsedGroupIds = new Set( allGroups @@ -1410,7 +1412,7 @@ /> - + {#if leftHeader}
    diff --git a/frontend/src/lib/components/graph/NoteTool.svelte b/frontend/src/lib/components/graph/NoteTool.svelte index ad41bd974a..d6f05244d1 100644 --- a/frontend/src/lib/components/graph/NoteTool.svelte +++ b/frontend/src/lib/components/graph/NoteTool.svelte @@ -86,7 +86,7 @@ // Create the actual note using NoteEditor context if (noteEditorContext?.noteEditor) { noteEditorContext.noteEditor.addNote({ - text: '### Free note\nDouble click to edit me', + text: '## Note\nDouble click to edit me', position, size, color: DEFAULT_NOTE_COLOR, @@ -107,7 +107,7 @@ if (!noteEditorContext?.noteEditor || !contextMenuPosition) return noteEditorContext.noteEditor.addNote({ - text: '### Free note\nDouble click to edit me', + text: '## Note\nDouble click to edit me', position: contextMenuPosition, size: { width: 300, height: 200 }, color: DEFAULT_NOTE_COLOR, diff --git a/frontend/src/lib/components/graph/PaneContextMenu.svelte b/frontend/src/lib/components/graph/PaneContextMenu.svelte index 37be64bffc..7f3409812b 100644 --- a/frontend/src/lib/components/graph/PaneContextMenu.svelte +++ b/frontend/src/lib/components/graph/PaneContextMenu.svelte @@ -68,7 +68,7 @@ function handleAddStickyNote() { if (noteEditorContext?.noteEditor && pendingFlowPosition) { noteEditorContext.noteEditor.addNote({ - text: '### Free note\nDouble click to edit me', + text: '## Note\nDouble click to edit me', position: { x: pendingFlowPosition.x, y: pendingFlowPosition.y - (graphContext?.yOffset || 0) diff --git a/frontend/src/lib/components/graph/SelectionTool.svelte b/frontend/src/lib/components/graph/SelectionTool.svelte index dd5033bf55..5fd6147c71 100644 --- a/frontend/src/lib/components/graph/SelectionTool.svelte +++ b/frontend/src/lib/components/graph/SelectionTool.svelte @@ -4,16 +4,18 @@ import type { SelectionManager } from './selectionUtils.svelte' interface Props { selectionManager: SelectionManager - clearGraphSelection: () => void } - let { selectionManager, clearGraphSelection }: Props = $props() + let { selectionManager }: Props = $props() - untrack(() => selectionManager).setClearGraphSelection(untrack(() => clearGraphSelection)) - - // Get store to access selectionRect const store = useStore() + // Clear through xyflow's store, never by handing it fresh node objects: replacing them + // re-creates every node's DOM, and a click whose node is rebuilt between pointerdown and + // release retargets to the pane, which clears the selection that same gesture just made. + // While xyflow holds a selection, useOnSelectionChange below re-broadcasts it over ours. + untrack(() => selectionManager).setClearGraphSelection(() => store.unselectNodesAndEdges()) + // Handle selection changes from SvelteFlow useOnSelectionChange(({ nodes: selectedNodes, edges: _selectedEdges }) => { // Notes are already non-selectable, so no filtering needed diff --git a/frontend/src/lib/components/graph/graphContext.ts b/frontend/src/lib/components/graph/graphContext.ts index a6245769cf..567bfb6348 100644 --- a/frontend/src/lib/components/graph/graphContext.ts +++ b/frontend/src/lib/components/graph/graphContext.ts @@ -12,6 +12,8 @@ export type GraphContext = { showAssets: Writable noteManager?: NoteManager moveManager?: MoveManager + /** Clears xyflow's selection by replacing every node object, so only for callers that rebuild + * the graph anyway. A selection change must use `selectionManager` instead. */ clearFlowSelection?: () => void yOffset?: number diffManager: FlowDiffManager diff --git a/frontend/src/lib/components/graph/noteEditor.svelte.ts b/frontend/src/lib/components/graph/noteEditor.svelte.ts index 50c2df7bfa..e3f17f576c 100644 --- a/frontend/src/lib/components/graph/noteEditor.svelte.ts +++ b/frontend/src/lib/components/graph/noteEditor.svelte.ts @@ -6,6 +6,7 @@ import { DEFAULT_GROUP_NOTE_COLOR, getNextAvailableColor } from './noteColors' import { generateId } from './util' import { getContext, setContext } from 'svelte' import { completeAndSplitGroup } from './groupDetectionUtils' +import { forEachFlowModule } from '../flows/dfs' /** * Utility class for editing flow notes via direct flowStore mutations @@ -158,10 +159,7 @@ export class NoteEditor { /** * Create a group note containing the specified node IDs */ - createGroupNote( - nodeIds: string[], - text: string = '### Group note\nDouble click to edit me' - ): string { + createGroupNote(nodeIds: string[], text: string = '## Note\nDouble click to edit me'): string { // Filter ids in case they contain subflow nodes let filteredNodeIds: string[] = nodeIds let subflowIds: string[] = [] @@ -219,10 +217,7 @@ export class NoteEditor { /** * Clean up group notes using DAG path completion */ - cleanupGroupNotes( - flowNodes: { id: string; parentIds?: string[] }[], - collapsedModuleIds?: Set - ): void { + cleanupGroupNotes(flowNodes: { id: string; parentIds?: string[] }[]): void { if (!this.isAvailable()) { return } @@ -232,19 +227,23 @@ export class NoteEditor { if (groupNotes.length === 0) return let hasChanges = false - const nodeSet = new Set(flowNodes.map((n) => n.id)) + const renderedIds = new Set(flowNodes.map((n) => n.id)) - // Include collapsed module IDs as valid — they are hidden but still exist - if (collapsedModuleIds) { - for (const id of collapsedModuleIds) { - nodeSet.add(id) - } - } + // A note's members are module ids, and the flow's own modules are what say whether one + // still exists. The rendered nodes are a view of them: a live module is absent from it + // while its group is collapsed, and again in the pass after its id changed, so pruning + // against the render alone deletes steps out of notes that are perfectly valid. + const moduleIds = new Set() + forEachFlowModule(this.flowStore.val.value?.modules ?? [], (mod) => { + moduleIds.add(mod.id) + }) // Step 1: Clean invalid nodes from existing group notes for (const note of groupNotes) { const originalIds = note.contained_node_ids || [] - const validIds = originalIds.filter((id) => nodeSet.has(id)) + // Path completion below can add ids that exist only in the graph (group + // boundaries), so a rendered node counts as valid alongside a live module. + const validIds = originalIds.filter((id) => moduleIds.has(id) || renderedIds.has(id)) if (validIds.length !== originalIds.length) { note.contained_node_ids = validIds @@ -259,9 +258,9 @@ export class NoteEditor { const originalNodes = note.contained_node_ids || [] if (originalNodes.length === 0) continue - // Skip path completion for notes that reference collapsed modules, - // since the DAG is incomplete when groups are collapsed - if (collapsedModuleIds && originalNodes.some((id) => collapsedModuleIds.has(id))) { + // Path completion walks the rendered edges, so it can only place members it can + // see; one it cannot would come back as unreachable and be dropped from the note. + if (originalNodes.some((id) => !renderedIds.has(id))) { continue } diff --git a/frontend/src/lib/components/graph/noteEditor.test.ts b/frontend/src/lib/components/graph/noteEditor.test.ts new file mode 100644 index 0000000000..5ec38601ef --- /dev/null +++ b/frontend/src/lib/components/graph/noteEditor.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect, vi } from 'vitest' + +// Mock modules that transitively import CSS/Monaco +vi.mock('monaco-editor', () => ({})) +vi.mock('@xyflow/svelte', () => ({})) + +import type { FlowModule, OpenFlow } from '$lib/gen' +import type { StateStore } from '$lib/utils' +import type { ExtendedOpenFlow } from '../flows/types' +import { NoteEditor } from './noteEditor.svelte' + +function makeFlowStore( + moduleIds: string[], + containedNodeIds: string[] +): StateStore { + const modules: FlowModule[] = moduleIds.map((id) => ({ + id, + value: { type: 'rawscript', content: '', language: 'bun' } as any + })) + const flow: OpenFlow = { + summary: '', + value: { + modules, + notes: [ + { + id: 'note', + text: 'note', + color: 'yellow', + type: 'group', + contained_node_ids: containedNodeIds + } + ] + }, + schema: {} + } + return { val: flow as ExtendedOpenFlow } as StateStore +} + +function containedIds(flowStore: StateStore): string[] | undefined { + return flowStore.val.value.notes?.[0]?.contained_node_ids +} + +describe('cleanupGroupNotes', () => { + it('keeps a module the graph has not rendered', () => { + // Both the pass after a module id changed and a collapsed group leave a live module + // out of the rendered nodes. + const flowStore = makeFlowStore(['renamed', 'b'], ['renamed', 'b']) + new NoteEditor(flowStore).cleanupGroupNotes([{ id: 'b' }]) + + expect(containedIds(flowStore)).toEqual(['renamed', 'b']) + }) + + it('drops a module that no longer exists in the flow', () => { + const flowStore = makeFlowStore(['b'], ['deleted', 'b']) + new NoteEditor(flowStore).cleanupGroupNotes([{ id: 'b' }]) + + expect(containedIds(flowStore)).toEqual(['b']) + }) +}) diff --git a/frontend/src/lib/components/graph/noteUtils.svelte.ts b/frontend/src/lib/components/graph/noteUtils.svelte.ts index 577b421a9e..7cb6e17ece 100644 --- a/frontend/src/lib/components/graph/noteUtils.svelte.ts +++ b/frontend/src/lib/components/graph/noteUtils.svelte.ts @@ -264,7 +264,7 @@ export function computeNoteNodes( if (editMode) { if (noteEditorContext?.noteEditor?.isAvailable()) { - noteEditorContext.noteEditor.cleanupGroupNotes(nodes, collapsedModuleIds) + noteEditorContext.noteEditor.cleanupGroupNotes(nodes) } } diff --git a/frontend/src/lib/components/home/CreateActionsMenu.svelte b/frontend/src/lib/components/home/CreateActionsMenu.svelte index 4d8adb88e9..2cf1f28d54 100644 --- a/frontend/src/lib/components/home/CreateActionsMenu.svelte +++ b/frontend/src/lib/components/home/CreateActionsMenu.svelte @@ -15,6 +15,7 @@ Loader2, Workflow, Import, + Store, PanelLeftClose } from 'lucide-svelte' import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte' @@ -26,6 +27,25 @@ import { conditionalMelt, getLocalSetting, storeLocalSetting } from '$lib/utils' import { createDropdownMenu, melt } from '@melt-ui/svelte' import YAML from 'yaml' + import type { Snippet } from 'svelte' + import { logFeatureUsage } from '$lib/utils/featureUsage' + + interface Props { + /** Replaces the default `New` button, e.g. with an inline text link. */ + trigger?: Snippet + /** The node `trigger` renders: what the menu anchors to and what opens it. */ + triggerElement?: HTMLElement + /** Which entry point this menu hangs off, for telemetry. */ + source?: 'toolbar' | 'empty_state' + /** + * Opens the hub project picker. The menu only offers the entry; the picker and the + * import dialog belong to the host, which is the one place a single import modal can + * serve both this menu and the empty state's own link. + */ + onImportHubProject?: () => void + } + + let { trigger, triggerElement, source = 'toolbar', onImportHubProject }: Props = $props() type Variant = { label: string @@ -228,8 +248,15 @@ } let activeKey = $state(allOptions[0]?.key) - // every option's import action, surfaced together under the bottom "Import" submenu - const importActions: Extra[] = allOptions.flatMap((o) => o.extras ?? []) + // every option's import action, surfaced together under the bottom "Import" submenu. + // The hub project leads and is separated below: the others each paste one artifact the + // user already holds, while this one brings a whole project in from somewhere else. + const importActions: Extra[] = $derived([ + ...(onImportHubProject + ? [{ label: 'Import a hub project', onSelect: onImportHubProject }] + : []), + ...allOptions.flatMap((o) => o.extras ?? []) + ]) // melt dropdown menu: arrow-key nav, typeahead, focus management and outside/escape // close all come for free; we only drive the doc panel off the highlighted item. @@ -306,7 +333,7 @@ // styling — melt element stores are callable on a node, exactly like `use:melt`. let triggerEl: HTMLButtonElement | HTMLAnchorElement | undefined = $state(undefined) $effect(() => { - const el = triggerEl + const el = triggerElement ?? triggerEl if (!el) return const applied = conditionalMelt(el, menuTrigger as any) as { destroy?: () => void @@ -314,6 +341,18 @@ return applied?.destroy }) + // Which entry point people actually create from: the toolbar button, or the inline + // link in the empty state. Only the open edge counts — melt writes the store on + // close and on every re-render of the menu. + let wasOpen = false + $effect(() => { + const isOpen = $open + if (isOpen && !wasOpen) { + logFeatureUsage('home', 'new_menu_open', { key: source }) + } + wasOpen = isOpen + }) + const SHOW_DOC_SETTING = 'home_create_show_doc' let showDoc = $state(getLocalSetting(SHOW_DOC_SETTING) !== 'false') function setShowDoc(value: boolean) { @@ -366,191 +405,204 @@ } -
    - - - {#if $open && active} -
    +
    + {#if trigger} + {@render trigger()} + {:else} + + {/if} +
    -

    {active.description}

    - -
      - {#each active.bullets as bullet (bullet)} -
    • - - {bullet} -
      • - {/each} -
    - - +
    + {/if} + + +
    + {#snippet rowBody(option: Option, ac: (typeof accentClasses)[string])} +
    + +
    + + {option.label} + + {#if option.badge} + + {option.badge.label} + + {/if} + {/snippet} + {#each allOptions as option (option.key)} + {@const ac = accentClasses[option.accent]} + {@const rowClass = + 'w-full flex flex-row items-center gap-2.5 rounded-md px-2 py-1.5 text-left cursor-pointer transition-colors focus:outline-none data-[highlighted]:bg-surface-hover hover:bg-surface-hover'} + {#if option.variants} + + {#if $wacSubOpen} +
    + {#each option.variants ?? [] as variant (variant.label)} + {@const VariantIcon = variant.icon} + + {/each} +
    + {/if} + {:else} + + {/if} + {/each} + + +
    + + {#if $importSubOpen} +
    + {#each importActions as action, i (action.label)} + + {#if onImportHubProject && i === 0} +
    + {/if} + {/each}
    {/if} - -
    - {#snippet rowBody(option: Option, ac: (typeof accentClasses)[string])} -
    - -
    - - {option.label} - - {#if option.badge} - - {option.badge.label} - - {/if} - {/snippet} - {#each allOptions as option (option.key)} - {@const ac = accentClasses[option.accent]} - {@const rowClass = - 'w-full flex flex-row items-center gap-2.5 rounded-md px-2 py-1.5 text-left cursor-pointer transition-colors focus:outline-none data-[highlighted]:bg-surface-hover hover:bg-surface-hover'} - {#if option.variants} - - {#if $wacSubOpen} -
    - {#each option.variants ?? [] as variant (variant.label)} - {@const VariantIcon = variant.icon} - - {/each} -
    - {/if} - {:else} - - {/if} - {/each} - - -
    + {#if !showDoc} - {#if $importSubOpen} -
    - {#each importActions as action (action.label)} - - {/each} -
    - {/if} - - {#if !showDoc} - - {/if} -
    + {/if}
    - {/if} -
    +
    +{/if} diff --git a/frontend/src/lib/components/home/HubProjectPickerModal.svelte b/frontend/src/lib/components/home/HubProjectPickerModal.svelte new file mode 100644 index 0000000000..6a353ea79d --- /dev/null +++ b/frontend/src/lib/components/home/HubProjectPickerModal.svelte @@ -0,0 +1,48 @@ + + + + + +
    + +
    +     diff --git a/frontend/src/lib/components/home/HubTemplatePicker.svelte b/frontend/src/lib/components/home/HubTemplatePicker.svelte new file mode 100644 index 0000000000..422de9bb55 --- /dev/null +++ b/frontend/src/lib/components/home/HubTemplatePicker.svelte @@ -0,0 +1,151 @@ + + + +
    + +

    + Working projects from + + {hubHost} + + — imported as a folder in this workspace. +

    + +
    + + + {#snippet customRow({ item }: { item: HubProjectPick })} + {@const Icon = hubAppIcon(item.iconApps[0] ?? '')} + + + + + + {/snippet} + + {#snippet empty()} +

    + {#if loadFailed} + Could not reach the hub. You can still browse its projects in a new tab. + {:else} + This hub has no projects yet. + {/if} +

    + {/snippet} +     +
    +
    diff --git a/frontend/src/lib/components/home/ImportProjectModal.svelte b/frontend/src/lib/components/home/ImportProjectModal.svelte new file mode 100644 index 0000000000..d9ac59d3b3 --- /dev/null +++ b/frontend/src/lib/components/home/ImportProjectModal.svelte @@ -0,0 +1,363 @@ + + +{#snippet importPage()} +
    + {#if project} + + + {/if} + + (folder = f)} + onFinish={() => (setup.needed ? (onSetupStep = true) : finish('none'))} + onBack={onClose} + onExecution={(e) => (execution = e)} + resume={execution} + /> +
    +{/snippet} + +{#snippet setupPlaceholder()} + +
    + +
    +{/snippet} + +{#snippet setupPage()} +
    + finish('skipped', outstanding)} + onFinish={(checked) => finish(checked ? 'filled' : 'unchecked')} + onBack={execution ? () => (onSetupStep = false) : undefined} + /> +
    +{/snippet} + + + + {#if slug} + + (onSetupStep = s === 2)} + /> + + { + if (key === IMPORT_PAGE && execution) onSetupStep = false + else if (key === SETUP_PAGE && setup.needed) onSetupStep = true + }} + pages={[ + { key: IMPORT_PAGE, content: importPage }, + { key: SETUP_PAGE, content: setupPage, placeholder: setupPlaceholder } + ]} + /> + {/if} + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 84432112a3..35b1c5a6f3 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -16,7 +16,7 @@ } from '$lib/gen' import { resource } from 'runed' import { getDraftItems } from '$lib/workspaceDrafts.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { disableHubStore, userStore, workspaceStore } from '$lib/stores' import type uFuzzy from '@leeoniya/ufuzzy' import { ArrowDownUp, @@ -40,6 +40,10 @@ type FilterSchemaRec } from '$lib/components/FilterSearchbar.svelte' import NoItemFound from './NoItemFound.svelte' + import WorkspaceEmptyState from './WorkspaceEmptyState.svelte' + import HubProjectPickerModal from './HubProjectPickerModal.svelte' + import ImportProjectModal from './ImportProjectModal.svelte' + import type { HubProjectPick } from '$lib/hubProject' import ListFilters from './ListFilters.svelte' import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' @@ -696,6 +700,10 @@ // runnables an owner holds. A scope change (sort/archive/kind/…) doesn't go // through here: the counts resource keys on those itself. async function reloadItemsAndCounts(): Promise { + // The answer can change with the rows: archiving the last item leaves the listing empty + // with something archived behind it, and a cached "nothing archived" would then call + // the workspace empty and hide the way to it until a page load. + archivedProbe = undefined // A mutated row can be gone, or sit at a new path, afterwards: snapshot what // was on screen so the selection can drop what this reload removes instead of // keeping a dead path. `tick` lets the reloaded rows re-register first. @@ -986,6 +994,105 @@ treeLazyMode && ownerCountsRes.current == undefined && ownerCountsRes.loading ) + // An import just landed, so the rows about to replace the empty state are all new: they + // fade in one after another rather than appearing as a finished list. Cleared on a timer + // because nothing else marks the end — the reload resolves before the rows animate. + let justImported = $state(false) + let justImportedTimer: ReturnType | undefined + function onImported() { + reloadItemsAndCounts() + justImported = true + clearTimeout(justImportedTimer) + justImportedTimer = setTimeout(() => (justImported = false), 2500) + } + + // The hub import, owned here rather than by either entry point: the empty state's link and + // the create menu's Import section open the same dialog, and mounting one per entry point + // would put two of them on the page at once while the workspace is still empty. + let hubPick = $state(undefined) + let hubPickerOpen = $state(false) + + /** + * Whether a workspace the default listing found empty is empty at all, or just has nothing + * unarchived — two different states that want two different things said about them. Asked + * only in that case, and once per workspace: one request for one row, never on a workspace + * with something in it. `hasArchived` is undefined when the request failed — see the catch + * for what that leaves standing. + */ + let archivedProbe = $state<{ workspace: string; hasArchived: boolean | undefined } | undefined>( + undefined + ) + $effect(() => { + const ws = $workspaceStore + if (!ws || !workspaceEmpty || archivedProbe?.workspace === ws) return + untrack(() => void probeArchived(ws)) + }) + async function probeArchived(workspace: string) { + try { + // `includeWithoutMain` to match the listing: the backend drops `auto_kind = 'lib'` + // without it, so a workspace holding only archived library scripts would answer + // "nothing archived". Always true here — hiding library scripts puts a filter in + // `activeFilters`, which `workspaceEmpty` requires to be empty. + const res = await ScriptService.listRunnables({ + workspace, + showArchived: true, + includeWithoutMain: true, + perPage: 1 + }) + archivedProbe = { workspace, hasArchived: (res.items?.length ?? 0) > 0 } + } catch (error) { + // Undefined, not false: false would say the workspace is empty and — since the + // toolbar is inert on the strength of the placeholder carrying the way to archived + // items — leave no way to them at all. Unknown keeps the ordinary caption, which + // promises nothing, and leaves the searchbar live as the fallback it used to be. + console.error('Could not check for archived items:', error) + archivedProbe = { workspace, hasArchived: undefined } + } + } + let emptyStateAnswered = $derived(archivedProbe?.workspace === $workspaceStore) + /** + * The probe could not tell. The toolbar stays usable in that case: `inert` is only right + * while the placeholder is the way to archived items, and here it cannot be. + */ + let archivedUnknown = $derived(emptyStateAnswered && archivedProbe?.hasArchived === undefined) + /** + * Whether this user may be offered the create actions. The empty state's template import + * and create menu do no permission check of their own, so an operator — or a workspace + * whose direct-deploy protection cleared `showEditButtons` — must not be shown them. + * Reading archived items is not a write, so it is not gated on this. + */ + let canCreateHere = $derived(!$userStore?.operator && showEditButtons) + + // The workspace itself holds nothing — no filter is narrowing the list away. It stays + // false until the first load resolves: a skeleton already means "loading", and the + // empty state must not be mistaken for one. The controls it dims stay mounted, so + // nothing moves when the first item lands. + let workspaceEmpty = $derived( + !loading && + !treeCountsPending && + !contentActive && + activeFilters.length === 0 && + filteredItems != undefined && + filteredItems.length === 0 && + visiblePipelineFolders.size === 0 && + !hasMoreServer + ) + /** + * Whether the placeholder below takes the toolbar's job over — it renders under the same + * conditions. Standing the toolbar down depends on something else offering a way onwards: + * where the placeholder holds back, as it does for an operator in a workspace that is + * simply empty, these controls are all there is and stay live. + */ + let placeholderTakesOver = $derived( + workspaceEmpty && emptyStateAnswered && (archivedProbe?.hasArchived === true || canCreateHere) + ) + /** + * The toolbar is dimmed either way; `inert` also takes it off the pointer, which is only + * right while the placeholder carries the way to archived items. A probe that could not + * tell leaves it live as the fallback. + */ + let toolbarInert = $derived(placeholderTakesOver && !archivedUnknown) + // Owners the counts found the user has something in, split by kind. They cover // what the folder/username lists miss: an item shared individually out of a // folder or user space the user is otherwise not a member of. @@ -1651,7 +1758,12 @@ }} > {#if !contentActive} -
    + +
    { @@ -1692,9 +1804,10 @@
    {/if} - {#if !loading && !contentActive} + {#if !loading && !contentActive && !workspaceEmpty} + view, expand/collapse (tree only), sort. Nothing to select, group or order on + an empty workspace, so the whole row goes. -->
    {#if homeSelection.available && !homeSelection.active}
    {/if} - {#if filteredItems?.length == 0} + {#if filteredItems?.length == 0 && !workspaceEmpty}
    {/if}
    @@ -1820,7 +1941,28 @@ - + {#if workspaceEmpty} + + {#if emptyStateAnswered} + {#if archivedProbe?.hasArchived || canCreateHere} + + (hubPick = project)} + onShowArchived={() => (filterValues.val = { ...filterValues.val, archived: true })} + /> + {:else} + + {/if} + {/if} + {:else} + + {/if} {#if hasMoreServer && !searching} @@ -1849,6 +1991,7 @@ allUsers={treeInjectUsers} ownerCounts={!searching && labelFilter == undefined ? ownerCounts : undefined} selfUsername={$userStore?.username} + groupOtherUsers={treeLazyMode} ownerLoad={treeLazyMode ? ownerLoad : undefined} onExpandOwner={treeLazyMode ? loadOwnerItems : undefined} onCollapseOwner={treeLazyMode ? collapseOwner : undefined} @@ -1863,7 +2006,7 @@ /> {/key} {:else} -
    +
    {#if filter === ''} {#each [...visiblePipelineFolders].sort() as folder (folder)} {/if} + + (hubPickerOpen = false)} + onPick={(project) => { + hubPickerOpen = false + hubPick = project + }} +/> + (hubPick = undefined)} {onImported} /> + + diff --git a/frontend/src/lib/components/home/TreeView.svelte b/frontend/src/lib/components/home/TreeView.svelte index 31eae20b16..0581700316 100644 --- a/frontend/src/lib/components/home/TreeView.svelte +++ b/frontend/src/lib/components/home/TreeView.svelte @@ -32,8 +32,9 @@ // `all` pages the prefix to the end in one call instead of fetching a single page. onExpandOwner?: (prefix: string, more?: boolean, opts?: { all?: boolean }) => void onCollapseOwner?: (prefix: string) => void - // Position of this node among the rendered root nodes; "expand all" only - // auto-loads the first EXPAND_ALL_LOAD_LIMIT of them (see the effect below). + // This root owner's place in line for "expand all", which only auto-loads the first + // EXPAND_ALL_LOAD_LIMIT (see the effect below). Not always its rendered position: + // owners nested under a grouping row are ranked after the rest. rootIndex?: number showEditButton?: boolean // Path prefix of the parent node, so this one can name its own (`ownerLoad` and @@ -43,6 +44,10 @@ // is grouped under this node is only part of it: counts render as "N+" and the // node offers to load the rest of itself. ancestorHasMore?: boolean + // Visual nesting on top of `depth`. `depth` stays semantic (0 is a top-level owner + // that loads lazily), so an owner shown inside a grouping row is indented through + // this rather than by raising its depth. + indent?: number } let { @@ -59,9 +64,12 @@ rootIndex = 0, showEditButton = true, parentPrefix, - ancestorHasMore = false + ancestorHasMore = false, + indent = 0 }: Props = $props() + let visualDepth = $derived(depth + indent) + // Bounds the request burst from "expand all": however many root owners the tree // renders (its slice grows as you scroll), it fetches at most this many. Lazy owners // past the cap stay collapsed and load on a single click (see the effect). @@ -256,7 +264,7 @@ >
    0 ? `padding-left: ${depth * 16}px;` : ''} + style={visualDepth > 0 ? `padding-left: ${visualDepth * 16}px;` : ''} >
    {#if isUser(item)} @@ -310,7 +318,7 @@ Pipeline @@ -335,12 +343,13 @@ {showCode} {showEditButton} depth={depth + 1} + {indent} /> {/each} {#if effectiveMax < item.items.length}
    @@ -377,7 +386,7 @@ as rows still missing. -->
    Showing {loadedHere}{ownerTotal != undefined ? ` of ${ownerTotal}` : ''} items in {nodePrefix} @@ -427,6 +436,6 @@ on:appChanged on:rawAppChanged on:reload - {depth} + depth={visualDepth} /> {/if} diff --git a/frontend/src/lib/components/home/TreeViewRoot.svelte b/frontend/src/lib/components/home/TreeViewRoot.svelte index 39c2188e59..062dfb69f8 100644 --- a/frontend/src/lib/components/home/TreeViewRoot.svelte +++ b/frontend/src/lib/components/home/TreeViewRoot.svelte @@ -1,8 +1,10 @@ +{#snippet ownerNode(node: RootNode, loadRank: number, indent: number)} + +{/snippet} + {#if groupedItems === 'loading'}
    {:else}
    - {#each groupedItems.slice(0, nbDisplayed) as item, rootIndex ('folderName' in item ? `f__${item.folderName}` : 'username' in item ? `u__${item.username}` : `i__${item.type}__${item.path}`)} - {#if item} - + {#each rows.slice(0, nbDisplayed) as row (row.kind === 'otherUsers' ? 'other_users' : 'folderName' in row.node ? `f__${row.node.folderName}` : 'username' in row.node ? `u__${row.node.username}` : `i__${row.node.type}__${row.node.path}`)} + {#if row.kind === 'otherUsers'} + + + +
    +
    + +
    + Other users +
    + ({pluralize(otherUsers.length, 'user')}{otherUsersItemCount != undefined + ? ` · ${pluralize(otherUsersItemCount, 'item')}` + : ''}) +
    +
    +
    +
    + {#if otherUsersOpen} + + {#each otherUsers.slice(0, nbOtherUsersDisplayed) as user, i (user.username)} + {@render ownerNode(user, ownerRowCount + i, 1)} + {/each} + {#if nbOtherUsersDisplayed < otherUsers.length} +
    + + Showing {nbOtherUsersDisplayed} of {otherUsers.length} users + + +
    + {/if} + {/if} + {:else} + {@render ownerNode(row.node, row.loadRank, 0)} {/if} {/each} - {#if nbDisplayed < groupedItems.length || hasMoreServer} + {#if nbDisplayed < rows.length || hasMoreServer}
    - {#if nbDisplayed < groupedItems.length} - Showing {nbDisplayed} of {groupedItems.length} folders and users + {#if nbDisplayed < rows.length} + Showing {shownOwnerRowCount} of {ownerRowCount} folders and users {:else} @@ -231,12 +355,12 @@ unifiedSize="sm" variant="subtle" on:click={() => { - if (nbDisplayed < groupedItems.length) - nbDisplayed = Math.min(nbDisplayed + ROOT_PAGE, groupedItems.length) + if (nbDisplayed < rows.length) + nbDisplayed = Math.min(nbDisplayed + ROOT_PAGE, rows.length) else onLoadMore?.() }} > - {nbDisplayed < groupedItems.length ? 'Show more' : 'Load more'} + {nbDisplayed < rows.length ? 'Show more' : 'Load more'}
    {/if} diff --git a/frontend/src/lib/components/home/TutorialBanner.svelte b/frontend/src/lib/components/home/TutorialBanner.svelte deleted file mode 100644 index ef7b30c27e..0000000000 --- a/frontend/src/lib/components/home/TutorialBanner.svelte +++ /dev/null @@ -1,178 +0,0 @@ - - -{#if !isDismissed} - -
    - - {#if hasCompletedAny} - New tutorial available! - {:else} - First time? - {/if} - - - -
    -{/if} diff --git a/frontend/src/lib/components/home/TutorialButton.svelte b/frontend/src/lib/components/home/TutorialButton.svelte deleted file mode 100644 index 6a31ae9569..0000000000 --- a/frontend/src/lib/components/home/TutorialButton.svelte +++ /dev/null @@ -1,124 +0,0 @@ - - - - diff --git a/frontend/src/lib/components/home/WorkspaceEmptyState.svelte b/frontend/src/lib/components/home/WorkspaceEmptyState.svelte new file mode 100644 index 0000000000..fc36cc334b --- /dev/null +++ b/frontend/src/lib/components/home/WorkspaceEmptyState.svelte @@ -0,0 +1,146 @@ + + +
    + {#each rowOpacities as opacity, i (i)} + + {/each} + + +
    + {#if archivedOnly} + + + Everything in this workspace is archived. + . + + {:else} + Your scripts, flows and apps will show up here. + {/if} + {#if canCreate} + + {#if !$disableHubStore} + + + e.detail && logFeatureUsage('home', 'template_picker_open', { key: 'empty_state' })} + > + {#snippet trigger()}Start from a template{/snippet} + {#snippet content({ close })} + { + close() + onPick(project) + }} + /> + {/snippet} + + or + {/if} + + {#snippet trigger()} + + . + {/snippet} + + {/if} +
    +
    diff --git a/frontend/src/lib/components/icons/BRAND_COLORS.md b/frontend/src/lib/components/icons/BRAND_COLORS.md index 1cff77a3d3..11d4df5c3b 100644 --- a/frontend/src/lib/components/icons/BRAND_COLORS.md +++ b/frontend/src/lib/components/icons/BRAND_COLORS.md @@ -179,6 +179,7 @@ repeatedly — check the brand's own page. | `KeycloakIcon` | `keycloak` | fixed | #00B8E3 | #00B8E3 | 8.18 | 10.65 | keycloak.org's own mark, https://www.keycloak.org/resources/images/icon.svg (cyan #00B8E3/#33C6E9/#008AAA over greys #4D4D4D–#EDEDED, single theme) | | `KlaviyoIcon` | `klaviyo` | pair | #1D1E20 | #FFFFFF | 16.14 | 12.47 | klaviyo.com --color-core-charcoal; the flag mark is the standalone logomark the site header collapses to, and the shape of klaviyo.com/icons/icon-512x512.png | | `KoboToolboxIcon` | `kobotoolbox` | fixed | #2095F3 | #2095F3 | 3.05 | 3.95 | the kobotoolbox.org header logo and $kobo-blue in kobotoolbox/kpi jsapp/scss/colors.scss | +| `KubernetesIcon` | — | fixed | #326CE5 | #326CE5 | 4.61 | 2.62 | the CNCF artwork repo (github.com/cncf/artwork/projects/kubernetes/icon/color/kubernetes-icon-color.svg, CC-BY-4.0) | | `KustomerIcon` | `kustomer` | fixed | #FBEC2A | #FBEC2A | 14.08 | 12.47 | kustomer.com/images/kustomer/Kusty.svg | | `LangfuseIcon` | `langfuse` | fixed | #FF5D5F | #FF5D5F | 2.91 | 4.47 | langfuse.com/brand "Icon - Color (SVG)", used unmodified | | `LessIcon` | — | pair | #274F82 | #FFFFFF | 8.04 | 12.47 | github.com/less/logo (MIT) | @@ -443,6 +444,7 @@ Constraints that would otherwise be broken by a well-meaning change. - **JsonIcon** — JSON itself has no brand owner or published colours — json.org states none — so this is a Material palette pick, not a brand colour. - **KanidmIcon** — Kanidm's artwork is CC-BY-NC-ND — no recolouring or other derivatives. - **KlaviyoIcon** — Klaviyo draws it in currentColor, hence the white swap on dark. +- **KubernetesIcon** — The Linux Foundation trademark guidelines allow only the published variants (colour, all-blue, black, white); the colour mark's white knockouts are part of the artwork, not a theme swap. - **LangfuseIcon** — Langfuse's trademark terms forbid modifying the assets. - **LineIcon** — LINE forbids any change to the logo's colour, so there is no reversed variant. - **LinearIcon** — Guidelines ship a light/dark logomark pair and forbid altering the assets in any other way. @@ -571,7 +573,7 @@ Not brands. These inherit `currentColor` on purpose and must not be given a pair ## Coverage -- brand icons: **314**, of which **310** carry a recorded source +- brand icons: **315**, of which **311** carry a recorded source - per-theme pairs applied: **136** (5 of them by inversion or a two-SVG swap, see above) - concept icons: **34** - effectively invisible on light: **1** (AbstractApiIcon) diff --git a/frontend/src/lib/components/icons/KubernetesIcon.svelte b/frontend/src/lib/components/icons/KubernetesIcon.svelte new file mode 100644 index 0000000000..76279e002b --- /dev/null +++ b/frontend/src/lib/components/icons/KubernetesIcon.svelte @@ -0,0 +1,35 @@ + + + + + + + + + diff --git a/frontend/src/lib/components/instanceBanner.test.ts b/frontend/src/lib/components/instanceBanner.test.ts new file mode 100644 index 0000000000..bc93af18a1 --- /dev/null +++ b/frontend/src/lib/components/instanceBanner.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from 'vitest' +import { + INSTANCE_BANNER_MESSAGE_MAX_LEN, + isInstanceBannerVisible, + resolveInstanceBanner +} from './instanceBanner' + +const BANNER = { + enabled: true, + message: 'Scheduled maintenance on Saturday.', + severity: 'warning', + dismissible: true +} + +describe('resolveInstanceBanner', () => { + it('drops a link that is not an absolute http(s) URL', () => { + // Declarative instance config writes global_settings rows directly, so the API's + // scheme check is not the only thing standing between a stored value and an href. + for (const link of ['javascript:alert(1)', 'data:text/html,x', 'status.example.com', 123]) { + expect(resolveInstanceBanner({ ...BANNER, link })?.link).toBeUndefined() + } + expect(resolveInstanceBanner({ ...BANNER, link: 'https://status.example.com' })?.link).toBe( + 'https://status.example.com' + ) + }) + + it('keeps a message the backend accepted whole', () => { + // The backend caps at INSTANCE_BANNER_MESSAGE_MAX_LEN code points (`chars().count()`). + // Truncating with `slice` here would count UTF-16 units and halve an all-emoji message + // that passed validation, so the two sides must measure the same way. + const emoji = '\u{1F6A7}'.repeat(INSTANCE_BANNER_MESSAGE_MAX_LEN) + expect([...resolveInstanceBanner({ ...BANNER, message: emoji })!.message]).toHaveLength( + INSTANCE_BANNER_MESSAGE_MAX_LEN + ) + }) + + it('shows nothing when disabled or without a message', () => { + // An enabled banner with no message is a writable state (the backend accepts it so a + // half-typed announcement cannot fail an admin's whole settings save), so this is the + // only thing keeping it off everyone's screen. + expect(resolveInstanceBanner({ ...BANNER, enabled: false })).toBeUndefined() + expect(resolveInstanceBanner({ ...BANNER, message: ' ' })).toBeUndefined() + expect(resolveInstanceBanner({ ...BANNER, message: 42 })).toBeUndefined() + }) +}) + +describe('isInstanceBannerVisible', () => { + it('honours a dismissal only while the announcement is dismissible', () => { + const dismissible = resolveInstanceBanner(BANNER)! + expect(isInstanceBannerVisible(dismissible, dismissible.fingerprint)).toBe(false) + expect(isInstanceBannerVisible(dismissible, 'some other announcement')).toBe(true) + + // Escalating the same announcement to mandatory must reach the people who already + // dismissed it — the fingerprint does not change, so nothing else would bring it back. + const mandatory = resolveInstanceBanner({ ...BANNER, dismissible: false })! + expect(mandatory.fingerprint).toBe(dismissible.fingerprint) + expect(isInstanceBannerVisible(mandatory, mandatory.fingerprint)).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/instanceBanner.ts b/frontend/src/lib/components/instanceBanner.ts new file mode 100644 index 0000000000..f1693024b8 --- /dev/null +++ b/frontend/src/lib/components/instanceBanner.ts @@ -0,0 +1,129 @@ +import { SettingService } from '$lib/gen' + +/** Drives the banner palette and icon; the subset of `AlertType` that fits an announcement. */ +export type InstanceBannerSeverity = 'info' | 'warning' | 'error' + +/** Stored shape of the `instance_banner` global setting. Every field is optional: a + * stored value can predate a field this code knows about. */ +export interface InstanceBanner { + enabled?: boolean + message?: string + severity?: InstanceBannerSeverity + /** Whether a viewer may dismiss the banner for themselves. Absent means yes. */ + dismissible?: boolean + link?: string + link_label?: string +} + +export const INSTANCE_BANNER_SETTING = 'instance_banner' + +/** Mirror `INSTANCE_BANNER_MESSAGE_MAX_LEN` / `INSTANCE_BANNER_LINK_LABEL_MAX_LEN` in + * backend/windmill-common/src/global_settings.rs, which reject longer values at write time. */ +export const INSTANCE_BANNER_MESSAGE_MAX_LEN = 500 +export const INSTANCE_BANNER_LINK_LABEL_MAX_LEN = 60 + +export type ResolvedInstanceBanner = { + message: string + severity: InstanceBannerSeverity + dismissible: boolean + link?: string + linkLabel: string + /** Dismissal token: a viewer who dismissed one announcement sees the next one, + * because editing any displayed part of the banner changes this string. */ + fingerprint: string +} + +/** + * Read a stored banner field as a string. + * + * The setting is a raw `global_settings` row, so its shape is only ever as good as the + * writer that last touched it — and anything here that calls `.trim()` on a number throws, + * taking the whole settings form down with it. + */ +export function bannerString(value: unknown): string { + return typeof value === 'string' ? value : '' +} + +/** + * Truncate to `max` code points, matching the backend's `chars().count()` cap. + * + * `String.slice` counts UTF-16 code units, so it would cut a 500-emoji message the backend + * accepted in half — and the two sides must agree on what "500 characters" means. + */ +function truncateChars(value: string, max: number): string { + const chars = [...value] + return chars.length > max ? chars.slice(0, max).join('') : value +} + +export function isHttpUrl(value: string): boolean { + try { + const url = new URL(value) + return url.protocol === 'http:' || url.protocol === 'https:' + } catch { + return false + } +} + +/** + * Turn the raw setting into what the banner renders, or `undefined` for "show nothing". + * + * The scheme check repeats the one the writers run on purpose. The link becomes the href of + * an anchor shown to every user of the instance, and this is the last place that can refuse + * it — a row predating the validator, or written straight to the table, reaches here having + * passed nothing. + */ +export function resolveInstanceBanner(raw: unknown): ResolvedInstanceBanner | undefined { + if (!raw || typeof raw !== 'object') return undefined + const banner = raw as InstanceBanner + const message = bannerString(banner.message).trim() + if (banner.enabled !== true || message === '') return undefined + + const severity: InstanceBannerSeverity = + banner.severity === 'warning' || banner.severity === 'error' ? banner.severity : 'info' + const rawLink = bannerString(banner.link).trim() + const link = isHttpUrl(rawLink) ? rawLink : undefined + const linkLabel = + truncateChars(bannerString(banner.link_label).trim(), INSTANCE_BANNER_LINK_LABEL_MAX_LEN) || + 'Learn more' + + return { + message: truncateChars(message, INSTANCE_BANNER_MESSAGE_MAX_LEN), + severity, + dismissible: banner.dismissible !== false, + link, + linkLabel, + fingerprint: JSON.stringify([message, severity, link ?? '', link ? linkLabel : '']) + } +} + +/** + * The reason the banner form cannot be saved, or `undefined` when it can. + * + * Shared with the setting's `isValid` so the Save button and the inline message agree: the + * backend refuses a bad link, and a category save fires its settings concurrently, so a Save + * that got this far would persist the other Core settings and fail only the banner. + */ +export function instanceBannerFormError(value: unknown): string | undefined { + if (!value || typeof value !== 'object') return undefined + const link = bannerString((value as InstanceBanner).link).trim() + return link !== '' && !isHttpUrl(link) ? 'Link must be an absolute http(s) URL' : undefined +} + +/** + * Whether a viewer holding `dismissedFingerprint` should see this announcement. + * + * A non-dismissible announcement ignores stored dismissals entirely: an admin escalating an + * existing notice to mandatory must reach the people who already dismissed it, and the + * fingerprint deliberately does not cover `dismissible`, so nothing else would bring it back. + */ +export function isInstanceBannerVisible( + banner: ResolvedInstanceBanner | undefined, + dismissedFingerprint: string +): banner is ResolvedInstanceBanner { + if (banner == undefined) return false + return !banner.dismissible || dismissedFingerprint !== banner.fingerprint +} + +export async function fetchInstanceBanner(): Promise { + return resolveInstanceBanner(await SettingService.getGlobal({ key: INSTANCE_BANNER_SETTING })) +} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 8bbca5b4be..0548d3ded5 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -1,5 +1,6 @@ import type { ButtonType } from './common/button/model' import { z } from 'zod' +import { instanceBannerFormError } from './instanceBanner' import { writable } from 'svelte/store' /** @@ -68,6 +69,7 @@ export interface Setting { | 'webhook_base_url' | 'ws_connectivity' | 'retention_overrides' + | 'instance_banner' storage: SettingStorage advancedToggle?: { label: string @@ -236,6 +238,19 @@ export const settings: Record = { placeholder: 'only for EE', storage: 'setting' }, + { + label: 'Announcement banner', + description: + 'Message shown above every page of the instance, for maintenance windows and incidents.', + key: 'instance_banner', + fieldType: 'instance_banner', + storage: 'setting', + // The banner only renders on the managed cloud, so only offer it there. + cloudonly: true, + hideInQuickSetup: true, + // Gates Save. The card renders the specific message itself, so no `error` here. + isValid: (value: any) => instanceBannerFormError(value) == undefined + }, { label: 'Non-prod instance', description: diff --git a/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte b/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte index 98fb15c4bd..d55c2f9ce6 100644 --- a/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte +++ b/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte @@ -65,33 +65,46 @@ description="People your identity provider authenticated who opened an app set to Guests without a Windmill account. One email is one guest, however many workspaces it opened." /> -
    - {#key usage} - setInstanceSwitch(e.detail)} - options={{ - right: 'Allow guests on this instance', - rightTooltip: - 'Off, no guest can sign in anywhere, whatever a workspace or an app says, and sessions already issued stop on their next request.' - }} - /> - {/key} -
    +{#if !usage.available} +
    + + No guest can sign in here, whatever a workspace or an app says. Guests require a self-hosted + instance or a dedicated Windmill Cloud deployment. + +
    +{:else} +
    + {#key usage} + setInstanceSwitch(e.detail)} + options={{ + right: 'Allow guests on this instance', + rightTooltip: + 'Off, no guest can sign in anywhere, whatever a workspace or an app says, and sessions already issued stop on their next request.' + }} + /> + {/key} +
    -
    - - {#if usage.metered} - Beyond the allowance, every four guests count as one seat{usage.guest_seats > 0 - ? `: ${usage.billable_guests} guests past it take ${usage.guest_seats} ${usage.guest_seats === 1 ? 'seat' : 'seats'} now` - : ''}. - {:else} - Beyond the allowance, new guests are refused until the count drops below it; an - Enterprise license meters them instead. - {/if} - -
    +
    + + {#if usage.metered} + Beyond the allowance, every four guests count as one seat{usage.guest_seats > 0 + ? `: ${usage.billable_guests} guests past it take ${usage.guest_seats} ${usage.guest_seats === 1 ? 'seat' : 'seats'} now` + : ''}. + {:else} + Beyond the allowance, new guests are refused until the count drops below it; an Enterprise + license meters them instead. + {/if} + +
    +{/if} + import type { Writable } from 'svelte/store' + import Toggle from '../Toggle.svelte' + import TextInput from '../text_input/TextInput.svelte' + import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' + import { classes as alertClasses, icons as alertIcons } from '../common/alert/model' + import { + bannerString, + INSTANCE_BANNER_LINK_LABEL_MAX_LEN, + INSTANCE_BANNER_MESSAGE_MAX_LEN, + INSTANCE_BANNER_SETTING, + instanceBannerFormError, + resolveInstanceBanner, + type InstanceBanner, + type InstanceBannerSeverity + } from '../instanceBanner' + + interface Props { + values: Writable> + disabled?: boolean + } + + let { values, disabled = false }: Props = $props() + + // Writes go through `$values[...]` member assignments: only those push the nested + // mutation back into the store, which is what the unsaved-changes check and the Save + // button watch. Reads go through a *snapshot* because the form mutates that object in + // place — a derived returning the object itself keeps its identity across edits, and + // Svelte then stops propagating to whatever depends on it. Every getter below applies + // the same default `resolveInstanceBanner` does, so a value missing a field displays + // as it will actually render. + let banner: InstanceBanner = $derived( + $state.snapshot($values[INSTANCE_BANNER_SETTING] ?? {}) as InstanceBanner + ) + + const severities: { value: InstanceBannerSeverity; label: string }[] = [ + { value: 'info', label: 'Info' }, + { value: 'warning', label: 'Warning' }, + { value: 'error', label: 'Critical' } + ] + + // Runs the resolver the banner itself uses, so this shows what the instance gets — + // including the "nothing is shown" cases (disabled, or an empty message). + let preview = $derived(resolveInstanceBanner(banner)) + let linkError = $derived(instanceBannerFormError(banner)) + + +
    + banner.enabled === true, (v) => ($values[INSTANCE_BANNER_SETTING].enabled = v) + } + options={{ right: 'Show the banner to every user' }} + /> + +
    + Message + bannerString(banner.message), + (v) => ($values[INSTANCE_BANNER_SETTING].message = String(v)) + } + /> +
    + +
    + Severity + bannerString(banner.severity) || 'info', + (v) => ($values[INSTANCE_BANNER_SETTING].severity = v) + } + > + {#snippet children({ item })} + {#each severities as severity (severity.value)} + + {/each} + {/snippet} + +
    + +
    + Link (optional) +
    + bannerString(banner.link), + (v) => ($values[INSTANCE_BANNER_SETTING].link = String(v)) + } + /> + bannerString(banner.link_label), + (v) => ($values[INSTANCE_BANNER_SETTING].link_label = String(v)) + } + /> +
    + {#if linkError} + {linkError} + {/if} +
    + + banner.dismissible !== false, (v) => ($values[INSTANCE_BANNER_SETTING].dismissible = v) + } + options={{ + right: 'Let users dismiss it', + rightTooltip: + 'Dismissal is remembered per browser and only for this exact announcement: editing the message, severity or link brings it back for everyone. Turning this off also shows it again to everyone who had dismissed it.' + }} + /> + +
    + Preview + {#if preview} + {@const palette = alertClasses[preview.severity]} + {@const Icon = alertIcons[preview.severity]} +
    + + {preview.message} + {#if preview.link} + {preview.linkLabel} + {/if} +
    + {:else} + + {banner.enabled === true + ? 'Nothing is shown until the message is filled in.' + : 'Nothing is shown while the banner is off.'} + + {/if} +
    +
    diff --git a/frontend/src/lib/components/job_args.test.ts b/frontend/src/lib/components/job_args.test.ts new file mode 100644 index 0000000000..cf5818ff0f --- /dev/null +++ b/frontend/src/lib/components/job_args.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from 'vitest' +import { + coerceArgsToSchema, + enforceDisabledDefaults, + redactFileArgs, + redactSecretArgs +} from './job_args' + +describe('coerceArgsToSchema', () => { + // A scalar widget renders its own reading of a wrong-typed value and never writes that + // reading back, so an untouched form submits something it never displayed: a number + // input paints `"12"` as a filled-looking 12, and a toggle shows `"false"` as on. + it('converts a value its widget would read, so the form shows what runs', () => { + const schema = { + properties: { + count: { type: 'number' }, + flag: { type: 'boolean' }, + label: { type: 'string' }, + name: { type: 'string' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { count: '12', flag: 'false', label: 3, name: 'ada' }, + schema + ) + expect(args).toEqual({ count: 12, flag: false, label: '3', name: 'ada' }) + expect(clearedKeys).toEqual([]) + }) + + // Cleared, not carried: the widget shows nothing for these, so nothing is what an + // untouched form should send. + it('empties a value with no reading in its declared type', () => { + const schema = { + properties: { + count: { type: 'number' }, + flag: { type: 'boolean' }, + label: { type: 'string' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { count: 'abc', flag: 'maybe', label: { a: 1 } }, + schema + ) + expect(args).toEqual({}) + expect(clearedKeys.sort()).toEqual(['count', 'flag', 'label']) + }) + + // A field the schema does not name is a field no run surface in the product draws, so a + // value under that name would reach the job without anyone having been able to see it. + // `constructor` is declared by every object through its prototype and by no schema. + it('drops arguments the schema does not declare, naming them', () => { + const kept = coerceArgsToSchema({ a: 'keep', b: 2, constructor: 'x' }, { + properties: { a: { type: 'string' } } + } as any) + expect(kept.args).toEqual({ a: 'keep' }) + expect(kept.undeclaredKeys).toEqual(['b', 'constructor']) + // Declaring nothing is declaring no arguments, which is what a `**kwargs` script and a + // schema that failed to infer both look like. + expect(coerceArgsToSchema({ a: 1 }, undefined).args).toEqual({}) + }) + + // Resolved by the job, so the declared type describes what it receives and never the + // string standing in for it. `Number('$var:…')` is NaN, so coercing would destroy it. + it('leaves a variable or resource reference in any slot', () => { + const schema = { + properties: { + size: { type: 'number' }, + on: { type: 'boolean' }, + db: { type: 'object', format: 'resource-postgresql' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { size: '$var:u/admin/size', on: '$var:u/admin/on', db: '$res:u/admin/pg' }, + schema + ) + expect(args).toEqual({ + size: '$var:u/admin/size', + on: '$var:u/admin/on', + db: '$res:u/admin/pg' + }) + expect(clearedKeys).toEqual([]) + }) + + // Not merely unreadable: `MultiSelect` maps over the value as it renders, so anything + // else throws and takes the whole card down, Cancel with it. A reference is no + // exception — the widget draws before anything resolves — so this slot is the one + // place the reference rule above does not hold. + it('empties a non-array in a dyn-multiselect slot, reference included', () => { + const schema = { properties: { tags: { type: 'object', format: 'dynmultiselect-list' } } } + expect(coerceArgsToSchema({ tags: ['a'] }, schema).args).toEqual({ tags: ['a'] }) + for (const bad of [{ a: 1 }, '$var:u/admin/watchlist']) { + const { args, clearedKeys } = coerceArgsToSchema({ tags: bad }, schema) + expect(args).toEqual({}) + expect(clearedKeys).toEqual(['tags']) + } + }) + + // Below the top the form has the same limitations as everywhere else in the product, + // and descending means resolving `oneOf` branches — where being wrong rewrites what the + // user typed into the branch they did open. + it('leaves nested and container values to the widget that renders them', () => { + const schema = { + properties: { + obj: { type: 'object', properties: { known: { type: 'string' } } }, + rows: { type: 'array', items: { type: 'object' } } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { obj: { known: 1, extra: 'b' }, rows: { id: 'x' } }, + schema + ) + expect(args).toEqual({ obj: { known: 1, extra: 'b' }, rows: { id: 'x' } }) + expect(clearedKeys).toEqual([]) + }) + + // Both sides parsed, never written as literals: `__proto__:` in an object literal is + // the prototype setter, so a literal declares nothing to coerce in the first place. + it('keeps a declared __proto__ instead of losing it to the setter', () => { + const { args } = coerceArgsToSchema( + JSON.parse('{"__proto__":"legit","keep":1}'), + JSON.parse('{"properties":{"__proto__":{"type":"string"},"keep":{"type":"number"}}}') + ) + expect(Object.hasOwn(args, '__proto__')).toBe(true) + expect(args['__proto__']).toBe('legit') + }) +}) + +describe('enforceDisabledDefaults', () => { + const schema = { + properties: { + locked: { type: 'string', disabled: true, default: 'fixed' }, + open: { type: 'string' } + } + } + + it('overwrites a disabled field and reports only what it changed', () => { + expect(enforceDisabledDefaults({ locked: 'mine', open: 'ok' }, schema)).toEqual({ + args: { locked: 'fixed', open: 'ok' }, + resetKeys: ['locked'] + }) + // Never supplied is not overwritten: the field shows the default either way, and a + // caller told otherwise would try to correct what it never sent. + expect(enforceDisabledDefaults({ open: 'ok' }, schema)).toEqual({ + args: { locked: 'fixed', open: 'ok' }, + resetKeys: [] + }) + }) + + it('reports no reset for an object default the caller already matched', () => { + const objSchema = { + properties: { conf: { type: 'object', disabled: true, default: { a: 1 } } } + } + expect(enforceDisabledDefaults({ conf: { a: 1 } }, objSchema).resetKeys).toEqual([]) + }) +}) + +describe('secret args at every level the form nests', () => { + const schema = { + properties: { + top: { type: 'string', password: true }, + obj: { properties: { inner: { type: 'string', password: true } } }, + list: { items: { properties: { secret: { type: 'string', password: true } } } }, + either: { + oneOf: [ + { title: 'a', properties: { key: { type: 'string', password: true } } }, + { title: 'b', properties: { other: { type: 'string', password: true } } } + ] + } + } + } + const args = { + top: 'hunter2', + obj: { inner: '$var:u/ada/prod', keep: 1 }, + list: [{ secret: 'one', name: 'a' }, { secret: 'two' }], + // Tagged as branch 'a', but 'b' is stripped too: the tag is runtime state. + either: { kind: 'a', key: 'k', other: 'o' } + } + + it('redacts every value and keeps every reference', () => { + const redacted = JSON.stringify(redactSecretArgs(args, schema)) + for (const secret of ['hunter2', 'one', 'two', '"k"', '"o"']) { + expect(redacted).not.toContain(secret) + } + expect(redacted).toContain('') + expect(redacted).toContain('"name":"a"') + expect(redacted).toContain('$var:u/ada/prod') + }) + + // ArgInput synthesises '' for every untouched string, so marking one would put a hidden + // value on the card for a field nobody filled in — and mint nothing to back it. + it('leaves an empty secret empty', () => { + expect( + redactSecretArgs({ tok: '' }, { properties: { tok: { type: 'string', password: true } } }) + ).toEqual({ tok: '' }) + }) + + it('reaches a secret under a oneOf branch of an array element', () => { + const oneOfItems = { + properties: { + steps: { + type: 'array', + items: { + oneOf: [{ title: 'push', properties: { token: { type: 'string', password: true } } }] + } + } + } + } + expect(redactSecretArgs({ steps: [{ token: 'hunter2', name: 'a' }] }, oneOfItems)).toEqual({ + steps: [{ token: '', name: 'a' }] + }) + }) + + // The walk descends on the value's shape: routing an array down `properties` because the + // declaration carries that key would visit none of its elements, leaving the secret in + // the persisted card verbatim. + it('reaches through a declaration carrying both items and properties', () => { + const both = { + properties: { + creds: { + type: 'array', + items: { properties: { token: { type: 'string', password: true } } }, + properties: { token: { type: 'string', password: true } } + } + } + } + expect(redactSecretArgs({ creds: [{ token: 'hunter2' }] }, both)).toEqual({ + creds: [{ token: '' }] + }) + }) + + // A container shaped unlike its declaration is kept, so the walk has to reach in through + // the half the declaration does carry — on the value's shape alone it stops at the + // mismatch, leaving the secret there for the persisted card and the model to read. + it('reaches through a container shaped unlike its declaration', () => { + const declaresArray = { + properties: { + rows: { type: 'array', items: { properties: { token: { password: true } } } } + } + } + expect(redactSecretArgs({ rows: { token: 'hunter2' } }, declaresArray)).toEqual({ + rows: { token: '' } + }) + + const declaresObject = { + properties: { cfg: { type: 'object', properties: { token: { password: true } } } } + } + expect(redactSecretArgs({ cfg: [{ token: 'hunter2' }] }, declaresObject)).toEqual({ + cfg: [{ token: '' }] + }) + }) +}) + +describe('redactFileArgs', () => { + const schema = { + properties: { + doc: { type: 'string', contentEncoding: 'base64' }, + pics: { type: 'array', items: { type: 'string', contentEncoding: 'base64' } }, + wrap: { properties: { inner: { type: 'string', contentEncoding: 'base64' } } }, + note: { type: 'string' } + } + } + + it('replaces the bytes with a size marker at every level, and keeps the rest', () => { + const oneMeg = 'A'.repeat(1024 * 1024 * 2) + const redacted = redactFileArgs( + { doc: oneMeg, pics: ['B'.repeat(4096)], wrap: { inner: 'C'.repeat(2048) }, note: 'hi' }, + schema + ) + expect(redacted).toEqual({ + doc: '', + pics: [''], + wrap: { inner: '' }, + note: 'hi' + }) + }) +}) diff --git a/frontend/src/lib/components/job_args.ts b/frontend/src/lib/components/job_args.ts index ef3377081f..9698c03cd4 100644 --- a/frontend/src/lib/components/job_args.ts +++ b/frontend/src/lib/components/job_args.ts @@ -1,5 +1,303 @@ +/** + * A job's arguments prepared for a run form, its readers, and a result view. Coercing must + * not lose what the caller meant to send, so it is exact and shallow; stripping and + * redacting only blank a field, so they go to any depth and err towards visiting too much. + */ import { deepEqual } from 'fast-equals' +const isLockedProp = (prop: any) => !!prop?.disabled && 'default' in prop + +/** + * A field the schema disables is not the caller's to set: the run sends the schema's + * default whatever it holds. Top-level only, like every filter here. Returns the keys it + * overwrote; notifying is the caller's job. + */ +export function enforceDisabledDefaults( + args: Record, + schema: { properties?: Record } | undefined +): { args: Record; resetKeys: string[] } { + // Null prototype: assigning a declared `__proto__` into a plain `{}` reaches the + // inherited setter and the default vanishes. Always copied — callers bind the result to + // a form that edits in place, so returning the input would write through to theirs. + const result: Record = Object.assign(Object.create(null), args) + if (!schema?.properties) return { args: { ...result }, resetKeys: [] } + const resetKeys: string[] = [] + for (const [key, prop] of Object.entries(schema.properties)) { + if (!isLockedProp(prop)) continue + // Never supplied is not overwritten, and compared by value: a default can be an + // object, where identity would report every correct run as overridden. + if (result[key] !== undefined && !deepEqual(result[key], prop.default)) resetKeys.push(key) + result[key] = prop.default + } + return { args: { ...result }, resetKeys } +} + +/** How a form says what {@link enforceDisabledDefaults} overwrote, shared by the two that + * run it so the wording cannot drift apart. */ +export const resetKeysToast = (resetKeys: string[]): string => + `Disabled field${resetKeys.length > 1 ? 's' : ''} ${resetKeys + .map((k) => `'${k}'`) + .join(', ')} reset to default value${resetKeys.length > 1 ? 's' : ''}` + +/** Types `setInputCat` routes to a widget bound to a scalar. */ +const SCALAR_TYPES = new Set(['string', 'number', 'integer', 'boolean']) + +/** + * Declares an array though its `type` says `object`. A mismatch here throws rather than + * reading wrong: `MultiSelect` maps over the value as it renders, so anything else takes + * the form down, Cancel with it — a reference included, since it draws before resolving. + */ +const declaresDynMultiselect = (prop: any) => + typeof prop?.format === 'string' && prop.format.startsWith('dynmultiselect-') + +const fitsScalarType = (value: any, type: string): boolean => + type === 'integer' ? typeof value === 'number' : typeof value === type + +/** + * Resolved at run time, so the declared type describes what the job receives and never the + * string standing in for it. `ArgInput.validateInput` blesses these ahead of every type check. + */ +const REFERENCE_PREFIXES = ['$var:', '$res:', '$jsonvar:'] +const isReference = (value: any): boolean => + typeof value === 'string' && REFERENCE_PREFIXES.some((prefix) => value.startsWith(prefix)) + +/** No plain reading in the declared type; distinct from a value that reads as `undefined`. */ +const UNCOERCIBLE = Symbol('uncoercible') + +/** + * The value a scalar widget would stand for, or {@link UNCOERCIBLE}. Only conversions with + * one plain reading: a number input shows `"7"` as 7 and a toggle shows any non-empty + * string as on, so guessing past this would put a value on screen that nobody wrote. + */ +function coerceScalar(value: any, type: string): any { + if (typeof value === 'object') return UNCOERCIBLE + if (type === 'string') { + return typeof value === 'number' || typeof value === 'boolean' ? String(value) : UNCOERCIBLE + } + if (typeof value !== 'string') return UNCOERCIBLE + const trimmed = value.trim() + if (type === 'number' || type === 'integer') { + if (trimmed === '') return UNCOERCIBLE + const parsed = Number(trimmed) + return Number.isFinite(parsed) ? parsed : UNCOERCIBLE + } + if (type === 'boolean') { + if (trimmed.toLowerCase() === 'true') return true + if (trimmed.toLowerCase() === 'false') return false + } + return UNCOERCIBLE +} + +/** + * Drop every argument the schema does not declare, naming them. The schema is what every run + * surface builds its fields from, so a value under a name it never declares has no widget + * anywhere: sending one is sending what nobody could see or edit before the run. + */ +export function dropUndeclaredArgs( + args: Record, + schema: { properties?: Record } | undefined +): { args: Record; undeclaredKeys: string[] } { + const properties = schema?.properties ?? {} + // hasOwn, not `in`: every object inherits `constructor` and `toString`, so `in` would + // hand an inherited declaration to an argument the schema never named. + const kept: Record = Object.create(null) + const undeclaredKeys: string[] = [] + for (const [key, value] of Object.entries(args ?? {})) { + if (Object.hasOwn(properties, key)) kept[key] = value + else undeclaredKeys.push(key) + } + return { args: { ...kept }, undeclaredKeys } +} + +/** + * Make arguments say what the run form will show, then apply {@link enforceDisabledDefaults}. + * A scalar widget renders its own reading of a wrong-typed value and never writes it back, so + * an untouched form would submit what it never displayed; a value with no reading is cleared. + * Top-level only: descending means resolving `oneOf`, where being wrong rewrites user input. + */ +export function coerceArgsToSchema( + args: Record, + schema: { properties?: Record } | undefined +): { + args: Record + resetKeys: string[] + clearedKeys: string[] + undeclaredKeys: string[] +} { + const properties = schema?.properties ?? {} + const clearedKeys: string[] = [] + const { args: declared, undeclaredKeys } = dropUndeclaredArgs(args, schema) + const kept: Record = Object.create(null) + for (const [key, value] of Object.entries(declared)) { + // Declared, but a declaration can still be nothing, and reading `.type` off it throws. + const prop = properties[key] + if ( + prop === undefined || + value == null || + (isReference(value) && !declaresDynMultiselect(prop)) + ) { + kept[key] = value + continue + } + if (declaresDynMultiselect(prop)) { + if (Array.isArray(value)) kept[key] = value + else clearedKeys.push(key) + continue + } + if (!SCALAR_TYPES.has(prop.type) || fitsScalarType(value, prop.type)) { + kept[key] = value + continue + } + const coerced = coerceScalar(value, prop.type) + if (coerced === UNCOERCIBLE) clearedKeys.push(key) + else kept[key] = coerced + } + const { args: result, resetKeys } = enforceDisabledDefaults({ ...kept }, schema) + return { args: result, resetKeys, clearedKeys, undeclaredKeys } +} + +/** + * Every bag of `properties` a declaration can show a value's keys through, including every + * `oneOf` branch rather than the selected one: a secret under a variant nobody opened + * leaves the form just the same. + */ +function declarationBags(prop: any): Record[] { + const bags: Record[] = [] + if (prop?.properties) bags.push(prop.properties) + if (Array.isArray(prop?.oneOf)) + for (const branch of prop.oneOf) if (branch?.properties) bags.push(branch.properties) + return bags +} + +/** + * Apply `visit` to every value whose declaration matches `isLeaf`, at any depth; returning + * `undefined` removes it. Recursive because the form is, so a level left unvisited is one a + * secret can sit at. Descends on the value's shape, never on the declaration's keys: one + * carrying both `items` and `properties` must not route a shape down the other's branch. + */ +function mapLeaves( + value: any, + prop: any, + isLeaf: (prop: any) => boolean, + visit: (value: unknown, prop: any, path: (string | number)[]) => unknown, + path: (string | number)[] +): any { + if (value == null || typeof value !== 'object') return value + // A container shaped unlike its declaration is kept rather than dropped, since the widget + // is the one that reports it — so the walk has to reach in through whichever half the + // declaration does carry, or a secret under one leaves the form verbatim. + if (Array.isArray(value)) + return value.map((item, i) => mapLeaves(item, prop?.items ?? prop, isLeaf, visit, [...path, i])) + const bags = declarationBags(prop) + if (bags.length === 0) + return prop?.items ? mapLeaves(value, prop.items, isLeaf, visit, path) : value + // Null prototype, and keyed off the value rather than the declaration: a key is only + // ever rewritten where it already exists, so no branch of a `oneOf` can add one. + const result: Record = Object.assign(Object.create(null), value) + for (const key of Object.keys(result)) { + const declared = bags.filter((bag) => Object.hasOwn(bag, key)).map((bag) => bag[key]) + // Segments, never a joined name: a key can itself hold a dot, and two leaves reported + // under one name let a caller correlating by it take the one for the other. + const keyPath = [...path, key] + // A matching object is a leaf, not a level: a password object is stored whole as a + // single $jsonvar: reference, and a file is one opaque base64 string. + const leaf = declared.find(isLeaf) + if (leaf) { + const mapped = visit(result[key], leaf, keyPath) + if (mapped === undefined) delete result[key] + else result[key] = mapped + continue + } + for (const declaration of declared) + result[key] = mapLeaves(result[key], declaration, isLeaf, visit, keyPath) + } + return { ...result } +} + +export const isSecretProp = (prop: any) => !!prop?.password + +const isFileProp = (prop: any) => + prop?.contentEncoding === 'base64' || prop?.items?.contentEncoding === 'base64' + +function fileMarker(base64: string): string { + const bytes = Math.floor((base64.length * 3) / 4) + return bytes < 1024 * 1024 + ? `` + : `` +} + +/** {@link mapLeaves} over a whole argument object, against a schema that may declare + * nothing to match. Copies either way, for the reason {@link enforceDisabledDefaults} + * copies. */ +export function mapArgLeaves( + args: Record, + schema: { properties?: Record } | undefined, + isLeaf: (prop: any) => boolean, + visit: (value: unknown, prop: any, path: (string | number)[]) => unknown +): Record { + return mapLeaves(args ?? {}, { properties: schema?.properties ?? {} }, isLeaf, visit, []) +} + +/** A leaf's path as the lines naming it to a reader read: `creds[0].secret`. */ +const formatArgPath = (path: (string | number)[]): string => + path.reduce( + (acc, segment) => + typeof segment === 'number' + ? `${acc}[${segment}]` + : acc + ? `${acc}.${segment}` + : String(segment), + '' + ) + +/** + * Drop every file argument, so a caller cannot propose file bytes on the user's behalf: + * the field opens empty and the user attaches the file. Bytes a form is prefilled with + * are bytes the stored transcript carries, unbounded, for a value no caller can produce. + * Reports the path of each one removed, or the caller reads the absence as the user having + * deleted the value. + */ +export function stripFileArgs( + args: Record, + schema: { properties?: Record } | undefined, + strippedKeys?: string[] +): Record { + return mapArgLeaves(args, schema, isFileProp, (value, _prop, path) => { + if (value !== undefined) strippedKeys?.push(formatArgPath(path)) + return undefined + }) +} + +/** + * Replace a sensitive value with a fixed marker, for text that leaves the form. A reference is + * kept: it names a variable rather than holding one, and the run page shows the same job's + * arguments that way. An empty field is kept for the reason `processSecretArgs` mints nothing + * for one — marking it would describe a secret the run never carried. + */ +export function redactSecretArgs( + args: Record, + schema: { properties?: Record } | undefined +): Record { + return mapArgLeaves(args, schema, isSecretProp, (value) => + value == null ? undefined : value === '' || isReference(value) ? value : '' + ) +} + +/** + * Replace every file argument with a marker naming its size. The base64 belongs in the + * job request and nowhere else: rendered it is unreadable, persisted it is unbounded, and + * a file small enough to survive truncation reaches the model whole. + */ +export function redactFileArgs( + args: Record, + schema: { properties?: Record } | undefined +): Record { + const mark = (value: unknown) => (typeof value === 'string' ? fileMarker(value) : value) + return mapArgLeaves(args, schema, isFileProp, (value) => + Array.isArray(value) ? value.map(mark) : mark(value) + ) +} + export function isWindmillTooBigObject(obj: any): boolean { return ( typeof obj === 'object' && diff --git a/frontend/src/lib/components/markdownProse.ts b/frontend/src/lib/components/markdownProse.ts index aedd2f45b1..3f79ab79a5 100644 --- a/frontend/src/lib/components/markdownProse.ts +++ b/frontend/src/lib/components/markdownProse.ts @@ -4,10 +4,14 @@ * call site with layout-only classes (padding, width, bg); anything typographic * belongs here. * - * - 'xs': micro scale for dense secondary panes (chat reasoning blocks) + * - 'xs': micro scale for dense secondary panes (chat reasoning blocks, group notes) * - 'sm': compact chat-bubble scale (assistant messages, flow/app chat, settings) * - 'doc': same rhythm and body size as 'sm', with a taller heading ramp * (lg/base/sm) and semibold headings for document-like surfaces (artifacts) + * + * h1 and h2 step above the body size in every preset, so `#` and `##` read as + * headings rather than bold body text. Below that the tight 'xs' and 'sm' scales + * run out of room, and h3 down differentiates by weight and colour alone. */ // Kept as literal template parts: Tailwind's scanner reads class names verbatim @@ -28,8 +32,8 @@ const bodyXs = 'text-primary prose-p:text-primary prose-li:text-primary prose-p:text-xs prose-li:text-xs prose-code:text-xs prose-pre:text-xs prose-table:text-xs' export const markdownProse = { - xs: `${base} prose-sm leading-snug prose-p:text-2xs prose-li:text-2xs prose-code:text-2xs prose-pre:text-2xs prose-headings:font-medium prose-headings:text-secondary prose-headings:mt-2 prose-headings:mb-1 prose-h1:text-2xs prose-h2:text-2xs prose-h3:text-2xs prose-h4:text-2xs prose-h5:text-2xs prose-h6:text-2xs prose-strong:text-secondary`, - sm: `${base} ${rhythm} ${bodyXs} prose-headings:mt-3 prose-headings:mb-1 prose-headings:font-medium prose-headings:text-emphasis prose-h1:text-sm prose-h2:text-xs prose-h3:text-xs prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs`, + xs: `${base} prose-sm leading-snug prose-p:text-2xs prose-li:text-2xs prose-code:text-2xs prose-pre:text-2xs prose-headings:font-medium prose-headings:text-secondary prose-headings:mt-2 prose-headings:mb-1 prose-h1:text-sm prose-h2:text-xs prose-h3:text-2xs prose-h4:text-2xs prose-h5:text-2xs prose-h6:text-2xs prose-strong:text-secondary`, + sm: `${base} ${rhythm} ${bodyXs} prose-headings:mt-3 prose-headings:mb-1 prose-headings:font-medium prose-headings:text-emphasis prose-h1:text-base prose-h2:text-sm prose-h3:text-xs prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs`, doc: `${base} ${rhythm} ${bodyXs} prose-headings:mt-8 prose-headings:mb-2 prose-headings:font-semibold prose-headings:text-emphasis prose-h1:text-lg prose-h2:text-base prose-h3:text-sm prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs prose-pre:bg-transparent prose-pre:p-0` } as const diff --git a/frontend/src/lib/components/mcp/enabledServers.ts b/frontend/src/lib/components/mcp/enabledServers.ts index b49b788c67..6dbeb997c1 100644 --- a/frontend/src/lib/components/mcp/enabledServers.ts +++ b/frontend/src/lib/components/mcp/enabledServers.ts @@ -1,11 +1,12 @@ -import { createEnabledPathsPreference } from '$lib/components/copilot/chat/enabledPathsPreference' +import { createPathsPreference } from '$lib/components/copilot/chat/enabledPathsPreference' /** Which MCP servers the chat may act through, per workspace and per account. A * server's tools both reach an external system and put their descriptions in the * model's context, so one is off until it is turned on; connecting one through the * chat turns it on for the person who connected it. */ -const preference = createEnabledPathsPreference('wm_mcp_enabled') +const preference = createPathsPreference('wm_mcp_enabled', false) -export const enabledMcpPaths = preference.enabledPaths +/** Servers are off by default, so what is stored as on is the whole enabled set. */ +export const enabledMcpPaths = preference.explicitlyEnabledPaths export const isMcpEnabled = preference.isEnabled export const setMcpEnabled = preference.setEnabled diff --git a/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts b/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts index 444e03a0a3..6a18a28e46 100644 --- a/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts +++ b/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts @@ -1,5 +1,6 @@ import { extToLang } from '$lib/editorLangUtils' -import { cleanValueProperties, orderedYamlStringify, replaceFalseWithUndefined } from '$lib/utils' +import { cleanValueProperties, replaceFalseWithUndefined } from '$lib/utils' +import { orderedYamlStringify } from '$lib/utils/orderedYaml' // A raw app rendered as a *folder of files* for diffing. Each entry is one // virtual file: real `files` keep their natural path, runnables become diff --git a/frontend/src/lib/components/runs/useJobsLoader.svelte.ts b/frontend/src/lib/components/runs/useJobsLoader.svelte.ts index 5d2f878bfd..828f060972 100644 --- a/frontend/src/lib/components/runs/useJobsLoader.svelte.ts +++ b/frontend/src/lib/components/runs/useJobsLoader.svelte.ts @@ -18,6 +18,9 @@ import { CancelablePromiseUtils } from '$lib/cancelable-promise-utils' import type { Timeframe } from './timeframes' import { allowWildcards as _allowWildcards, type RunsFilterInstance } from './runsFilter' +// windmill_common::utils::MAX_PER_PAGE: the server silently caps per_page at this value +const MAX_PER_PAGE = 10000 + export function computeJobKinds(jobKindsCat: string | null): string { if (jobKindsCat == 'all') { return '' @@ -75,6 +78,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { let label = $derived(filters?.label ?? null) let worker = $derived(filters?.worker ?? null) let success = $derived(filters?.status ?? null) + let isQueueOnly = $derived(success == 'running' || success == 'suspended' || success == 'waiting') let showSkipped = $derived(filters?.show_skipped ?? false) let resolutionFilter = $derived(filters?.resolved ?? 'all') let showSchedules = $derived(!filters?.job_trigger_kind?.includes('!schedule')) @@ -126,9 +130,14 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { let promise = loadJobsIntern(true) if (perPage > 25) { promise = CancelablePromiseUtils.onTimeout(promise, 4000, () => { - sendUserToast('Loading jobs is taking longer than expected...', 'warning', [ - { label: 'Stream by batches of 25', callback: () => restreamWithSmallBatches() } - ]) + const noStartDate = timeframe?.computeMinMax().minTs == null + sendUserToast( + (success == 'failure' || success == 'canceled') && noStartDate + ? `Loading ${success == 'failure' ? 'failed' : 'canceled'} jobs with no start date scans the full job history. Set a time range to speed it up.` + : 'Loading jobs is taking longer than expected...', + 'warning', + [{ label: 'Stream by batches of 25', callback: () => restreamWithSmallBatches() }] + ) }) } promise = CancelablePromiseUtils.finallyDo(promise, () => { @@ -191,21 +200,40 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { loadingExtra = false } + // Mirrors when list_completed_jobs_query sorts by completed_at. A created_at cursor does not + // bound that index scan, so each batch would rescan from the newest job, and skip jobs created + // after the cursor but completed before it. + function sortsByCompletedAt(minTs: string | null, maxTs: string | null): boolean { + return minTs != null || maxTs != null || success == 'failure' || success == 'canceled' + } + function loadExtraJobsBatch(batchSize: number): CancelablePromise { if (!jobs || jobs.length === 0) { lastFetchWentToEnd = true return CancelablePromiseUtils.pure(undefined as void) } - const lastJob = jobs[jobs.length - 1] - const ts = lastJob.created_at - if (!ts) { + const { minTs, maxTs } = timeframe?.computeMinMax() ?? { minTs: null, maxTs: null } + const byCompletedAt = + jobs[jobs.length - 1].type === 'CompletedJob' && sortsByCompletedAt(minTs, maxTs) + const sortKey = (j: Job) => + byCompletedAt ? (j.type === 'CompletedJob' ? j.completed_at : undefined) : j.created_at + const cursorTs = sortKey(jobs[jobs.length - 1]) + if (!cursorTs) { lastFetchWentToEnd = true return CancelablePromiseUtils.pure(undefined as void) } - const cursorTs = new Date(new Date(ts).getTime() - 1).toISOString() - const minTs = timeframe?.computeMinMax().minTs ?? null + // Inclusive cursor at the API's microsecond precision: jobs sharing the boundary timestamp (e.g. + // a bulk cancel) are refetched rather than skipped, and the page grows by those already listed, + // up to the server's MAX_PER_PAGE. Once the listed part of the group fills that cap, the cursor + // steps just below the group, dropping its remainder instead of ending the list early. + const tied = jobs.filter((j) => sortKey(j) === cursorTs).length + const stepOver = tied >= MAX_PER_PAGE + const cursor = stepOver ? new Date(new Date(cursorTs).getTime() - 1).toISOString() : cursorTs + const pageSize = stepOver ? batchSize : Math.min(batchSize + tied, MAX_PER_PAGE) return CancelablePromiseUtils.map( - fetchJobs(null, minTs, undefined, cursorTs, batchSize), + byCompletedAt + ? fetchJobs(cursor, minTs, undefined, undefined, pageSize) + : fetchJobs(null, minTs, undefined, cursor, pageSize), (olderJobs) => { jobs = updateWithNewJobs(olderJobs ?? [], jobs ?? []) if (extendedJobs) { @@ -213,7 +241,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { extendedJobs = extendedJobs } computeCompletedJobs() - lastFetchWentToEnd = (olderJobs?.length ?? 0) < batchSize + lastFetchWentToEnd = (olderJobs?.length ?? 0) < pageSize loading = false } ) @@ -230,7 +258,6 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { loadingFetch = true let scriptPathStart = folder == null || folder === '' ? undefined : `f/${folder}/` let scriptPathExact = path == null || path === '' ? undefined : path - let isQueueOnly = success == 'running' || success == 'suspended' || success == 'waiting' let isCompletedOnly = success == 'success' || success == 'failure' || success == 'canceled' let promise = JobService.listJobs({ workspace: currentWorkspace, @@ -289,7 +316,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { }) promise = CancelablePromiseUtils.catchErr(promise, (e) => { if (e instanceof CancelError) return CancelablePromiseUtils.err(e) - sendUserToast('There was an issue loading jobs, see browser console for more details', true) + sendUserToast(`Could not load jobs: ${e.body ?? e.message}`, true) console.error(e) return CancelablePromiseUtils.pure([]) }) @@ -394,6 +421,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { overrideBatchSize?: number ): CancelablePromise { const { minTs, maxTs } = timeframe?.computeMinMax() ?? { minTs: null, maxTs: null } + listLoadedAt = new Date(Date.now() - 5 * 60_000).toISOString() if (shouldGetCount) { getCount() } @@ -529,6 +557,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { } let lastQueueTs: string | undefined = undefined + let listLoadedAt: string | null = null async function syncer() { if (loadingFetch) { @@ -575,7 +604,15 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { loading = true let newJobs: Job[] if (concurrencyKey == null || concurrencyKey === '') { - newJobs = await fetchJobs(maxTs, minTs ?? completedTs, queueTs) + // With no completed job to anchor on, each refresh would repeat the initial + // unbounded scan of completed jobs, possibly the one that just timed out. Not for + // queue-only views: fetchJobs turns this into the queue's created_at bound, hiding + // older jobs that suspend or come due. The margin absorbs browser/database skew. + newJobs = await fetchJobs( + maxTs, + minTs ?? completedTs ?? (isQueueOnly ? null : listLoadedAt), + queueTs + ) } else { // Obscured jobs have no ids, so we have to do the full request extendedJobs = await fetchExtendedJobs(concurrencyKey, maxTs, minTs ?? completedTs) diff --git a/frontend/src/lib/components/secretArgUtils.test.ts b/frontend/src/lib/components/secretArgUtils.test.ts new file mode 100644 index 0000000000..d455b4ad62 --- /dev/null +++ b/frontend/src/lib/components/secretArgUtils.test.ts @@ -0,0 +1,130 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const created: { path: string; value: string; is_secret?: boolean }[] = [] + +vi.mock('$lib/gen', () => ({ + VariableService: { + createVariable: vi.fn(async ({ requestBody }: any) => { + created.push(requestBody) + return requestBody.path + }) + } +})) + +vi.mock('$lib/stores', async () => { + const { writable } = await import('svelte/store') + return { workspaceStore: writable('test-ws'), userStore: writable({ username: 'ada' }) } +}) + +import { processSecretArgs } from './secretArgUtils' + +describe('processSecretArgs', () => { + beforeEach(() => (created.length = 0)) + + const schema = { + properties: { + token: { type: 'string', password: true }, + creds: { type: 'object', password: true, properties: { user: { type: 'string' } } }, + nested: { type: 'object', properties: { inner: { type: 'string', password: true } } }, + plain: { type: 'string' } + } + } as any + + // Nothing else turns a proposed secret into a reference when no form mounts, so a literal + // left alone here is a plaintext credential stored on the job for anyone who can see it. + it('mints a reference for a literal at every level, leaving other arguments alone', async () => { + const out = await processSecretArgs( + { token: 'hunter2', creds: { user: 'ada' }, nested: { inner: 'deep' }, plain: 'kept' }, + schema + ) + expect(out.token).toMatch(/^\$var:u\/ada\/secret_arg\//) + expect(out.creds).toMatch(/^\$jsonvar:u\/ada\/secret_arg\//) + expect(out.nested.inner).toMatch(/^\$var:u\/ada\/secret_arg\//) + expect(out.plain).toBe('kept') + // The object goes into the variable as JSON, which is what `$jsonvar:` parses back. + expect(created.map((c) => c.value).sort()).toEqual(['deep', 'hunter2', '{"user":"ada"}']) + expect(created.every((c) => c.is_secret)).toBe(true) + }) + + it('leaves a reference the caller already named alone', async () => { + const out = await processSecretArgs( + { token: '$var:f/team/api_token', creds: '$jsonvar:u/ada/existing' }, + schema + ) + expect(out).toEqual({ token: '$var:f/team/api_token', creds: '$jsonvar:u/ada/existing' }) + expect(created).toEqual([]) + }) + + // `$var:` hands the job the variable's text; a field declaring an object needs it parsed, + // which is the same variable read the other way rather than a secret the caller cannot see. + it('reads a plain variable as JSON where the field cannot hold a string', async () => { + const out = await processSecretArgs({ creds: '$var:u/ada/stripe' }, schema) + expect(out.creds).toBe('$jsonvar:u/ada/stripe') + expect(created).toEqual([]) + }) + + // Reached by every run form in the product, not only the ones a chat opens. + it('leaves an absent, null or empty secret alone', async () => { + expect(await processSecretArgs({ token: null, plain: 'kept' }, schema)).toEqual({ + token: null, + plain: 'kept' + }) + expect(await processSecretArgs({ token: '', plain: 'kept' }, schema)).toEqual({ + token: '', + plain: 'kept' + }) + expect(created).toEqual([]) + }) + + // A property name can itself contain a dot. Reported under one label these two leaves + // would share a mint, and the flat field would run on the nested field's secret. + it("tells apart a key that spells another key's path", async () => { + const out = await processSecretArgs({ 'db.password': 'FLAT', db: { password: 'NESTED' } }, { + properties: { + 'db.password': { type: 'string', password: true }, + db: { + type: 'object', + properties: { password: { type: 'string', password: true } } + } + } + } as any) + const flat = out['db.password'].slice('$var:'.length) + const nested = out.db.password.slice('$var:'.length) + expect(flat).not.toBe(nested) + expect(created.find((c) => c.path === flat)?.value).toBe('FLAT') + expect(created.find((c) => c.path === nested)?.value).toBe('NESTED') + }) + + // Callers bind a form that stays editable while the mints are in flight, and a leaf is + // addressed by its path: a row moved between the two walks would take the other row's + // reference and the job would run it on the wrong credentials. + it('ignores the caller mutating the arguments while minting', async () => { + const rows = { + creds: [ + { name: 'alpha', secret: 'FIRST' }, + { name: 'beta', secret: 'SECOND' } + ] + } + const arraySchema = { + properties: { + creds: { + type: 'array', + items: { properties: { name: {}, secret: { password: true } } } + } + } + } as any + + const pending = processSecretArgs(rows, arraySchema) + rows.creds.reverse() + const out = await pending + + // Keyed by the row's own name, not its index: a substitution by position lands the + // first-minted reference on index 0 either way. + const secretOf = (name: string) => { + const row = out.creds.find((c: any) => c.name === name) + return created.find((c) => c.path === row.secret.slice('$var:'.length))?.value + } + expect(secretOf('alpha')).toBe('FIRST') + expect(secretOf('beta')).toBe('SECOND') + }) +}) diff --git a/frontend/src/lib/components/secretArgUtils.ts b/frontend/src/lib/components/secretArgUtils.ts index 91101d4816..a330d3bcd9 100644 --- a/frontend/src/lib/components/secretArgUtils.ts +++ b/frontend/src/lib/components/secretArgUtils.ts @@ -3,11 +3,54 @@ import { VariableService } from '$lib/gen' import { get } from 'svelte/store' import { userStore, workspaceStore } from '$lib/stores' import { generateRandomString } from '$lib/utils' +import { stateSnapshot } from '$lib/stateSnapshot.svelte' +import { isSecretProp, mapArgLeaves } from './job_args' + +/** Where a caller's own ephemeral secrets live, so a field can tell one it minted from a + * workspace variable someone linked by hand. */ +export function ephemeralSecretPrefix(username: string): string { + return `u/${username}/secret_arg/` +} /** - * Process args before job submission: for non-string fields marked as password/sensitive, - * create ephemeral secret variables and replace values with $jsonvar:path references. - * String password fields are already handled by PasswordArgInput (uses $var:). + * Mint the ephemeral secret variable a sensitive argument is submitted as, and return its path. + * It expires on its own, so a run that is abandoned leaves no permanent secret behind. + */ +export async function mintEphemeralSecret( + workspace: string, + username: string, + value: string +): Promise { + const path = ephemeralSecretPrefix(username) + generateRandomString(12) + await VariableService.createVariable({ + workspace, + requestBody: { + value, + is_secret: true, + path, + description: 'Ephemeral secret variable', + expires_at: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7).toISOString() + } + }) + return path +} + +/** `$var:` hands the job the variable's text and `$jsonvar:` hands it the parsed value, so a + * field that cannot hold a string needs the second one whichever the caller named. */ +function referencePrefix(prop: any, value: unknown): '$var:' | '$jsonvar:' { + return typeof value === 'string' && prop?.type !== 'object' && prop?.type !== 'array' + ? '$var:' + : '$jsonvar:' +} + +/** + * Turn every sensitive argument into a reference before the job is submitted: a plaintext value + * is minted into an ephemeral secret variable, so what is stored on the job — and readable by + * anyone who can see its run — names a secret instead of holding one. + * + * The single place that decides how a secret reaches a job: {@link PasswordArgInput} mints + * through it while the user types, and a run the autonomy posture starts without a form calls it + * in the widget's stead. */ export async function processSecretArgs( args: Record, @@ -24,29 +67,53 @@ export async function processSecretArgs( const username = (user.username ?? user.email)?.split('@')[0] if (!username) return args - const userPrefix = `u/${username}/secret_arg/` - const result = { ...args } + // Detached from the caller: every one binds a form that stays editable across the awaits + // below, and the two walks address a leaf by its path — an array reordered between them + // would hand a row the reference minted for another row's secret. + args = stateSnapshot(args) - for (const [key, prop] of Object.entries(schema.properties)) { - if (!prop.password) continue - if (prop.type !== 'object') continue // only object types; strings handled by PasswordArgInput - if (result[key] == null || result[key] === undefined) continue - if (typeof result[key] === 'string' && result[key].startsWith('$jsonvar:')) continue // already processed + // A value that already names a variable is one; anything else is the secret itself. An empty + // field holds nothing to mint, and ArgInput synthesises '' for every untouched string. + const holdsSecret = (value: unknown) => + value != null && + value !== '' && + !( + typeof value === 'string' && + (value.startsWith('$var:') || value.startsWith('$jsonvar:') || value.startsWith('$res:')) + ) - const path = userPrefix + generateRandomString(12) - await VariableService.createVariable({ + // Collected first and substituted after, because the walk is synchronous and minting is not. + // Keyed by the whole path the walk reports, which is what tells two same-named leaves apart. + const pending: { key: string; prop: any; value: unknown }[] = [] + mapArgLeaves(args, schema as any, isSecretProp, (value, prop, path) => { + if (holdsSecret(value)) pending.push({ key: JSON.stringify(path), prop, value }) + return value + }) + + const minted = new Map() + for (const { key, prop, value } of pending) { + const reference = referencePrefix(prop, value) + const variable = await mintEphemeralSecret( workspace, - requestBody: { - value: JSON.stringify(result[key]), - is_secret: true, - path, - description: 'Ephemeral secret variable', - expires_at: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7).toISOString() - } - }) - result[key] = '$jsonvar:' + path + username, + reference === '$var:' ? String(value) : JSON.stringify(value) + ) + minted.set(key, reference + variable) } - return result + return mapArgLeaves(args, schema as any, isSecretProp, (value, prop, path) => { + const replacement = minted.get(JSON.stringify(path)) + if (replacement !== undefined) return replacement + // A plain variable named for a field that cannot hold a string: the caller meant that + // variable's contents, which is the same secret read the way the field needs it. + if ( + typeof value === 'string' && + value.startsWith('$var:') && + referencePrefix(prop, value) === '$jsonvar:' + ) { + return '$jsonvar:' + value.slice('$var:'.length) + } + return value + }) } diff --git a/frontend/src/lib/components/sessions/PreviewTabHost.svelte b/frontend/src/lib/components/sessions/PreviewTabHost.svelte index bf26605ac2..3dee7dd1b3 100644 --- a/frontend/src/lib/components/sessions/PreviewTabHost.svelte +++ b/frontend/src/lib/components/sessions/PreviewTabHost.svelte @@ -19,6 +19,7 @@ } from './previewRouter' import { withMenuHidden } from './sessionMode.svelte' import ArtifactViewer from '../copilot/chat/artifacts/ArtifactViewer.svelte' + import RunFormPreviewSlot from './RunFormPreviewSlot.svelte' import { setOverlayHost } from '../common/overlayHost.svelte' let { @@ -327,6 +328,20 @@
    This artifact is no longer available.
    {/if}
    +{:else if slot.kind === 'runform' && mounted} +
    + + {#if runtime && overlayHostEl} + + {/if} +
    {:else if mounted}