diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 4473c00f0a..88275f204b 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -28,7 +28,7 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv ENV TZ=Etc/UTC diff --git a/.github/change-versions-mac.sh b/.github/change-versions-mac.sh index 8a1d1c5cfc..2f23b21eb8 100755 --- a/.github/change-versions-mac.sh +++ b/.github/change-versions-mac.sh @@ -7,7 +7,7 @@ VERSION=$1 echo "Updating versions to: $VERSION" sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/backend/Cargo.toml -sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/cli/src/main.ts +sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/cli/src/core/constants.ts sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/benchmarks/lib.ts sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml diff --git a/.github/change-versions.sh b/.github/change-versions.sh index 57fe28b343..ddebbbca87 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -7,7 +7,7 @@ VERSION=$1 echo "Updating versions to: $VERSION" sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/backend/Cargo.toml -sed -i -e "/^export const VERSION =/s/= .*/= \"$VERSION\";/" ${root_dirpath}/cli/src/main.ts +sed -i -e "/^export const VERSION =/s/= .*/= \"$VERSION\";/" ${root_dirpath}/cli/src/core/constants.ts sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/benchmarks/lib.ts sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index f7c49654d1..1c73e5d429 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -74,7 +74,7 @@ jobs: - uses: astral-sh/setup-uv@v6.2.1 with: - version: "0.9.24" + version: "0.9.25" - uses: shivammathur/setup-php@v2 with: diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 9009b47e9d..8f1f15447c 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -62,7 +62,7 @@ jobs: node-version: "20" - uses: astral-sh/setup-uv@v6.2.1 with: - version: "0.9.24" + version: "0.9.25" - uses: shivammathur/setup-php@v2 with: php-version: "8.3" diff --git a/AGENTS.md b/AGENTS.md index 5dda12ab70..6cf4e3d7f3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,3 +106,4 @@ $NAV --root backend callees "X" # what does X call? - Search for existing code to reuse before writing new code - Follow established patterns in the codebase - Keep changes focused — don't refactor beyond what's asked +- **Never attribute work to a specific customer, account, or "requested by a customer" in repo-tracked content** (PR descriptions, commit messages, code comments, docs). Describe changes by their technical motivation instead. diff --git a/CHANGELOG.md b/CHANGELOG.md index 193472c590..639d1ad7ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,95 @@ # Changelog +## [1.719.0](https://github.com/windmill-labs/windmill/compare/v1.718.0...v1.719.0) (2026-06-06) + + +### Features + +* **otel:** connect jobs to the inbound distributed trace ([#9456](https://github.com/windmill-labs/windmill/issues/9456)) ([fad1a54](https://github.com/windmill-labs/windmill/commit/fad1a549d95c00d0746a48163c4f95fc69733e1a)) + + +### Bug Fixes + +* authenticate slack callback payload with per-workspace hmac ([#9461](https://github.com/windmill-labs/windmill/issues/9461)) ([fbdf81b](https://github.com/windmill-labs/windmill/commit/fbdf81ba5f77d282c025360ecee14138dd4cb4a2)) +* prevent token label collision bypassing job read access control ([#9462](https://github.com/windmill-labs/windmill/issues/9462)) ([e1e7af6](https://github.com/windmill-labs/windmill/commit/e1e7af6a25a44eb06b67332ce1efeae2a21e0c6d)) +* **python:** escape reserved-keyword step ids in wrapper codegen ([#9460](https://github.com/windmill-labs/windmill/issues/9460)) ([6a15a9b](https://github.com/windmill-labs/windmill/commit/6a15a9b152ad20be4b5c3de6000516da231e41e0)), closes [#8893](https://github.com/windmill-labs/windmill/issues/8893) + +## [1.718.0](https://github.com/windmill-labs/windmill/compare/v1.717.1...v1.718.0) (2026-06-05) + + +### Features + +* **flows:** opt-in to include the stopping step's result in early-stop errors ([#9446](https://github.com/windmill-labs/windmill/issues/9446)) ([f2f0812](https://github.com/windmill-labs/windmill/commit/f2f0812a04c9256cfc8eba5e0dcf38d71d971410)) +* make C# dotnet target framework configurable via DOTNET_TARGET_FRAMEWORK ([#9454](https://github.com/windmill-labs/windmill/issues/9454)) ([9a609bf](https://github.com/windmill-labs/windmill/commit/9a609bf08ac1b6157dbdfb827fc01e771d71262e)) +* sandboxed daemonless container runtime via '# sandbox <image>' ([#9453](https://github.com/windmill-labs/windmill/issues/9453)) ([1727271](https://github.com/windmill-labs/windmill/commit/1727271e197b34026efeaf1b6561bb404a440baa)) +* **sandbox:** pull/extract images with crane instead of podman ([#9455](https://github.com/windmill-labs/windmill/issues/9455)) ([7590b28](https://github.com/windmill-labs/windmill/commit/7590b281085afd1fc2774e8fb37a4c0af3aedbad)) + + +### Bug Fixes + +* distinguish canceled jobs in runs ([#9452](https://github.com/windmill-labs/windmill/issues/9452)) ([9067787](https://github.com/windmill-labs/windmill/commit/90677872f6185eb0c81e0e84a426a54653818457)) + +## [1.717.1](https://github.com/windmill-labs/windmill/compare/v1.717.0...v1.717.1) (2026-06-04) + + +### Bug Fixes + +* invalidate relative-import cache when imported script changes ([#9443](https://github.com/windmill-labs/windmill/issues/9443)) ([f595787](https://github.com/windmill-labs/windmill/commit/f595787409a3fcda9278bbcf2cfcc80092f16460)) + +## [1.717.0](https://github.com/windmill-labs/windmill/compare/v1.716.0...v1.717.0) (2026-06-04) + + +### Features + +* let flow AI chat create and edit sticky notes ([#9412](https://github.com/windmill-labs/windmill/issues/9412)) ([e4e0984](https://github.com/windmill-labs/windmill/commit/e4e0984e55afd3c73f1c365cd0608493a9fd87ed)) + + +### Bug Fixes + +* **cli:** push whole raw app instead of treating frontend files as scripts ([#9442](https://github.com/windmill-labs/windmill/issues/9442)) ([b5a6a1e](https://github.com/windmill-labs/windmill/commit/b5a6a1eeab663c2d6aaec2c89eab7a550cb0bb6b)) +* read latest db draft for scripts/flows in global mode read tool ([#9441](https://github.com/windmill-labs/windmill/issues/9441)) ([819ba5e](https://github.com/windmill-labs/windmill/commit/819ba5e150ec9f5199919fbea50874fc156d0189)) + +## [1.716.0](https://github.com/windmill-labs/windmill/compare/v1.715.0...v1.716.0) (2026-06-03) + + +### Features + +* add metadata generation model setting ([#9418](https://github.com/windmill-labs/windmill/issues/9418)) ([cf5fefb](https://github.com/windmill-labs/windmill/commit/cf5fefb521479170b9dc64b884630c4dac789931)) +* auto-generate AI session names ([#9399](https://github.com/windmill-labs/windmill/issues/9399)) ([26b7270](https://github.com/windmill-labs/windmill/commit/26b727041830c9b741668a9ab73e2eb90c7cec74)) +* support $f/ and $u/ import path aliases for scripts ([#9378](https://github.com/windmill-labs/windmill/issues/9378)) ([220cd35](https://github.com/windmill-labs/windmill/commit/220cd35cf799c42ebf588bc97a6d8e6f4e97c2e3)) +* use metadata model for small AI tasks ([#9431](https://github.com/windmill-labs/windmill/issues/9431)) ([79178f6](https://github.com/windmill-labs/windmill/commit/79178f6f5a7c606a2e05677c6efcbdd84c608325)) + + +### Bug Fixes + +* **apps:** relock no longer reverts raw app to a stale version ([#9432](https://github.com/windmill-labs/windmill/issues/9432)) ([073857a](https://github.com/windmill-labs/windmill/commit/073857ac0a9ed54bdeac8f373f7c855fe34eb0ac)) +* **security:** scope variable and resource value caches by caller identity ([#9427](https://github.com/windmill-labs/windmill/issues/9427)) ([0ba128a](https://github.com/windmill-labs/windmill/commit/0ba128afe797bd016da60563949ac3abbbfe1978)) + +## [1.715.0](https://github.com/windmill-labs/windmill/compare/v1.714.1...v1.715.0) (2026-06-03) + + +### Features + +* **frontend:** add rebuild dependency map button to workspace settings ([#9424](https://github.com/windmill-labs/windmill/issues/9424)) ([3b2e748](https://github.com/windmill-labs/windmill/commit/3b2e748daf0a8ec4447c30423068df803f3f9ca2)) + + +### Bug Fixes + +* **auth:** filter script/flow listings by token scope (GHSA-2ppx-66jv-wpw5) ([#9426](https://github.com/windmill-labs/windmill/issues/9426)) ([7edf3f0](https://github.com/windmill-labs/windmill/commit/7edf3f02122e20fde1e95e0252e7bda641075326)) +* **backend:** authorize single-job read endpoints by job/flow visibility ([#9416](https://github.com/windmill-labs/windmill/issues/9416)) ([89a7a37](https://github.com/windmill-labs/windmill/commit/89a7a377764086911db18252f2478f42f0e1e3ea)) +* **mcp:** resolve MCP resource token via caller RLS + SSRF-guard url ([#9428](https://github.com/windmill-labs/windmill/issues/9428)) ([8053266](https://github.com/windmill-labs/windmill/commit/8053266f88bd4c94fc86278412df5a0beeed5e77)) +* **nsjail:** precompile python stdlib + raise download rlimit_as ([#9429](https://github.com/windmill-labs/windmill/issues/9429)) ([7031744](https://github.com/windmill-labs/windmill/commit/7031744a199f0bf8b8e35043afa959977e5ecdbd)) +* omit temperature for gpt-5+ and o-series models on all providers ([#9422](https://github.com/windmill-labs/windmill/issues/9422)) ([11d1ad9](https://github.com/windmill-labs/windmill/commit/11d1ad9a872d2ec2f14cde35708c84a0c7bdc172)) + +## [1.714.1](https://github.com/windmill-labs/windmill/compare/v1.714.0...v1.714.1) (2026-06-02) + + +### Bug Fixes + +* **backend:** route //native TypeScript previews to native workers (WIN-2007) ([#9407](https://github.com/windmill-labs/windmill/issues/9407)) ([73edebc](https://github.com/windmill-labs/windmill/commit/73edebc833a981488a8ea116f4f13c020a011a6f)) +* **nsjail:** raise python download fd limit for --compile-bytecode (WIN-2009) ([#9414](https://github.com/windmill-labs/windmill/issues/9414)) ([9e6559a](https://github.com/windmill-labs/windmill/commit/9e6559a6f688cc8d982277b19920219ea6d0fd8e)) +* **triggers:** prevent Zoom challenge handler from being used as a signing oracle ([#9413](https://github.com/windmill-labs/windmill/issues/9413)) ([ab2a15b](https://github.com/windmill-labs/windmill/commit/ab2a15b2a859096eabde718bf6e60289ae187118)) + ## [1.714.0](https://github.com/windmill-labs/windmill/compare/v1.713.1...v1.714.0) (2026-06-02) diff --git a/Dockerfile b/Dockerfile index 9062a4d9d8..d327c9b394 100644 --- a/Dockerfile +++ b/Dockerfile @@ -233,11 +233,14 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtimes to temp build location (will copy with world-writable perms later) -RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install 3.11 -RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY +# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run +# under the read-only nsjail runtime mount (uv >= 0.9.25). The copy below MUST preserve +# timestamps or Python's mtime-based .pyc invalidation discards these compiled files. +RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install 3.11 --compile-bytecode +RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - @@ -259,7 +262,7 @@ RUN export GOCACHE=/tmp/build_cache/go && \ # chmod a+rw adds read+write WITHOUT removing execute bits (755->777, 644->666) # Note: uv python install only creates py_runtime, not uv cache - we create uv/go dirs for runtime RUN mkdir -p /tmp/windmill/cache && \ - cp -r /tmp/build_cache/* /tmp/windmill/cache/ && \ + cp -r --preserve=timestamps /tmp/build_cache/* /tmp/windmill/cache/ && \ chmod -R a+rw /tmp/windmill/cache && \ rm -rf /tmp/build_cache && \ mkdir -p -m 777 /tmp/windmill/cache/uv /tmp/windmill/cache/go /tmp/windmill/cache/rustup /tmp/windmill/cache/cargo @@ -300,10 +303,20 @@ ENV CARGO_HOME="/tmp/windmill/cache/cargo" ENV LD_LIBRARY_PATH="." # nsjail runtime deps and binary -RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev \ +RUN apt-get update && apt-get install -y --no-install-recommends libprotobuf32 libnl-route-3-200 libnl-3-200 \ && apt-get clean && rm -rf /var/lib/apt/lists/* COPY --from=nsjail /nsjail/nsjail /bin/nsjail +# crane: pulls + flattens images for the sandboxed container runtime (`# sandbox `). +# Single static binary — no daemon/store/root needed. See docs/docker-v2-runtime.md. +ARG CRANE_VERSION=v0.20.6 +RUN arch="$(dpkg --print-architecture)"; \ + case "$arch" in amd64) crane_arch=x86_64 ;; arm64) crane_arch=arm64 ;; *) echo >&2 "error: unsupported arch '$arch' for crane"; exit 1 ;; esac; \ + wget -O /tmp/crane.tgz "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${crane_arch}.tar.gz" \ + && tar -xzf /tmp/crane.tgz -C /usr/local/bin crane \ + && rm /tmp/crane.tgz \ + && chmod +x /usr/local/bin/crane + WORKDIR ${APP} RUN ln -s ${APP}/windmill /usr/local/bin/windmill diff --git a/ai_evals/AGENTS.md b/ai_evals/AGENTS.md index d26e6d60ea..af5427abae 100644 --- a/ai_evals/AGENTS.md +++ b/ai_evals/AGENTS.md @@ -86,6 +86,31 @@ Global prompts should exercise workspace-level drafting behavior: Keep deterministic validation focused on the draft contract: required draft type/path, required content snippets, forbidden draft paths, and forbidden mutating tools such as deploy/delete unless the case explicitly asks for them. +Datatable cases should set `skipJudge: true` and validate through tool-use +(`requiredToolsUsed` / `forbiddenToolsUsed`) and SQL-argument assertions +(`toolCallArgs` with `stringIncludesAnyOf`, e.g. `['select']`, `['create table']`, +`['update', 'insert into']`). Two reasons the judge is unreliable here: + +- `list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql` + produce no drafts, and the global judge only sees the drafts artifact — it + scores a no-draft conversational answer as empty (same as the + `askUserQuestion` cases). +- Even a case that *does* produce a draft (a script reading the data table via + `wmill.datatable()` at runtime) is mis-judged: the judge has no datatable SDK + reference and penalizes correct `wmill.datatable()` usage as wrong. Verify the + SDK call deterministically instead — `requiredDrafts.valueIncludes: ['wmill.datatable(']` + plus forbidding `exec_datatable_sql` (keeping chat-time SQL distinct from + runtime SDK use). + +`stringIncludesAnyOf` is existential over calls (at least one matching call), so a +mutation case still passes when the model mixes its UPDATE/INSERT with +verification SELECTs. The in-memory engine (`datatableSqlEngine.ts`) is stateful +within a case — writes persist, so a model that re-queries to verify its +CREATE/UPDATE sees the change and does not loop. But the engine is best-effort +(SELECT returns all rows of the referenced/first table with no WHERE/projection), +so still never assert specific returned row values. Seed data via +`workspace.datatables` in the `initial` fixture (see README). + ## Deterministic validation Use deterministic validation only for hard failures such as: diff --git a/ai_evals/README.md b/ai_evals/README.md index d5fe1661a2..88825f4ee7 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -148,6 +148,23 @@ Global initial fixtures can also seed `liveEditorDrafts` with `type`, currently open script, flow, or raw app editor so cases can test prompts that refer to "this" or the "current" item. +Global (and flow) initial fixtures can seed `workspace.datatables` so the +`list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql` tools +return seeded data during evals. Each entry is +`{ datatable_name, schemas: { : { : { columns, rows? } } } }`. +SQL runs through a small in-memory engine (`datatableSqlEngine.ts`), not a real +database. Writes are **stateful within a case**: `CREATE`/`DROP`/`INSERT`/`UPDATE`/ +`DELETE` mutate the seeded datatable in place, so a later `list_datatables`, +`get_datatable_table_schema`, `SELECT`, or `information_schema` query reflects them +— this is what stops a model from looping when it re-queries to verify a write. +The engine is best-effort: `SELECT` returns all rows of the referenced (or first) +table with no WHERE filtering/projection/joins, `WHERE` on UPDATE/DELETE supports +`col = value` predicates joined by `AND`, and anything unparseable is a no-op +success. So validate datatable cases through tool-use and SQL-argument assertions +(`requiredToolsUsed`, `stringIncludesAnyOf`) — not through exact returned row +values. An empty/absent `datatables` seed makes `list_datatables` return `[]`, +which is what the "no datatable configured" blocking cases rely on. + Set `WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT=1` to run those cases with the old behavior where the live editor is only discoverable through `list_workspace_items`. diff --git a/ai_evals/adapters/frontend/datatableSqlEngine.test.ts b/ai_evals/adapters/frontend/datatableSqlEngine.test.ts new file mode 100644 index 0000000000..e0398b407b --- /dev/null +++ b/ai_evals/adapters/frontend/datatableSqlEngine.test.ts @@ -0,0 +1,262 @@ +import { describe, expect, it } from 'bun:test' +import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine' + +function makeDatatable(): BenchmarkDatatableSeed { + return { + datatable_name: 'main', + schemas: { + public: { + orders: { + columns: { id: 'int4', customer_id: 'int4', total: 'numeric', status: 'text' }, + rows: [ + { id: 1, customer_id: 1, total: 42.5, status: 'shipped' }, + { id: 2, customer_id: 2, total: 19.99, status: 'pending' }, + { id: 3, customer_id: 1, total: 88, status: 'shipped' } + ] + }, + customers: { + columns: { id: 'int4', name: 'text' }, + rows: [{ id: 1, name: 'Alice' }] + } + } + } + } +} + +describe('SELECT', () => { + it('returns the referenced table rows', () => { + const dt = makeDatatable() + expect(applyDatatableSql(dt, 'SELECT id, name FROM customers').rows).toEqual([ + { id: 1, name: 'Alice' } + ]) + }) + + it('falls back to the first table when no known table is referenced', () => { + const dt = makeDatatable() + expect(applyDatatableSql(dt, 'select 1').rows).toHaveLength(3) + }) + + it('resolves a schema-qualified table', () => { + const dt = makeDatatable() + expect(applyDatatableSql(dt, 'SELECT * FROM public.customers').rows).toEqual([ + { id: 1, name: 'Alice' } + ]) + }) +}) + +describe('CREATE TABLE', () => { + it('adds a table with parsed columns, skipping table constraints and FK clauses', () => { + const dt = makeDatatable() + const result = applyDatatableSql( + dt, + 'CREATE TABLE public.refunds (\n order_id int4 NOT NULL REFERENCES public.orders(id),\n amount numeric(10,2),\n PRIMARY KEY (order_id)\n)' + ) + expect(result.rows).toEqual([]) + expect(dt.schemas.public.refunds).toEqual({ + columns: { order_id: 'int4', amount: 'numeric(10,2)' }, + rows: [] + }) + }) + + it('defaults an unqualified table to the public schema', () => { + const dt = makeDatatable() + applyDatatableSql(dt, 'CREATE TABLE notes (id int4, body text)') + expect(dt.schemas.public.notes.columns).toEqual({ id: 'int4', body: 'text' }) + }) + + it('is a no-op for an existing table with IF NOT EXISTS', () => { + const dt = makeDatatable() + applyDatatableSql(dt, 'CREATE TABLE IF NOT EXISTS public.orders (x int4)') + expect(Object.keys(dt.schemas.public.orders.columns)).toContain('status') + }) +}) + +describe('DROP TABLE', () => { + it('removes the table', () => { + const dt = makeDatatable() + applyDatatableSql(dt, 'DROP TABLE IF EXISTS public.customers') + expect(dt.schemas.public.customers).toBeUndefined() + }) +}) + +describe('INSERT', () => { + it('appends a row using an explicit column list', () => { + const dt = makeDatatable() + applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (2, 'Bob')") + expect(dt.schemas.public.customers.rows).toContainEqual({ id: 2, name: 'Bob' }) + }) + + it('infers columns from the table when none are given, and appends multiple tuples', () => { + const dt = makeDatatable() + applyDatatableSql(dt, "INSERT INTO customers VALUES (2, 'Bob'), (3, 'Carol')") + expect(dt.schemas.public.customers.rows).toHaveLength(3) + }) + + it('returns the inserted rows when RETURNING is present', () => { + const dt = makeDatatable() + const result = applyDatatableSql( + dt, + "INSERT INTO customers (id, name) VALUES (2, 'Bob') RETURNING *" + ) + expect(result.rows).toEqual([{ id: 2, name: 'Bob' }]) + }) +}) + +describe('UPDATE', () => { + it('updates only the rows matching an equality WHERE', () => { + const dt = makeDatatable() + const result = applyDatatableSql( + dt, + "UPDATE public.orders SET status = 'shipped' WHERE id = 2" + ) + expect(result.rows).toEqual([]) + expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('shipped') + expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped') + }) + + it('strips a Postgres cast in the WHERE value', () => { + const dt = makeDatatable() + applyDatatableSql(dt, "UPDATE orders SET status = 'done' WHERE id = 2::int4") + expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('done') + }) + + it('matches multiple AND predicates including a numeric literal', () => { + const dt = makeDatatable() + applyDatatableSql( + dt, + "UPDATE orders SET status = 'done' WHERE customer_id = 2 AND total = 19.99" + ) + expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('done') + expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped') + }) + + it('updates every row when there is no WHERE', () => { + const dt = makeDatatable() + applyDatatableSql(dt, "UPDATE orders SET status = 'archived'") + expect(dt.schemas.public.orders.rows?.every((r) => r.status === 'archived')).toBe(true) + }) + + it('returns the affected rows when RETURNING is present', () => { + const dt = makeDatatable() + const result = applyDatatableSql( + dt, + "UPDATE orders SET status = 'shipped' WHERE id = 2 RETURNING *" + ) + expect(result.rows).toHaveLength(1) + expect(result.rows[0]).toMatchObject({ id: 2, status: 'shipped' }) + }) + + it('affects no rows when the WHERE clause cannot be parsed', () => { + const dt = makeDatatable() + applyDatatableSql(dt, "UPDATE orders SET status = 'x' WHERE total > 20") + expect(dt.schemas.public.orders.rows?.some((r) => r.status === 'x')).toBe(false) + }) +}) + +describe('DELETE', () => { + it('removes only the matching rows', () => { + const dt = makeDatatable() + applyDatatableSql(dt, 'DELETE FROM orders WHERE id = 2') + expect(dt.schemas.public.orders.rows?.map((r) => r.id)).toEqual([1, 3]) + }) + + it('returns the removed rows when RETURNING is present', () => { + const dt = makeDatatable() + const result = applyDatatableSql(dt, 'DELETE FROM orders WHERE id = 2 RETURNING *') + expect(result.rows).toEqual([{ id: 2, customer_id: 2, total: 19.99, status: 'pending' }]) + }) +}) + +describe('writes are reflected by later reads', () => { + it('UPDATE then SELECT sees the new value (the verify-loop fix)', () => { + const dt = makeDatatable() + applyDatatableSql(dt, "UPDATE orders SET status = 'shipped' WHERE id = 2") + const seen = applyDatatableSql(dt, 'SELECT * FROM orders').rows + expect(seen.find((r) => r.id === 2)?.status).toBe('shipped') + }) + + it('INSERT then SELECT sees the new row', () => { + const dt = makeDatatable() + applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (9, 'Zed')") + const seen = applyDatatableSql(dt, 'SELECT * FROM customers').rows + expect(seen).toContainEqual({ id: 9, name: 'Zed' }) + }) + + it('CREATE then SELECT on the new table returns its (empty) rows', () => { + const dt = makeDatatable() + applyDatatableSql(dt, 'CREATE TABLE public.refunds (order_id int4, amount numeric)') + expect(applyDatatableSql(dt, 'SELECT * FROM refunds').rows).toEqual([]) + }) +}) + +describe('system-catalog queries reflect the current tables/columns', () => { + it('lists current tables (including a freshly created one) via information_schema.tables', () => { + const dt = makeDatatable() + applyDatatableSql(dt, 'CREATE TABLE public.refunds (order_id int4)') + const rows = applyDatatableSql( + dt, + "SELECT table_name FROM information_schema.tables WHERE table_name = 'refunds'" + ).rows + expect(rows.map((r) => r.table_name)).toContain('refunds') + }) + + it('does not list a dropped table', () => { + const dt = makeDatatable() + applyDatatableSql(dt, 'DROP TABLE public.customers') + const rows = applyDatatableSql(dt, 'SELECT table_name FROM information_schema.tables').rows + expect(rows.map((r) => r.table_name)).not.toContain('customers') + }) + + it('reports columns via information_schema.columns', () => { + const dt = makeDatatable() + const rows = applyDatatableSql( + dt, + "SELECT column_name FROM information_schema.columns WHERE table_name = 'orders'" + ).rows + expect(rows.map((r) => r.column_name)).toContain('status') + }) +}) + +describe('parser robustness (string/paren-aware splitting)', () => { + it('does not treat the word "returning" inside a string value as a RETURNING clause', () => { + const dt = makeDatatable() + const result = applyDatatableSql( + dt, + "INSERT INTO customers (id, name) VALUES (5, 'is returning soon')" + ) + expect(result.rows).toEqual([]) + expect(dt.schemas.public.customers.rows).toContainEqual({ id: 5, name: 'is returning soon' }) + }) + + it('does not split on the word "where" inside a SET string value', () => { + const dt = makeDatatable() + applyDatatableSql(dt, "UPDATE orders SET status = 'ship where ordered' WHERE id = 2") + expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('ship where ordered') + expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped') + }) + + it('keeps INSERT tuples intact when a value contains a function call', () => { + const dt = makeDatatable() + applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (6, coalesce(NULL, 'x'))") + expect(dt.schemas.public.customers.rows).toHaveLength(2) + expect(dt.schemas.public.customers.rows?.[1]).toMatchObject({ id: 6 }) + }) + + it('CREATE TABLE ignores a trailing semicolon-separated statement', () => { + const dt = makeDatatable() + applyDatatableSql( + dt, + 'CREATE TABLE public.refunds (id int4, amount numeric); INSERT INTO refunds VALUES (1, 5)' + ) + expect(dt.schemas.public.refunds.columns).toEqual({ id: 'int4', amount: 'numeric' }) + expect(dt.schemas.public.refunds.rows).toEqual([]) + }) +}) + +describe('unparseable statements are a safe no-op', () => { + it('returns [] and does not throw', () => { + const dt = makeDatatable() + expect(applyDatatableSql(dt, 'VACUUM ANALYZE').rows).toEqual([]) + expect(applyDatatableSql(dt, 'GRANT SELECT ON orders TO someone').rows).toEqual([]) + }) +}) diff --git a/ai_evals/adapters/frontend/datatableSqlEngine.ts b/ai_evals/adapters/frontend/datatableSqlEngine.ts new file mode 100644 index 0000000000..453eca286e --- /dev/null +++ b/ai_evals/adapters/frontend/datatableSqlEngine.ts @@ -0,0 +1,541 @@ +/** + * A deliberately small, best-effort SQL engine for the benchmark datatable mock. + * + * This is NOT a real SQL implementation — it exists only so that writes a model + * issues during an eval (`CREATE TABLE`, `INSERT`, `UPDATE`, `DELETE`, `DROP`) + * become visible to its later reads (`list_datatables`, `get_datatable_table_schema`, + * `SELECT`). Without that, a model that re-queries to verify a write sees stale + * seed data, concludes the write failed, and loops until it exhausts its turns. + * + * It parses only the common statement shapes models produce. Anything it cannot + * parse is a no-op success (it never throws) — behavioral evals assert that the + * right statement was issued, not its exact data effects. Notable limits: + * - `SELECT` returns all rows of the referenced (or first) table — no WHERE + * filtering, projection, joins, or aggregation. + * - `WHERE` supports `col = value` predicates joined by `AND` only; an + * unparseable WHERE on UPDATE/DELETE affects zero rows (never the whole table). + */ + +/** One seeded datatable table: its columns (col -> compact_type) and optional rows. */ +export interface BenchmarkDatatableTableSeed { + columns: Record + rows?: Record[] +} + +/** A seeded datatable: `datatable_name` plus a `schema -> table -> seed` map. */ +export interface BenchmarkDatatableSeed { + datatable_name: string + schemas: { + [schema: string]: { + [table: string]: BenchmarkDatatableTableSeed + } + } +} + +export interface DatatableSqlResult { + rows: Record[] +} + +const DEFAULT_SCHEMA = 'public' + +type ParsedRef = { schema: string; table: string } +type Predicate = { column: string; value: unknown } + +/** + * Apply one SQL statement to `datatable` IN PLACE and return the result rows. + * SELECT returns the referenced/first table's rows; a mutation returns its + * affected rows when it has a RETURNING clause, otherwise `[]`. + */ +export function applyDatatableSql( + datatable: BenchmarkDatatableSeed, + sql: string +): DatatableSqlResult { + const statement = stripTrailingSemicolon(sql.trim()) + if (/^\s*(with|select)\b/i.test(statement)) { + return { rows: selectRows(datatable, statement) } + } + if (/^\s*create\s+table\b/i.test(statement)) { + return { rows: applyCreateTable(datatable, statement) } + } + if (/^\s*drop\s+table\b/i.test(statement)) { + return { rows: applyDropTable(datatable, statement) } + } + if (/^\s*insert\s+into\b/i.test(statement)) { + return { rows: applyInsert(datatable, statement) } + } + if (/^\s*update\b/i.test(statement)) { + return { rows: applyUpdate(datatable, statement) } + } + if (/^\s*delete\s+from\b/i.test(statement)) { + return { rows: applyDelete(datatable, statement) } + } + return { rows: [] } +} + +// ============= Reads ============= + +function selectRows( + datatable: BenchmarkDatatableSeed, + sql: string +): Record[] { + const fromRef = sql.match(/\bfrom\s+([a-zA-Z_"][\w."]*)/i)?.[1] + if (fromRef) { + const catalog = catalogRows(datatable, fromRef) + if (catalog) { + return catalog + } + } + const table = fromRef ? resolveTable(datatable, fromRef) : undefined + const seed = table ?? firstTable(datatable) + return seed?.rows ?? [] +} + +/** + * Synthesize rows for a system-catalog query so a model verifying a `CREATE`/`DROP` + * via `information_schema.tables` / `.columns` (or `pg_tables`) sees the current + * tables/columns instead of fallback data. WHERE is not applied, so the model gets + * the full set and finds (or no longer finds) the table it just changed. + * Returns `undefined` for non-catalog refs so normal table resolution proceeds. + */ +function catalogRows( + datatable: BenchmarkDatatableSeed, + ref: string +): Record[] | undefined { + const normalized = ref.toLowerCase().replace(/"/g, '') + const name = normalized.split('.').pop() + const isCatalog = normalized.includes('information_schema.') || normalized.startsWith('pg_') + if (!isCatalog) { + return undefined + } + const tables = allTables(datatable) + if (name === 'tables' || name === 'pg_tables') { + return tables.map(({ schema, table }) => ({ + table_schema: schema, + table_name: table, + schemaname: schema, + tablename: table + })) + } + if (name === 'columns') { + return tables.flatMap(({ schema, table, seed }) => + Object.entries(seed.columns).map(([column, type]) => ({ + table_schema: schema, + table_name: table, + column_name: column, + data_type: type + })) + ) + } + return undefined +} + +function allTables( + datatable: BenchmarkDatatableSeed +): { schema: string; table: string; seed: BenchmarkDatatableTableSeed }[] { + return Object.entries(datatable.schemas).flatMap(([schema, tables]) => + Object.entries(tables).map(([table, seed]) => ({ schema, table, seed })) + ) +} + +// ============= DDL ============= + +function applyCreateTable( + datatable: BenchmarkDatatableSeed, + sql: string +): Record[] { + const head = sql.match( + /^\s*create\s+table\s+(?:if\s+not\s+exists\s+)?([a-zA-Z_"][\w."]*)/i + ) + // The first top-level paren group is the column-definition list; using it (rather + // than a greedy `(...)` capture) ignores any trailing `;`-separated statement. + const columnText = extractParenGroups(sql)[0] + if (!head || columnText === undefined) { + return [] + } + const { schema, table } = parseRef(head[1]) + const existing = datatable.schemas[schema]?.[table] + if (existing) { + return [] + } + const columns: Record = {} + for (const rawDef of splitTopLevel(columnText)) { + const def = rawDef.trim() + if (!def || isTableConstraint(def)) { + continue + } + const tokens = def.split(/\s+/) + const column = unquoteIdentifier(tokens[0]) + if (!column) { + continue + } + columns[column] = tokens[1] ?? 'text' + } + if (!datatable.schemas[schema]) { + datatable.schemas[schema] = {} + } + datatable.schemas[schema][table] = { columns, rows: [] } + return [] +} + +function applyDropTable( + datatable: BenchmarkDatatableSeed, + sql: string +): Record[] { + const match = sql.match( + /^\s*drop\s+table\s+(?:if\s+exists\s+)?([a-zA-Z_"][\w."]*)/i + ) + if (!match) { + return [] + } + const { schema, table } = parseRef(match[1]) + if (datatable.schemas[schema]?.[table]) { + delete datatable.schemas[schema][table] + } + return [] +} + +// ============= DML ============= + +function applyInsert( + datatable: BenchmarkDatatableSeed, + sql: string +): Record[] { + const { body, returning } = splitOffReturning(sql) + const match = body.match( + /^\s*insert\s+into\s+([a-zA-Z_"][\w."]*)\s*(?:\(([^)]*)\))?\s*values\s*([\s\S]+)$/i + ) + if (!match) { + return [] + } + const table = resolveTable(datatable, match[1]) + if (!table) { + return [] + } + const columns = match[2] + ? splitTopLevel(match[2]).map((entry) => unquoteIdentifier(entry.trim())) + : Object.keys(table.columns) + const inserted: Record[] = [] + for (const tuple of extractParenGroups(match[3])) { + const values = splitTopLevel(tuple).map((entry) => parseValue(entry)) + const row: Record = {} + columns.forEach((column, index) => { + row[column] = values[index] + }) + inserted.push(row) + } + table.rows ??= [] + table.rows.push(...inserted) + return returning ? inserted : [] +} + +function applyUpdate( + datatable: BenchmarkDatatableSeed, + sql: string +): Record[] { + const { body, returning } = splitOffReturning(sql) + const match = body.match(/^\s*update\s+([a-zA-Z_"][\w."]*)\s+set\s+([\s\S]+)$/i) + if (!match) { + return [] + } + const table = resolveTable(datatable, match[1]) + if (!table) { + return [] + } + let assignmentText = match[2] + let whereText: string | undefined + const whereMatch = maskForClauseScan(assignmentText).match(/\swhere\s/i) + if (whereMatch && whereMatch.index !== undefined) { + whereText = assignmentText.slice(whereMatch.index + whereMatch[0].length) + assignmentText = assignmentText.slice(0, whereMatch.index) + } + const predicates = parsePredicates(whereText) + if (predicates === null) { + return [] + } + const assignments: Record = {} + for (const entry of splitTopLevel(assignmentText)) { + const pair = entry.match(/^\s*([a-zA-Z_"][\w."]*)\s*=\s*([\s\S]+?)\s*$/) + if (pair) { + assignments[lastIdentifier(pair[1])] = parseValue(pair[2]) + } + } + const affected = (table.rows ?? []).filter((row) => rowMatches(row, predicates)) + for (const row of affected) { + Object.assign(row, assignments) + } + return returning ? affected : [] +} + +function applyDelete( + datatable: BenchmarkDatatableSeed, + sql: string +): Record[] { + const { body, returning } = splitOffReturning(sql) + const match = body.match(/^\s*delete\s+from\s+([a-zA-Z_"][\w."]*)\s*([\s\S]*)$/i) + if (!match) { + return [] + } + const table = resolveTable(datatable, match[1]) + if (!table) { + return [] + } + const whereText = match[2].replace(/^\s*where\s+/i, '').trim() || undefined + const predicates = parsePredicates(whereText) + if (predicates === null) { + return [] + } + const rows = table.rows ?? [] + const removed = rows.filter((row) => rowMatches(row, predicates)) + table.rows = rows.filter((row) => !rowMatches(row, predicates)) + return returning ? removed : [] +} + +// ============= Parsing helpers ============= + +function resolveTable( + datatable: BenchmarkDatatableSeed, + ref: string +): BenchmarkDatatableTableSeed | undefined { + const { schema, table } = parseRef(ref) + const direct = datatable.schemas[schema]?.[table] + if (direct) { + return direct + } + // Bare table name: fall back to searching every schema for a matching table. + if (!ref.includes('.')) { + for (const tables of Object.values(datatable.schemas)) { + if (tables[table]) { + return tables[table] + } + } + } + return undefined +} + +function firstTable( + datatable: BenchmarkDatatableSeed +): BenchmarkDatatableTableSeed | undefined { + for (const tables of Object.values(datatable.schemas)) { + for (const seed of Object.values(tables)) { + return seed + } + } + return undefined +} + +function parseRef(ref: string): ParsedRef { + const parts = ref.split('.').map(unquoteIdentifier) + if (parts.length >= 2) { + return { schema: parts[parts.length - 2], table: parts[parts.length - 1] } + } + return { schema: DEFAULT_SCHEMA, table: parts[0] } +} + +/** A WHERE clause with no parseable form returns `null`; absent WHERE returns `[]` (match all). */ +function parsePredicates(whereText: string | undefined): Predicate[] | null { + if (whereText === undefined || whereText.trim() === '') { + return [] + } + const predicates: Predicate[] = [] + for (const part of whereText.split(/\s+and\s+/i)) { + const match = part.match(/^\s*([a-zA-Z_"][\w."]*)\s*=\s*([\s\S]+?)\s*$/) + if (!match) { + return null + } + predicates.push({ column: lastIdentifier(match[1]), value: parseValue(match[2]) }) + } + return predicates +} + +function rowMatches(row: Record, predicates: Predicate[]): boolean { + return predicates.every((predicate) => looseEquals(row[predicate.column], predicate.value)) +} + +function looseEquals(left: unknown, right: unknown): boolean { + if (left === null || left === undefined) { + return right === null || right === undefined + } + if (typeof left === 'number' && typeof right === 'number') { + return left === right + } + return String(left) === String(right) +} + +function parseValue(raw: string): unknown { + // Drop a trailing Postgres cast (e.g. `2::int4`) before interpreting the literal. + const token = raw.trim().replace(/::\s*[a-zA-Z_][\w]*(\([^)]*\))?\s*$/, '').trim() + const stringMatch = token.match(/^'([\s\S]*)'$/) + if (stringMatch) { + return stringMatch[1].replace(/''/g, "'") + } + if (/^-?\d+(\.\d+)?$/.test(token)) { + return Number(token) + } + if (/^true$/i.test(token)) { + return true + } + if (/^false$/i.test(token)) { + return false + } + if (/^null$/i.test(token)) { + return null + } + return token +} + +function splitOffReturning(sql: string): { body: string; returning: boolean } { + const match = maskForClauseScan(sql).match(/\sreturning\s/i) + if (!match || match.index === undefined) { + return { body: sql, returning: false } + } + return { body: sql.slice(0, match.index), returning: true } +} + +/** + * A same-length copy of `sql` with the contents of single-quoted strings and + * parenthesized groups blanked to spaces, so a top-level keyword scan + * (WHERE / RETURNING) cannot match inside a string literal or a subquery. Index + * positions in the result map 1:1 back onto the original. + */ +function maskForClauseScan(sql: string): string { + let masked = '' + let depth = 0 + let inString = false + for (let i = 0; i < sql.length; i++) { + const char = sql[i] + if (inString) { + if (char === "'") { + if (sql[i + 1] === "'") { + masked += ' ' + i++ + continue + } + inString = false + } + masked += ' ' + continue + } + if (char === "'") { + inString = true + masked += ' ' + } else if (char === '(') { + depth++ + masked += ' ' + } else if (char === ')') { + depth = Math.max(0, depth - 1) + masked += ' ' + } else { + masked += depth > 0 ? ' ' : char + } + } + return masked +} + +/** + * Inner text of each top-level `( ... )` group in `input`, honoring nested parens + * (e.g. `now()`, `numeric(10,2)`) and single-quoted strings. Used for the CREATE + * column-definition group and INSERT value tuples. + */ +function extractParenGroups(input: string): string[] { + const groups: string[] = [] + let depth = 0 + let inString = false + let current = '' + for (let i = 0; i < input.length; i++) { + const char = input[i] + if (inString) { + current += char + if (char === "'") { + if (input[i + 1] === "'") { + current += input[++i] + } else { + inString = false + } + } + continue + } + if (char === "'") { + inString = true + current += char + } else if (char === '(') { + depth++ + if (depth === 1) { + current = '' + } else { + current += char + } + } else if (char === ')') { + depth = Math.max(0, depth - 1) + if (depth === 0) { + groups.push(current) + current = '' + } else { + current += char + } + } else if (depth > 0) { + current += char + } + } + return groups +} + +/** Split on commas that are not inside parentheses or single-quoted strings. */ +function splitTopLevel(input: string): string[] { + const parts: string[] = [] + let depth = 0 + let inString = false + let current = '' + for (let i = 0; i < input.length; i++) { + const char = input[i] + if (inString) { + current += char + if (char === "'") { + if (input[i + 1] === "'") { + current += input[++i] + } else { + inString = false + } + } + continue + } + if (char === "'") { + inString = true + current += char + } else if (char === '(') { + depth++ + current += char + } else if (char === ')') { + depth = Math.max(0, depth - 1) + current += char + } else if (char === ',' && depth === 0) { + parts.push(current) + current = '' + } else { + current += char + } + } + if (current.trim() !== '') { + parts.push(current) + } + return parts +} + +function isTableConstraint(def: string): boolean { + return /^(primary\s+key|foreign\s+key|constraint|unique|check|exclude|like)\b/i.test(def) +} + +function unquoteIdentifier(identifier: string): string { + const trimmed = identifier.trim() + const quoted = trimmed.match(/^"([\s\S]*)"$/) + return quoted ? quoted[1] : trimmed +} + +/** For a qualified reference like `orders.id`, keep only the final identifier. */ +function lastIdentifier(reference: string): string { + const parts = reference.split('.') + return unquoteIdentifier(parts[parts.length - 1]) +} + +function stripTrailingSemicolon(sql: string): string { + return sql.replace(/;\s*$/, '') +} diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index be4868d8d4..2750058a68 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -1,7 +1,14 @@ import { randomUUID } from 'node:crypto' import type { CompletedJob, Flow, Script } from '../../../frontend/src/lib/gen' -import type { ScriptLang } from '../../../frontend/src/lib/gen/types.gen' +import type { + DataTableTables, + DataTableTableSchema, + ScriptLang +} from '../../../frontend/src/lib/gen/types.gen' import { buildScriptLintResult } from './core/script/preview' +import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine' + +export type { BenchmarkDatatableSeed, BenchmarkDatatableTableSeed } from './datatableSqlEngine' const BENCHMARK_TIMESTAMP = '1970-01-01T00:00:00.000Z' @@ -25,6 +32,7 @@ export interface BenchmarkWorkspaceFlow { export interface BenchmarkWorkspaceRunnables { scripts?: BenchmarkWorkspaceScript[] flows?: BenchmarkWorkspaceFlow[] + datatables?: BenchmarkDatatableSeed[] } type BenchmarkCompletedJob = CompletedJob & { type: 'CompletedJob' } @@ -48,7 +56,12 @@ export function registerBenchmarkWorkspaceRunnables( runnables: BenchmarkWorkspaceRunnables ): void { benchmarkWorkspaces.add(workspace) - benchmarkWorkspaceRunnables.set(workspace, runnables) + // Datatables are mutated in place by exec_datatable_sql (a write must be visible + // to later reads), so store an isolated deep copy — never mutate the caller's seed. + benchmarkWorkspaceRunnables.set(workspace, { + ...runnables, + datatables: runnables.datatables ? structuredClone(runnables.datatables) : undefined + }) } export function unregisterBenchmarkWorkspace(workspace: string): void { @@ -161,6 +174,99 @@ export function getBenchmarkCompletedJob( return structuredClone(entry.job) } +// ============= Datatables (best-effort in-memory SQL) ============= + +/** + * Project the seeded datatables down to the `list_datatable_tables` response: + * `datatable_name` + `schema -> table_names`, with no column detail. + * Returns `null` for a non-benchmark workspace so callers can fall through to + * the real backend; an empty seed yields `[]`. + */ +export function listBenchmarkDatatables(workspace: string): DataTableTables[] | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + if (!runnables) { + return null + } + return (runnables.datatables ?? []).map((datatable) => ({ + datatable_name: datatable.datatable_name, + schemas: Object.fromEntries( + Object.entries(datatable.schemas).map(([schema, tables]) => [schema, Object.keys(tables)]) + ) + })) +} + +export function getBenchmarkDatatableSchema(input: { + workspace: string + datatableName: string + schemaName: string + tableName: string +}): DataTableTableSchema { + const runnables = benchmarkWorkspaceRunnables.get(input.workspace) + const datatable = (runnables?.datatables ?? []).find( + (entry) => entry.datatable_name === input.datatableName + ) + if (!datatable) { + // Message MUST match the production `isDatatableNotConfiguredError` regex + // (/datatable\s+\S+\s+not found/i in datatableTools.ts) so the + // get_datatable_table_schema not-configured mapping is actually exercised. + throw new Error(`datatable "${input.datatableName}" not found`) + } + const table = datatable.schemas?.[input.schemaName]?.[input.tableName] + if (!table) { + throw new Error( + `table "${input.schemaName}.${input.tableName}" not found in datatable "${input.datatableName}"` + ) + } + return { + datatable_name: input.datatableName, + schema_name: input.schemaName, + table_name: input.tableName, + columns: table.columns + } +} + +/** + * Execute SQL against a seeded datatable through the best-effort in-memory engine + * (`applyDatatableSql`). Writes (CREATE/INSERT/UPDATE/DELETE/DROP) mutate the + * stored datatable in place so a later list/schema/SELECT reflects them; SELECT + * (and RETURNING) yield rows, other statements yield `[]`. Creates a benchmark + * completed job and returns its id, like `runBenchmarkScriptPreview`. + */ +export function runBenchmarkDatatableSql(input: { + workspace: string + datatableName: string + sql: string +}): string { + const runnables = benchmarkWorkspaceRunnables.get(input.workspace) + const datatable = (runnables?.datatables ?? []).find( + (entry) => entry.datatable_name === input.datatableName + ) + const rows = datatable ? applyDatatableSql(datatable, input.sql).rows : [] + return createBenchmarkCompletedJob({ + workspace: input.workspace, + jobKind: 'preview', + success: true, + args: { database: `datatable://${input.datatableName}` }, + result: rows + }) +} + +/** + * Mirror `JobService.getCompletedJobResultMaybe` for benchmark workspaces — the + * shape `pollJobResult` consumes. The job is created synchronously before + * polling, so it is always present and completed. + */ +export function getBenchmarkCompletedJobResultMaybe(input: { + workspace: string + id: string +}): { success: boolean; completed: boolean; result: unknown } { + const job = getBenchmarkCompletedJob(input.workspace, input.id) + if (!job) { + throw new Error(`Job "${input.id}" not found in benchmark workspace`) + } + return { success: job.success, completed: true, result: job.result } +} + export function runBenchmarkScriptPreview(input: { workspace: string requestBody: { diff --git a/ai_evals/adapters/frontend/mockBackendDatatables.test.ts b/ai_evals/adapters/frontend/mockBackendDatatables.test.ts new file mode 100644 index 0000000000..5d12ebb661 --- /dev/null +++ b/ai_evals/adapters/frontend/mockBackendDatatables.test.ts @@ -0,0 +1,175 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { + getBenchmarkCompletedJobResultMaybe, + getBenchmarkDatatableSchema, + listBenchmarkDatatables, + registerBenchmarkWorkspaceRunnables, + resetBenchmarkMockBackend, + runBenchmarkDatatableSql, + type BenchmarkWorkspaceRunnables +} from './mockBackend' + +const WORKSPACE = 'benchmark-datatable-ws' + +// Mirrors the production `isDatatableNotConfiguredError` regex in +// datatableTools.ts. The schema mock's "not configured" message MUST match it, +// otherwise the not-configured mapping in get_datatable_table_schema is silently +// untested. +const NOT_CONFIGURED_RE = /datatable\s+\S+\s+not found/i + +const SEED: BenchmarkWorkspaceRunnables = { + datatables: [ + { + datatable_name: 'main', + schemas: { + public: { + orders: { + columns: { id: 'int', total: 'numeric' }, + rows: [ + { id: 1, total: 10 }, + { id: 2, total: 20 } + ] + }, + customers: { + columns: { id: 'int', name: 'text' }, + rows: [{ id: 1, name: 'alice' }] + } + } + } + } + ] +} + +beforeEach(() => resetBenchmarkMockBackend()) +afterEach(() => resetBenchmarkMockBackend()) + +describe('listBenchmarkDatatables', () => { + it('returns null for a non-benchmark workspace (caller falls through to real backend)', () => { + expect(listBenchmarkDatatables('unregistered')).toBeNull() + }) + + it('returns [] for a registered workspace with no datatables seed', () => { + registerBenchmarkWorkspaceRunnables(WORKSPACE, {}) + expect(listBenchmarkDatatables(WORKSPACE)).toEqual([]) + }) + + it('projects seeded datatables to schema -> table names only (no columns)', () => { + registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED) + expect(listBenchmarkDatatables(WORKSPACE)).toEqual([ + { datatable_name: 'main', schemas: { public: ['orders', 'customers'] } } + ]) + }) +}) + +describe('getBenchmarkDatatableSchema', () => { + beforeEach(() => registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED)) + + it('returns the columns for a seeded table', () => { + expect( + getBenchmarkDatatableSchema({ + workspace: WORKSPACE, + datatableName: 'main', + schemaName: 'public', + tableName: 'orders' + }) + ).toEqual({ + datatable_name: 'main', + schema_name: 'public', + table_name: 'orders', + columns: { id: 'int', total: 'numeric' } + }) + }) + + it('throws a not-configured error matching the production regex for an unknown datatable', () => { + let error: Error | undefined + try { + getBenchmarkDatatableSchema({ + workspace: WORKSPACE, + datatableName: 'ghost', + schemaName: 'public', + tableName: 'orders' + }) + } catch (e) { + error = e as Error + } + expect(error).toBeDefined() + expect(error!.message).toMatch(NOT_CONFIGURED_RE) + }) + + it('throws a table-not-found error that does NOT match the datatable-not-configured regex', () => { + // The datatable IS configured; only the table is missing. Production maps + // this to a generic "error getting schema", not the blocking message. + let error: Error | undefined + try { + getBenchmarkDatatableSchema({ + workspace: WORKSPACE, + datatableName: 'main', + schemaName: 'public', + tableName: 'ghost' + }) + } catch (e) { + error = e as Error + } + expect(error).toBeDefined() + expect(error!.message).not.toMatch(NOT_CONFIGURED_RE) + }) +}) + +describe('runBenchmarkDatatableSql + getBenchmarkCompletedJobResultMaybe', () => { + beforeEach(() => registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED)) + + function exec(sql: string): { success: boolean; completed: boolean; result: unknown } { + const jobId = runBenchmarkDatatableSql({ workspace: WORKSPACE, datatableName: 'main', sql }) + return getBenchmarkCompletedJobResultMaybe({ workspace: WORKSPACE, id: jobId }) + } + + it('returns the canned rows of the table named in a SELECT FROM clause', () => { + expect(exec('SELECT * FROM customers')).toEqual({ + success: true, + completed: true, + result: [{ id: 1, name: 'alice' }] + }) + }) + + it('falls back to the first seeded table when the SELECT references no known table', () => { + expect(exec('select 1').result).toEqual([ + { id: 1, total: 10 }, + { id: 2, total: 20 } + ]) + }) + + it('returns [] success for DDL and DML statements without RETURNING', () => { + expect(exec('CREATE TABLE foo (id int)').result).toEqual([]) + expect(exec('INSERT INTO orders VALUES (3, 30)').result).toEqual([]) + expect(exec('update orders set total = 0').result).toEqual([]) + }) + + it('reflects a write in a later SELECT, isolated from the shared seed', () => { + exec('UPDATE orders SET total = 999 WHERE id = 1') + expect((exec('SELECT * FROM orders').result as Record[])).toContainEqual({ + id: 1, + total: 999 + }) + // Registration deep-clones the seed, so the shared SEED const stays pristine. + expect(SEED.datatables![0].schemas.public.orders.rows).toContainEqual({ id: 1, total: 10 }) + }) + + it('reflects a CREATE in list_datatables and get_datatable_table_schema', () => { + exec('CREATE TABLE public.refunds (order_id int4, amount numeric)') + expect(listBenchmarkDatatables(WORKSPACE)?.[0].schemas.public).toContain('refunds') + expect( + getBenchmarkDatatableSchema({ + workspace: WORKSPACE, + datatableName: 'main', + schemaName: 'public', + tableName: 'refunds' + }).columns + ).toEqual({ order_id: 'int4', amount: 'numeric' }) + }) + + it('throws for an unknown job id', () => { + expect(() => + getBenchmarkCompletedJobResultMaybe({ workspace: WORKSPACE, id: 'does-not-exist' }) + ).toThrow() + }) +}) diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 92c43aab06..a84f990483 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -34,15 +34,19 @@ vi.mock('$lib/gen', async () => { const actual = await vi.importActual('$lib/gen') const { getBenchmarkCompletedJob, + getBenchmarkCompletedJobResultMaybe, + getBenchmarkDatatableSchema, getBenchmarkFlowByPath, getBenchmarkScriptByHash, getBenchmarkScriptByPath, hasBenchmarkWorkspace, + listBenchmarkDatatables, listBenchmarkFlows, listBenchmarkScripts, createBenchmarkHttpTrigger, createBenchmarkSchedule, previewBenchmarkSchedule, + runBenchmarkDatatableSql, runBenchmarkFlowByPath, runBenchmarkScriptPreview } = await import('./mockBackend') @@ -149,13 +153,27 @@ vi.mock('$lib/gen', async () => { args?: Record path?: string } - }) => - hasBenchmarkWorkspace(data.workspace) - ? runBenchmarkScriptPreview({ - workspace: data.workspace, - requestBody: data.requestBody ?? {} - }) - : actual.JobService.runScriptPreview(data), + }) => { + if (!hasBenchmarkWorkspace(data.workspace)) { + return actual.JobService.runScriptPreview(data) + } + const requestBody = data.requestBody ?? {} + const database = requestBody.args?.database + // Datatable SQL runs as a `postgresql` preview against `datatable://`. + // Execute it through the canned-SQL mock instead of linting it as a script. + if ( + requestBody.language === 'postgresql' && + typeof database === 'string' && + database.startsWith('datatable://') + ) { + return runBenchmarkDatatableSql({ + workspace: data.workspace, + datatableName: database.slice('datatable://'.length), + sql: requestBody.content ?? '' + }) + } + return runBenchmarkScriptPreview({ workspace: data.workspace, requestBody }) + }, runFlowByPath: async (data: { workspace: string path: string @@ -177,7 +195,31 @@ vi.mock('$lib/gen', async () => { return job } return actual.JobService.getJob(data) - } + }, + getCompletedJobResultMaybe: async (data: { workspace: string; id: string }) => + hasBenchmarkWorkspace(data.workspace) + ? getBenchmarkCompletedJobResultMaybe({ workspace: data.workspace, id: data.id }) + : actual.JobService.getCompletedJobResultMaybe(data) + }), + WorkspaceService: wrapService(actual.WorkspaceService, { + listDataTableTables: async (data: { workspace: string }) => + hasBenchmarkWorkspace(data.workspace) + ? (listBenchmarkDatatables(data.workspace) ?? []) + : actual.WorkspaceService.listDataTableTables(data), + getDataTableTableSchema: async (data: { + workspace: string + datatableName: string + schemaName: string + tableName: string + }) => + hasBenchmarkWorkspace(data.workspace) + ? getBenchmarkDatatableSchema({ + workspace: data.workspace, + datatableName: data.datatableName, + schemaName: data.schemaName, + tableName: data.tableName + }) + : actual.WorkspaceService.getDataTableTableSchema(data) }), ScheduleService: wrapService(actual.ScheduleService, { existsSchedule: async (data: { workspace: string; path: string }) => diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 732ca7f2e9..8d58248d62 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -618,3 +618,206 @@ - creates a Slack resource draft that references the secret variable instead of embedding the token - keeps both drafts under a sensible eval/global Slack-related path - does not create schedules, triggers, or deployed workspace changes + +- id: global-test19-datatable-not-configured-asks-to-set-up + prompt: |- + Here are two newsletter signups: alice@example.com and bob@example.com. + Save them into a workspace data table for me. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - list_datatables + forbiddenToolsUsed: + - exec_datatable_sql + - write_script + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - checks which data tables exist in the workspace before acting + - recognizes that no data table is configured in this workspace + - explains that a data table must first be set up by the user in the workspace settings (Data Tables) and is not created via SQL + - does not run SQL, write a script, or invent a data table to work around the missing configuration + - tells the user to configure a data table and then try again + +- id: global-test20-datatable-no-hallucinated-main + prompt: |- + Pull the latest rows from the orders table in our data table so I can see recent orders. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - list_datatables + forbiddenToolsUsed: + - exec_datatable_sql + - write_script + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - checks which data tables exist in the workspace before querying + - recognizes that no data table is configured in this workspace + - does not assume a data table named "main" (or any other name) exists + - does not run SQL against a guessed data table or fabricate order rows + - tells the user they need to set up a data table in the workspace settings first + +- id: global-test21-datatable-list-summarize + prompt: |- + What tables do we have in our workspace data table? Just give me the list. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - list_datatables + forbiddenToolsUsed: + - get_datatable_table_schema + - exec_datatable_sql + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - lists the tables available in the workspace data table (orders and customers) + - answers from the data table listing rather than fabricating table names + - does not fetch column details or run SQL just to produce a table list + +- id: global-test22-datatable-inspect-columns + prompt: |- + What columns does the orders table have in our workspace data table? + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - get_datatable_table_schema + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - inspects the orders table schema in the workspace data table + - reports the orders columns (such as id, customer_id, total, status, created_at) + - answers from the retrieved schema rather than guessing the columns + +- id: global-test23-datatable-query-select + prompt: |- + Show me the orders in our workspace data table, including their status and total. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - exec_datatable_sql + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: exec_datatable_sql + field: sql + stringIncludesAnyOf: + - select + skipJudge: true + judgeChecklist: + - runs a SELECT query against the orders table in the workspace data table + - reports the orders returned by the query back to the user instead of fabricating data + - does not tell the user to set up a data table, since one already exists + +- id: global-test24-datatable-create-table + prompt: |- + Add a new table called refunds to our workspace data table, with an order id and a refund amount. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - exec_datatable_sql + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: exec_datatable_sql + field: sql + stringIncludesAnyOf: + - create table + skipJudge: true + judgeChecklist: + - creates the refunds table with a plain CREATE TABLE statement on the data table + - includes an order id and a refund amount column + - treats creating the table as a normal SQL statement and does not claim a separate registration step is needed + - does not write a script to create the table + +- id: global-test25-datatable-mutate-rows + prompt: |- + Mark order number 2 as shipped in our workspace data table. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json + runtime: + # Headroom for inspect-schema -> UPDATE -> verify; the in-memory engine now + # persists the write, so verification confirms on the first try (no loop). + maxTurns: 12 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - exec_datatable_sql + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: exec_datatable_sql + field: sql + stringIncludesAnyOf: + - update + - insert into + skipJudge: true + judgeChecklist: + - runs an UPDATE on the orders table setting the status of order id 2 to shipped + - targets only order number 2 rather than rewriting the whole table + - confirms the change back to the user + +- id: global-test26-datatable-script-sdk + prompt: |- + Write a script that reads our workspace data table and returns the total revenue across all orders. + Leave it as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + valueIncludes: + - wmill.datatable( + toolExpect: + requiredToolsUsed: + - get_instructions + - write_script + forbiddenToolsUsed: + - exec_datatable_sql + - deploy_workspace_item + - delete_workspace_item + # The judge has no datatable SDK reference and wrongly penalizes correct + # wmill.datatable() tagged-template usage, so rely on the deterministic checks: + # required get_instructions + write_script, forbidden exec_datatable_sql, and a + # draft that contains wmill.datatable(. + skipJudge: true + judgeChecklist: + - writes a script (not a chat-time SQL execution) that reads the workspace data table at runtime + - uses the wmill.datatable() SDK to query the orders table and sum the order totals + - returns the total revenue from the script + - leaves the result as an AI draft and does not deploy or save it diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index ecc46591fc..27c2fcddac 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -155,6 +155,15 @@ export interface ToolCallArgumentRule { field: string; stringStartsWithAnyOf?: string[]; stringMustNotStartWithAnyOf?: string[]; + /** + * Case-insensitive "contains", existential over calls: at least one recorded + * call to `tool` must have `field` containing one of these substrings. Other + * calls to the same tool may do anything. Use instead of `stringStartsWithAnyOf` + * (which is universal over calls) when the meaningful token can appear anywhere + * in the value and the model may make additional, unrelated calls to the same + * tool — e.g. SQL where a mutation is mixed with verification SELECTs. + */ + stringIncludesAnyOf?: string[]; } export interface ToolValidationSpec { diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index 6010f6351a..7f0e841366 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -140,6 +140,111 @@ describe("validateToolExpectations", () => { details: "tools used: write_script, deploy_workspace_item", }); }); + + it("accepts a stringIncludesAnyOf substring regardless of case or position", () => { + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["exec_datatable_sql"], + toolCallDetails: [ + { + name: "exec_datatable_sql", + arguments: { + sql: "WITH recent AS (SELECT * FROM orders) SELECT count(*) FROM recent", + }, + }, + ], + skillsInvoked: [], + }, + toolExpect: { + requiredToolsUsed: ["exec_datatable_sql"], + toolCallArgs: [ + { + tool: "exec_datatable_sql", + field: "sql", + stringIncludesAnyOf: ["select"], + }, + ], + }, + }); + + expect(checks.every((check) => check.passed)).toBe(true); + }); + + it("accepts stringIncludesAnyOf when only one of several calls matches", () => { + // Existential: a mutation mixed with verification SELECTs still passes. + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 2, + toolsUsed: ["exec_datatable_sql"], + toolCallDetails: [ + { + name: "exec_datatable_sql", + arguments: { sql: "UPDATE orders SET status = 'shipped' WHERE id = 2" }, + }, + { + name: "exec_datatable_sql", + arguments: { sql: "SELECT * FROM orders WHERE id = 2" }, + }, + ], + skillsInvoked: [], + }, + toolExpect: { + toolCallArgs: [ + { + tool: "exec_datatable_sql", + field: "sql", + stringIncludesAnyOf: ["insert into", "update"], + }, + ], + }, + }); + + expect(checks.every((check) => check.passed)).toBe(true); + }); + + it("rejects stringIncludesAnyOf when no call matches any substring", () => { + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["exec_datatable_sql"], + toolCallDetails: [ + { + name: "exec_datatable_sql", + arguments: { + sql: "DROP TABLE orders", + }, + }, + ], + skillsInvoked: [], + }, + toolExpect: { + toolCallArgs: [ + { + tool: "exec_datatable_sql", + field: "sql", + stringIncludesAnyOf: ["insert into", "update"], + }, + ], + }, + }); + + expect(checks).toContainEqual({ + name: "exec_datatable_sql.sql includes a required substring", + passed: false, + details: + 'accepted substrings: insert into, update; values: "DROP TABLE orders"', + }); + }); }); describe("validateGlobalState", () => { diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 693d34a013..e7a7641c00 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -222,6 +222,25 @@ export function validateToolExpectations(input: { ) ); } + + if (rule.stringIncludesAnyOf && rule.stringIncludesAnyOf.length > 0) { + // Existential: at least one call must contain one of the substrings. + // Other calls to the same tool may do anything — this suits SQL, where a + // model mixes the requested statement (e.g. an UPDATE) with verification + // SELECTs that would otherwise fail an "all calls" check. + const needles = rule.stringIncludesAnyOf.map((needle) => needle.toLowerCase()); + const hasMatch = values.some( + (value) => + typeof value === "string" && needles.some((needle) => value.toLowerCase().includes(needle)) + ); + checks.push( + check( + `${rule.tool}.${rule.field} includes a required substring`, + hasMatch, + `accepted substrings: ${rule.stringIncludesAnyOf.join(", ")}; values: ${summarizeToolValues(values)}` + ) + ); + } } return checks; diff --git a/ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json b/ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json new file mode 100644 index 0000000000..23300ca41e --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json @@ -0,0 +1,39 @@ +{ + "workspace": { + "datatables": [ + { + "datatable_name": "main", + "schemas": { + "public": { + "orders": { + "columns": { + "id": "int4", + "customer_id": "int4", + "total": "numeric", + "status": "text", + "created_at": "timestamptz" + }, + "rows": [ + { "id": 1, "customer_id": 1, "total": 42.5, "status": "shipped", "created_at": "2026-05-01T10:00:00Z" }, + { "id": 2, "customer_id": 2, "total": 19.99, "status": "pending", "created_at": "2026-05-02T11:30:00Z" }, + { "id": 3, "customer_id": 1, "total": 88, "status": "shipped", "created_at": "2026-05-03T09:15:00Z" } + ] + }, + "customers": { + "columns": { + "id": "int4", + "name": "text", + "email": "text", + "tier": "text" + }, + "rows": [ + { "id": 1, "name": "Alice", "email": "alice@example.com", "tier": "gold" }, + { "id": 2, "name": "Bob", "email": "bob@example.com", "tier": "silver" } + ] + } + } + } + } + ] + } +} diff --git a/backend/.sqlx/query-19513c4158267cc7fe10d999ad571052c112e6bbb3cf834f16176cbb7e1ac319.json b/backend/.sqlx/query-19513c4158267cc7fe10d999ad571052c112e6bbb3cf834f16176cbb7e1ac319.json new file mode 100644 index 0000000000..8944a6001f --- /dev/null +++ b/backend/.sqlx/query-19513c4158267cc7fe10d999ad571052c112e6bbb3cf834f16176cbb7e1ac319.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT id AS \"id!\" FROM chain", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "19513c4158267cc7fe10d999ad571052c112e6bbb3cf834f16176cbb7e1ac319" +} diff --git a/backend/.sqlx/query-8e8933fc6648a88dc35cd81559a31d10678d6c68fc920c876914e71324d5e460.json b/backend/.sqlx/query-8e8933fc6648a88dc35cd81559a31d10678d6c68fc920c876914e71324d5e460.json new file mode 100644 index 0000000000..2ad0791d3c --- /dev/null +++ b/backend/.sqlx/query-8e8933fc6648a88dc35cd81559a31d10678d6c68fc920c876914e71324d5e460.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM v2_job WHERE id = $1 AND workspace_id = $2 AND tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8e8933fc6648a88dc35cd81559a31d10678d6c68fc920c876914e71324d5e460" +} diff --git a/backend/.sqlx/query-afb0762c88d9232b79090f2e5966e78437a5e4d3b5e2341ec5f7725a28870270.json b/backend/.sqlx/query-afb0762c88d9232b79090f2e5966e78437a5e4d3b5e2341ec5f7725a28870270.json new file mode 100644 index 0000000000..f0c91daf16 --- /dev/null +++ b/backend/.sqlx/query-afb0762c88d9232b79090f2e5966e78437a5e4d3b5e2341ec5f7725a28870270.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app SET versions = array_append(versions, $1::bigint) WHERE path = $2 AND workspace_id = $3 AND versions[array_upper(versions, 1)] = $1::bigint", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "afb0762c88d9232b79090f2e5966e78437a5e4d3b5e2341ec5f7725a28870270" +} \ No newline at end of file diff --git a/backend/.sqlx/query-c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5.json b/backend/.sqlx/query-c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5.json new file mode 100644 index 0000000000..774b47f825 --- /dev/null +++ b/backend/.sqlx/query-c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n ))\n RETURNING token_prefix", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token_prefix", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5" +} diff --git a/backend/.sqlx/query-c533691be8136c5ed6835c2fcdb016c257a5e4ee271d221fa961c24bd119d98e.json b/backend/.sqlx/query-c533691be8136c5ed6835c2fcdb016c257a5e4ee271d221fa961c24bd119d98e.json new file mode 100644 index 0000000000..a0cb83bf14 --- /dev/null +++ b/backend/.sqlx/query-c533691be8136c5ed6835c2fcdb016c257a5e4ee271d221fa961c24bd119d98e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT permissioned_as_email FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "permissioned_as_email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c533691be8136c5ed6835c2fcdb016c257a5e4ee271d221fa961c24bd119d98e" +} diff --git a/backend/.sqlx/query-ca5bb402834502432f3d7260fdd5b9fb568a4c77e2a91f55575a93461d5a7f50.json b/backend/.sqlx/query-ca5bb402834502432f3d7260fdd5b9fb568a4c77e2a91f55575a93461d5a7f50.json new file mode 100644 index 0000000000..a84b022930 --- /dev/null +++ b/backend/.sqlx/query-ca5bb402834502432f3d7260fdd5b9fb568a4c77e2a91f55575a93461d5a7f50.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM v2_job WHERE id = ANY($1) AND workspace_id = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ca5bb402834502432f3d7260fdd5b9fb568a4c77e2a91f55575a93461d5a7f50" +} diff --git a/backend/.sqlx/query-cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037.json b/backend/.sqlx/query-cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037.json new file mode 100644 index 0000000000..29f31c62eb --- /dev/null +++ b/backend/.sqlx/query-cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037.json @@ -0,0 +1,60 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag, script_lang AS \"script_lang: ScriptLang\" FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037" +} diff --git a/backend/.sqlx/query-e88e1009f5359e205a523a32e4e8e72605971a3f417cb2b032c4d43652aca056.json b/backend/.sqlx/query-e88e1009f5359e205a523a32e4e8e72605971a3f417cb2b032c4d43652aca056.json new file mode 100644 index 0000000000..3db3b0c977 --- /dev/null +++ b/backend/.sqlx/query-e88e1009f5359e205a523a32e4e8e72605971a3f417cb2b032c4d43652aca056.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT hash, content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "content", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "e88e1009f5359e205a523a32e4e8e72605971a3f417cb2b032c4d43652aca056" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e39f97cd1f..80b89abee7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1521,7 +1521,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1541,7 +1541,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1584,9 +1584,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" dependencies = [ "serde_core", ] @@ -1924,9 +1924,9 @@ dependencies = [ [[package]] name = "bytes-str" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c60b5ce37e0b883c37eb89f79a1e26fbe9c1081945d024eee93e8d91a7e18b3" +checksum = "577d2bf5650f8554d5a372af5ac93535110a0fc75b3e702bb853369febf227c2" dependencies = [ "bytes", "serde", @@ -2112,9 +2112,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -4585,7 +4585,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "rustc_version 0.4.1", ] @@ -6099,7 +6099,7 @@ version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "libc", ] @@ -6372,6 +6372,16 @@ version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" +[[package]] +name = "kstat-rs" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27964e4632377753acb0898ce6f28770d50cbca1339200ae63d700cff97b5c2b" +dependencies = [ + "libc", + "thiserror 1.0.69", +] + [[package]] name = "kube" version = "1.1.0" @@ -6587,7 +6597,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e668df13f2e97f3eed52d9301f6b1c4c1ccfccc30eab9e6628e4a8c1fc3546" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "bytes", "lazy_static", "libgssapi-sys", @@ -6636,7 +6646,7 @@ version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "libc", "plain", "redox_syscall 0.8.1", @@ -6720,9 +6730,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.30" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "loom" @@ -6817,6 +6827,12 @@ dependencies = [ "libc", ] +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + [[package]] name = "macro_rules_attribute" version = "0.2.2" @@ -7240,7 +7256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b42ced54aa8ac97226486337973f9bc3956e24f03a23e88a6e18f640959d6e2" dependencies = [ "base64 0.22.1", - "bitflags 2.11.1", + "bitflags 2.13.0", "btoi", "byteorder", "bytes", @@ -7299,7 +7315,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "libc", ] @@ -7310,7 +7326,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "cfg_aliases", "libc", @@ -7322,7 +7338,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "cfg_aliases", "libc", @@ -7334,7 +7350,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "cfg_aliases", "libc", @@ -7487,7 +7503,7 @@ dependencies = [ "libc", "libproc", "log", - "mach2", + "mach2 0.4.3", "nix 0.29.0", "ntapi", "procfs", @@ -7784,7 +7800,7 @@ version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "libc", "once_cell", "onig_sys", @@ -7843,7 +7859,7 @@ version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "foreign-types", "libc", @@ -8792,7 +8808,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "chrono", "flate2", "hex", @@ -8806,7 +8822,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "chrono", "hex", ] @@ -8893,7 +8909,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "getopts", "memchr", "unicase", @@ -8950,9 +8966,9 @@ dependencies = [ [[package]] name = "quick_cache" -version = "0.6.22" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1c821816e9b928e20e92ed59bb3ac4aab321d16ca2316871c9fe7ca739cd477" +checksum = "3a3db184a8b66cfe87f0263a1de147a6b554c864d1767c6f7fa4eb0e5497b565" dependencies = [ "ahash 0.8.12", "equivalent", @@ -9177,7 +9193,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -9291,7 +9307,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -9300,7 +9316,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b44b894f2a6e36457d665d1e08c3866add6ed5e70050c1b4ba8a8ddedb02ce7" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -9867,7 +9883,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -9880,7 +9896,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys 0.12.1", @@ -10333,7 +10349,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -10346,7 +10362,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -10555,9 +10571,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64 0.22.1", "bs58", @@ -10575,9 +10591,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -11062,7 +11078,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.11.1", + "bitflags 2.13.0", "byteorder", "bytes", "chrono", @@ -11107,7 +11123,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.11.1", + "bitflags 2.13.0", "byteorder", "chrono", "crc", @@ -11382,7 +11398,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c25af97d53cf8aab66a6c68f3418663313fc969ad267fc2a4d19402c329be1" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "is-macro", "num-bigint", "once_cell", @@ -11438,7 +11454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "017d06ea85008234aa9fb34d805c7dc563f2ea6e03869ed5ac5a2dc27d561e4d" dependencies = [ "arrayvec", - "bitflags 2.11.1", + "bitflags 2.13.0", "either", "num-bigint", "phf 0.11.3", @@ -11760,7 +11776,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "byteorder", "enum-as-inner", "libc", @@ -11788,7 +11804,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -11805,13 +11821,15 @@ dependencies = [ [[package]] name = "systemstat" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e89b75de097d0c52a1dc2114e19439d55f0e2e42d32168c6df44f139dfb66f" +checksum = "a583abe520746270ffdbdaf0e3039a806f29be9d7034d66466a4839a01de0610" dependencies = [ "bytesize", + "kstat-rs", "lazy_static", "libc", + "mach2 0.6.0", "nom", "time", "winapi", @@ -12703,7 +12721,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.11.1", + "bitflags 2.13.0", "bytes", "futures-core", "futures-util", @@ -13343,7 +13361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" dependencies = [ "bindgen 0.71.1", - "bitflags 2.11.1", + "bitflags 2.13.0", "fslock", "gzip-header", "home", @@ -13598,7 +13616,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "hashbrown 0.15.5", "indexmap 2.14.0", "semver 1.0.28", @@ -13764,7 +13782,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-nats", @@ -13815,6 +13833,7 @@ dependencies = [ "windmill-api-agent-workers", "windmill-api-auth", "windmill-api-client", + "windmill-api-scripts", "windmill-api-settings", "windmill-autoscaling", "windmill-common", @@ -13845,7 +13864,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.714.0" +version = "1.719.0" dependencies = [ "async-stream", "async-trait", @@ -13878,7 +13897,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13891,7 +13910,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "argon2", @@ -14029,7 +14048,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14052,7 +14071,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14065,7 +14084,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14091,7 +14110,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.714.0" +version = "1.719.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14101,7 +14120,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14118,7 +14137,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14140,7 +14159,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14163,7 +14182,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14179,7 +14198,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14200,7 +14219,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14221,7 +14240,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14235,7 +14254,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-nats", @@ -14245,6 +14264,8 @@ dependencies = [ "axum 0.8.9", "base64 0.22.1", "futures", + "hex", + "hmac", "rand 0.9.0", "rdkafka", "reqwest 0.13.1", @@ -14252,6 +14273,7 @@ dependencies = [ "rumqttc", "serde", "serde_json", + "sha2 0.10.9", "sqlx", "tokio", "uuid", @@ -14267,7 +14289,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14292,7 +14314,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14310,7 +14332,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14332,7 +14354,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14352,7 +14374,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14361,6 +14383,7 @@ dependencies = [ "hyper 1.10.1", "itertools 0.14.0", "lazy_static", + "prometheus", "quick_cache", "reqwest 0.13.1", "serde", @@ -14382,7 +14405,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14410,7 +14433,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.714.0" +version = "1.719.0" dependencies = [ "lazy_static", "serde", @@ -14422,7 +14445,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.714.0" +version = "1.719.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14447,7 +14470,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14461,7 +14484,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.714.0" +version = "1.719.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14494,7 +14517,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.714.0" +version = "1.719.0" dependencies = [ "chrono", "lazy_static", @@ -14508,7 +14531,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14527,7 +14550,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.714.0" +version = "1.719.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14544,7 +14567,7 @@ dependencies = [ "axum 0.8.9", "backon", "base64 0.22.1", - "bitflags 2.11.1", + "bitflags 2.13.0", "bytes", "chrono", "chrono-tz", @@ -14628,7 +14651,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.714.0" +version = "1.719.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14647,7 +14670,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.714.0" +version = "1.719.0" dependencies = [ "regex", "serde", @@ -14662,7 +14685,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14686,7 +14709,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "futures", @@ -14703,7 +14726,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.714.0" +version = "1.719.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14719,7 +14742,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -14740,7 +14763,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -14771,7 +14794,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "arc-swap", @@ -14796,7 +14819,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-stream", @@ -14830,7 +14853,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "futures", @@ -14848,7 +14871,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.714.0" +version = "1.719.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14857,7 +14880,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "lazy_static", @@ -14869,7 +14892,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde_json", @@ -14881,7 +14904,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "gosyn", @@ -14893,7 +14916,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "lazy_static", @@ -14905,7 +14928,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde_json", @@ -14917,7 +14940,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "nu-parser", @@ -14928,7 +14951,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14939,7 +14962,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14951,7 +14974,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14962,7 +14985,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-recursion", @@ -14984,7 +15007,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde_json", @@ -14996,7 +15019,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "lazy_static", @@ -15010,7 +15033,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15027,7 +15050,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "lazy_static", @@ -15040,7 +15063,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde", @@ -15052,7 +15075,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "lazy_static", @@ -15070,7 +15093,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15086,7 +15109,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15102,7 +15125,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde", @@ -15113,7 +15136,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-recursion", @@ -15151,7 +15174,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "const_format", @@ -15189,7 +15212,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.714.0" +version = "1.719.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15200,13 +15223,14 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-recursion", "axum 0.8.9", "chrono", "futures", + "hex", "http 1.4.1", "hyper 1.10.1", "lazy_static", @@ -15214,6 +15238,7 @@ dependencies = [ "reqwest 0.13.1", "serde", "serde_json", + "sha2 0.10.9", "sql-builder", "sqlx", "tokio", @@ -15230,7 +15255,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15254,7 +15279,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15287,7 +15312,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15320,7 +15345,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15340,7 +15365,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15374,7 +15399,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15410,7 +15435,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15433,7 +15458,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15457,7 +15482,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-nats", @@ -15481,7 +15506,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15516,7 +15541,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15544,7 +15569,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-trait", @@ -15569,10 +15594,10 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", - "bitflags 2.11.1", + "bitflags 2.13.0", "chrono", "hex", "itertools 0.14.0", @@ -15588,7 +15613,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-once-cell", @@ -15698,7 +15723,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.714.0" +version = "1.719.0" dependencies = [ "bytes", "futures", @@ -15949,7 +15974,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "widestring", "windows-sys 0.52.0", ] @@ -16375,7 +16400,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.1", + "bitflags 2.13.0", "indexmap 2.14.0", "log", "serde", @@ -16489,9 +16514,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 32ae3ce1fc..df637c3f75 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.714.0" +version = "1.719.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.714.0" +version = "1.719.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -251,6 +251,7 @@ windmill-object-store.workspace = true windmill-git-sync.workspace = true windmill-api = { workspace = true, default-features = false } windmill-api-agent-workers = { workspace = true, optional = true } +windmill-api-scripts.workspace = true windmill-api-settings.workspace = true windmill-worker.workspace = true windmill-indexer = { workspace = true, optional = true } diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md new file mode 100644 index 0000000000..8cc71ec71c --- /dev/null +++ b/backend/THREAT_MODEL.md @@ -0,0 +1,172 @@ +# Threat Model: Windmill Backend + +## 1. System context + +Windmill is an open-source (AGPLv3) developer platform for internal tools, +workflows, background jobs, API integrations, and UIs — a self-hostable +alternative to Retool / Pipedream / Airplane. The backend is a Rust workspace +(~60 crates: `windmill-api`, `windmill-worker`, `windmill-queue`, +`windmill-common`, a family of `windmill-trigger-*` crates, `windmill-mcp`, +`windmill-sandbox`, etc.) fronting a PostgreSQL database. A Svelte 5 frontend +(not in scope here, but referenced where stored-XSS threats originate) is +served by the same instance. The product ships in a Community Edition (CE, +public Docker images) and an Enterprise Edition (EE, `*_ee.rs` files gated by +`enterprise`/`private`/`license` cargo features). + +The defining characteristic for threat modeling is that **Windmill executes +arbitrary user-supplied code** (Python, TypeScript via Bun/Deno, Go, Bash, +SQL, GraphQL, PowerShell, Rust, …) on its workers, and **stores the +credentials to every system its users connect to** (databases, cloud +accounts, SaaS APIs, OAuth tokens). It is therefore simultaneously an +arbitrary-code-execution engine and a credential vault — compromising one +instance can pivot into an organization's entire connected estate. Crucially, +the owner confirms `nsjail` is **off by default everywhere** (`ENABLE_NSJAIL` +is opt-in) and network isolation (`clone_newnet`) is separately gated: the +*only* job isolation present in a default install is PID-namespace `unshare`. +Filesystem and outbound-network isolation are therefore absent unless an +operator deliberately enables them, which makes "weak-by-default isolation" a +more accurate frame than "sandbox escape" for typical deployments. Cross-tenant +separation is enforced in software via workspace IDs, token scopes, folder +ACLs, and Postgres row-level security; on the managed offering, sensitive +customers can opt into dedicated DB / worker / namespace infrastructure, but +the shared tier relies entirely on that software boundary. Administrators are +strongly encouraged to use nsjail sandboxing and are reminded that if they don't, +their security model is that they trust their developers that write code ran on windmill +to not do anything TOO malicious on the workers. When the default +database secret backend is used, only per-workspace secret *variables* are +encrypted at rest — instance-level `global_settings` (OAuth client secrets, +SMTP, object-store keys, license) are stored plaintext, so a database read +yields the instance-wide credential set. Internet-facing instances are +typically exposed directly with no built-in rate limiting or WAF. + +It is deployed self-hosted (Docker Compose, Kubernetes/Helm, bare metal), on +cloud providers, and as a Windmill-Labs-managed multi-tenant service. The API +server is internet-facing in most deployments; workers pull jobs from the +Postgres queue. The large public attack surface (a sprawling authenticated +HTTP API, unauthenticated public-app and webhook/trigger endpoints, outbound +HTTP from user code and proxies) combined with the high-value assets makes +authorization-enforcement bugs, SSRF, SQL injection, and sandbox escape the +dominant risk categories — a pattern strongly confirmed by the project's +published advisory history (73 GHSA advisories, several rated 9.9 critical). + +## 2. Assets + +| asset | description | sensitivity | +|---|---|---| +| Workspace encryption keys | Per-workspace key (`workspace_key`) used to encrypt secret variables (MagicCrypt256); decrypts all secrets in the workspace | critical | +| Secret variables | User secrets stored encrypted in `variable` (is_secret) | critical | +| Resource credentials | DB passwords, cloud creds, API keys, connection strings in `resource` JSONB | critical | +| OAuth / external-account tokens | Refresh/access tokens in `account`, MCP OAuth tables | critical | +| User password hashes | Argon2 hashes in `password` table | critical | +| API tokens & session cookies | Bearer tokens / cookies in `token`; superadmin & scoped tokens | critical | +| Instance global settings | License key, JWT secret, SUPERADMIN_SECRET, SMTP, object-store + secret-backend (Vault/KMS/SM) creds in `global_settings` | critical | +| Worker host & process integrity | The host that runs untrusted user code | critical | +| Cross-tenant / cross-workspace isolation | The software boundary separating workspaces, folders, and tenants | critical | +| Downstream connected systems | Windmill is a credential vault: stored creds reach external DBs, cloud accounts, SaaS | critical | +| Script / flow / app source | Customer IP & business logic in `script`, `flow`, `app`, `raw_app` | high | +| Job arguments, results & logs | `queue`/`completed_job` args+result, `job_logs`; routinely contain secrets | high | +| Object store / S3 data | Files uploaded/produced by jobs | high | +| Audit logs | `audit`/`audit_partitioned` action trail | high | +| Service availability | API server + worker fleet uptime | high | +| PII | User emails, group membership | medium | + +## 3. Entry points & trust boundaries + +| entry_point | description | trust_boundary | reachable_assets | +|---|---|---|---| +| EP1 Authenticated job-execution API | `jobs/run/preview`, `run/h/{hash}`, `run_flow/run_script` — runs user code on workers | authenticated user → arbitrary code on worker | Worker host, downstream systems, isolation, job args/results/logs | +| EP2 Unauthenticated public endpoints | `apps_u/*`, `jobs_u/getupdate*`, `scripts_u`, `settings_u`, `resources_u` (`public_app_layer.rs`) | unauth HTTP → app logic & job data | Job results, scripts, secrets, PII | +| EP3 HTTP-trigger & webhook ingestion | `/api/r/*`, GCP/Azure push, Slack callback, `capture_u/*` | untrusted webhook → job queue | Job execution integrity, worker host | +| EP4 Message-queue / native triggers | kafka, postgres, mqtt, websocket, nats, sqs, email triggers | external broker/message → job queue | Job execution integrity, availability | +| EP5 HTTP API authorization layer | Token/scope/RLS/folder-ACL enforcement across all workspaced routes (`windmill-api-auth`) | scoped token / low-priv user → other users' & workspaces' data | Scripts, job data, secrets, isolation | +| EP6 AI proxy & MCP endpoints | `ai/proxy/*`, `mcp` — resolve `$var:`/resources, proxy to LLM APIs, `X-Resource-Path` | authenticated user → outbound HTTP + secret resolution | Secrets, resource creds, internal network, downstream | +| EP7 Outbound HTTP from executors/resources | GraphQL/HTTP/Postgres executors, webhook delivery, `test_object_storage_config`, git clone, npm tarball fetch | user-controlled URL → server-side request | Cloud metadata, internal network, downstream creds | +| EP8 SQL query builders & contextual-var substitution | App DB query builder (`whereClause`/`tags`), Postgres-trigger `where_clause`, `%%WM_*%%` interpolation, `WM_INTERNAL_DB` | user input → raw SQL | Database, connected DBs | +| EP9 Worker sandbox | nsjail / unshare / dind / rootless podman isolating user code | user code → host & cross-tenant filesystem/network | Worker host, isolation, downstream | +| EP10 Worker code generation / wrappers | Entrypoint override, env-var names, workspace env interpolated into generated wrapper code | user-controlled identifier → executable code | Worker host, isolation | +| EP11 OAuth / OIDC / SAML / MCP-OAuth / logout | Login callbacks, MCP OAuth client registration, logout `rd` redirect | untrusted IdP / redirect input → session | Session tokens, accounts | +| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers | stored user content → admin browser (same origin) | Admin session, account takeover | +| EP13 Log/file reading & export endpoints | `service_logs`, `jobs_u/getupdate` log file read (symlinks), workspace/tarball export | authed/unauth request → arbitrary file or admin-only config | Arbitrary files, global settings | +| EP14 Secret-value & resource-value caches | In-memory caches in `windmill-store` keyed (historically un-keyed) by path | cache lookup crossing identity/folder boundary | Secret variables, resource creds | +| EP15 Deployment & runtime config | docker-compose defaults: dind, debugger (`REQUIRE_SIGNED_DEBUG_REQUESTS=false`), CORS `Any`, default admin/`changeme`, exposed Postgres, `SUPERADMIN_SECRET`, `ENABLE_NSJAIL=false`, privileged containers | operator/infra default → full instance | All assets | +| EP16 Supply chain | Cached hub scripts, GitHub workflow actions, vendored deps, Docker base image | build/update-time input → host & build integrity | Worker host, build integrity | +| EP17 Token lifecycle | Token create/rescope/refresh, script-issued JWTs | scoped caller → broader privilege | Tokens, accounts, isolation | + +## 4. Threats + +| id | threat | actor | surface | asset | impact | likelihood | status | controls | evidence | +|---|---|---|---|---|---|---|---|---|---| +| T1 | SQL injection in app/internal query builders and trigger clauses compromises the metadata DB and connected databases | remote_auth | EP8 | Database, downstream connected systems | critical | almost_certain | partially_mitigated | sqlx parameterized queries elsewhere; query-builder safety reviews | GHSA-225c-j3xq-g6x6, GHSA-78p7-jc72-gv66, GHSA-hvc7-f67h-jx3g, GHSA-wrrg-f89m-f84q, GHSA-79vf-3qwm-2w64, GHSA-55p6-fxj4-v983, GHSA-5g4v-49rj-r52r, GHSA-x6cq-7xr8-53x3, 2cf4bb180b | +| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; MCP private URL access requires the instance-wide `ALLOW_PRIVATE_MCP_SERVER_URLS` opt-in; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 | +| T3 | Broken authorization / IDOR lets a scoped token or low-privilege member read scripts, job data, and secrets across folders and workspaces | remote_auth | EP5, EP2, EP1 | Scripts, job data, secrets, isolation | critical | almost_certain | partially_mitigated | RLS, token scopes, folder ACLs, view-token HMAC (added incrementally); on managed, sensitive tenants can opt into dedicated DB/worker/namespace, but the shared tier IS the software boundary | GHSA-qfg7-x243-5hg4, GHSA-8x8x-88qc-qp4r, GHSA-2ppx-66jv-wpw5, GHSA-x3x7-g97v-mp59, GHSA-j276-g4h8-g6h5, GHSA-8mv7-hmrg-96xv, GHSA-x2wf-f962-7frq, GHSA-qc7c-gcw6-h4xp, GHSA-vxc5-w28p-m9xw, GHSA-2g34-wfvr-5qqj, GHSA-w7p6-wpxm-pp66, 7edf3f0212, 89a7a37776, ab11c7747a, 664edcdfb7 | +| T4 | Remote code execution by injecting attacker-controlled identifiers into generated worker wrappers | remote_auth | EP10 | Worker host, isolation, downstream | critical | likely | partially_mitigated | entrypoint/env-var-name validation added | GHSA-wxjq-w5pj-jqhx, GHSA-5f5q-2vg2-r2x4, GHSA-8q8j-mm3g-5c2q (CVE-2026-33881), bf93657fee, bd05bcadde, 22ec4da5f0 | +| T5 | Worker compromise & cross-tenant access via weak-by-default isolation (nsjail off by default → user code runs with only PID-ns `unshare`); sandbox escape where nsjail/dind/podman is enabled | remote_auth | EP9, EP15 | Worker host, isolation, downstream | critical | likely | unmitigated | nsjail off by default everywhere (`DISABLE_NSJAIL=true`); shipped compose gives PID-ns `unshare` only (`FAVOR_UNSHARE_PID=true`), bare installs get no isolation. Where nsjail enabled: read-only remounts, jail-tmp refusal, podman socket gating | GHSA-6qr8-xhg4-453q, GHSA-3vpp-vf62-wqp6, f8467f38c8, df5aec0f5d, f1b6746e0e | +| T6 | Disclosure of secrets, resource credentials, and workspace encryption keys across the authorization boundary (AI proxy, MCP, caches, export); database read additionally yields plaintext instance-level `global_settings` secrets | remote_auth | EP6, EP14, EP13 | Secret variables, encryption keys, resource creds, global settings | critical | likely | partially_mitigated | RLS on `$var:`, cache scoping by caller, admin checks on export; per-workspace secret *variables* encrypted at rest, but `global_settings` is plaintext under the default DB secret backend | GHSA-jwg4-v3cj-rvfm, GHSA-8m2p-2crh-9h3w, GHSA-6635-6fch-v8px, GHSA-437f-725p-7w84, GHSA-f27g-j463-q85w (CVE-2026-26964), GHSA-j679-v6vj-jfxc, GHSA-6vrr-fq33-qpfp, 0ba128afe7, 7836a4e733, ff8e39c69b | +| T7 | Full instance compromise from insecure deployment defaults (dind control, default admin/`changeme`, exposed Postgres, publicly readable SUPERADMIN_SECRET) | remote_unauth | EP15 | All assets | critical | likely | partially_mitigated | first-time-setup warning on default admin; docs recommend hardening | GHSA-3vpp-vf62-wqp6, GHSA-24fr-44f8-fqwg (CVE-2026-29059), GHSA-6q36-5p3h-766j | +| T8 | Unauthenticated RCE via the Debugger WebSocket in the default `windmill_extra` configuration | remote_unauth | EP15 | Worker host, all assets | critical | possible | unmitigated | `REQUIRE_SIGNED_DEBUG_REQUESTS` exists but defaults to false | GHSA-725h-99vx-9xr4 | +| T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | +| T10 | Unauthenticated disclosure of job results, args, logs, and admin config via missing-authz public endpoints | remote_unauth | EP2, EP13 | Job results/args/logs, global settings, scripts | high | likely | partially_mitigated | anonymous-job checks, log-endpoint authz hardening | GHSA-qfg7-x243-5hg4, GHSA-v448-fmm4-52fp, 108a88a180, bb90f4ce83 | +| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, or S3 download content-type | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0 | +| T12 | Webhook authentication bypass / signature replay forges trigger invocations and approvals | remote_unauth | EP3 | Job execution integrity, approvals | high | likely | partially_mitigated | HMAC verification on some triggers; signing-oracle fix | GHSA-jw8c-h45c-xpjw, GHSA-hh9x-rcf8-xjr2, GHSA-q9g3-q6fj-hc2x, GHSA-8jc4-wj2p-2vmp, ab2a15b2a8 | +| T13 | Path traversal / arbitrary file read via log-reading and MCP path endpoints (incl. symlink following) | remote_auth | EP13 | Arbitrary files on server, global settings | high | likely | partially_mitigated | traversal checks + no-symlink-follow added | GHSA-4hrf-mgvv-xp9x, bb90f4ce83, df451aa64f, ad5ec293b5, 5f2d3e6812 | +| T14 | Privilege escalation via token rescope/refresh, script-issued JWTs, or operator-permission gaps | remote_auth | EP17, EP5 | Tokens, isolation, accounts | high | likely | partially_mitigated | monotonic-privilege enforcement on token lifecycle; SECURITY DEFINER triggers | GHSA-p62p-67xp-v775, GHSA-vv9w-wx3c-q3x2, 2ddf93de96, 865ab70c89, 33fb08cf3d | +| T15 | Credential leakage via worker `/proc` environment and unmasked secrets in job logs | remote_auth | EP9, EP1 | DB creds, secrets, downstream | high | likely | partially_mitigated | Aho-Corasick secret masking in logs | GHSA-pmp9-9924-f9cx, 0885d8c986 | +| T16 | Denial of service via resource exhaustion: unbounded uploads, runaway jobs, queue flooding, or trigger-message storms | remote_auth | EP1, EP3, EP4 | Service availability, worker fleet | high | likely | risk_accepted | Per-job rlimits/timeouts exist; instance-wide DoS by an authenticated tenant is largely accepted on shared self-host (operator's job to add global quotas). Hard requirement only for managed multi-tenant | | +| T17 | Account/credential theft via unauthenticated MCP-OAuth client registration and open redirect on logout | remote_unauth | EP11 | Accounts, session tokens | high | possible | partially_mitigated | redirect-URI handling / registration hardening | GHSA-q9xg-f2v2-695g, GHSA-53xj-pvqf-wpm9, GHSA-rr8j-ffc4-pf7h, GHSA-6c5w-777m-8rv5 | +| T18 | Account takeover via missing rate limiting / brute force on auth endpoints | remote_unauth | EP11 | Accounts | medium | likely | unmitigated | none built-in; owner confirms instances are typically exposed directly with no app-level rate limiting or WAF | GHSA-cmv6-m7wc-c87p | +| T19 | Enterprise license bypass and account impersonation | remote_auth | EP5 | Global settings, accounts | medium | possible | unmitigated | license validation gated by `license` feature | GHSA-48j5-p323-4mpx, GHSA-pv35-65rq-w29h, GHSA-2qx7-634r-qj6r | +| T20 | Trigger spoofing: an actor with broker/queue access injects messages that execute jobs without app-level auth | adjacent_network | EP4 | Job execution integrity, downstream | medium | possible | risk_accepted | Owner confirms trust is delegated to broker ACLs by design; no app-level message authenticity check. Anyone able to publish to a subscribed topic/queue can cause job execution | | +| T21 | Data-in-transit interception/tampering from TLS-disabled defaults (DB `sslmode=disable`, HTTP-only Caddy) | adjacent_network | EP15 | DB creds, secrets, session tokens | medium | possible | unmitigated | docs recommend TLS; not default | | +| T22 | Repudiation / incident blind spots from gaps in audit coverage of sensitive actions | remote_auth | EP5 | Audit logs | medium | possible | partially_mitigated | `windmill-audit` records many actions | | + +## 5. Deprioritized + +| threat | reason | +|---|---| +| Physical access to the host / cold-boot key extraction | Out of scope; deployment-environment responsibility, not addressable in this codebase | +| Memory-safety RCE in the Rust backend itself | Rust's safety model makes this rare; no evidence in history. Note: `unsafe` FFI (duckdb) is a narrow exception folded into supply-chain/T9 | +| Client-side-only nuisance bugs (CSS, layout) with no security impact | No asset compromised | +| Insider with legitimate superadmin / DB-root access | Trusted role; mitigations are operational (least privilege, audit), not technical controls in scope | +| Spoofing of a fully-trusted upstream IdP that has itself been compromised | Out of model; Windmill trusts the configured IdP by design | +| Instance-wide DoS by an authenticated tenant on shared self-host (T16) | Risk accepted (owner): per-job rlimits/timeouts are in place; global concurrency/queue quotas are the operator's responsibility on self-host. Remains a hard requirement for the managed multi-tenant fleet | +| Job execution triggered by an actor with legitimate broker/queue publish access (T20) | Risk accepted (owner): trigger authenticity is delegated to broker ACLs by design; consuming from a configured source and acting on its messages is the intended behavior | + +## 6. Open questions + +Facts that drove the score changes above. Two were confirmed in code during +the interview (`[Code-verified]`); the rest remain `[Owner-states]` pending a +check. + +- [Code-verified] nsjail is off by default in every configuration: `DISABLE_NSJAIL` defaults to `true` (`windmill-worker/src/worker.rs:346`), and `is_sandboxing_enabled()` requires `DISABLE_NSJAIL=false` or the `job_isolation` global setting = `nsjail_sandboxing` (`worker.rs:890`). PID-ns `unshare` is also off at the code level (`is_unshare_enabled()`, `worker.rs:903`); the shipped `docker-compose.yml` sets `FAVOR_UNSHARE_PID=true` (line 91), so the official compose gives PID-ns unshare only, nsjail off — a bare install gets no isolation at all. No separate `clone_newnet` flag exists; network isolation is an nsjail feature, so outbound network from user code is unrestricted by default. Affects: T2 controls/likelihood, T5 status (unmitigated), T8. +- [Code-verified] `global_settings` is plaintext at rest under the default DB backend: `set_value_in_global_settings` stores the raw JSON value with no encryption (`windmill-common/src/global_settings.rs:259`); the encrypting secret backend (`secret_backend/database.rs:66`) only encrypts per-workspace `variable` rows with `is_secret=true`. Instance-level SMTP/OAuth/AI/object-store secrets are therefore plaintext. Affects: T6 impact/controls, T7. +- [Owner-states] Internet-facing instances are typically exposed directly with no built-in rate limiting / WAF. Affects: T16, T18 likelihood. Verify by: confirm absence of a rate-limit layer in `windmill-api/src/lib.rs` middleware stack. +- [Owner-states] Managed offering provides an optional dedicated DB/worker/namespace tier for sensitive tenants; the shared tier relies solely on the software authz boundary. Affects: T3 controls. Verify by: deployment topology (not in this repo) — out-of-tree. +- [Owner-states] Per-job rlimits/timeouts exist; instance-wide DoS by an authed tenant is risk-accepted on shared self-host. Affects: T16 status. Verify by: locate the rlimit/timeout enforcement in the worker execution path and confirm there is no global queue/concurrency cap. +- [Owner-states] Message-queue trigger authenticity is delegated to broker ACLs only. Affects: T20 status. Verify by: review `windmill-trigger-{kafka,sqs,nats,mqtt,postgres}` consume paths for any payload authentication. + +## 7. Provenance + +- mode: bootstrap-then-interview +- date: 2026-06-05 +- target: /home/rfiszel/windmill/backend @ 819ba5e150 +- inputs: git-log mined + GitHub security advisories (gh api, 73 advisories) + CHANGELOG; seed: THREAT_MODEL.md (bootstrap pass) +- owner: Ruben Fiszel (Windmill core dev) + +## 8. Recommended mitigations + +| mitigation | threat_ids | closes_class | effort | +|---|---|---|---| +| Centralize a single audited query-builder that forbids string-interpolated SQL; ban `format!`-built queries via lint/CI | T1 | yes | M | +| Route all outbound requests through one SSRF-guarded HTTP client (allowlist/denylist of private+metadata ranges, redirects disabled, re-validated per hop) | T2 | yes | M | +| Enforce authorization centrally in middleware (scope + RLS + folder ACL) with deny-by-default and a per-route coverage test, instead of per-handler checks | T3, T10, T14, T22 | yes | L | +| Treat all user-supplied identifiers as data: pass via argv/env/structured params, never splice into generated wrapper source; validate against strict allowlists at the boundary | T4 | yes | M | +| Make `nsjail` + network-namespace isolation default-on / fail-closed (flip `ENABLE_NSJAIL` and `clone_newnet` defaults) and remove privileged/dind defaults from shipped compose; default-deny debugger | T2, T5, T7, T8 | partial | L | +| Encrypt `global_settings` at rest under the workspace/instance key even on the default DB secret backend, so a DB read no longer yields plaintext instance-wide credentials | T6, T7 | partial | M | +| Ship hardened defaults: random per-install secrets, no default admin password, Postgres not exposed, CORS locked to configured origin, TLS-on | T7, T18, T21 | partial | M | +| Resolve secrets/resources only with the caller's identity and scope every cache entry by (caller, scope); apply uniformly to AI proxy, MCP, and exports | T6 | yes | M | +| Output-encode/sanitize all stored content at render and force `nosniff` + restrictive CSP on every user-content response | T11 | yes | M | +| Verify webhook authenticity uniformly (constant-time HMAC + timestamp/nonce anti-replay) in a shared trigger-auth helper | T12 | yes | S | +| Canonicalize + confine all file-path inputs to a base dir and never follow symlinks in log/file readers | T13 | yes | S | +| Mask secrets at the log sink and keep secrets out of worker process env (`/proc`) — pass via files/pipes scrubbed after use | T15 | partial | M | +| Add global rate limiting and per-tenant resource/queue quotas at the edge | T16, T18 | partial | M | +| Pin and integrity-verify hub scripts and CI actions; SBOM + automated base-image CVE scanning in release | T9 | partial | M | diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 4e1f516570..84d5414fa2 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -3742e0659c5e97aab03b9efeea14cd94a3ac658a +481ea7f28dc5af6b72390c82f494f34cb9809546 diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 36f565fc2b..01becd80f3 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -154,29 +154,34 @@ "zoho": { "auth_url": "https://accounts.zoho.com/oauth/v2/auth", "token_url": "https://accounts.zoho.com/oauth/v2/token", - "scopes": [ - "ZohoAssist.sessionapi.ALL" - ], + "scopes": ["ZohoAssist.sessionapi.ALL"], "extra_params": { "access_type": "offline" } }, - "snowflake_oauth": {}, + "snowflake_oauth": { + "connect_config_template": { + "display_name": "Snowflake", + "label": "Snowflake Account Identifier", + "placeholder": "-", + "help_url": "https://docs.snowflake.com/en/user-guide/admin-account-identifier#using-an-account-name-as-an-identifier", + "auth_url": "https://{instance}.snowflakecomputing.com/oauth/authorize", + "token_url": "https://{instance}.snowflakecomputing.com/oauth/token-request", + "req_body_auth": false, + "extra_params_key": "account_identifier", + "resource_mapping": { "account_identifier": "{instance}" } + } + }, "apify": { "auth_url": "https://console.apify.com/authorize/oauth", "token_url": "https://console-backend.apify.com/oauth/apps/token", - "scopes": [ - "profile", - "full_api_access" - ], + "scopes": ["profile", "full_api_access"], "extra_params": {} }, "docusign": { "auth_url": "https://account.docusign.com/oauth/auth", "token_url": "https://account.docusign.com/oauth/token", - "scopes": [ - "signature" - ], + "scopes": ["signature"], "sandbox": { "auth_url": "https://account-d.docusign.com/oauth/auth", "token_url": "https://account-d.docusign.com/oauth/token" @@ -185,14 +190,25 @@ "salesforce": { "auth_url": "https://login.salesforce.com/services/oauth2/authorize", "token_url": "https://login.salesforce.com/services/oauth2/token", - "scopes": [ - "api", - "refresh_token", - "offline_access" - ], + "scopes": ["api", "refresh_token", "offline_access"], "sandbox": { "auth_url": "https://test.salesforce.com/services/oauth2/authorize", "token_url": "https://test.salesforce.com/services/oauth2/token" } + }, + "servicenow": { + "connect_config_template": { + "display_name": "ServiceNow", + "label": "ServiceNow Instance", + "placeholder": " (e.g. dev12345)", + "help_url": "https://www.servicenow.com/docs/bundle/zurich-platform-security/page/administer/security/concept/c_OAuthApplications.html", + "auth_url": "https://{instance}.service-now.com/oauth_auth.do", + "token_url": "https://{instance}.service-now.com/oauth_token.do", + "req_body_auth": true, + "strip_suffix": ".service-now.com", + "resource_mapping": { + "instance_url": "https://{instance}.service-now.com" + } + } } } diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 1e78ec8665..b0ed12fb1a 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -129,7 +129,10 @@ impl Visit for ImportsFinder { /// See also: [`parse_relative_imports`] for resolved absolute paths. pub fn parse_expr_for_imports(code: &str, skip_type_only: bool) -> anyhow::Result> { let cm: Lrc = Default::default(); - let fm = cm.new_source_file(FileName::Custom("main.d.ts".into()).into(), code.to_string()); + let fm = cm.new_source_file( + FileName::Custom("main.d.ts".into()).into(), + code.to_string(), + ); let mut tss = TsSyntax::default(); tss.disallow_ambiguous_jsx_like; tss.tsx = true; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index aa0b618798..2a144ff4c3 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.714.0" +version = "1.719.0" dependencies = [ "aho-corasick", "anyhow", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.714.0" +version = "1.719.0" dependencies = [ "proc-macro2", "quote", @@ -6275,7 +6275,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.714.0" +version = "1.719.0" dependencies = [ "convert_case", "serde", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "lazy_static", @@ -6296,7 +6296,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde_json", @@ -6308,7 +6308,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "gosyn", @@ -6320,7 +6320,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "lazy_static", @@ -6332,7 +6332,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde_json", @@ -6344,7 +6344,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "nu-parser", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "async-recursion", @@ -6411,7 +6411,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde_json", @@ -6423,7 +6423,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "lazy_static", @@ -6437,7 +6437,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "convert_case", @@ -6454,7 +6454,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "lazy_static", @@ -6467,7 +6467,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "lazy_static", @@ -6497,7 +6497,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6513,7 +6513,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6529,7 +6529,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6561,7 +6561,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "serde", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.714.0" +version = "1.719.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index ec9a764a03..d87cd6ee7d 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.714.0" +version = "1.719.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index 29bea050ed..b4d3cef4f9 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -57,11 +57,14 @@ use windmill_common::{ PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, - SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, - TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, - UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, - WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, - WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_REGISTRIES_SETTING, + SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, + SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, + SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, + STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, + UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, + WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, + WORKSPACE_REGISTRIES_SETTING, }, scripts::ScriptLang, stats_oss::schedule_stats, @@ -134,8 +137,11 @@ use crate::monitor::{ reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, - reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, - reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, + reload_pip_index_url_setting, reload_retention_period_setting, + reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, + reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting, + reload_sandbox_registry_auth_setting, reload_scim_token_setting, reload_smtp_config, + reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, }; @@ -1664,6 +1670,12 @@ async fn process_notify_event( match *source_type { "script" => { windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key); + // Evict the relative-import latest-hash cache so a redeployed + // imported script flips the content cache to its new version + // across all replicas within a poll interval (see #6769). Keyed + // by the bare path, matching this event's payload. + windmill_api_scripts::scripts::RAW_SCRIPT_LATEST_HASH_CACHE + .remove(&format!("{workspace_id}:{path}")); if *kind == "preprocessor" { match sqlx::query_scalar::<_, i64>( "SELECT fv.id @@ -1821,6 +1833,19 @@ async fn process_notify_event( JOB_ISOLATION_SETTING => reload_job_isolation_setting(conn).await, NSJAIL_TMPFS_SIZE_MB_SETTING => reload_nsjail_tmpfs_size_setting(conn).await, NSJAIL_TMP_BACKING_SETTING => reload_nsjail_tmp_backing_setting(conn).await, + SANDBOX_IMAGE_MAX_SIZE_MB_SETTING => { + reload_sandbox_image_max_size_setting(conn).await + } + SANDBOX_IMAGE_CACHE_MAX_MB_SETTING => { + reload_sandbox_image_cache_max_setting(conn).await + } + SANDBOX_IMAGE_PULL_POLICY_SETTING => { + reload_sandbox_image_pull_policy_setting(conn).await + } + SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING => { + reload_sandbox_image_default_registry_setting(conn).await + } + SANDBOX_REGISTRY_AUTH_SETTING => reload_sandbox_registry_auth_setting(conn).await, #[cfg(feature = "parquet")] OBJECT_STORE_CONFIG_SETTING => { if !disable_s3_store { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index f902a72826..b1706d9ce4 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -66,7 +66,9 @@ use windmill_common::{ OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, + RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, + SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, @@ -112,8 +114,10 @@ use windmill_worker::{ JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, MAVEN_REPOS, MAVEN_SETTINGS_XML, NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NSJAIL_TMPFS_SIZE_MB, NSJAIL_TMP_BACKING, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, - UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES, + PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, SANDBOX_IMAGE_CACHE_MAX_MB, + SANDBOX_IMAGE_DEFAULT_REGISTRY, SANDBOX_IMAGE_MAX_SIZE_MB, SANDBOX_IMAGE_PULL_POLICY, + SANDBOX_REGISTRY_AUTH, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, + UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES, }; #[cfg(feature = "parquet")] @@ -407,6 +411,11 @@ pub async fn initial_load( reload_job_isolation_setting(&conn).await; reload_nsjail_tmpfs_size_setting(&conn).await; reload_nsjail_tmp_backing_setting(&conn).await; + reload_sandbox_image_max_size_setting(&conn).await; + reload_sandbox_image_cache_max_setting(&conn).await; + reload_sandbox_image_pull_policy_setting(&conn).await; + reload_sandbox_image_default_registry_setting(&conn).await; + reload_sandbox_registry_auth_setting(&conn).await; reload_extra_pip_index_url_setting(&conn).await; reload_pip_index_url_setting(&conn).await; reload_uv_index_strategy_setting(&conn).await; @@ -1095,24 +1104,8 @@ struct TokenRow { workspace_id: Option, } -/// When updating this filter, also update: -/// - `register_token_expiry_notification` in windmill-api-auth/src/lib.rs -/// - `isUserToken` in frontend/src/lib/components/settings/TokensTable.svelte -fn is_user_token(label: Option<&str>) -> bool { - match label { - None => true, - Some(l) => { - l != "session" - && !l.starts_with("ephemeral") - && !l.starts_with("Ephemeral") - && l != "debugger-token" - && !l.starts_with("mcp-oauth-") - } - } -} - async fn report_token_expiration(db: &DB, token: &TokenRow, expired: bool) { - if !is_user_token(token.label.as_deref()) { + if !windmill_common::auth::is_user_token(token.label.as_deref()) { return; } let prefix = token.token_prefix.as_deref().unwrap_or("??????????"); @@ -2049,6 +2042,66 @@ pub async fn reload_nsjail_tmp_backing_setting(conn: &Connection) { .await; } +pub async fn reload_sandbox_image_max_size_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, + "SANDBOX_IMAGE_MAX_SIZE_MB", + SANDBOX_IMAGE_MAX_SIZE_MB.clone(), + ) + .await; +} + +pub async fn reload_sandbox_image_cache_max_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + "SANDBOX_IMAGE_CACHE_MAX_MB", + SANDBOX_IMAGE_CACHE_MAX_MB.clone(), + ) + .await; +} + +pub async fn reload_sandbox_image_pull_policy_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + SANDBOX_IMAGE_PULL_POLICY_SETTING, + "SANDBOX_IMAGE_PULL_POLICY", + SANDBOX_IMAGE_PULL_POLICY.clone(), + ) + .await; +} + +pub async fn reload_sandbox_image_default_registry_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, + "SANDBOX_IMAGE_DEFAULT_REGISTRY", + SANDBOX_IMAGE_DEFAULT_REGISTRY.clone(), + ) + .await; +} + +pub async fn reload_sandbox_registry_auth_setting(conn: &Connection) { + // Secret-aware: the value is a raw docker/podman auth.json with credentials, so + // it must never be logged. Load directly (the generic reload_option_setting path + // logs the value via load_option_setting_value) and only log a redacted message. + let q = + match load_value_from_global_settings_with_conn(conn, SANDBOX_REGISTRY_AUTH_SETTING, true) + .await + { + Ok(q) => q, + Err(e) => { + tracing::error!("Error reloading setting SANDBOX_REGISTRY_AUTH: {e:?}"); + return; + } + }; + let value = q.and_then(|q| serde_json::from_value::(q).ok()); + let configured = value.as_ref().is_some_and(|v| !v.trim().is_empty()); + *SANDBOX_REGISTRY_AUTH.write().await = value; + tracing::info!("Loaded setting SANDBOX_REGISTRY_AUTH (redacted), configured={configured}"); +} + pub async fn reload_job_isolation_setting(conn: &Connection) { let value = match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await { diff --git a/backend/tests/fixtures/jobs_read_auth.sql b/backend/tests/fixtures/jobs_read_auth.sql new file mode 100644 index 0000000000..e6b28fba0c --- /dev/null +++ b/backend/tests/fixtures/jobs_read_auth.sql @@ -0,0 +1,192 @@ +-- Fixture for the single-job read authorization regression test +-- (see tests/jobs_read_auth.rs). +-- +-- Users available from `base`: +-- test-user (admin, token SECRET_TOKEN) +-- test-user-2 (User, token SECRET_TOKEN_2) -- owner of the secret script +-- test-user-3 (User, token SECRET_TOKEN_3) -- the unprivileged "viewer" +-- +-- test-user-3 is NOT a member of any folder/group granting access to +-- `u/test-user-2/...`, so under the same RLS as `jobs/list` they cannot see any +-- of these jobs unless they created them. + +-- A tag-scoped token for test-user-2 (who can read both VICTIM (tag 'deno') and +-- the flow (tag 'flow')). The `if_jobs:filter_tags:deno` modifier restricts it to +-- the 'deno' tag, so it must NOT be able to mint a share token for the 'flow' job. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('SCOPED_DENO_TOKEN'::bytea), 'hex'), 'SCOPED_DEN', 'SCOPED_DENO_TOKEN', + 'test2@windmill.dev', 'scoped deno token', false, + ARRAY['jobs:read', 'if_jobs:filter_tags:deno'] +); + +-- RUNNING job: queued (no completed row) and owned by test-user-2. Used to check +-- that `completed/get_result_maybe?get_started=true` authorizes before disclosing +-- running-state to a non-reader. +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner +) VALUES ( + '77777777-7777-7777-7777-777777777777', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'script', 'deno', 'u/test-user-2/running_secret', 'deno', true +); +INSERT INTO public.v2_job_queue (id, workspace_id, scheduled_for, running, tag) VALUES + ('77777777-7777-7777-7777-777777777777', 'test-workspace', '2023-01-01 00:00:00', true, 'deno'); + +-- 1. VICTIM job: a completed run of test-user-2's private script, e.g. produced +-- by a public HTTP trigger. `created_by` is the route identity (test-user-2), +-- NOT the viewer; `permissioned_as`/`runnable_path` sit in test-user-2's +-- namespace; `visible_to_owner` is true. Its args + result carry secrets. +-- Pre-fix, test-user-3 could read all of these by UUID. +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, args +) VALUES ( + 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'script', 'deno', 'u/test-user-2/secret_script', 'deno', true, + '{"secret": "LEAK_TEST_ARGS"}' +); +INSERT INTO public.v2_job_completed ( + id, workspace_id, duration_ms, status, result +) VALUES ( + 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'test-workspace', 1000, + 'success'::job_status, '{"secret": "RESULT_SECRET"}' +); +INSERT INTO public.job_logs (job_id, workspace_id, logs) VALUES + ('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'test-workspace', 'secret logs LEAK_TEST_LOGS'); + +-- 2. APP-style job: run by the viewer (test-user-3) on behalf of an app whose +-- policy executes as test-user-2. `created_by` is the launching viewer, but +-- `permissioned_as`/`runnable_path` are the app owner's and +-- `visible_to_owner` is false (apps hide their component runs from the runs +-- list). This is the case that must KEEP working after the fix: the viewer +-- polls their own component result by UUID. +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, args +) VALUES ( + 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'test-workspace', 'test-user-3', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'script', 'deno', 'u/test-user-2/app_component', 'deno', false, + '{"app_arg": "ok"}' +); +INSERT INTO public.v2_job_completed ( + id, workspace_id, duration_ms, status, result +) VALUES ( + 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'test-workspace', 1000, + 'success'::job_status, '{"app_result": "visible_to_launcher"}' +); + +-- 3. ANONYMOUS job: a public-trigger run whose creator is `anonymous`. Reading +-- it without authentication must keep working (unchanged behavior). +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, args +) VALUES ( + 'cccccccc-cccc-cccc-cccc-cccccccccccc', 'test-workspace', 'anonymous', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'script', 'deno', 'u/test-user-2/public_trigger', 'deno', true, + '{"public": "arg"}' +); +INSERT INTO public.v2_job_completed ( + id, workspace_id, duration_ms, status, result +) VALUES ( + 'cccccccc-cccc-cccc-cccc-cccccccccccc', 'test-workspace', 1000, + 'success'::job_status, '{"public": "result"}' +); + +-- 4. FLOW + STEP: test-user-3 has *read* access to folder `shared` (extra_perms), +-- so they can see flow `f/shared/flow1` (run by test-user-2) even though they +-- did not launch it. The flow's STEP job runs the inner script +-- `u/test-user-2/inner_secret` (test-user-3 has NO direct ACL on it) and is +-- not in their list. Visibility must be INHERITED from the flow root: being +-- able to see the flow means being able to inspect its steps (the flow-run UI +-- fetches each step by id). This guards against the fix over-blocking. +INSERT INTO public.folder (workspace_id, name, display_name, owners, extra_perms, created_by) +VALUES ('test-workspace', 'shared', 'Shared Folder', '{"u/test-user-2"}', + '{"u/test-user-3": false}', 'test-user-2'); + +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner +) VALUES ( + 'dddddddd-dddd-dddd-dddd-dddddddddddd', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'flow', 'deno', 'f/shared/flow1', 'flow', true +); +INSERT INTO public.v2_job_completed ( + id, workspace_id, duration_ms, status, result +) VALUES ( + 'dddddddd-dddd-dddd-dddd-dddddddddddd', 'test-workspace', 1000, + 'success'::job_status, '{"flow": "done"}' +); + +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, + parent_job, root_job, flow_innermost_root_job, args +) VALUES ( + 'eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'script', 'deno', 'u/test-user-2/inner_secret', 'deno', true, + 'dddddddd-dddd-dddd-dddd-dddddddddddd', 'dddddddd-dddd-dddd-dddd-dddddddddddd', + 'dddddddd-dddd-dddd-dddd-dddddddddddd', '{"step_arg": "x"}' +); +INSERT INTO public.v2_job_completed ( + id, workspace_id, duration_ms, status, result +) VALUES ( + 'eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee', 'test-workspace', 1000, + 'success'::job_status, '{"step": "STEP_RESULT_INHERITED"}' +); + +-- 5. DEEP NESTING / MIDDLE-LAYER VISIBILITY: top flow `f/secret/top` is NOT +-- visible to test-user-3; it has a sub-flow step `f/shared/mid` that IS visible +-- (folder `shared`); and that sub-flow has its own leaf step running +-- `u/test-user-2/deep_secret` (not visible). The leaf's `root_job` points at the +-- *outermost* top (not visible), so visibility must come from the *intermediate* +-- sub-flow the user can see — which requires walking the full parent chain, not +-- just [self, root]. +INSERT INTO public.folder (workspace_id, name, display_name, owners, extra_perms, created_by) +VALUES ('test-workspace', 'secret', 'Secret Folder', '{"u/test-user-2"}', '{}', 'test-user-2'); + +-- top flow (not visible to test-user-3) +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner +) VALUES ( + 'ffffffff-ffff-ffff-ffff-ffffffffffff', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'flow', 'deno', 'f/secret/top', 'flow', true +); +-- intermediate sub-flow (visible via folder `shared`), child of top +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, + parent_job, root_job, flow_innermost_root_job +) VALUES ( + '99999999-9999-9999-9999-999999999999', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'flow', 'deno', 'f/shared/mid', 'flow', true, + 'ffffffff-ffff-ffff-ffff-ffffffffffff', 'ffffffff-ffff-ffff-ffff-ffffffffffff', + 'ffffffff-ffff-ffff-ffff-ffffffffffff' +); +-- leaf step of the sub-flow; runnable not visible, root_job = outermost top (not visible) +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, + parent_job, root_job, flow_innermost_root_job +) VALUES ( + '88888888-8888-8888-8888-888888888888', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'script', 'deno', 'u/test-user-2/deep_secret', 'deno', true, + '99999999-9999-9999-9999-999999999999', 'ffffffff-ffff-ffff-ffff-ffffffffffff', + '99999999-9999-9999-9999-999999999999' +); +INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES + ('ffffffff-ffff-ffff-ffff-ffffffffffff', 'test-workspace', 1000, 'success'::job_status, + '{"top": "TOP_SECRET_RESULT"}'), + ('99999999-9999-9999-9999-999999999999', 'test-workspace', 1000, 'success'::job_status, + '{"mid": "MID_RESULT"}'), + ('88888888-8888-8888-8888-888888888888', 'test-workspace', 1000, 'success'::job_status, + '{"deep": "DEEP_STEP_INHERITED"}'); diff --git a/backend/tests/fixtures/mcp_token_exfil.sql b/backend/tests/fixtures/mcp_token_exfil.sql new file mode 100644 index 0000000000..edf1113137 --- /dev/null +++ b/backend/tests/fixtures/mcp_token_exfil.sql @@ -0,0 +1,29 @@ +-- Fixture for the MCP token-exfiltration regression test. +-- +-- Models a malicious developer (test-user-3, a plain workspace member) who: +-- - owns an MCP resource they are allowed to read, and +-- - points that resource's `token` field at a secret variable living in a +-- folder they have NO access to (`f/locked`, only test-user/admin owns it). +-- +-- The secret variable `f/locked/secret_token` itself is inserted by the test in +-- Rust (so it is encrypted with the real workspace key); this fixture only sets +-- up the locked folder, the resource, and their permissions. + +-- Folder the developer cannot read (empty extra_perms, owned by admin only). +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by) +VALUES ('test-workspace', 'locked', 'Locked Folder', '{"u/test-user"}', '{}', 'test-user'); + +-- MCP resource owned by the developer (so RLS lets them read the resource), +-- whose token references the locked secret. The URL is a non-resolvable public +-- host so that, for an authorized caller, resolution succeeds but the later +-- connection/SSRF step fails deterministically without network access. +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ( + 'test-workspace', + 'u/test-user-3/evil_mcp', + '{"name": "evil", "url": "https://mcp.invalid.windmill.test", "token": "$var:f/locked/secret_token"}', + 'MCP resource whose token points at a locked secret', + 'mcp', + '{}', + 'test-user-3' +); diff --git a/backend/tests/flow_engine_parity.rs b/backend/tests/flow_engine_parity.rs index bae776e7f1..78e45e404b 100644 --- a/backend/tests/flow_engine_parity.rs +++ b/backend/tests/flow_engine_parity.rs @@ -2916,6 +2916,7 @@ export function main() { expr: "flow_env.STOP === true".to_string(), skip_if_stopped: true, error_message: None, + error_include_result: false, }); m }; @@ -2966,6 +2967,92 @@ export function main() { Ok(()) } +// stop_after_if with `error_message` + `error_include_result` should fail the +// flow but preserve the stopping step's own result inside the raised error +// object, i.e. `{ "error": { .., "result": } }`. With the flag off +// (the default) the error object carries no `result`. Regression for the +// early-stop branch in `update_flow_status_after_job_completion_internal`. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_stop_after_if_error_include_result(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + let make_flow = |include_result: bool| { + let mut m = flow_module( + "step", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" +export function main() { + return { userErrors: ["email taken"], ok: false }; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + m.stop_after_if = Some(windmill_common::flows::StopAfterIf { + expr: "true".to_string(), + skip_if_stopped: false, + error_message: Some("API returned userErrors".to_string()), + error_include_result: include_result, + }); + FlowValue { modules: vec![m], same_worker: false, ..Default::default() } + }; + + // include_result = true: result preserves both the error and the step output + let job = RunJob::from(JobPayload::RawFlow { + value: make_flow(true), + path: None, + restarted_from: None, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!( + !job.success, + "flow with raised early-stop error should fail" + ); + let result = job.json_result().unwrap(); + assert_eq!( + result["error"]["name"], "EarlyStopError", + "expected EarlyStopError; got {result:?}" + ); + assert_eq!(result["error"]["message"], "API returned userErrors"); + assert_eq!( + result["error"]["result"], + json!({ "userErrors": ["email taken"], "ok": false }), + "step result should be preserved under `error.result`; got {result:?}" + ); + + // include_result = false (default behavior): result is the bare error object + let job = RunJob::from(JobPayload::RawFlow { + value: make_flow(false), + path: None, + restarted_from: None, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!( + !job.success, + "flow with raised early-stop error should fail" + ); + let result = job.json_result().unwrap(); + assert_eq!(result["error"]["name"], "EarlyStopError"); + assert!( + result["error"].get("result").is_none(), + "without the flag the error must not embed the step result; got {result:?}" + ); + + Ok(()) +} + // retry_if predicate sees flow_env. Regression for the two evaluate_retry // call sites in `update_flow_status_after_job_completion_internal` (lines // 1194 and 1576) which used to pass `None` for flow_env. @@ -3093,6 +3180,7 @@ export function main(i: number) { expr: "flow_env.STOP === true".to_string(), skip_if_stopped: true, error_message: None, + error_include_result: false, }); m }; @@ -3143,3 +3231,84 @@ export function main() { Ok(()) } + +// stop_after_all_iters_if with `error_message` + `error_include_result` fails the +// flow and embeds the loop's aggregated iteration results under `error.result`. +// Covers the loop/branch-all path where `nresult` is already populated with the +// aggregated results (distinct from the per-step fallback to `result`). +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_stop_after_all_iters_if_error_includes_result( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + let inner = flow_module( + "iter_step", + FlowModuleValue::RawScript { + input_transforms: [js_input("i", "flow_input.iter.value")].into(), + language: ScriptLang::Deno, + content: r#" +export function main(i: number) { + return { iter: i }; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + + let loop_module = { + let mut m = flow_module( + "loop", + FlowModuleValue::ForloopFlow { + iterator: InputTransform::Javascript { expr: "[1, 2, 3]".to_string() }, + modules: vec![inner], + modules_node: None, + skip_failures: false, + parallel: false, + parallelism: None, + squash: None, + }, + ); + m.stop_after_all_iters_if = Some(windmill_common::flows::StopAfterIf { + expr: "true".to_string(), + skip_if_stopped: false, + error_message: Some("loop failed".to_string()), + error_include_result: true, + }); + m + }; + + let flow = FlowValue { modules: vec![loop_module], same_worker: false, ..Default::default() }; + + let job = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await; + + assert!( + !job.success, + "loop with a raised early-stop error should fail" + ); + let result = job.json_result().unwrap(); + assert_eq!(result["error"]["name"], "EarlyStopError", "got {result:?}"); + assert_eq!(result["error"]["message"], "loop failed"); + // error.result holds the aggregated iteration results (one per iteration) + let iters = result["error"]["result"].as_array().unwrap_or_else(|| { + panic!("error.result should be an array of iteration results; got {result:?}") + }); + let iter_values: Vec<_> = iters.iter().map(|r| r["iter"].clone()).collect(); + assert_eq!( + iter_values, + vec![json!(1), json!(2), json!(3)], + "error.result should contain each iteration's output; got {result:?}" + ); + + Ok(()) +} diff --git a/backend/tests/jobs_read_auth.rs b/backend/tests/jobs_read_auth.rs new file mode 100644 index 0000000000..f19daf0f92 --- /dev/null +++ b/backend/tests/jobs_read_auth.rs @@ -0,0 +1,512 @@ +//! Regression test for the single-job read authorization bypass. +//! +//! The single-job read endpoints (`/jobs_u/get`, `/completed/get`, +//! `/completed/get_result`, `/get_args`, `/get_logs`, `/getupdate`, ...) fetch a +//! job through the root DB handle, filtered only by job id + workspace. That is +//! required for the unauthenticated approval / public-trigger / anonymous-job +//! flows, but for a *logged-in* user it meant any workspace member — including a +//! plain viewer with no ACL on the runnable — could read another user's job +//! args/result/logs simply by obtaining the job UUID, even though the same job is +//! hidden from them in `jobs/list` (RLS-filtered) and the underlying script +//! returns 404. +//! +//! The fix (`require_job_read_access`) gates the authenticated case: a caller may +//! read a job they created (covers app components / webhooks / their own runs) +//! or one visible to them under the same RLS as `jobs/list` (admins bypass); +//! otherwise 404. Unauthenticated access is unchanged (anonymous jobs only). +//! +//! This test pins down, against the `jobs_read_auth` fixture: +//! - a viewer is denied the victim job's full record / result / result_maybe / +//! args / logs / live update by UUID, and the secret never appears in the +//! body (the core fix; pre-fix these returned 200 with the secret), +//! - the job's owner and an admin can still read it (no over-blocking), +//! - the "app component" affordance survives: a viewer who *launched* a job +//! (created_by) running as someone else's identity can still read its result, +//! - unauthenticated behavior is unchanged: anonymous jobs readable, the +//! non-anonymous victim job rejected. + +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const VICTIM: &str = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; +const APP_JOB: &str = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"; +const ANON_JOB: &str = "cccccccc-cccc-cccc-cccc-cccccccccccc"; +const FLOW_JOB: &str = "dddddddd-dddd-dddd-dddd-dddddddddddd"; +const STEP_JOB: &str = "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee"; +// Deep nesting: top (not visible) -> mid (visible via folder) -> deep leaf. +const TOP_SECRET_FLOW: &str = "ffffffff-ffff-ffff-ffff-ffffffffffff"; +const DEEP_LEAF_JOB: &str = "88888888-8888-8888-8888-888888888888"; +// A queued/running job (no completed row) owned by test-user-2. +const RUNNING_JOB: &str = "77777777-7777-7777-7777-777777777777"; + +// Secrets that must never leak to an unauthorized viewer. +const RESULT_SECRET: &str = "RESULT_SECRET"; +const ARGS_SECRET: &str = "LEAK_TEST_ARGS"; +const LOGS_SECRET: &str = "LEAK_TEST_LOGS"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +async fn get(base: &str, path: &str, token: Option<&str>) -> (reqwest::StatusCode, String) { + let mut req = client().get(format!("{base}/{path}")); + if let Some(token) = token { + req = req.header("Authorization", format!("Bearer {token}")); + } + let resp = req.send().await.expect("request"); + let status = resp.status(); + let body = resp.text().await.expect("body"); + (status, body) +} + +#[sqlx::test(fixtures("base", "jobs_read_auth"))] +async fn test_single_job_read_authorization(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/jobs_u"); + // result_by_id / get_otel_traces live on the authed `/jobs` service, not `/jobs_u`. + let authed_base = format!("http://localhost:{port}/api/w/test-workspace/jobs"); + + // The endpoints that return the victim job's sensitive data by UUID. + let endpoints = [ + ("get", format!("get/{VICTIM}")), + ("completed/get", format!("completed/get/{VICTIM}")), + ( + "completed/get_result", + format!("completed/get_result/{VICTIM}"), + ), + ( + "completed/get_result_maybe", + format!("completed/get_result_maybe/{VICTIM}"), + ), + ("get_args", format!("get_args/{VICTIM}")), + ("get_logs", format!("get_logs/{VICTIM}")), + ( + "get_completed_logs_tail", + format!("get_completed_logs_tail/{VICTIM}"), + ), + ("get_flow_all_logs", format!("get_flow_all_logs/{VICTIM}")), + ( + "completed/get_timing", + format!("completed/get_timing/{VICTIM}"), + ), + ("getupdate", format!("getupdate/{VICTIM}?only_result=true")), + ]; + + // ---- CORE REGRESSION: the viewer (test-user-3) is denied on every endpoint + // and no secret ever appears in the body. Pre-fix these returned 200 + // and leaked the secret. + for (name, path) in &endpoints { + let (status, body) = get(&base, path, Some("SECRET_TOKEN_3")).await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "viewer must get 403 on {name} (got {status}): {body}" + ); + for secret in [RESULT_SECRET, ARGS_SECRET, LOGS_SECRET] { + assert!( + !body.contains(secret), + "viewer response for {name} leaked `{secret}`: {body}" + ); + } + } + + // The 403 for an existing-but-forbidden job carries actionable guidance + // (request a share link), distinguishing it from a plain not-found. + let (status, body) = get( + &base, + &format!("completed/get_result/{VICTIM}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!( + body.to_lowercase().contains("share"), + "403 body should guide the user to request a share link: {body}" + ); + + // A genuinely non-existent job is a 404, not a 403 — existence is only disclosed + // for jobs that actually exist in the workspace. + let missing = "00000000-0000-4000-8000-000000000000"; + let (status, _) = get( + &base, + &format!("completed/get_result/{missing}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a non-existent job must be 404, not 403 (got {status})" + ); + + // ---- NO OVER-BLOCKING: the job's owner (test-user-2) can read its result. + let (status, body) = get( + &base, + &format!("completed/get_result/{VICTIM}"), + Some("SECRET_TOKEN_2"), + ) + .await; + assert!( + status.is_success(), + "owner must still read their own job result (got {status}): {body}" + ); + assert!( + body.contains(RESULT_SECRET), + "owner result must contain the value: {body}" + ); + + // ---- ADMIN BYPASS: an admin (test-user) can read any job in the workspace. + let (status, body) = get( + &base, + &format!("completed/get_result/{VICTIM}"), + Some("SECRET_TOKEN"), + ) + .await; + assert!( + status.is_success(), + "admin must read any job (got {status}): {body}" + ); + assert!(body.contains(RESULT_SECRET), "admin result body: {body}"); + + // ---- APP AFFORDANCE: a viewer who LAUNCHED a job (created_by = viewer) that + // runs as another identity (permissioned_as = test-user-2, + // visible_to_owner = false) can still read its result. This is the app + // component-polling path; the fix must not break it. + let (status, body) = get( + &base, + &format!("completed/get_result/{APP_JOB}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert!( + status.is_success(), + "launcher must read a job they created even without ACL on the runnable (got {status}): {body}" + ); + assert!( + body.contains("visible_to_launcher"), + "launcher should get the result they polled: {body}" + ); + + // ---- AUTHED `/jobs` endpoints in the same class: result_by_id (flow node + // result) and get_otel_traces (job telemetry). The viewer must be denied + // the victim by UUID. The auth gate runs before result/trace resolution, + // so 404 here is the gate, not incidental resolution failure. + let (status, body) = get( + &authed_base, + &format!("result_by_id/{VICTIM}/somenode"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "viewer must get 403 on result_by_id (got {status}): {body}" + ); + assert!(!body.contains(RESULT_SECRET), "result_by_id leaked: {body}"); + + let (status, body) = get( + &authed_base, + &format!("get_otel_traces/{VICTIM}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "viewer must get 403 on get_otel_traces (got {status}): {body}" + ); + + // ---- FLOW VISIBILITY INHERITANCE: test-user-3 has folder ACL on the flow + // `f/shared/flow1` (run by test-user-2) but did NOT launch it, and has no + // ACL on the step's inner runnable `u/test-user-2/inner_secret`. They must + // still be able to (a) read the flow they can see, and (b) inspect its + // step result — visibility is inherited from the flow root. A naive + // "same as list" gate would 404 the step and break the flow-run UI. + let (status, body) = get( + &base, + &format!("completed/get_result/{FLOW_JOB}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert!( + status.is_success(), + "viewer with folder ACL must read the flow they can see (got {status}): {body}" + ); + let (status, body) = get( + &base, + &format!("completed/get_result/{STEP_JOB}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert!( + status.is_success(), + "viewer must inspect a step of a flow they can see, even without ACL on the step's runnable (got {status}): {body}" + ); + assert!( + body.contains("STEP_RESULT_INHERITED"), + "step result should be returned via flow-root inheritance: {body}" + ); + + // ---- DEEP NESTING / MIDDLE-LAYER VISIBILITY: the deep leaf's root_job is the + // top flow (NOT visible to test-user-3), but an intermediate sub-flow + // (f/shared/mid) IS visible. Reading the leaf must succeed via that middle + // ancestor — i.e. the full parent chain is walked, not just [self, root]. + let (status, body) = get( + &base, + &format!("completed/get_result/{DEEP_LEAF_JOB}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert!( + status.is_success(), + "deep leaf must be readable via a visible intermediate sub-flow (got {status}): {body}" + ); + assert!( + body.contains("DEEP_STEP_INHERITED"), + "deep leaf result should be returned via mid-ancestor visibility: {body}" + ); + // ...but the top flow itself, in a folder the viewer cannot read, stays denied. + let (status, body) = get( + &base, + &format!("completed/get_result/{TOP_SECRET_FLOW}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "top flow in an unreadable folder must stay denied (got {status}): {body}" + ); + + // ---- UNAUTHENTICATED, unchanged: an anonymous-created job is readable + // without a token (public trigger / public app result polling). + let (status, body) = get(&base, &format!("completed/get_result/{ANON_JOB}"), None).await; + assert!( + status.is_success(), + "anonymous job must remain readable unauthenticated (got {status}): {body}" + ); + + // ---- UNAUTHENTICATED, unchanged: the non-anonymous victim job is rejected + // for an unauthenticated caller (400, the pre-existing guard). + let (status, body) = get(&base, &format!("completed/get_result/{VICTIM}"), None).await; + assert_eq!( + status, + reqwest::StatusCode::BAD_REQUEST, + "unauthenticated access to a non-anonymous job must stay rejected (got {status}): {body}" + ); + assert!( + !body.contains(RESULT_SECRET), + "unauth body must not leak: {body}" + ); + + // ---- SHARE READ LINK (view_token) ---- + // The owner (test-user-2) mints a share token for the victim job. + let (status, mint_body) = get( + &authed_base, + &format!("job_view_token/{VICTIM}"), + Some("SECRET_TOKEN_2"), + ) + .await; + assert!( + status.is_success(), + "owner must be able to mint a share token (got {status}): {mint_body}" + ); + let token = mint_body.trim().trim_matches('"').to_string(); + assert!( + token.starts_with(VICTIM), + "token must encode the job id: {token}" + ); + + // The viewer (no ACL) can now read the victim job via the share link. + let (status, body) = get( + &base, + &format!("completed/get_result/{VICTIM}?view_token={token}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert!( + status.is_success(), + "view_token must grant the viewer read of the shared job (got {status}): {body}" + ); + assert!( + body.contains(RESULT_SECRET), + "shared job result must be returned with a valid view_token: {body}" + ); + // ...and its args/logs too (whole detail page). + let (status, _) = get( + &base, + &format!("get_args/{VICTIM}?view_token={token}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert!( + status.is_success(), + "view_token must also grant args (got {status})" + ); + + // The token is scoped: it does NOT authorize an unrelated job. + let (status, _) = get( + &base, + &format!("completed/get_result/{ANON_JOB}?view_token={token}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "a victim-scoped token must not authorize a different job (got {status})" + ); + + // A garbage token is rejected (falls through to the normal 404). + let (status, _) = get( + &base, + &format!("completed/get_result/{VICTIM}?view_token={VICTIM}.deadbeef"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "an invalid view_token must not grant access (got {status})" + ); + + // A share token authorizes the shared job's whole flow subtree: the owner mints + // for the top secret flow, and the viewer can then read its deep leaf. + let (status, mint_body) = get( + &authed_base, + &format!("job_view_token/{TOP_SECRET_FLOW}"), + Some("SECRET_TOKEN_2"), + ) + .await; + assert!( + status.is_success(), + "owner mints token for top flow (got {status}): {mint_body}" + ); + let top_token = mint_body.trim().trim_matches('"').to_string(); + let (status, body) = get( + &base, + &format!("completed/get_result/{DEEP_LEAF_JOB}?view_token={top_token}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert!( + status.is_success(), + "a flow's share token must authorize its deep descendants (got {status}): {body}" + ); + + // A viewer who cannot read a job cannot mint a share token for it. + let (status, _) = get( + &authed_base, + &format!("job_view_token/{TOP_SECRET_FLOW}"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "a non-reader must not be able to mint a share token (got {status})" + ); + + // ---- TAG-SCOPED token must not mint a token outside its allowed tags ---- + // SCOPED_DENO_TOKEN (test-user-2, scope `if_jobs:filter_tags:deno`) can read both + // VICTIM (tag deno) and FLOW_JOB (tag flow) by RLS, but minting must honor the + // tag scope: allowed for the deno job, denied for the flow job. + let (status, body) = get( + &authed_base, + &format!("job_view_token/{VICTIM}"), + Some("SCOPED_DENO_TOKEN"), + ) + .await; + assert!( + status.is_success(), + "tag-scoped token may mint for an in-scope (deno) job (got {status}): {body}" + ); + let (status, _) = get( + &authed_base, + &format!("job_view_token/{FLOW_JOB}"), + Some("SCOPED_DENO_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "tag-scoped token must NOT mint for an out-of-scope (flow) job (got {status})" + ); + + // ---- USE side: a tag-scoped token must not use someone else's valid view_token + // to read an out-of-scope job, even via handlers that don't tag-filter their + // data query (result_by_id, get_otel_traces, get_flow_debug_info). ---- + // An unscoped owner mints a valid token for the flow (tag 'flow'). + let (status, mint_body) = get( + &authed_base, + &format!("job_view_token/{FLOW_JOB}"), + Some("SECRET_TOKEN_2"), + ) + .await; + assert!( + status.is_success(), + "owner mints flow token (got {status}): {mint_body}" + ); + let flow_token = mint_body.trim().trim_matches('"').to_string(); + + // The deno-scoped token presents that valid flow token to the non-tag-filtered + // endpoints — must still be denied (flow tag is out of its scope). + for path in [ + format!("get_otel_traces/{FLOW_JOB}?view_token={flow_token}"), + format!("result_by_id/{FLOW_JOB}/somenode?view_token={flow_token}"), + ] { + let (status, _) = get(&authed_base, &path, Some("SCOPED_DENO_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "tag-scoped token must not use a view_token to read an out-of-scope job ({path}, got {status})" + ); + } + + // ...but the deno-scoped token CAN use an in-scope (deno) view_token. + let (status, body) = get( + &base, + &format!("completed/get_result/{VICTIM}?view_token={token}"), + Some("SCOPED_DENO_TOKEN"), + ) + .await; + assert!( + status.is_success(), + "tag-scoped token may use a view_token for an in-scope (deno) job (got {status}): {body}" + ); + + // ---- get_result_maybe?get_started=true must authorize before disclosing the + // running-state of a queued (not-yet-completed) private job. ---- + // Viewer (no ACL) must be denied rather than told the job is started. + let (status, body) = get( + &base, + &format!("completed/get_result_maybe/{RUNNING_JOB}?get_started=true"), + Some("SECRET_TOKEN_3"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "viewer must be denied the running-state of a private queued job (got {status}): {body}" + ); + assert!( + !body.contains("\"started\""), + "denied response must not disclose started-state: {body}" + ); + // The owner still gets the in-progress response. + let (status, body) = get( + &base, + &format!("completed/get_result_maybe/{RUNNING_JOB}?get_started=true"), + Some("SECRET_TOKEN_2"), + ) + .await; + assert!( + status.is_success() && body.contains("\"started\":true"), + "owner must see the running job as started (got {status}): {body}" + ); + + Ok(()) +} diff --git a/backend/tests/mcp_token_exfil.rs b/backend/tests/mcp_token_exfil.rs new file mode 100644 index 0000000000..ce278f3c53 --- /dev/null +++ b/backend/tests/mcp_token_exfil.rs @@ -0,0 +1,111 @@ +//! Regression test for the MCP token-exfiltration vulnerability. +//! +//! `GET /api/w/{w}/resources/mcp_tools/{path}` builds an MCP client from a +//! resource whose `token` field is a `$var:` reference. Before the fix the token +//! was resolved with `get_secret_value_as_admin` on the bare DB pool — no RLS, +//! no audit — so any workspace member who could read an MCP *resource* could +//! point its token at *any* secret variable in the workspace (e.g. one in an +//! admin-only folder) and have it decrypted and shipped as a bearer token. +//! +//! The fix resolves the token through the caller's permissioned path +//! (`get_value_internal` over the authed `user_db`), so the variable RLS — the +//! same gate as `variables/get_value` — applies and the secret read is audited. +//! +//! This test pins, against the `mcp_token_exfil` fixture: +//! - a plain developer (test-user-3) who can read the MCP resource but has no +//! access to the locked secret is DENIED (401) at token resolution, before +//! any connection is attempted, and the secret never leaks; +//! - an admin (test-user) clears the variable-RLS gate, the token resolves, +//! and the request only fails later at the connect/SSRF step — proving the +//! legitimate path still resolves the token (no over-blocking). +//! +//! SSRF rejection of an author-controlled URL is covered by the unit test in +//! `windmill-mcp` (`from_resource_rejects_ssrf_url`). +#![cfg(feature = "mcp")] + +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const SECRET_VALUE: &str = "S3CRET-MCP-TOKEN-VALUE"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +async fn get(base: &str, path: &str, token: &str) -> (reqwest::StatusCode, String) { + let resp = client() + .get(format!("{base}/{path}")) + .header("Authorization", format!("Bearer {token}")) + .send() + .await + .expect("request"); + let status = resp.status(); + let body = resp.text().await.expect("body"); + (status, body) +} + +#[sqlx::test(fixtures("base", "mcp_token_exfil"))] +async fn test_mcp_token_not_exfiltrated(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Insert the locked secret variable with a real, workspace-key-encrypted + // value so an authorized read genuinely decrypts it. + let mc = windmill_common::variables::build_crypt(&db, "test-workspace").await?; + let encrypted = windmill_common::variables::encrypt(&mc, SECRET_VALUE); + // Runtime-checked query (not the `query!` macro) so no offline `.sqlx` cache + // entry is needed for this test-only insert. + sqlx::query( + "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) + VALUES ('test-workspace', 'f/locked/secret_token', $1, true, 'Locked secret', '{}')", + ) + .bind(&encrypted) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/resources/mcp_tools"); + let path = "u/test-user-3/evil_mcp"; + + // ---- CORE REGRESSION: the developer can read the resource but must NOT be + // able to resolve the locked secret. They are denied (401) at the + // variable-RLS gate, before any MCP connection is attempted, and the + // secret never appears in the response. + let (status, body) = get(&base, path, "SECRET_TOKEN_3").await; + assert_eq!( + status, + reqwest::StatusCode::UNAUTHORIZED, + "developer must be denied resolving a secret they can't read (got {status}): {body}" + ); + assert!( + !body.contains(SECRET_VALUE), + "the locked secret must never leak to the developer: {body}" + ); + assert!( + body.contains("don't have access"), + "denial should come from the variable-RLS gate, not a connection error: {body}" + ); + // Pre-fix, the token was decrypted as admin and the handler proceeded to the + // connection step; that path must no longer be reached for the developer. + assert!( + !body.contains("Failed to connect to MCP server"), + "developer must be blocked before the connection step (would mean the token was resolved): {body}" + ); + + // ---- NO OVER-BLOCKING: an admin clears the variable-RLS gate, so the token + // resolves and the request only fails later at the connect/SSRF step. + // A different failure mode (not 401, reaches the connection) proves the + // legitimate read still works. + let (status, body) = get(&base, path, "SECRET_TOKEN").await; + assert_ne!( + status, + reqwest::StatusCode::UNAUTHORIZED, + "admin must clear the variable-RLS gate (got {status}): {body}" + ); + assert!( + body.contains("Failed to connect to MCP server"), + "admin should resolve the token and only fail at the connect/SSRF step: {body}" + ); + + Ok(()) +} diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index 2cf10b52d1..3a81f31021 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -507,3 +507,123 @@ async fn test_root_job_span_attributes_values() { assert_eq!(get_attr("workspace_id"), "test-workspace"); assert_eq!(get_attr("script_path"), "f/test/script"); } + +// ═══════════════════════════════════════════════════════════════════════ +// INBOUND TRACE CONTEXT (W3C traceparent → span link) +// ═══════════════════════════════════════════════════════════════════════ + +const SAMPLE_TRACEPARENT: &str = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + +fn sample_trace_id() -> opentelemetry::trace::TraceId { + opentelemetry::trace::TraceId::from_hex("0af7651916cd43dd8448eb211c80319c").unwrap() +} + +fn sample_span_id() -> opentelemetry::trace::SpanId { + opentelemetry::trace::SpanId::from_hex("b7ad6b7169203331").unwrap() +} + +#[test] +fn test_span_cx_from_traceparent_valid() { + let cx = span_cx_from_traceparent(SAMPLE_TRACEPARENT).expect("valid traceparent"); + assert_eq!(cx.trace_id(), sample_trace_id()); + assert_eq!(cx.span_id(), sample_span_id()); + assert!(cx.is_remote()); + assert!(cx.is_sampled()); +} + +#[test] +fn test_span_cx_from_traceparent_unsampled_flag() { + let cx = span_cx_from_traceparent("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-00") + .expect("valid traceparent"); + assert!(!cx.is_sampled()); +} + +#[test] +fn test_span_cx_from_traceparent_malformed() { + for bad in [ + "", + "garbage", + "00-tooshort-b7ad6b7169203331-01", + // missing flags field + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331", + // trailing extra field + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01-extra", + // all-zero trace id / span id are invalid per the spec + "00-00000000000000000000000000000000-b7ad6b7169203331-01", + "00-0af7651916cd43dd8448eb211c80319c-0000000000000000-01", + // non-hex + "00-zzf7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", + ] { + assert!( + span_cx_from_traceparent(bad).is_none(), + "expected None for {bad:?}" + ); + } +} + +fn job_with_traceparent(tp: Option<&str>) -> windmill_queue::MiniPulledJob { + let mut job = make_test_job(uuid::Uuid::new_v4(), None); + if let Some(tp) = tp { + let mut args = std::collections::HashMap::new(); + args.insert( + windmill_common::jobs::WM_TRACEPARENT.to_string(), + windmill_common::worker::to_raw_value(&tp), + ); + job.args = Some(sqlx::types::Json(args)); + } + job +} + +#[test] +fn test_inbound_span_cx_from_job_present() { + let job = job_with_traceparent(Some(SAMPLE_TRACEPARENT)); + let cx = windmill_worker::otel_ee::inbound_span_cx_from_job(&job).expect("link expected"); + assert_eq!(cx.trace_id(), sample_trace_id()); + assert_eq!(cx.span_id(), sample_span_id()); +} + +#[test] +fn test_inbound_span_cx_from_job_absent_or_malformed() { + // No reserved key (e.g. a flow step or internally-created job) → no link. + assert!( + windmill_worker::otel_ee::inbound_span_cx_from_job(&job_with_traceparent(None)).is_none() + ); + // Malformed header is ignored rather than producing a bogus link. + assert!( + windmill_worker::otel_ee::inbound_span_cx_from_job(&job_with_traceparent(Some("garbage"))) + .is_none() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_root_job_span_relocated_to_inbound_trace() { + let state = ensure_setup().await; + state.span_exporter.reset(); + + let job = job_with_traceparent(Some(SAMPLE_TRACEPARENT)); + let job_id = job.id; + windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true); + + let spans = state.span_exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|s| s.name == "full_job") + .expect("full_job span not found"); + + // Relocated into the inbound trace, keeping the job-UUID-derived span id and + // parented on the inbound caller span. + assert_eq!(span.span_context.trace_id(), sample_trace_id()); + let expected_span_id = + opentelemetry::trace::SpanId::from_bytes(job_id.as_u64_pair().1.to_be_bytes()); + assert_eq!(span.span_context.span_id(), expected_span_id); + assert_eq!(span.parent_span_id, sample_span_id()); + + // Linked back to the UUID-derived context so trace-by-job-id still resolves. + assert_eq!(span.links.links.len(), 1); + let expected_uuid_trace = + opentelemetry::trace::TraceId::from_bytes(job_id.as_u128().to_be_bytes()); + assert_eq!( + span.links.links[0].span_context.trace_id(), + expected_uuid_trace + ); +} diff --git a/backend/tests/preview_native_tag.rs b/backend/tests/preview_native_tag.rs new file mode 100644 index 0000000000..29aefa588a --- /dev/null +++ b/backend/tests/preview_native_tag.rs @@ -0,0 +1,122 @@ +/* + * Regression tests for WIN-2007. + * + * Previewing a TypeScript script carrying the `//native` annotation used to be + * pushed with `language = bun` (what the editor sends), so the job was tagged + * `bun` and routed to a regular bun worker. A native-mode worker neither matches + * the `bun` tag nor accepts a non-native `script_lang`, so previewing a `//native` + * script on a native-only worker setup failed even though the *deployed* version + * of the same script runs fine (as `bunnative` / tag `nativets`). + * + * `push` now reconciles the preview language with the `//native` annotation, + * mirroring the deploy-time logic in `worker_lockfiles`. These tests assert the + * queued job ends up with the right `script_lang` and `tag` for every combination + * of declared language and annotation. No worker is spawned — we only inspect the + * row `push` writes. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::{ + jobs::{JobPayload, RawCode}, + scripts::ScriptLang, +}; +use windmill_queue::PushIsolationLevel; + +async fn push_preview_and_get_row( + db: &Pool, + content: &str, + language: ScriptLang, +) -> (String, Option) { + let hm_args = std::collections::HashMap::new(); + + let job = JobPayload::Code(RawCode { + hash: None, + content: content.to_string(), + path: None, + language, + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + modules: None, + tag: None, + }); + + let tx = PushIsolationLevel::IsolatedRoot(db.clone()); + let (uuid, tx) = windmill_queue::push( + db, + tx, + "test-workspace", + job, + windmill_queue::PushArgs::from(&hm_args), + /* user */ "test-user", + /* email */ "test@windmill.dev", + /* permissioned_as */ "u/test-user".to_string(), + /* token_prefix */ None, + /* scheduled_for */ None, + /* schedule_path */ None, + /* parent_job */ None, + /* root_job */ None, + /* flow_innermost_root_job */ None, + /* job_id */ None, + /* is_flow_step */ false, + /* same_worker */ false, + None, + true, + None, + None, + None, + None, + None, + false, + None, + None, + None, + ) + .await + .expect("push must succeed"); + tx.commit().await.unwrap(); + + let row = sqlx::query!( + r#"SELECT tag, script_lang AS "script_lang: ScriptLang" FROM v2_job WHERE id = $1"#, + uuid + ) + .fetch_one(db) + .await + .unwrap(); + (row.tag, row.script_lang) +} + +const NATIVE_CONTENT: &str = r#"//native + +export function main(x: number) { + return x; +} +"#; + +const PLAIN_CONTENT: &str = r#"export function main(x: number) { + return x; +} +"#; + +/// The reported case: editor sends `bun`, content has `//native`. The preview +/// must be promoted to `bunnative` so it tags `nativets` and a native worker +/// (which rejects non-native `script_lang`) can run it. +#[sqlx::test(fixtures("base"))] +async fn test_bun_with_native_annotation_becomes_nativets(db: Pool) { + let (tag, lang) = push_preview_and_get_row(&db, NATIVE_CONTENT, ScriptLang::Bun).await; + assert_eq!(lang, Some(ScriptLang::Bunnative)); + assert_eq!(tag, "nativets"); +} + +/// Guard: a plain bun preview (no `//native`) must stay `bun` / tag `bun`, so +/// the promotion above doesn't broadly retag normal previews. +#[sqlx::test(fixtures("base"))] +async fn test_bun_without_native_annotation_stays_bun(db: Pool) { + let (tag, lang) = push_preview_and_get_row(&db, PLAIN_CONTENT, ScriptLang::Bun).await; + assert_eq!(lang, Some(ScriptLang::Bun)); + assert_eq!(tag, "bun"); +} diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index 734753a0e0..b9e6a748d4 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -871,9 +871,6 @@ pub async fn create_token_internal( /// Insert a pending expiry notification row for user tokens that have an expiration. /// Stores the token_hash so the join in check_expiring_tokens works even when /// the plaintext token column is NULL (after hash migration). -/// When updating this filter, also update: -/// - `is_user_token` in src/monitor.rs -/// - `isUserToken` in frontend/src/lib/components/settings/TokensTable.svelte pub async fn register_token_expiry_notification( tx: &mut sqlx::PgConnection, token_hash: &str, @@ -881,14 +878,8 @@ pub async fn register_token_expiry_notification( expiration: Option>, ) { let Some(expiration) = expiration else { return }; - if label == Some("session") - || label.is_some_and(|l| { - l.starts_with("ephemeral") - || l.starts_with("Ephemeral") - || l == "debugger-token" - || l.starts_with("mcp-oauth-") - }) - { + // System tokens don't get expiry notifications. + if !windmill_common::auth::is_user_token(label) { return; } if let Err(e) = sqlx::query!( diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 546bda32f0..c63d53171c 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -16,7 +16,8 @@ use axum::{ }; use windmill_api_auth::{ auth::{list_tokens_internal, TruncatedTokenWithEmail}, - check_scopes, maybe_refresh_folders, require_owner_of_path, ApiAuthed, + build_scope_path_predicate, check_scopes, maybe_refresh_folders, require_owner_of_path, + ApiAuthed, }; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use windmill_common::{ @@ -108,9 +109,10 @@ async fn list_search_flows( let n = 3; let mut tx = user_db.begin(&authed).await?; + let allowed = build_scope_path_predicate(&authed, "flows", "read"); let rows = sqlx::query_as::<_, SearchFlow>( "SELECT flow.path, flow_version.value - FROM flow + FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.workspace_id = $1 LIMIT $2", ) @@ -119,6 +121,7 @@ async fn list_search_flows( .fetch_all(&mut *tx) .await? .into_iter() + .filter(|r| allowed(&r.path)) .collect::>(); tx.commit().await?; Ok(Json(rows)) @@ -212,9 +215,13 @@ async fn list_flows( let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; + let allowed = build_scope_path_predicate(&authed, "flows", "read"); let rows = sqlx::query_as::<_, ListableFlow>(&sql) .fetch_all(&mut *tx) - .await?; + .await? + .into_iter() + .filter(|r| allowed(&r.path)) + .collect::>(); tx.commit().await?; Ok(Json(rows)) } diff --git a/backend/windmill-api-integration-tests/Cargo.toml b/backend/windmill-api-integration-tests/Cargo.toml index a73d3d3c99..3aee7e766c 100644 --- a/backend/windmill-api-integration-tests/Cargo.toml +++ b/backend/windmill-api-integration-tests/Cargo.toml @@ -35,6 +35,9 @@ anyhow.workspace = true uuid.workspace = true futures.workspace = true rand.workspace = true +hmac.workspace = true +sha2.workspace = true +hex.workspace = true rumqttc.workspace = true rdkafka.workspace = true async-nats.workspace = true diff --git a/backend/windmill-api-integration-tests/tests/fixtures/resource_cache_rls.sql b/backend/windmill-api-integration-tests/tests/fixtures/resource_cache_rls.sql new file mode 100644 index 0000000000..8c30eab5b2 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fixtures/resource_cache_rls.sql @@ -0,0 +1,23 @@ +-- Fixture for the resource-value interpolation cache RLS regression test. +-- Extends base.sql (which defines test-user [admin], test-user-2, test-user-3 +-- and their tokens). +-- +-- A folder `secret` is readable ONLY by test-user-2 (via extra_perms). It holds a +-- variable and a resource that interpolates it. test-user-3 has no access to the +-- folder, so a cache entry warmed by test-user-2 with allow_cache=true must never +-- be served back to test-user-3. + +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by) +VALUES ('test-workspace', 'secret', 'Secret Folder', '{}', + '{"u/test-user-2": true}', 'test-user'); + +-- A (non-secret) variable gated to the `secret` folder; its value gets interpolated +-- into the resource value below and ends up in the cached, already-resolved blob. +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES ('test-workspace', 'f/secret/db_password', 'LEAKED_FOLDER_SECRET', false, + 'Folder-gated secret', '{}'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'f/secret/cache_target', + '{"host": "db.internal", "password": "$var:f/secret/db_password"}', + 'Folder-gated resource referencing a folder-gated variable', 'object', '{}', 'test-user'); diff --git a/backend/windmill-api-integration-tests/tests/fixtures/variable_cache_rls.sql b/backend/windmill-api-integration-tests/tests/fixtures/variable_cache_rls.sql new file mode 100644 index 0000000000..69a6810b7b --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fixtures/variable_cache_rls.sql @@ -0,0 +1,15 @@ +-- Fixture for the variable-value cache RLS regression test. +-- Extends base.sql (which defines test-user [admin], test-user-2, test-user-3 +-- and their tokens). +-- +-- A folder `secret` is readable ONLY by test-user-2 (via extra_perms). It holds a +-- variable that test-user-2 can read but test-user-3 cannot. A cache entry warmed +-- by test-user-2 with allow_cache=true must never be served back to test-user-3. + +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by) +VALUES ('test-workspace', 'secret', 'Secret Folder', '{}', + '{"u/test-user-2": true}', 'test-user'); + +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES ('test-workspace', 'f/secret/cache_target_var', 'LEAKED_VAR_SECRET', false, + 'Folder-gated variable', '{}'); diff --git a/backend/windmill-api-integration-tests/tests/flows.rs b/backend/windmill-api-integration-tests/tests/flows.rs index ff3f86bf2d..b6075c8e69 100644 --- a/backend/windmill-api-integration-tests/tests/flows.rs +++ b/backend/windmill-api-integration-tests/tests/flows.rs @@ -259,12 +259,10 @@ async fn test_flow_endpoints(db: Pool) -> anyhow::Result<()> { // ===== Hub endpoints (require external network, expect 500 or 200) ===== // --- hub/list --- - let resp = authed(client().get(format!( - "http://localhost:{port}/api/flows/hub/list" - ))) - .send() - .await - .unwrap(); + let resp = authed(client().get(format!("http://localhost:{port}/api/flows/hub/list"))) + .send() + .await + .unwrap(); assert!( resp.status() == 200 || resp.status() == 500, "hub/list: unexpected status {}", @@ -272,12 +270,10 @@ async fn test_flow_endpoints(db: Pool) -> anyhow::Result<()> { ); // --- hub/get --- - let resp = authed(client().get(format!( - "http://localhost:{port}/api/flows/hub/get/1" - ))) - .send() - .await - .unwrap(); + let resp = authed(client().get(format!("http://localhost:{port}/api/flows/hub/get/1"))) + .send() + .await + .unwrap(); assert!( resp.status() == 200 || resp.status() == 500, "hub/get: unexpected status {}", @@ -286,3 +282,98 @@ async fn test_flow_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// Regression test for GHSA-2ppx-66jv-wpw5: a path-scoped token must only see +/// the flows within its scope when listing, even though the route-level scope +/// check only validates `domain:action`. Before the fix, `list_search` returned +/// `path` + the full flow `value` for every flow the underlying user could see, +/// leaking out-of-scope flow definitions to narrowly-scoped tokens. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_list_search_scope_filtering(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/flows"); + + // Create two folders and one flow in each, as the (super-admin) test user. + for folder in ["allowed", "private"] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/folders/create" + ))) + .json(&json!({ "name": folder })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create folder: {}", resp.text().await?); + } + + for path in ["f/allowed/foo", "f/private/bar"] { + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_flow(path, "summary")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create {path}: {}", resp.text().await?); + } + + // Helper: GET /list_search with an arbitrary bearer token, returning the set + // of flow paths visible to that token. + async fn list_search_paths(port: u16, token: &str) -> Vec { + let resp = client() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/flows/list_search" + )) + .header("Authorization", format!("Bearer {token}")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>() + .await + .unwrap() + .into_iter() + .map(|s| s["path"].as_str().unwrap().to_string()) + .collect() + } + + // Insert three tokens for the same super-admin user, differing only by scope. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES + (encode(sha256('SCOPED_TOKEN'::bytea), 'hex'), 'SCOPED_TOK', 'SCOPED_TOKEN', 'test@windmill.dev', 'scoped', true, ARRAY['flows:read:f/allowed/*']), + (encode(sha256('BROAD_TOKEN'::bytea), 'hex'), 'BROAD_TOK', 'BROAD_TOKEN', 'test@windmill.dev', 'broad', true, ARRAY['flows:read']), + (encode(sha256('TAG_TOKEN'::bytea), 'hex'), 'TAG_TOK', 'TAG_TOKEN', 'test@windmill.dev', 'tag-only', true, ARRAY['if_jobs:filter_tags:default'])", + ) + .execute(&db) + .await?; + + // Path-scoped token: only sees flows within `f/allowed/*`. + let scoped = list_search_paths(port, "SCOPED_TOKEN").await; + assert!( + scoped.contains(&"f/allowed/foo".to_string()), + "scoped token should see f/allowed/foo, got: {scoped:?}" + ); + assert!( + !scoped.contains(&"f/private/bar".to_string()), + "scoped token must NOT see f/private/bar, got: {scoped:?}" + ); + + // Broad `flows:read` token: still sees every RLS-visible flow. + let broad = list_search_paths(port, "BROAD_TOKEN").await; + assert!(broad.contains(&"f/allowed/foo".to_string())); + assert!( + broad.contains(&"f/private/bar".to_string()), + "broad flows:read token should see all flows, got: {broad:?}" + ); + + // Tag-filter-only token is not scope-restricted: unchanged, sees all. + let tag_only = list_search_paths(port, "TAG_TOKEN").await; + assert!(tag_only.contains(&"f/allowed/foo".to_string())); + assert!(tag_only.contains(&"f/private/bar".to_string())); + + // Unscoped token (no scopes column set): unchanged, sees all. + let unscoped = list_search_paths(port, "SECRET_TOKEN").await; + assert!(unscoped.contains(&"f/allowed/foo".to_string())); + assert!(unscoped.contains(&"f/private/bar".to_string())); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/native_triggers.rs b/backend/windmill-api-integration-tests/tests/native_triggers.rs index 159236ed4b..1d809ae526 100644 --- a/backend/windmill-api-integration-tests/tests/native_triggers.rs +++ b/backend/windmill-api-integration-tests/tests/native_triggers.rs @@ -17,8 +17,8 @@ use windmill_native_triggers::{ decrypt_oauth_data, delete_native_trigger, delete_workspace_integration, get_workspace_integration, google::{parse_stop_channel_params, should_renew_channel}, - store_native_trigger, store_workspace_integration, NativeTriggerConfig, OAuthConfig, - ServiceName, + require_native_integration_use, store_native_trigger, store_workspace_integration, + NativeTriggerConfig, OAuthConfig, ServiceName, }; // ============================================================================ @@ -329,6 +329,26 @@ async fn test_token_update_persists(db: Pool) -> anyhow::Result<()> { // 3. Channel Expiration Renewal — should_renew_channel // ============================================================================ +#[test] +fn test_require_native_integration_use_blocks_operators() { + // Regression: the integration *use* routes (calendar/drive/repo/event pickers) + // must reject read-only operators, who cannot create native triggers and so + // must not be able to drive the admin-configured integration's upstream API. + let mut operator = test_authed(); + operator.is_admin = false; + operator.is_operator = true; + assert!(require_native_integration_use(&operator).is_err()); + + // A regular non-admin author (the population that configures triggers) is allowed. + let mut author = test_authed(); + author.is_admin = false; + author.is_operator = false; + assert!(require_native_integration_use(&author).is_ok()); + + // Admins are allowed. + assert!(require_native_integration_use(&test_authed()).is_ok()); +} + #[test] fn test_should_renew_drive_channel_expired() { let config = json!({ diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index 363217712f..cc78056176 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -477,6 +477,117 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } +/// Regression test: the resource-value interpolation cache +/// (`get_value_interpolated?allow_cache=true`) must be identity-scoped. test-user-2 +/// (folder access) warms the cache; test-user-3 (no access) must then be denied rather +/// than served the cached, already-decrypted value. Pre-fix the unscoped key returned +/// a 200 with the secret here. +#[sqlx::test(migrations = "../migrations", fixtures("base", "resource_cache_rls"))] +async fn test_resource_value_cache_is_identity_scoped(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let url = format!( + "{}?allow_cache=true", + resource_url(port, "get_value_interpolated", "f/secret/cache_target") + ); + let get = |token: &str| { + client() + .get(url.as_str()) + .header("Authorization", format!("Bearer {token}")) + }; + + // test-user-2 has folder access and WARMS the cache. + let resp = get("SECRET_TOKEN_2").send().await?; + assert_eq!(resp.status(), 200); + assert!(resp.text().await?.contains("LEAKED_FOLDER_SECRET")); + + // test-user-3 has no folder access: must miss the cache and be denied (401), not leak. + let resp = get("SECRET_TOKEN_3").send().await?; + assert_eq!(resp.status(), 401); + assert!(!resp.text().await?.contains("LEAKED_FOLDER_SECRET")); + + Ok(()) +} + +/// A resource whose value contains a `$WM_*` contextual variable (e.g. `$WM_TOKEN`) is +/// job-dependent and must NEVER be cached — even when first read WITHOUT a `job_id`, where the +/// placeholder is left unresolved (caching that would serve a stale placeholder to a later job +/// read). Any other value — plain, or a non-`$WM_` `$`-string like `$HOME` (which is NOT +/// interpolated, so it's constant) — is job-independent and IS cached, with the entry shared +/// across job contexts (a read carrying a `job_id` still hits it, keeping the hit ratio up). +/// We prove all three by warming each (no job_id), deleting the row directly (cache survives), +/// then re-reading: the job-independent ones are still served from cache — even under a +/// `job_id` — while the `$WM_*` one was never cached and 404s. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_resource_cache_handles_job_context(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/resources"); + + let plain = "u/test-user/plain_res"; + let dollar = "u/test-user/dollar_res"; // non-$WM_ `$`-string: not interpolated, cacheable + let jobctx = "u/test-user/jobctx_res"; + for (path, value) in [ + (plain, json!({"v": 1})), + (dollar, json!({"d": "$HOME"})), + (jobctx, json!({"j": "$WM_JOB_ID"})), + ] { + let resp = authed(client().post(format!("{base}/create"))) + .json( + &json!({ "path": path, "value": value, "description": "", "resource_type": "object" }), + ) + .send() + .await?; + assert_eq!(resp.status(), 201); + } + + let get = |path: &str, query: &str| { + let url = format!("{base}/get_value_interpolated/{path}?{query}"); + async move { authed(client().get(url)).send().await.unwrap() } + }; + + // Warm all three WITHOUT a job context (the placeholder is left unresolved for `jobctx`). + for path in [plain, dollar, jobctx] { + assert_eq!(get(path, "allow_cache=true").await.status(), 200); + } + + // Delete the rows directly — bypasses the API/NOTIFY, so the in-memory cache survives. + for path in [plain, dollar, jobctx] { + sqlx::query("DELETE FROM resource WHERE workspace_id = 'test-workspace' AND path = $1") + .bind(path) + .execute(&db) + .await?; + } + + // Job-independent values are cached and still served even under a job_id (a random uuid is + // fine: a cache hit short-circuits before any job lookup). `$HOME` is a non-`$WM_` string, + // so it's not interpolated and stays cacheable. + for path in [plain, dollar] { + let resp = get( + path, + "allow_cache=true&job_id=11111111-1111-4111-8111-111111111111", + ) + .await; + assert_eq!( + resp.status(), + 200, + "job-independent resource ({path}) must stay cached and be served under a job_id" + ); + } + + // The `$WM_*` resource was never cached → the (now deleted) row is not found. + let resp = get(jobctx, "allow_cache=true").await; + assert_ne!( + resp.status(), + 200, + "resource with a $WM_* contextual variable must not be cached" + ); + + Ok(()) +} + #[cfg(feature = "mcp")] #[sqlx::test(migrations = "../migrations", fixtures("base", "resources_test"))] async fn test_mcp_tools(db: Pool) -> anyhow::Result<()> { diff --git a/backend/windmill-api-integration-tests/tests/scripts.rs b/backend/windmill-api-integration-tests/tests/scripts.rs index f5e78f880f..c374b757a4 100644 --- a/backend/windmill-api-integration-tests/tests/scripts.rs +++ b/backend/windmill-api-integration-tests/tests/scripts.rs @@ -463,3 +463,107 @@ async fn test_auto_parent_resolves_parent_hash(db: Pool) -> anyhow::Re Ok(()) } + +/// Regression test for GHSA-2ppx-66jv-wpw5: a path-scoped token must only see +/// the scripts within its scope when listing, even though the route-level scope +/// check only validates `domain:action`. Before the fix, `list_search` (and +/// `list`) returned `path` + full `content` for every script the underlying +/// user could see, leaking out-of-scope script source to narrowly-scoped tokens. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_list_search_scope_filtering(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/scripts"); + + // Create two folders and one script in each, as the (super-admin) test user. + for folder in ["allowed", "private"] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/folders/create" + ))) + .json(&json!({ "name": folder })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create folder: {}", resp.text().await?); + } + + for (path, content) in [ + ( + "f/allowed/foo", + "export async function main() { return 'allowed'; }", + ), + ( + "f/private/bar", + "export async function main() { return 'secret'; }", + ), + ] { + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_script(path, "summary", content)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create {path}: {}", resp.text().await?); + } + + // Helper: GET /list_search with an arbitrary bearer token, returning the set + // of script paths visible to that token. + async fn list_search_paths(port: u16, token: &str) -> Vec { + let resp = client() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/list_search" + )) + .header("Authorization", format!("Bearer {token}")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>() + .await + .unwrap() + .into_iter() + .map(|s| s["path"].as_str().unwrap().to_string()) + .collect() + } + + // Insert three tokens for the same super-admin user, differing only by scope. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES + (encode(sha256('SCOPED_TOKEN'::bytea), 'hex'), 'SCOPED_TOK', 'SCOPED_TOKEN', 'test@windmill.dev', 'scoped', true, ARRAY['scripts:read:f/allowed/*']), + (encode(sha256('BROAD_TOKEN'::bytea), 'hex'), 'BROAD_TOK', 'BROAD_TOKEN', 'test@windmill.dev', 'broad', true, ARRAY['scripts:read']), + (encode(sha256('TAG_TOKEN'::bytea), 'hex'), 'TAG_TOK', 'TAG_TOKEN', 'test@windmill.dev', 'tag-only', true, ARRAY['if_jobs:filter_tags:default'])", + ) + .execute(&db) + .await?; + + // Path-scoped token: only sees scripts within `f/allowed/*`. + let scoped = list_search_paths(port, "SCOPED_TOKEN").await; + assert!( + scoped.contains(&"f/allowed/foo".to_string()), + "scoped token should see f/allowed/foo, got: {scoped:?}" + ); + assert!( + !scoped.contains(&"f/private/bar".to_string()), + "scoped token must NOT see f/private/bar, got: {scoped:?}" + ); + + // Broad `scripts:read` token: still sees every RLS-visible script. + let broad = list_search_paths(port, "BROAD_TOKEN").await; + assert!(broad.contains(&"f/allowed/foo".to_string())); + assert!( + broad.contains(&"f/private/bar".to_string()), + "broad scripts:read token should see all scripts, got: {broad:?}" + ); + + // Tag-filter-only token is not scope-restricted: unchanged, sees all. + let tag_only = list_search_paths(port, "TAG_TOKEN").await; + assert!(tag_only.contains(&"f/allowed/foo".to_string())); + assert!(tag_only.contains(&"f/private/bar".to_string())); + + // Unscoped token (no scopes column set): unchanged, sees all. + let unscoped = list_search_paths(port, "SECRET_TOKEN").await; + assert!(unscoped.contains(&"f/allowed/foo".to_string())); + assert!(unscoped.contains(&"f/private/bar".to_string())); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/slack_approvals_unauthed.rs b/backend/windmill-api-integration-tests/tests/slack_approvals_unauthed.rs new file mode 100644 index 0000000000..f07e8b81e1 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/slack_approvals_unauthed.rs @@ -0,0 +1,230 @@ +//! Regression tests for GHSA-vm75-gmpw-rvp9: the unauthenticated `/api/slack` callback must +//! not be drivable into decrypting arbitrary workspace variables. +//! +//! The OpenModal branch reaches `get_slack_token` (a privileged, RLS-bypassing variable +//! decryption). It is now gated by a per-workspace HMAC over (w_id, job_id, path) — the same +//! workspace key used to sign resume URLs. Without a valid signature the request is rejected +//! with 401 before any decryption, even when `SLACK_SIGNING_SECRET` is unset (the default). + +use hmac::{Hmac, Mac}; +use serde_json::json; +use sha2::Sha256; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +/// Re-implementation of the server's `sign_slack_payload` for the positive-control test. +/// The fixture sets `workspace_key.key = 'test-key'` for `test-workspace`. +fn sign(w_id: &str, parts: &[&[u8]]) -> String { + let mut mac = Hmac::::new_from_slice(b"test-key").unwrap(); + mac.update(b"slack_payload_v1\0"); // SLACK_PAYLOAD_HMAC_DOMAIN + mac.update(w_id.as_bytes()); + for p in parts { + mac.update(b"\0"); + mac.update(p); + } + hex::encode(mac.finalize().into_bytes()) +} + +/// POST an `open_modal` block action to the unauthenticated `/api/slack` callback. +async fn post_open_modal(port: u16, value: serde_json::Value) -> reqwest::Response { + let payload = json!({ + "type": "block_actions", + "trigger_id": "trigger-123", + "container": { "message_ts": "0", "channel_id": "C1" }, + "actions": [ { "action_id": "open_modal", "value": value.to_string() } ], + }); + client() + .post(format!("http://localhost:{port}/api/slack")) + .form(&[("payload", payload.to_string())]) + .send() + .await + .unwrap() +} + +/// POST a `view_submission` to the unauthenticated `/api/slack` callback with the given +/// private_metadata. +async fn post_view_submission(port: u16, private_metadata: serde_json::Value) -> reqwest::Response { + let payload = json!({ + "type": "view_submission", + "view": { + "state": { "values": {} }, + "private_metadata": private_metadata.to_string(), + }, + }); + client() + .post(format!("http://localhost:{port}/api/slack")) + .form(&[("payload", payload.to_string())]) + .send() + .await + .unwrap() +} + +/// A submission with an unsigned (or tampered) `private_metadata` must be rejected with 401 +/// BEFORE the resume/cancel action runs — the signature gate is checked first. The resume_url +/// here is well-formed (so it parses) but never acted upon. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_view_submission_without_signature_is_rejected( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let job_id = Uuid::new_v4(); + let resume_url = format!("/api/w/test-workspace/jobs_u/resume/{job_id}/1/deadbeef"); + + let resp = post_view_submission( + port, + json!({ + "resume_url": resume_url, + "resource_path": "u/admin/secret", + "container": { "message_ts": "0", "channel_id": "C1" }, + "hide_cancel": false, + }), + ) + .await; + assert_eq!( + resp.status(), + 401, + "unsigned submission must be rejected before the resume action" + ); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_open_modal_without_signature_is_rejected(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let job_id = Uuid::new_v4(); + + // No signature → must be rejected with 401 before any variable lookup. Before the fix + // this reached `get_slack_token` and forced decryption of `u/admin/secret`. + let resp = post_open_modal( + port, + json!({ + "w_id": "test-workspace", + "job_id": job_id.to_string(), + "path": "u/admin/secret", + "flow_step_id": "a", + }), + ) + .await; + assert_eq!( + resp.status(), + 401, + "unsigned OpenModal callback must be rejected" + ); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_open_modal_with_wrong_signature_is_rejected( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let job_id = Uuid::new_v4(); + + let resp = post_open_modal( + port, + json!({ + "w_id": "test-workspace", + "job_id": job_id.to_string(), + "path": "u/admin/secret", + "flow_step_id": "a", + "signature": "deadbeef", + }), + ) + .await; + assert_eq!( + resp.status(), + 401, + "OpenModal callback with an invalid signature must be rejected" + ); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_open_modal_with_tampered_path_is_rejected(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let job_id = Uuid::new_v4(); + + // A signature legitimately minted for one path cannot be reused to decrypt another: the + // path is bound into the HMAC. + let signature = sign( + "test-workspace", + &[job_id.to_string().as_bytes(), b"u/admin/legit_resource"], + ); + let resp = post_open_modal( + port, + json!({ + "w_id": "test-workspace", + "job_id": job_id.to_string(), + "path": "u/admin/some_other_secret", + "flow_step_id": "a", + "signature": signature, + }), + ) + .await; + assert_eq!( + resp.status(), + 401, + "a signature bound to a different path must not authorize decryption" + ); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_open_modal_with_valid_signature_passes_the_gate( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let job_id = Uuid::new_v4(); + let path = "u/admin/nonexistent_resource"; + + // A correctly signed payload passes the authorization gate and proceeds to resolve the + // slack resource. The resource does not exist, so the handler returns a generic 400 + // ("Invalid Slack callback request") rather than 401 — proving the gate accepted the + // signature (so the fix does not simply reject everything) without echoing the path. + let signature = sign( + "test-workspace", + &[job_id.to_string().as_bytes(), path.as_bytes()], + ); + let resp = post_open_modal( + port, + json!({ + "w_id": "test-workspace", + "job_id": job_id.to_string(), + "path": path, + "flow_step_id": "a", + "signature": signature, + }), + ) + .await; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "validly signed callback should pass the gate and 400 on the missing resource, got {status}: {body}" + ); + assert!( + !body.contains("nonexistent_resource"), + "error must not echo the probed path: {body}" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/token_label_idor.rs b/backend/windmill-api-integration-tests/tests/token_label_idor.rs new file mode 100644 index 0000000000..92dada92c2 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/token_label_idor.rs @@ -0,0 +1,192 @@ +//! Regression tests for GHSA-8x8x-88qc-qp4r: token label collision bypassing job read +//! access control (IDOR). +//! +//! `username_override` is derived from a fully user-controlled token label, so a bare +//! `username_override == created_by` match in `require_job_read_access` is forgeable. The fix +//! binds that fast path to a non-forgeable attribute — the job's `permissioned_as_email` (the +//! token owner's email) must equal the caller's email. This: +//! - denies a colliding-label token created by a different principal, while +//! - still allowing a principal to re-read its own labeled-token jobs (incl. when RLS would +//! otherwise hide them), and +//! - leaving user-facing webhook/http/email trigger token creation untouched (those labels +//! are created through the public token API by design). + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn bearer(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {token}")) +} + +async fn create_token_with_label(port: u16, caller_token: &str, label: &str) -> reqwest::Response { + bearer( + client().post(format!("http://localhost:{port}/api/users/tokens/create")), + caller_token, + ) + .json(&json!({ "label": label })) + .send() + .await + .unwrap() +} + +/// Insert a completed job with a labeled-token `created_by`, running as `permissioned_as` +/// (email `permissioned_as_email`) with the given `runnable_path` (which governs RLS). +async fn insert_labeled_job( + db: &Pool, + created_by: &str, + runnable_path: &str, + permissioned_as: &str, + permissioned_as_email: &str, +) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, permissioned_as_email, runnable_path, kind, tag, args, visible_to_owner) + VALUES ($1, 'test-workspace', $2, $3, $4, $5, 'script', 'deno', '{}'::jsonb, true)", + ) + .bind(id) + .bind(created_by) + .bind(permissioned_as) + .bind(permissioned_as_email) + .bind(runnable_path) + .execute(db) + .await + .unwrap(); + sqlx::query( + "INSERT INTO v2_job_completed (id, workspace_id, duration_ms, result, status) + VALUES ($1, 'test-workspace', 100, '{\"secret\":\"super-secret-value\"}'::jsonb, 'success')", + ) + .bind(id) + .execute(db) + .await + .unwrap(); + id +} + +/// The core IDOR: an operator who mints a token whose label collides with another +/// principal's labeled-token identity must NOT be able to read that principal's job — the +/// `permissioned_as_email` of that job is the victim's, not the attacker's. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_label_collision_does_not_grant_job_read(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/jobs"); + + // A job submitted with a token labeled "collide", running as the admin (test-user). + let job_id = insert_labeled_job( + &db, + "label-collide", + "u/test-user/secret_script", + "u/test-user", + "test@windmill.dev", + ) + .await; + + // Sanity: the admin can read it, so the job exists and is otherwise readable. + let resp = bearer( + client().get(format!("{base}/completed/get/{job_id}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "admin must still read the job"); + + // The attacker (a different member, test-user-2) mints a colliding-label token. + let resp = create_token_with_label(port, "SECRET_TOKEN_2", "collide").await; + assert_eq!(resp.status(), 201); + let attacker_token = resp.text().await?; + + // Reading the admin's job with the colliding token must be denied. Before the fix the + // `username_override == created_by` fast path returned the full result here. + let resp = bearer( + client().get(format!("{base}/completed/get/{job_id}")), + &attacker_token, + ) + .send() + .await?; + assert!( + !resp.status().is_success(), + "colliding-label token must not read another principal's job (got {})", + resp.status() + ); + let body = resp.text().await?; + assert!( + !body.contains("super-secret-value"), + "job result must not leak to the colliding-label token" + ); + + Ok(()) +} + +/// The fix must not regress the legitimate case: a principal re-reading its own +/// labeled-token job is granted via the email-bound fast path, even when RLS would hide the +/// job (the runnable lives in another user's space the caller has no RLS path to). +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_legit_labeled_self_read_still_works(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/jobs"); + + // Created by test-user-2's labeled token, running as test-user-2, but the runnable lives + // under u/test-user so RLS alone would not reveal it to test-user-2 — the grant must come + // from the email-bound fast path. + let job_id = insert_labeled_job( + &db, + "label-mine", + "u/test-user/shared_script", + "u/test-user-2", + "test2@windmill.dev", + ) + .await; + + let resp = create_token_with_label(port, "SECRET_TOKEN_2", "mine").await; + assert_eq!(resp.status(), 201); + let token = resp.text().await?; + + let resp = bearer( + client().get(format!("{base}/completed/get/{job_id}")), + &token, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "owner must still read their own labeled-token job via the email-bound fast path" + ); + + Ok(()) +} + +/// P1 regression guard: the user-facing token API must keep accepting the labels that the +/// webhook / http-route / email trigger panels mint (e.g. `webhook--`). The fix +/// must not reserve those prefixes. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_trigger_token_labels_still_creatable(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + for label in [ + "webhook-test-user-2-ab12", + "http-test-user-2-cd34", + "email-test-user-2-ef56", + "my-ci-token", + ] { + let resp = create_token_with_label(port, "SECRET_TOKEN_2", label).await; + assert_eq!( + resp.status(), + 201, + "creating a token with label {label:?} must succeed" + ); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/variables.rs b/backend/windmill-api-integration-tests/tests/variables.rs index 0d4edaff91..e5018f4f97 100644 --- a/backend/windmill-api-integration-tests/tests/variables.rs +++ b/backend/windmill-api-integration-tests/tests/variables.rs @@ -108,12 +108,10 @@ async fn test_variable_endpoints(db: Pool) -> anyhow::Result<()> { assert_eq!(secret["value"], serde_json::Value::Null); // list with path_start filter - let resp = authed(client().get(format!( - "{base}/list?path_start=u/test-user/plain" - ))) - .send() - .await - .unwrap(); + let resp = authed(client().get(format!("{base}/list?path_start=u/test-user/plain"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); let list = resp.json::>().await?; assert_eq!(list.len(), 1); @@ -252,3 +250,91 @@ async fn test_variable_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// Regression test: the variable-value cache (`get_value?allow_cache=true`) must be +/// identity-scoped. test-user-2 (folder access) warms the cache; test-user-3 (no access) +/// must then be denied rather than served the cached value. +#[sqlx::test(migrations = "../migrations", fixtures("base", "variable_cache_rls"))] +async fn test_variable_value_cache_is_identity_scoped(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let url = format!( + "{}?allow_cache=true", + variable_url(port, "get_value", "f/secret/cache_target_var") + ); + let get = |token: &str| { + client() + .get(url.as_str()) + .header("Authorization", format!("Bearer {token}")) + }; + + // test-user-2 has folder access and WARMS the cache. + let resp = get("SECRET_TOKEN_2").send().await?; + assert_eq!(resp.status(), 200); + assert!(resp.text().await?.contains("LEAKED_VAR_SECRET")); + + // test-user-3 has no folder access: must miss the cache and be denied (401), not leak. + let resp = get("SECRET_TOKEN_3").send().await?; + assert_eq!(resp.status(), 401); + assert!(!resp.text().await?.contains("LEAKED_VAR_SECRET")); + + Ok(()) +} + +/// Secret variables ARE cached (with their per-read side effects — the EE +/// `variables.decrypt_secret` audit and running-job secret registration — re-run on every +/// hit; that re-emission is not observable in the OSS build since `audit_log` is a no-op). +/// We assert the caching itself: warm the cache, delete the row directly (no API/NOTIFY, so +/// the in-memory cache survives), and re-read with `allow_cache=true` — the value is still +/// returned from cache. A non-secret variable behaves identically (control). +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_variables_are_cached(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/variables"); + + let plain = "u/test-user/cache_plain_probe"; + let secret = "u/test-user/cache_secret_probe"; + + // Create one non-secret and one secret variable (the secret is stored encrypted). + for (path, value, is_secret) in [ + (plain, "PLAIN_PROBE", false), + (secret, "SECRET_PROBE", true), + ] { + let resp = authed(client().post(format!("{base}/create"))) + .json( + &json!({ "path": path, "value": value, "is_secret": is_secret, "description": "" }), + ) + .send() + .await?; + assert_eq!(resp.status(), 201); + } + + let read = |path: &str| { + let url = format!("{base}/get_value/{path}?allow_cache=true"); + async move { authed(client().get(url)).send().await.unwrap() } + }; + + // Warm the cache for both. + assert_eq!(read(plain).await.json::().await?, "PLAIN_PROBE"); + assert_eq!(read(secret).await.json::().await?, "SECRET_PROBE"); + + // Delete both rows directly — bypasses the API and its NOTIFY-based invalidation, so + // the in-memory cache survives. A subsequent read can only succeed from cache. + for path in [plain, secret] { + sqlx::query("DELETE FROM variable WHERE workspace_id = 'test-workspace' AND path = $1") + .bind(path) + .execute(&db) + .await?; + } + + // Both (secret included) are still served from the cache. + assert_eq!(read(plain).await.json::().await?, "PLAIN_PROBE"); + let resp = read(secret).await; + assert_eq!(resp.status(), 200, "secret must still be served from cache"); + assert_eq!(resp.json::().await?, "SECRET_PROBE"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index 131cfbbae5..3d208d4e25 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -709,7 +709,9 @@ async fn test_get_copilot_settings_state_reports_instance_ai_fallback_flags( "resource_path": "u/test-user/openai_instance", "models": ["gpt-4o-mini"] } - } + }, + "default_model": { "provider": "openai", "model": "gpt-4o-mini" }, + "metadata_model": { "provider": "openai", "model": "gpt-4o-mini" } }); let workspace_ai_config = json!({ "providers": { @@ -749,6 +751,10 @@ async fn test_get_copilot_settings_state_reports_instance_ai_fallback_flags( settings["instance_ai_summary"]["providers"][0]["models"][0], "gpt-4o-mini" ); + assert_eq!( + settings["instance_ai_summary"]["metadata_model"]["model"], + "gpt-4o-mini" + ); sqlx::query("UPDATE workspace_settings SET ai_config = $1 WHERE workspace_id = $2") .bind(workspace_ai_config) diff --git a/backend/windmill-api-jobs/src/concurrency_groups.rs b/backend/windmill-api-jobs/src/concurrency_groups.rs index d8e7a54112..8303788b8a 100644 --- a/backend/windmill-api-jobs/src/concurrency_groups.rs +++ b/backend/windmill-api-jobs/src/concurrency_groups.rs @@ -191,6 +191,7 @@ async fn get_concurrent_intervals( script_path_exact: None, script_hash: None, created_by: None, + status: None, success: None, running: None, parent_job: None, diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs index e6c57e6c5b..a95a12ee6a 100644 --- a/backend/windmill-api-jobs/src/query.rs +++ b/backend/windmill-api-jobs/src/query.rs @@ -430,7 +430,15 @@ pub fn filter_list_completed_query( sqlb.and_where_in("created_by", "ed); } } - if let Some(r) = &lq.success { + if let Some(status) = &lq.status { + let status = match status { + windmill_common::jobs::JobStatus::Success => "success", + windmill_common::jobs::JobStatus::Failure => "failure", + windmill_common::jobs::JobStatus::Canceled => "canceled", + windmill_common::jobs::JobStatus::Skipped => "skipped", + }; + sqlb.and_where_eq("v2_job_completed.status", quote(status)); + } else if let Some(r) = &lq.success { if *r { sqlb.and_where_eq("status", "'success'") .or_where_eq("status", "'skipped'"); @@ -572,6 +580,11 @@ pub fn list_completed_jobs_query( if lq.completed_before.is_some() || lq.completed_after.is_some() || lq.success == Some(false) + || matches!( + lq.status, + Some(windmill_common::jobs::JobStatus::Failure) + | Some(windmill_common::jobs::JobStatus::Canceled) + ) { "v2_job_completed.completed_at" } else { @@ -653,6 +666,7 @@ mod tests { created_after_queue: None, completed_after: None, completed_before: None, + status: None, success: None, running: None, parent_job: None, @@ -928,6 +942,23 @@ mod tests { assert!(sql.contains("'failure'")); } + #[test] + fn test_completed_filter_status_canceled() { + let lq = ListCompletedQuery { + status: Some(windmill_common::jobs::JobStatus::Canceled), + ..empty_completed_query() + }; + let sqlb = filter_list_completed_query( + SqlBuilder::select_from("v2_job_completed").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("v2_job_completed.status")); + assert!(sql.contains("'canceled'")); + } + #[test] fn test_completed_order_by_completed_at() { let lq = ListCompletedQuery { @@ -939,6 +970,25 @@ mod tests { assert!(sql.contains("completed_at")); } + #[test] + fn test_completed_order_by_completed_at_status_failure_canceled() { + // status=failure|canceled must order by v2_job_completed.completed_at so the + // partial index ix_v2_job_completed_failure_workspace serves both filtering + // and ordering in a single scan. + for status in [ + windmill_common::jobs::JobStatus::Failure, + windmill_common::jobs::JobStatus::Canceled, + ] { + let lq = ListCompletedQuery { status: Some(status), ..empty_completed_query() }; + let sqlb = list_completed_jobs_query("ws", Some(10), 0, &lq, &["id"], false, None); + let sql = build_sql(sqlb); + assert!( + sql.contains("ORDER BY v2_job_completed.completed_at"), + "expected order by completed_at, got: {sql}" + ); + } + } + #[test] fn test_completed_filter_label() { let lq = ListCompletedQuery { diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs index bc03a04289..6252cef9b7 100644 --- a/backend/windmill-api-jobs/src/types.rs +++ b/backend/windmill-api-jobs/src/types.rs @@ -19,7 +19,7 @@ use std::collections::HashMap; use uuid::Uuid; use windmill_common::{ error, - jobs::{CompletedJob, JobKind, JobTriggerKind, QueuedJob}, + jobs::{CompletedJob, JobKind, JobStatus, JobTriggerKind, QueuedJob}, scripts::{ScriptHash, ScriptLang}, utils::now_from_db, DB, @@ -142,6 +142,7 @@ pub struct ListCompletedQuery { pub created_after_queue: Option>, pub completed_after: Option>, pub completed_before: Option>, + pub status: Option, pub success: Option, pub running: Option, pub parent_job: Option, @@ -680,6 +681,7 @@ mod tests { created_after_queue: None, completed_after: None, completed_before: None, + status: None, success: None, running: Some(true), parent_job: None, @@ -752,6 +754,7 @@ mod tests { created_after_queue: Some(specific_time), completed_after: None, completed_before: None, + status: None, success: None, running: None, parent_job: None, diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 29fa215800..e2d495eaad 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -242,6 +242,23 @@ async fn create_schedule( let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; + // A git-sync/merge/create write into a fork never sets operational state: + // force `enabled = false` so a cloned / synced / merged / UI-created schedule + // can't fire alongside the parent's. The fork owner re-enables locally via + // `setenabled`. Schedule analog of the trigger rule in + // `windmill-trigger::handler::workspace_is_fork`; the read half (parent-value + // substitution on fork export) lives in `workspaces_export.rs`. Read fork-ness + // on the non-RLS `db` pool (like the other two sites) so the determination is + // complete regardless of the caller's folder perms. + let target_is_fork: bool = sqlx::query_scalar!( + "SELECT parent_workspace_id IS NOT NULL FROM workspace WHERE id = $1", + w_id + ) + .fetch_optional(&db) + .await? + .flatten() + .unwrap_or(false); + // Check schedule for error ScheduleType::from_str(&ns.schedule, ns.cron_version.as_deref(), true)?; @@ -341,7 +358,12 @@ async fn create_schedule( // flows (CLI merge, UI merge, `wmill push` of a fork tarball) — which // either send the source's actual flag (create case) or omit `enabled` // entirely (update case, where `EditSchedule` lacks the field). - ns.enabled.unwrap_or(true), + // A write into a fork always lands disabled regardless of the request. + if target_is_fork { + false + } else { + ns.enabled.unwrap_or(true) + }, resolved_email, resolved_permissioned_as, ns.on_failure, @@ -413,7 +435,7 @@ async fn create_schedule( .await?; } - if ns.enabled.unwrap_or(true) { + if !target_is_fork && ns.enabled.unwrap_or(true) { tx = push_scheduled_job(&db, tx, &schedule, Some(&authed.clone().into()), None).await? } tx.commit().await?; diff --git a/backend/windmill-api-scripts/Cargo.toml b/backend/windmill-api-scripts/Cargo.toml index 200a2aef64..5ea3088837 100644 --- a/backend/windmill-api-scripts/Cargo.toml +++ b/backend/windmill-api-scripts/Cargo.toml @@ -13,6 +13,7 @@ default = [] enterprise = ["windmill-common/enterprise"] private = ["windmill-common/private", "windmill-dep-map/private"] python = ["dep:windmill-parser-py"] +prometheus = ["dep:prometheus", "windmill-common/prometheus"] [dependencies] windmill-common = { workspace = true, default-features = false } windmill-object-store.workspace = true @@ -38,4 +39,5 @@ tracing.workspace = true chrono.workspace = true lazy_static.workspace = true tokio.workspace = true +prometheus = { workspace = true, optional = true } windmill-parser-py = { workspace = true, optional = true } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index b2ed92ad8f..3ad18e4dcf 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -9,7 +9,8 @@ use axum::extract::Multipart; use windmill_api_auth::{ auth::{list_tokens_internal, AuthCache, TruncatedTokenWithEmail}, - check_scopes, maybe_refresh_folders, require_owner_of_path, ApiAuthed, + build_scope_path_predicate, check_scopes, maybe_refresh_folders, require_owner_of_path, + ApiAuthed, }; use windmill_common::{ utils::{BulkDeleteRequest, WithStarredInfoQuery, HTTP_CLIENT}, @@ -277,6 +278,7 @@ async fn list_search_scripts( #[cfg(not(feature = "enterprise"))] let n = 10; + let allowed = build_scope_path_predicate(&authed, "scripts", "read"); let rows = sqlx::query_as!( SearchScript, "SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2", @@ -286,6 +288,7 @@ async fn list_search_scripts( .fetch_all(&mut *tx) .await? .into_iter() + .filter(|r| allowed(&r.path)) .collect::>(); tx.commit().await?; Ok(Json(rows)) @@ -440,9 +443,13 @@ async fn list_scripts( let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; + let allowed = build_scope_path_predicate(&authed, "scripts", "read"); let rows = sqlx::query_as::<_, ListableScript>(&sql) .fetch_all(&mut *tx) - .await?; + .await? + .into_iter() + .filter(|r| allowed(&r.path)) + .collect::>(); tx.commit().await?; Ok(Json(rows)) } @@ -2171,14 +2178,64 @@ async fn raw_script_by_path_unpinned( lazy_static::lazy_static! { static ref DEBUG_RAW_SCRIPT_ENDPOINTS: bool = std::env::var("DEBUG_RAW_SCRIPT_ENDPOINTS").is_ok(); + + /// Fallback freshness window (seconds) for [`RAW_SCRIPT_LATEST_HASH_CACHE`]. + /// Primary invalidation is event-driven: deploying a script writes a + /// `notify_runnable_version_change` row, and the server's polling-events handler + /// evicts the entry across all replicas (see `main.rs`). This TTL only bounds + /// staleness if that event is missed. Defaults to 60s (matches + /// `DEPLOYED_SCRIPT_HASH_CACHE`). Override with `RAW_SCRIPT_CACHE_TTL_SECONDS`. + static ref RAW_SCRIPT_CACHE_TTL_S: i64 = std::env::var("RAW_SCRIPT_CACHE_TTL_SECONDS") + .ok() + .and_then(|s| s.parse::().ok()) + .filter(|s| *s >= 0) + .unwrap_or(60); } lazy_static::lazy_static! { + // Imported-script content, keyed by + // `{ws}:{path}:{importer_cache_key}[:unpinned]:{latest_hash}`. Including the + // imported script's own latest hash makes each entry immutable, so no + // per-entry TTL is needed; staleness is bounded by RAW_SCRIPT_LATEST_HASH_CACHE. pub static ref RAW_SCRIPT_CACHE: Cache = Cache::new(1000); + // `{ws}:{path}` (bare path) -> (latest non-archived hash, unix_ts cached). + // Resolving the imported script's own hash and keying content by it is what + // fixes relative-import staleness for deployed scripts, whose importer hash + // never moves (see #6769). Evicted on deploy by the `notify_runnable_version_change` + // handler in main.rs (cross-replica, within a poll interval); RAW_SCRIPT_CACHE_TTL_S + // is a fallback bound. + pub static ref RAW_SCRIPT_LATEST_HASH_CACHE: Cache = Cache::new(1000); pub static ref CACHE_FOLDERS_PATH: Cache = Cache::new(1000); } +/// Records a [`RAW_SCRIPT_CACHE`] lookup outcome (`hit` / `expired` / `miss`) to +/// the `raw_script_cache_total` counter when the prometheus feature is enabled. +#[cfg(feature = "prometheus")] +fn record_raw_script_cache(result: &str) { + if let Some(c) = RAW_SCRIPT_CACHE_METRIC.as_ref() { + c.with_label_values(&[result]).inc(); + } +} + +#[cfg(not(feature = "prometheus"))] +fn record_raw_script_cache(_result: &str) {} + +#[cfg(feature = "prometheus")] +lazy_static::lazy_static! { + /// Raw relative-import cache lookups, labeled by `result` (hit/expired/miss). + static ref RAW_SCRIPT_CACHE_METRIC: Option = + if windmill_common::METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { + Some(prometheus::register_int_counter_vec!( + "raw_script_cache_total", + "Raw script relative-import cache lookups by result (hit/expired/miss)", + &["result"] + ).unwrap()) + } else { + None + }; +} + async fn raw_script_by_path_internal( path: StripPath, user_db: UserDB, @@ -2200,23 +2257,10 @@ async fn raw_script_by_path_internal( } } - let cache_path = query - .cache_key - .map(|x| format!("{w_id}:{path}:{x}{}", if unpin { ":unpinned" } else { "" })); - if let Some(cache_path) = cache_path.clone() { - let cached_content = RAW_SCRIPT_CACHE.get(&cache_path); - if let Some(cached_content) = cached_content { - if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!("Raw script by path request: {} (cached)", path); - } - return Ok(cached_content); - } - } - - if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!("Raw script by path request: {} (not cached)", path); - } - + // Validate + strip the language extension up front so cache keys use the bare + // script path. This matches the `notify_runnable_version_change` event payload + // (which carries the bare path), so a deploy can evict RAW_SCRIPT_LATEST_HASH_CACHE + // by key from the polling-events handler in the server binary. if !path.ends_with(".py") && !path.ends_with(".ts") && !path.ends_with(".go") @@ -2235,6 +2279,52 @@ async fn raw_script_by_path_internal( .trim_end_matches(".go") .trim_end_matches(".sh"); + // Content cache is keyed by the IMPORTED script's own latest hash, not by the + // importer's runnable hash (`query.cache_key`). The importer hash never moves + // when only an imported script's content changes (relock is in-place — see + // #6769), so keying solely on it served stale content indefinitely. The + // importer + unpin dimensions are kept to preserve per-runnable authorization + // scoping (a content-cache hit skips the authed RLS query, so an entry must + // stay scoped to the runnable that fetched it); the imported latest hash is + // appended for content correctness. + let cache_path_base = query + .cache_key + .as_ref() + .map(|x| format!("{w_id}:{path}:{x}{}", if unpin { ":unpinned" } else { "" })); + + // Resolve the imported script's latest hash from RAW_SCRIPT_LATEST_HASH_CACHE + // (keyed by the bare path so the deploy event can evict it). A fresh entry + // serves from the immutable content cache with no DB hit; a stale/absent entry + // falls through to the query below, which refreshes both caches. + let hash_cache_key = format!("{w_id}:{path}"); + let (fresh_hash, had_stale_hash) = match RAW_SCRIPT_LATEST_HASH_CACHE.get(&hash_cache_key) { + Some((hash, cached_at)) + if chrono::Utc::now().timestamp() - cached_at <= *RAW_SCRIPT_CACHE_TTL_S => + { + (Some(hash), false) + } + Some(_) => (None, true), + None => (None, false), + }; + + if let (Some(base), Some(latest_hash)) = (cache_path_base.as_ref(), fresh_hash) { + let content_key = format!("{base}:{latest_hash}"); + if let Some(cached_content) = RAW_SCRIPT_CACHE.get(&content_key) { + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + tracing::warn!("Raw script by path request: {path} (cached, key={content_key})"); + } + record_raw_script_cache("hit"); + return Ok(cached_content); + } + } + if cache_path_base.is_some() { + record_raw_script_cache(if had_stale_hash { "expired" } else { "miss" }); + } + + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + tracing::warn!("Raw script by path request: {} (not cached)", path); + } + // folder cache is only useful for python given it needs to recuse over all intermediate folders to find the package. // When a script exists in a folder, we can cache the fact that the folder exists to avoid extra db calls. let mut split_path = path.split("/").collect::>(); @@ -2267,8 +2357,10 @@ async fn raw_script_by_path_internal( let mut tx = user_db.begin(&authed).await?; - let content_o = sqlx::query_scalar!( - "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", + // Fetch the latest non-archived row's hash AND content in one query: the hash + // keys the (immutable) content cache and refreshes RAW_SCRIPT_LATEST_HASH_CACHE. + let row_o = sqlx::query!( + "SELECT hash, content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", path, w_id ) @@ -2276,6 +2368,10 @@ async fn raw_script_by_path_internal( .warn_after_seconds(5) .await?; tx.commit().await?; + let (db_hash, content_o) = match row_o { + Some(r) => (Some(r.hash), Some(r.content)), + None => (None, None), + }; if *DEBUG_RAW_SCRIPT_ENDPOINTS { tracing::warn!( "Raw script by path request: {} (content: {:?})", @@ -2339,8 +2435,14 @@ async fn raw_script_by_path_internal( } } - if let Some(cache_path) = cache_path { - RAW_SCRIPT_CACHE.insert(cache_path, content.clone()); + // content_o was Some, so db_hash is Some too (same row). Refresh the latest-hash + // cache and store the content under the hash-qualified key. + if let Some(db_hash) = db_hash { + RAW_SCRIPT_LATEST_HASH_CACHE + .insert(hash_cache_key, (db_hash, chrono::Utc::now().timestamp())); + if let Some(base) = cache_path_base { + RAW_SCRIPT_CACHE.insert(format!("{base}:{db_hash}"), content.clone()); + } } if *DEBUG_RAW_SCRIPT_ENDPOINTS { tracing::warn!("Raw script by path request: {} (content response)", path); diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 95b9527a8e..db9ecb118d 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -139,6 +139,10 @@ pub fn global_service() -> Router { "/tokens/update_scopes/{token_prefix}", post(update_token_scopes), ) + .route( + "/tokens/update_label/{token_prefix}", + post(update_token_label), + ) .route("/tokens/list", get(list_tokens)) .route("/tokens/impersonate", post(impersonate)) .route("/usage", get(get_usage)) @@ -2408,6 +2412,89 @@ async fn update_token_scopes( Ok(format!("updated scopes for token {prefix}")) } +#[derive(Deserialize)] +struct UpdateTokenLabelRequest { + label: Option, +} + +async fn update_token_label( + Extension(db): Extension, + authed: ApiAuthed, + Path(token_prefix): Path, + Json(req): Json, +) -> Result { + // The new label must not collide with a system-token namespace (`session`, + // `ephemeral*`, `debugger-token`, `mcp-oauth-*`): those labels are + // load-bearing, and a user-set collision would orphan the token — hidden + // from the UI (`isUserToken`) and rejected by the editability guard below — + // while it still authenticates. (`is_user_token(None)` is true, so clearing + // the label is allowed.) + if !windmill_common::auth::is_user_token(req.label.as_deref()) { + return Err(Error::BadRequest( + "label collides with a reserved system-token namespace".to_string(), + )); + } + + // Matches the `token.label VARCHAR(1000)` column — reject overlong labels with + // a 400 rather than letting Postgres raise a 500. + const MAX_TOKEN_LABEL_LEN: usize = 1000; + if req + .label + .as_deref() + .is_some_and(|l| l.chars().count() > MAX_TOKEN_LABEL_LEN) + { + return Err(Error::BadRequest(format!( + "label must be at most {MAX_TOKEN_LABEL_LEN} characters" + ))); + } + + let mut tx = db.begin().await?; + + // Only user-created tokens may be relabeled — system tokens carry the + // load-bearing labels described above. This SQL mirrors the canonical + // `windmill_common::auth::is_user_token`; keep the two in sync (note the + // case-insensitive `ephemeral` match). + let updated: Option = sqlx::query_scalar!( + "UPDATE token SET label = $1 + WHERE email = $2 AND token_prefix = $3 + AND (label IS NULL OR ( + label <> 'session' + AND lower(label) NOT LIKE 'ephemeral%' + AND label <> 'debugger-token' + AND label NOT LIKE 'mcp-oauth-%' + )) + RETURNING token_prefix", + req.label.as_deref(), + &authed.email, + &token_prefix, + ) + .fetch_optional(&mut *tx) + .await?; + + let prefix = updated.ok_or_else(|| { + Error::NotFound(format!( + "token {token_prefix} not found, not owned by user, or not editable" + )) + })?; + + audit_log( + &mut *tx, + &authed, + "users.token.update_label", + ActionKind::Update, + &"global", + Some(&prefix), + Some([("label", req.label.as_deref().unwrap_or(""))].into()), + ) + .await?; + + tx.commit().await?; + + windmill_api_auth::invalidate_token_from_cache(&prefix); + + Ok(format!("updated label for token {prefix}")) +} + async fn leave_workspace( Extension(db): Extension, Path(w_id): Path, diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index db60ae7fac..00cebc32ff 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -340,6 +340,8 @@ pub struct InstanceAISummary { #[serde(skip_serializing_if = "Option::is_none")] pub default_model: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub metadata_model: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub code_completion_model: Option, } @@ -825,6 +827,7 @@ pub fn build_instance_ai_summary(config: Option<&serde_json::Value>) -> Option- + Workspaces that reference this database via a ducklake + catalog or datatable database with resource_type + 'instance'. Computed at request time, not persisted. /settings/setup_custom_instance_pg_database/{name}: post: summary: >- @@ -4842,6 +4850,10 @@ paths: required: &ref_44 - model - provider + metadata_model: + type: object + properties: *ref_43 + required: *ref_44 code_completion_model: type: object properties: *ref_43 @@ -5828,6 +5840,10 @@ paths: type: object properties: *ref_43 required: *ref_44 + metadata_model: + type: object + properties: *ref_43 + required: *ref_44 code_completion_model: type: object properties: *ref_43 @@ -11364,6 +11380,13 @@ paths: description: >- If true, all steps run on the same worker for better performance + preserve_step_tags: + type: boolean + description: >- + If true and the flow runs on a custom worker tag, + steps that declare their own non-empty tag run on + it instead of inheriting the flow tag. Steps + without their own tag still inherit the flow tag. concurrent_limit: type: number description: >- @@ -12619,6 +12642,11 @@ paths: type: array items: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this script + does not delete an existing user draft at the same path. required: &ref_105 - path - summary @@ -15461,6 +15489,12 @@ paths: type: boolean deployment_message: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this + flow does not delete an existing user draft at the same + path. responses: '201': description: flow created @@ -15507,6 +15541,12 @@ paths: properties: deployment_message: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this + flow does not delete an existing user draft at the same + path. responses: '200': description: flow updated @@ -16244,6 +16284,11 @@ paths: type: array items: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this app + does not delete an existing user draft at the same path. required: - path - value @@ -16303,6 +16348,12 @@ paths: type: array items: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this + app does not delete an existing user draft at the same + path. required: - path - value @@ -16740,6 +16791,11 @@ paths: type: array items: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this app + does not delete an existing user draft at the same path. responses: '200': description: app updated @@ -16796,6 +16852,12 @@ paths: type: array items: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this + app does not delete an existing user draft at the same + path. js: type: string css: @@ -17954,6 +18016,13 @@ paths: description: >- If true, all steps run on the same worker for better performance + preserve_step_tags: + type: boolean + description: >- + If true and the flow runs on a custom worker tag, steps + that declare their own non-empty tag run on it instead + of inheriting the flow tag. Steps without their own tag + still inherit the flow tag. concurrent_limit: type: number description: Maximum number of concurrent executions of this flow @@ -30256,6 +30325,37 @@ paths: type: object additionalProperties: type: integer + /workers/workspace_fairness_events: + get: + summary: list last 100 workspace-fairness cap/uncap events (cloud-only) + operationId: getWorkspaceFairnessEvents + tags: + - worker + responses: + '200': + description: workspace fairness events (empty on non-cloud) + content: + application/json: + schema: + type: array + items: + type: object + properties: + timestamp: + type: string + format: date-time + operation: + type: string + workspace_id: + type: string + nullable: true + parameters: + type: object + nullable: true + additionalProperties: true + required: + - timestamp + - operation /configs/list_worker_groups: get: summary: list worker groups diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 22014e1229..f651692005 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.714.0 + version: 1.719.0 title: Windmill API contact: @@ -5120,6 +5120,37 @@ paths: schema: type: string + /users/tokens/update_label/{token_prefix}: + post: + summary: update label of an existing token (owner only) + operationId: updateTokenLabel + tags: + - user + parameters: + - name: token_prefix + in: path + required: true + schema: + type: string + requestBody: + description: new label (null or omitted = no label) + required: true + content: + application/json: + schema: + type: object + properties: + label: + type: string + nullable: true + responses: + "200": + description: label updated + content: + text/plain: + schema: + type: string + /users/tokens/list: get: summary: list token @@ -9318,6 +9349,33 @@ paths: application/json: schema: {} + /w/{workspace}/jobs/job_view_token/{id}: + get: + summary: mint a read-only share token for a job + description: > + Returns a stateless `{job_id}.{hmac}` token that grants an authenticated + workspace member read access to this job (and its flow subtree) via a + `view_token` query param or `X-View-Token` header. Only callable by a user + who can already read the job. + operationId: getJobViewToken + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: the share read token + content: + text/plain: + schema: + type: string + /w/{workspace}/flows/list_paths: get: summary: list all flow paths @@ -11794,6 +11852,16 @@ paths: in: query schema: type: boolean + - name: status + description: filter on the exact completed job status. Unlike `success=true` (which also matches `skipped`), `status=success` matches only `success`. + in: query + schema: + type: string + enum: + - success + - failure + - canceled + - skipped - name: all_workspaces description: get jobs from all workspaces (only valid if request come from the `admins` workspace) in: query @@ -12029,6 +12097,16 @@ paths: - $ref: "#/components/parameters/StartedBefore" - $ref: "#/components/parameters/StartedAfter" - $ref: "#/components/parameters/Success" + - name: status + description: filter on the exact completed job status. Unlike `success=true` (which also matches `skipped`), `status=success` matches only `success`. + in: query + schema: + type: string + enum: + - success + - failure + - canceled + - skipped - $ref: "#/components/parameters/JobKinds" - $ref: "#/components/parameters/ArgsFilter" - $ref: "#/components/parameters/ResultFilter" @@ -12236,6 +12314,16 @@ paths: in: query schema: type: boolean + - name: status + description: filter on the exact completed job status. Unlike `success=true` (which also matches `skipped`), `status=success` matches only `success`. + in: query + schema: + type: string + enum: + - success + - failure + - canceled + - skipped - name: all_workspaces description: get jobs from all workspaces (only valid if request come from the `admins` workspace) in: query @@ -19814,6 +19902,16 @@ paths: in: query schema: type: boolean + - name: status + description: filter on the exact completed job status. Unlike `success=true` (which also matches `skipped`), `status=success` matches only `success`. + in: query + schema: + type: string + enum: + - success + - failure + - canceled + - skipped - name: all_workspaces description: get jobs from all workspaces (only valid if request come from the `admins` workspace) in: query @@ -21611,6 +21709,8 @@ components: $ref: "#/components/schemas/AIProviderConfig" default_model: $ref: "#/components/schemas/AIProviderModel" + metadata_model: + $ref: "#/components/schemas/AIProviderModel" code_completion_model: $ref: "#/components/schemas/AIProviderModel" custom_prompts: @@ -21646,6 +21746,8 @@ components: $ref: "#/components/schemas/InstanceAIProviderSummary" default_model: $ref: "#/components/schemas/AIProviderModel" + metadata_model: + $ref: "#/components/schemas/AIProviderModel" code_completion_model: $ref: "#/components/schemas/AIProviderModel" required: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 0e096eb798..fb92d28cb3 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -377,6 +377,8 @@ pub struct AIConfig { #[serde(skip_serializing_if = "Option::is_none")] pub default_model: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub metadata_model: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub code_completion_model: Option, #[serde(skip_serializing_if = "Option::is_none")] pub custom_prompts: Option>, diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index ece28e51de..3793e94b02 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -13,9 +13,10 @@ use serde_json::value::RawValue; use sqlx::types::JsonRawValue; use windmill_common::{ error::Error, + jobs::WM_TRACEPARENT, triggers::{RunnableFormat, RunnableFormatVersion, TriggerKind}, worker::to_raw_value, - DB, + DB, OTEL_TRACING_ENABLED, }; use windmill_queue::PushArgsOwned; @@ -280,6 +281,13 @@ impl WebhookArgs { self, runnable_format: RunnableFormat, ) -> Result { + // Capture the inbound W3C `traceparent` before `self.metadata` is + // consumed below. Read back at root-job completion to link the job's + // OTLP span to the originating distributed trace. Deliberately bypasses + // the header whitelist, and is gated to OTel-enabled instances so others + // don't get a stray `_wm_traceparent` arg key. + let trace_context = inbound_traceparent(&self.metadata.headers); + let headers = build_headers( &self.metadata.headers, self.metadata.query_include_header, @@ -292,7 +300,7 @@ impl WebhookArgs { runnable_format.has_preprocessor, ); - match runnable_format { + let mut push_args = match runnable_format { RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { let mut args = HashMap::new(); @@ -307,7 +315,7 @@ impl WebhookArgs { }), ); - Ok(PushArgsOwned { args, extra: None }) + PushArgsOwned { args, extra: None } } RunnableFormat { has_preprocessor, .. } => { let mut extra = HashMap::new(); @@ -343,16 +351,40 @@ impl WebhookArgs { if query_wrap_body { body = HashMap::from([("body".to_string(), to_raw_value(&body))]); } - Ok(PushArgsOwned { args: body, extra }) + PushArgsOwned { args: body, extra } } Body::NoHashMap(args) => { let mut hm = HashMap::new(); hm.insert("body".to_string(), args); - Ok(PushArgsOwned { args: hm, extra }) + PushArgsOwned { args: hm, extra } } } } + }; + + // `_wm_traceparent` is Windmill-controlled: strip any caller-supplied + // value (e.g. smuggled through the request body) so only the header we + // captured above can become the job's inbound trace context. Then stash + // the captured value as a reserved arg key — it rides the `args` jsonb + // like `_ENTRYPOINT_OVERRIDE`; normal scripts never see it (args are + // bound by declared parameter name). + push_args.args.remove(WM_TRACEPARENT); + if let Some(ref mut extra) = push_args.extra { + extra.remove(WM_TRACEPARENT); } + if let Some(trace_context) = trace_context { + let raw = to_raw_value(&trace_context); + match push_args.extra { + Some(ref mut extra) => { + extra.insert(WM_TRACEPARENT.to_string(), raw); + } + None => { + push_args.args.insert(WM_TRACEPARENT.to_string(), raw); + } + } + } + + Ok(push_args) } } @@ -487,6 +519,23 @@ lazy_static::lazy_static! { .collect()).unwrap_or_default(); } +/// Extract the inbound W3C `traceparent` header so the enqueued job can be +/// linked back to the originating distributed trace. Returns `None` when OTel +/// tracing is disabled (so non-tracing instances don't accumulate a stray +/// reserved arg key) or when no `traceparent` header is present. The W3C format +/// is not validated here — it is checked later at use time +/// (`valid_w3c_traceparent` for the env, EE `span_cx_from_traceparent` for the +/// span). +fn inbound_traceparent(headers: &HeaderMap) -> Option { + if !OTEL_TRACING_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { + return None; + } + headers + .get("traceparent") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) +} + pub fn build_headers( headers: &HeaderMap, include_header: Option, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 71f9645272..891fc3c9bf 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -17,6 +17,7 @@ use itertools::Itertools; use quick_cache::sync::Cache; use serde_json::value::RawValue; use serde_json::Value; +use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; @@ -344,6 +345,10 @@ pub fn workspaced_service() -> Router { "/result_by_id/{job_id}/{node_id}", get(get_result_by_id).layer(cors.clone()), ) + .route( + "/job_view_token/{id}", + get(get_job_view_token).layer(cors.clone()), + ) .route("/run/dependencies", post(run_dependencies_job)) .route("/run/dependencies_async", post(run_dependencies_job_async)) .route("/run/flow_dependencies", post(run_flow_dependencies_job)) @@ -427,12 +432,27 @@ struct JsonPath { pub approver: Option, } async fn get_result_by_id( + OptViewToken(view_token): OptViewToken, authed: ApiAuthed, tokened: Tokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, flow_id, node_id)): Path<(String, Uuid, String)>, Query(JsonPath { json_path, .. }): Query, ) -> windmill_common::error::JsonResult> { + // Reading a node's result requires being able to read the flow itself (the node + // belongs to it). Gate on the flow's visibility (created_by / RLS / root + // inheritance) before resolving via the root DB. + require_job_update_read_access( + &db, + &user_db, + &authed, + &w_id, + &flow_id, + view_token.as_deref(), + ) + .await?; + let res = windmill_queue::get_result_by_id(db.clone(), w_id.clone(), flow_id, node_id, json_path) .await?; @@ -442,6 +462,25 @@ async fn get_result_by_id( Ok(Json(res)) } +/// Mint a stateless "share read link" token for a job. Only a caller who can already +/// read the job (creator / RLS / flow ancestor / admin) may mint it. The returned +/// `{job_id}.{hmac}` is passed back as the `view_token` query param on the run page's +/// reads, granting an authenticated member read of this job and its flow subtree. +async fn get_job_view_token( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> error::Result { + // No `view_token` here: minting requires the caller's own read access, so a share + // link cannot be used to mint further links. `require_job_read_access` also + // enforces the caller's `if_jobs:filter_tags` scope, so a tag-scoped token can't + // mint a transferable link for a job outside its allowed tags. + require_job_update_read_access(&db, &user_db, &authed, &w_id, &id, None).await?; + let hmac = generate_view_token(&w_id, id, &db).await?; + Ok(format!("{id}.{hmac}")) +} + async fn get_root_job( Extension(db): Extension, Path((w_id, id)): Path<(String, Uuid)>, @@ -691,9 +730,11 @@ async fn get_scheduled_for( } async fn get_flow_job_debug_info( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, tokened_o: OptTokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::Result { let job = GetQuery::new() @@ -701,6 +742,18 @@ async fn get_flow_job_debug_info( .fetch_queued((&db).into(), &id, &w_id) .await?; if let Some(job) = job { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &job.created_by, + view_token.as_deref(), + ) + .await?; + } let is_flow = job.is_flow(); if job.is_flow_step || !is_flow { return Err(error::Error::BadRequest( @@ -858,10 +911,338 @@ struct GetJobQuery { pub approval_token: Option, } +/// Authorize an *authenticated* caller to read a single job's data +/// (full job / args / result / logs / live updates). +/// +/// Single-job read endpoints query through the root `DB` (RLS-bypassing), filtered +/// only by job id + workspace (+ token scope tags). That is required for the +/// unauthenticated approval / public-trigger / anonymous-job flows, but for a +/// logged-in user it meant any workspace member — e.g. a viewer with no ACL on the +/// runnable — could read another user's job args/result/logs simply by obtaining the +/// job UUID, even though the same job is hidden from them in `jobs/list` +/// (RLS-filtered) and the underlying script returns 404. (WIN-2026-jobs-read) +/// +/// Unauthenticated callers are still handled by each handler's anonymous-job check; +/// this gate applies only when a user is authenticated. Access is granted when: +/// - the caller created the job (`created_by`) — covers app components, webhooks and +/// the caller's own runs, whose `permissioned_as` is the policy identity rather +/// than the caller, so they would otherwise fail the RLS probe; or +/// - the job is visible to the caller under the same RLS as `jobs/list`, probed on +/// `v2_job` via `user_db` (admins BYPASSRLS). +/// +/// Optional share-read-link token (validated by [`validate_view_token`]). Read from +/// the `view_token` query parameter — needed for `EventSource`/SSE and direct links, +/// which can't set headers — falling back to the `X-View-Token` header, which lets the +/// frontend attach it to every generated-client request via a single interceptor +/// instead of threading it through each call. Read independently of each handler's own +/// `Query` extractor (axum allows only one typed `Query`). +pub struct OptViewToken(pub Option); + +impl axum::extract::FromRequestParts for OptViewToken { + type Rejection = std::convert::Infallible; + + async fn from_request_parts( + parts: &mut axum::http::request::Parts, + _state: &S, + ) -> std::result::Result { + let from_query = parts.uri.query().and_then(|q| { + serde_urlencoded::from_str::>(q) + .ok() + .and_then(|pairs| { + pairs + .into_iter() + .find(|(k, _)| k == "view_token") + .map(|(_, v)| v) + }) + }); + let token = from_query.or_else(|| { + parts + .headers + .get("x-view-token") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) + }); + Ok(OptViewToken(token)) + } +} + +/// Otherwise returns 404 — matching `scripts/get` and avoiding existence disclosure. +async fn require_job_read_access( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + job_id: &Uuid, + created_by: &str, + view_token: Option<&str>, +) -> error::Result<()> { + // Tag scope (`if_jobs:filter_tags:`) is an orthogonal hard restriction on a + // scoped token: it must never read a job outside its allowed tags, regardless of + // how authorization is otherwise satisfied (created_by / view token / RLS). Most + // read handlers also tag-filter their data query, but some (result_by_id, + // get_flow_job_debug_info, get_otel_traces) do not, so enforce it here — before + // the grants below — so a share token can't be used to escape the tag scope. + // `get_scope_tags` is `None` for unscoped callers (the common case), so this adds + // no query for normal sessions/tokens. + if let Some(tags) = get_scope_tags(authed) { + let in_scope = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM v2_job WHERE id = $1 AND workspace_id = $2 AND tag = ANY($3))", + job_id, + w_id, + &tags.iter().map(|t| t.to_string()).collect::>(), + ) + .fetch_one(db) + .await? + == Some(true); + if !in_scope { + return Err(Error::NotFound(format!("Job {job_id} not found"))); + } + } + + // Fast path: you can always read a job you launched. This is also load-bearing + // for apps — a component job runs as the app policy's `permissioned_as`, but its + // `created_by` is the launching viewer, so the RLS probe below would hide it. + if created_by == authed.username { + return Ok(()); + } + + // `username_override` is derived from the token *label* (`username_override_from_label`), + // which is fully user-controlled with no uniqueness/ownership check (webhook-/http-/ + // email-/ws- trigger tokens, `ephemeral-script-end-user-*`, and the generic `label-*` + // all flow through it). A bare `username_override == created_by` match is therefore + // forgeable: any member can mint a token with a colliding label and read another + // principal's jobs (IDOR — results/args/logs with resolved secrets). Bind the grant to + // a non-forgeable attribute instead: the job must actually run as the caller's own + // identity, i.e. its `permissioned_as_email` (the token owner's email, never set from + // the label) equals `authed.email`. This still admits every legitimate same-owner + // re-read (trigger tokens reading their own webhook/http/email jobs, the + // ephemeral-script-end-user worker token, generic labeled tokens) while denying + // cross-principal collisions. The DB hit only happens when an override is present and + // matches, so the common session/token path stays query-free. + if authed + .username_override + .as_deref() + .is_some_and(|u| u == created_by) + { + let job_email = sqlx::query_scalar!( + "SELECT permissioned_as_email FROM v2_job WHERE id = $1 AND workspace_id = $2", + job_id, + w_id, + ) + .fetch_optional(db) + .await?; + if job_email.as_deref() == Some(authed.email.as_str()) { + return Ok(()); + } + } + + // Share read link: a valid view token minted by someone with read access grants + // this authenticated member read of the shared job and its flow subtree. + if let Some(token) = view_token { + if validate_view_token(db, w_id, job_id, token).await? { + return Ok(()); + } + } + + // The probe below (chain walk + an RLS-scoped transaction) is comparatively + // expensive and the same (caller, job) is hit repeatedly — e.g. `getupdate` + // polling of a run you can see but did not launch, or an admin watching many + // runs. Cache the boolean outcome. All job-side inputs to the decision + // (created_by, runnable_path, permissioned_as, visible_to_owner, flow lineage) + // are immutable after creation, and every mutable caller-side input + // (is_admin / username / username_override / groups / folders) is folded into + // the key — so a permission change yields a new key rather than a stale hit, and + // no TTL is needed (size-bounded LRU; mirrors apps' PERMIT_CACHE). + let cache_key = job_read_access_cache_key(authed, w_id, job_id); + let visible = if let Some(visible) = JOB_READ_ACCESS_CACHE.get(&cache_key) { + visible + } else { + // Visibility is inherited along the flow hierarchy: if you can read ANY flow + // that (transitively) contains this job, you can read the job. A step runs as + // its flow's `permissioned_as` but its `runnable_path` is the inner runnable's + // — which the caller may have no direct ACL on — and the flow-run UI fetches + // each step by id, so gating purely on the step's own RLS visibility would + // break inspecting a flow you can see but did not launch. We therefore probe + // RLS visibility of the job OR any of its `parent_job` ancestors (admins + // BYPASSRLS) — the same visibility as `jobs/list`. + let chain_ids = job_ancestor_chain_ids(db, w_id, job_id).await?; + + let mut tx = user_db.clone().begin(authed).await?; + let visible = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM v2_job WHERE id = ANY($1) AND workspace_id = $2)", + &chain_ids[..], + w_id, + ) + .fetch_one(&mut *tx) + .await? + == Some(true); + tx.commit().await?; + + JOB_READ_ACCESS_CACHE.insert(cache_key, visible); + visible + }; + + if visible { + return Ok(()); + } + + // Denied. Distinguish "the run exists but you lack access" (actionable: ask a + // colleague for a share link) from "no such run", so the UI can guide the user. + // Only authenticated members reach this point and job UUIDs are non-enumerable, + // so disclosing mere existence to a member is an acceptable trade-off for the UX. + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM v2_job WHERE id = ANY($1) AND workspace_id = $2)", + &[*job_id][..], + w_id, + ) + .fetch_one(db) + .await? + == Some(true); + if exists { + Err(Error::PermissionDenied(format!( + "You do not have access to run {job_id}. Ask a user who can see it to open the run and \ + share a read-only link with you (the \"Share\" button on the run page)." + ))) + } else { + Err(Error::NotFound(format!("Job {job_id} not found"))) + } +} + +/// Self + every `parent_job` ancestor (intermediate sub-flows up to the top-level +/// root) of `job_id`, resolved via the root DB (flow lineage is not sensitive). +/// Falls back to `[job_id]` if the row is absent so callers still run their probe. +async fn job_ancestor_chain_ids(db: &DB, w_id: &str, job_id: &Uuid) -> error::Result> { + let chain_ids = sqlx::query_scalar!( + r#"WITH RECURSIVE chain(id, parent_job) AS ( + SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2 + UNION ALL + SELECT j.id, j.parent_job FROM v2_job j + JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2 + ) + SELECT id AS "id!" FROM chain"#, + job_id, + w_id, + ) + .fetch_all(db) + .await?; + Ok(if chain_ids.is_empty() { + vec![*job_id] + } else { + chain_ids + }) +} + +/// A share read link token has the form `{shared_job_id}.{hmac}` where `hmac` is +/// [`windmill_common::variables::generate_view_token`] for `shared_job_id`. It grants +/// read of that job and its whole flow subtree, so the run page can present a single +/// link that also renders the flow's steps. Returns true iff the signature is valid +/// AND `accessed_job_id` is the shared job or one of its descendants. +async fn validate_view_token( + db: &DB, + w_id: &str, + accessed_job_id: &Uuid, + token: &str, +) -> error::Result { + let Some((shared_id_str, provided_hmac)) = token.split_once('.') else { + return Ok(false); + }; + let Ok(shared_id) = Uuid::parse_str(shared_id_str) else { + return Ok(false); + }; + let Ok(provided_bytes) = hex::decode(provided_hmac) else { + return Ok(false); + }; + // Constant-time verification (same domain as `generate_view_token`, mirroring + // `verify_suspended_secret`); avoids the timing side-channel of comparing the + // hex strings with `!=`. + let key = get_workspace_key(w_id, db).await?; + let mut mac = HmacSha256::new_from_slice(key.as_bytes()).map_err(to_anyhow)?; + mac.update(shared_id.as_bytes()); + mac.update(b"view_token"); + if mac.verify_slice(&provided_bytes).is_err() { + return Ok(false); + } + if accessed_job_id == &shared_id { + return Ok(true); + } + // The token authorizes the shared job's subtree: accessed must descend from it, + // i.e. the shared job is among accessed's ancestors. + let chain = job_ancestor_chain_ids(db, w_id, accessed_job_id).await?; + Ok(chain.contains(&shared_id)) +} + +lazy_static::lazy_static! { + /// Caches the result of the `require_job_read_access` RLS visibility probe, + /// keyed by the caller's authorization-relevant identity plus the job id (see + /// [`job_read_access_cache_key`]). No TTL: the cached decision is a pure function + /// of immutable job-side state and the caller-side state encoded in the key, so a + /// permission change re-keys rather than going stale. Size-bounded LRU. + static ref JOB_READ_ACCESS_CACHE: Cache<[u8; 32], bool> = Cache::new(50_000); +} + +/// Key for [`JOB_READ_ACCESS_CACHE`]: a SHA-256 over every caller-side input that +/// affects job-read visibility (admin flag, username, username override, the sorted +/// group set, and the sorted folder set the caller has any grant on — RLS reads from +/// all of them) plus the workspace and job id. Sorting makes the key order-independent; +/// each variable-length field is length-prefixed so no choice of input values can make +/// two distinct identities hash equal (e.g. `["a","bc"]` vs `["ab","c"]`). +fn job_read_access_cache_key(authed: &ApiAuthed, w_id: &str, job_id: &Uuid) -> [u8; 32] { + let mut hasher = Sha256::new(); + // Length-prefix every variable-length field (u32 BE) to make the encoding injective. + let field = |hasher: &mut Sha256, bytes: &[u8]| { + hasher.update((bytes.len() as u32).to_be_bytes()); + hasher.update(bytes); + }; + hasher.update([authed.is_admin as u8]); + field(&mut hasher, authed.username.as_bytes()); + field( + &mut hasher, + authed.username_override.as_deref().unwrap_or("").as_bytes(), + ); + let mut groups: Vec<&str> = authed.groups.iter().map(String::as_str).collect(); + groups.sort_unstable(); + hasher.update((groups.len() as u32).to_be_bytes()); + for g in groups { + field(&mut hasher, g.as_bytes()); + } + let mut folders: Vec<&str> = authed.folders.iter().map(|f| f.0.as_str()).collect(); + folders.sort_unstable(); + hasher.update((folders.len() as u32).to_be_bytes()); + for f in folders { + field(&mut hasher, f.as_bytes()); + } + field(&mut hasher, w_id.as_bytes()); + hasher.update(job_id.as_bytes()); + hasher.finalize().into() +} + +/// [`require_job_read_access`] for callers (job-update poll / SSE) that haven't +/// already loaded `created_by` — fetches it (root DB, by id+workspace) first. +async fn require_job_update_read_access( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + job_id: &Uuid, + view_token: Option<&str>, +) -> error::Result<()> { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2", + job_id, + w_id, + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job {job_id} not found")))?; + require_job_read_access(db, user_db, authed, w_id, job_id, &created_by, view_token).await +} + async fn get_job( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, opt_tokened: OptTokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, Query(GetJobQuery { no_logs, no_code, approval_token }): Query, ) -> error::Result { @@ -904,6 +1285,23 @@ async fn get_job( let mut job = get.fetch(&db, &id, &w_id).await?; job.fetch_outstanding_wait_time(&db).await?; + // A valid approval token is itself the capability; otherwise an authenticated + // caller must pass the same visibility as `jobs/list` (see `require_job_read_access`). + if !has_valid_approval_token { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + job.created_by(), + view_token.as_deref(), + ) + .await?; + } + } + log_job_view( &db, opt_authed.as_ref(), @@ -1478,8 +1876,10 @@ async fn get_logs_from_disk( } async fn get_completed_job_logs_tail( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::JsonResult { let tags = opt_authed @@ -1502,7 +1902,18 @@ async fn get_completed_job_logs_tail( .await?; if let Some(record) = record { - if opt_authed.is_none() && record.created_by != "anonymous" { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &record.created_by, + view_token.as_deref(), + ) + .await?; + } else if record.created_by != "anonymous" { return Err(Error::BadRequest( "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), )); @@ -1520,9 +1931,11 @@ struct QueryJobLogs { } async fn get_job_logs( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, opt_tokened: OptTokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, Query(query_job_logs): Query, ) -> error::Result { @@ -1553,7 +1966,18 @@ async fn get_job_logs( .await?; if let Some(record) = record { - if opt_authed.is_none() && record.created_by != "anonymous" { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &record.created_by, + view_token.as_deref(), + ) + .await?; + } else if record.created_by != "anonymous" { return Err(Error::BadRequest( "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), )); @@ -1681,9 +2105,11 @@ async fn resolve_logs_to_string( } async fn get_flow_all_logs( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, opt_tokened: OptTokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::Result { let tags = opt_authed @@ -1703,7 +2129,18 @@ async fn get_flow_all_logs( let root_job = not_found_if_none(root_job, "Job", id.to_string())?; - if opt_authed.is_none() && root_job.created_by != "anonymous" { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &root_job.created_by, + view_token.as_deref(), + ) + .await?; + } else if root_job.created_by != "anonymous" { return Err(Error::BadRequest( "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), )); @@ -1859,9 +2296,11 @@ async fn get_flow_all_logs( } async fn get_args( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, opt_tokened: OptTokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> JsonResult> { let tags = opt_authed @@ -1880,7 +2319,18 @@ async fn get_args( .await?; if let Some(record) = record { - if opt_authed.is_none() && record.created_by != "anonymous" { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &record.created_by, + view_token.as_deref(), + ) + .await?; + } else if record.created_by != "anonymous" { return Err(Error::BadRequest( "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), )); @@ -1908,7 +2358,18 @@ async fn get_args( .fetch_optional(&db) .await?; let record = not_found_if_none(record, "Job Args", id.to_string())?; - if opt_authed.is_none() && record.created_by != "anonymous" { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &record.created_by, + view_token.as_deref(), + ) + .await?; + } else if record.created_by != "anonymous" { return Err(Error::BadRequest( "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), )); @@ -2115,15 +2576,19 @@ async fn list_filtered_job_uuids( false, get_scope_tags(&authed), ); - let sqlb2 = list_queue_jobs_query( - w_id.as_str(), - &lq.into(), - &["v2_job.id"], - Pagination { page: None, per_page: None }, - false, - get_scope_tags(&authed), - ); - let query = sqlb.union_all(sqlb2.subquery()?).subquery()?; + let query = if lq.status.is_some() { + sqlb.subquery()? + } else { + let sqlb2 = list_queue_jobs_query( + w_id.as_str(), + &lq.into(), + &["v2_job.id"], + Pagination { page: None, per_page: None }, + false, + get_scope_tags(&authed), + ); + sqlb.union_all(sqlb2.subquery()?).subquery()? + }; let ids = sqlx::query_scalar(query.as_str()).fetch_all(&db).await?; Ok(Json(ids)) } @@ -2281,9 +2746,9 @@ async fn list_jobs( tracing::warn!("offset is not 0, but is ignored for list_jobs. Use created_before or completed_before instead."); } - if lq.success.is_some() && lq.running.is_some_and(|x| x) { + if (lq.success.is_some() || lq.status.is_some()) && lq.running.is_some_and(|x| x) { return Err(error::Error::BadRequest( - "cannot specify both success and running".to_string(), + "cannot specify success/status with running".to_string(), )); } @@ -2336,6 +2801,7 @@ async fn list_jobs( }; let sql = if lq.success.is_none() + && lq.status.is_none() && lq.label.is_none() && lq.result.is_none() && !lq.is_skipped.unwrap_or(false) @@ -2361,7 +2827,7 @@ async fn list_jobs( } else { if sqlc.is_none() { return Err(error::Error::BadRequest( - "cannot specify success, label, created_or_started_before, or starte + "cannot specify success, status, label, created_or_started_before, or starte d_before with running" .to_string(), )); @@ -2454,7 +2920,7 @@ pub async fn resume_suspended_flow_as_owner( // --- New approval system endpoints --- -use windmill_common::variables::generate_approval_token; +use windmill_common::variables::{generate_approval_token, generate_view_token}; /// Verify an approval token against the workspace key + job_id. async fn validate_approval_token( @@ -7108,8 +7574,10 @@ pub async fn run_job_by_hash_inner( } async fn get_log_file( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, file_p)): Path<(String, String)>, ) -> error::Result { if file_p.contains("..") { @@ -7148,7 +7616,18 @@ async fn get_log_file( .fetch_optional(&db) .await? .ok_or_else(|| error::Error::NotFound(format!("Job {job_id} not found")))?; - if opt_authed.is_none() && created_by != "anonymous" { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &job_id, + &created_by, + view_token.as_deref(), + ) + .await?; + } else if created_by != "anonymous" { return Err(error::Error::BadRequest( "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), )); @@ -7218,9 +7697,11 @@ async fn get_log_file( } async fn get_job_update( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, opt_tokened: OptTokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, job_id)): Path<(String, Uuid)>, Query(JobUpdateQuery { log_offset, @@ -7233,6 +7714,17 @@ async fn get_job_update( .. }): Query, ) -> JsonResult { + if let Some(authed) = opt_authed.as_ref() { + require_job_update_read_access( + &db, + &user_db, + authed, + &w_id, + &job_id, + view_token.as_deref(), + ) + .await?; + } Ok(Json( get_job_update_data( &opt_authed, @@ -7260,9 +7752,11 @@ async fn get_job_update( } async fn get_job_update_sse( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, opt_tokened: OptTokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, job_id)): Path<(String, Uuid)>, Query(JobUpdateQuery { log_offset, @@ -7276,6 +7770,20 @@ async fn get_job_update_sse( poll_delay_ms, }): Query, ) -> error::Result { + // Authorize once at connection time; `created_by` cannot change for a given job, + // mirroring the per-stream `anonymous_verified` latch in the streaming loop. + if let Some(authed) = opt_authed.as_ref() { + require_job_update_read_access( + &db, + &user_db, + authed, + &w_id, + &job_id, + view_token.as_deref(), + ) + .await?; + } + let (tx, rx) = tokio::sync::mpsc::channel(32); start_job_update_sse_stream( @@ -8035,9 +8543,11 @@ async fn list_completed_jobs( } async fn get_completed_job<'a>( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, opt_tokened: OptTokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::Result { let tags = opt_authed @@ -8052,6 +8562,20 @@ async fn get_completed_job<'a>( .await?; let cj = not_found_if_none(job_o, "Completed Job", id.to_string())?; + + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &cj.created_by, + view_token.as_deref(), + ) + .await?; + } + let response = Json(cj).into_response(); // let extra_log = query_scalar!( // "SELECT substr(logs, $1) as logs FROM large_logs WHERE workspace_id = $2 AND job_id = $3", @@ -8082,9 +8606,11 @@ pub struct RawResult { } async fn get_completed_job_result( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, opt_tokened: OptTokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, Query(JsonPath { json_path, suspended_job, approver, resume_id, secret }): Query, ) -> error::Result { @@ -8129,26 +8655,40 @@ async fn get_completed_job_result( let mut raw_result = not_found_if_none(result_o, "Completed Job", id.to_string())?; - if opt_authed.is_none() && raw_result.created_by.unwrap_or_default() != "anonymous" { - match (suspended_job, resume_id, approver, secret) { - (Some(suspended_job), Some(resume_id), approver, Some(secret)) => { - let mut parent_job = id; - while parent_job != suspended_job { - let p_job = sqlx::query_scalar!( - "SELECT parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2", - parent_job, - &w_id - ) - .fetch_optional(&db) - .await? - .flatten(); - if let Some(p_job) = p_job { - parent_job = p_job; - } else { - return Err(Error::BadRequest("Approval secret of suspended job is not a parent of the job whose id's is being searched not found".to_string())); + let created_by = raw_result.created_by.take().unwrap_or_default(); + + // A valid approval secret for the suspended parent flow grants access to this + // node's result for ANY caller — logged in or not — since the approval page + // renders its form from this result. Try it first. If the secret triple is absent, + // or present but invalid, fall through to normal authorization: an authenticated + // reader with ACL must NOT be blocked just because a stale/garbage secret was + // attached (pre-fix the secret branch was skipped entirely for authed callers), + // while an unauthenticated caller, for whom the secret is the only credential, + // still ends up rejected below. + let approval_secret_ok = match (suspended_job, resume_id, secret) { + (Some(suspended_job), Some(resume_id), Some(secret)) => { + // Walk from `id` up to the claimed suspended parent. + let mut parent_job = id; + let mut reached = true; + while parent_job != suspended_job { + let p_job = sqlx::query_scalar!( + "SELECT parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2", + parent_job, + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(); + match p_job { + Some(p_job) => parent_job = p_job, + None => { + reached = false; + break; } } - verify_suspended_secret( + } + reached + && verify_suspended_secret( &w_id, &db, suspended_job, @@ -8156,14 +8696,28 @@ async fn get_completed_job_result( &QueryApprover { approver, flow_level: None }, secret, ) - .await? - } - _ => { - return Err(Error::BadRequest( - "As a non logged in user, you can only see jobs ran by anonymous users" - .to_string(), - )) - } + .await + .is_ok() + } + _ => false, + }; + + if !approval_secret_ok { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &created_by, + view_token.as_deref(), + ) + .await?; + } else if created_by != "anonymous" { + return Err(Error::BadRequest( + "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), + )); } } @@ -8236,9 +8790,11 @@ struct GetCompletedJobQuery { } async fn get_completed_job_result_maybe( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, opt_tokened: OptTokened, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, Query(GetCompletedJobQuery { get_started }): Query, ) -> error::Result { @@ -8264,7 +8820,18 @@ async fn get_completed_job_result_maybe( if let Some(mut res) = result_o { format_result(res.result_columns.as_ref(), res.result.as_mut()); - if opt_authed.is_none() && res.created_by != "anonymous" { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &res.created_by, + view_token.as_deref(), + ) + .await?; + } else if res.created_by != "anonymous" { return Err(Error::BadRequest( "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), )); @@ -8287,6 +8854,36 @@ async fn get_completed_job_result_maybe( }) .into_response()) } else if get_started.is_some_and(|x| x) { + // No completed row yet — the job may be queued/running. Returning its + // running-state still discloses information about a (possibly private) job, so + // authorize first when the job exists. If it doesn't exist, fall through to a + // `started: false` response (which leaks nothing). + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2", + id, + &w_id + ) + .fetch_optional(&db) + .await?; + if let Some(created_by) = created_by { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &created_by, + view_token.as_deref(), + ) + .await?; + } else if created_by != "anonymous" { + return Err(Error::BadRequest( + "As a non logged in user, you can only see jobs ran by anonymous users" + .to_string(), + )); + } + } let started = sqlx::query_scalar!( "SELECT running AS \"running!\" FROM v2_job_queue WHERE id = $1 AND workspace_id = $2", id, @@ -8404,8 +9001,10 @@ async fn get_dispatch_events( } async fn get_completed_job_timing( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::JsonResult { let tags = opt_authed @@ -8431,7 +9030,18 @@ async fn get_completed_job_timing( let result = not_found_if_none(result, "Completed Job", id.to_string())?; - if opt_authed.is_none() && result.created_by != "anonymous" { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &result.created_by, + view_token.as_deref(), + ) + .await?; + } else if result.created_by != "anonymous" { return Err(Error::BadRequest( "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), )); @@ -8451,7 +9061,7 @@ async fn delete_completed_job<'a>( Extension(db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::Result { - let mut tx = user_db.begin(&authed).await?; + let mut tx = user_db.clone().begin(&authed).await?; require_admin(authed.is_admin, &authed.username)?; let tags = get_scope_tags(&authed); @@ -8495,17 +9105,21 @@ async fn delete_completed_job<'a>( tx.commit().await?; return get_completed_job( + OptViewToken(None), OptAuthed(Some(authed)), OptTokened { token: Some(token) }, Extension(db), + Extension(user_db), Path((w_id, id)), ) .await; } async fn get_otel_traces( + OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, + Extension(user_db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::Result>> { // Check job exists and user has permission to view it @@ -8519,7 +9133,18 @@ async fn get_otel_traces( match job { Some(created_by) => { - if opt_authed.is_none() && created_by != "anonymous" { + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &created_by, + view_token.as_deref(), + ) + .await?; + } else if created_by != "anonymous" { return Err(Error::BadRequest( "As a non logged in user, you can only see jobs ran by anonymous users" .to_string(), diff --git a/backend/windmill-api/src/mcp_tools.rs b/backend/windmill-api/src/mcp_tools.rs index 10397f0dee..bba945ac54 100644 --- a/backend/windmill-api/src/mcp_tools.rs +++ b/backend/windmill-api/src/mcp_tools.rs @@ -5,11 +5,11 @@ use axum::{ use serde_json::value::RawValue; use windmill_api_auth::{check_scopes, ApiAuthed}; use windmill_common::{ - db::{UserDB, DB}, + db::{DbWithOptAuthed, UserDB, DB}, error::{Error, JsonResult, Result}, utils::{not_found_if_none, StripPath}, }; -use windmill_store::resources::explain_resource_perm_error; +use windmill_store::{resources::explain_resource_perm_error, variables::get_value_internal}; pub(crate) async fn get_mcp_tools( authed: ApiAuthed, @@ -65,7 +65,7 @@ pub(crate) async fn get_mcp_tools( if let Some(info) = token_info { if let (Some(account_id), Some(true)) = (info.account_id, info.is_expired) { - let refresh_tx = user_db.begin(&authed).await?; + let refresh_tx = user_db.clone().begin(&authed).await?; if let Err(e) = crate::oauth2_oss::_refresh_token( refresh_tx, token_var_path, @@ -85,7 +85,23 @@ pub(crate) async fn get_mcp_tools( } } - let client = windmill_mcp::McpClient::from_resource(mcp_resource, &db, &w_id) + // Resolve the token through the caller's permissioned (RLS + audit) path so + // a developer cannot exfiltrate a secret they are not allowed to read by + // pointing an MCP resource's token at it. + let token = if let Some(token_path) = &mcp_resource.token { + let token_var_path = token_path.trim_start_matches("$var:"); + if token_var_path.trim().is_empty() { + None + } else { + let db_authed = + DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); + Some(get_value_internal(&db_authed, &w_id, token_var_path, false).await?) + } + } else { + None + }; + + let client = windmill_mcp::McpClient::from_resource(mcp_resource, token) .await .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e)))?; diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 28e327c1ca..593ea3b298 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -3,15 +3,17 @@ use axum::{ Extension, }; use bytes::Bytes; +use hmac::{Hmac, Mac}; use http::HeaderMap; use hyper::StatusCode; use reqwest::Client; use serde::{Deserialize, Serialize}; use serde_json::Value; +use sha2::Sha256; use sqlx::types::Uuid; use std::collections::HashMap; -use windmill_common::error::Error; -use windmill_common::variables::get_secret_value_as_admin; +use windmill_common::error::{to_anyhow, Error}; +use windmill_common::variables::{get_secret_value_as_admin, get_workspace_key}; use crate::db::{ApiAuthed, DB}; use crate::jobs::{QueryApprover, ResumeUrls}; @@ -111,6 +113,9 @@ struct ModalActionValue { dynamic_enums_json: Option, resume_button_text: Option, cancel_button_text: Option, + // HMAC over (w_id, job_id, path) keyed on the workspace key; minted by + // `send_slack_message`, required by the OpenModal callback branch. + signature: Option, } #[derive(Deserialize, Debug)] @@ -119,8 +124,16 @@ struct PrivateMetadata { resource_path: String, container: Container, hide_cancel: Option, + // HMAC over (w_id, resource_path) keyed on the workspace key; minted when the modal is + // built, required by `handle_submission` before the resource_path is decrypted. + signature: Option, } +// Opportunistic transport-level check: when `SLACK_SIGNING_SECRET` is configured we verify +// the Slack request signature (which also defeats replay). It is NOT the primary defense: +// the secret is unset in the default deployment, so authorization of the sensitive actions +// is instead anchored on a per-workspace HMAC over the callback payload itself (see +// `verify_slack_payload`), which holds even when this check is a no-op. #[cfg(feature = "oauth2")] fn verify_slack_callback_signature(headers: &HeaderMap, body: &str) -> Result<(), Error> { if let Some(sv) = crate::SLACK_SIGNING_SECRET.as_ref() { @@ -143,6 +156,66 @@ fn verify_slack_callback_signature(_headers: &HeaderMap, _body: &str) -> Result< Ok(()) } +/// HMAC keyed on the per-workspace encryption key (the same trust anchor as resume-URL +/// signatures). Used to authenticate the `/api/slack` callback payload itself so the +/// unauthenticated route cannot be driven into decrypting arbitrary workspace variables, +/// regardless of whether `SLACK_SIGNING_SECRET` is configured. +type SlackPayloadHmac = Hmac; + +/// Domain-separation tag prepended to every Slack-payload MAC. The workspace key is also used +/// for resume-secret signatures (`create_signature` in `jobs.rs`), and those secrets are +/// distributed to approvers in resume URLs — so a fixed, scheme-specific prefix makes the two +/// MAC families non-interchangeable by construction rather than relying on their byte layouts +/// happening to differ. Bump the version suffix if the signed layout ever changes. +const SLACK_PAYLOAD_HMAC_DOMAIN: &[u8] = b"slack_payload_v1\0"; + +/// Sign the security-sensitive fields of a Slack callback payload with the workspace key. +/// Parts are joined with a `\0` delimiter (absent from paths/UUIDs) so distinct field tuples +/// cannot collide into the same MAC. +async fn sign_slack_payload(db: &DB, w_id: &str, parts: &[&[u8]]) -> Result { + let key = get_workspace_key(w_id, db).await?; + let mut mac = SlackPayloadHmac::new_from_slice(key.as_bytes()).map_err(to_anyhow)?; + mac.update(SLACK_PAYLOAD_HMAC_DOMAIN); + mac.update(w_id.as_bytes()); + for part in parts { + mac.update(b"\0"); + mac.update(part); + } + Ok(hex::encode(mac.finalize().into_bytes())) +} + +/// Verify a signature produced by [`sign_slack_payload`] in constant time. A missing or +/// malformed signature is rejected: an attacker cannot forge one without the workspace key. +async fn verify_slack_payload( + db: &DB, + w_id: &str, + parts: &[&[u8]], + signature: Option<&str>, +) -> Result<(), Error> { + let signature = signature.ok_or_else(|| { + Error::NotAuthorized("Slack callback rejected: missing payload signature".to_string()) + })?; + let provided = hex::decode(signature).map_err(|_| { + Error::NotAuthorized("Slack callback rejected: malformed payload signature".to_string()) + })?; + // Map a missing workspace key (e.g. non-existent workspace) to the same generic 401 as a + // bad signature, so an unauthenticated caller cannot use the status code (500 vs 401) as a + // workspace-existence oracle. + let key = get_workspace_key(w_id, db).await.map_err(|_| { + Error::NotAuthorized("Slack callback rejected: invalid payload signature".to_string()) + })?; + let mut mac = SlackPayloadHmac::new_from_slice(key.as_bytes()).map_err(to_anyhow)?; + mac.update(SLACK_PAYLOAD_HMAC_DOMAIN); + mac.update(w_id.as_bytes()); + for part in parts { + mac.update(b"\0"); + mac.update(part); + } + mac.verify_slice(&provided).map_err(|_| { + Error::NotAuthorized("Slack callback rejected: invalid payload signature".to_string()) + }) +} + pub async fn slack_app_callback_handler( authed: Option, opt_tokened: OptTokened, @@ -188,7 +261,30 @@ pub async fn slack_app_callback_handler( let job_id = Uuid::parse_str(&parsed_value.job_id)?; let flow_step_id = parsed_value.flow_step_id.as_deref(); - let slack_token = get_slack_token(&db, path, w_id).await?; + // Authorize the request before any privileged read: the button + // payload was minted by `send_slack_message` with an HMAC over + // (w_id, job_id, path) keyed on the workspace key. Without a valid + // signature an unauthenticated caller cannot reach the decryption + // below for an arbitrary variable, even when SLACK_SIGNING_SECRET + // is unset. + verify_slack_payload( + &db, + w_id, + &[parsed_value.job_id.as_bytes(), path.as_bytes()], + parsed_value.signature.as_deref(), + ) + .await?; + + // Map any lookup/decryption failure to a generic error: the + // raw error echoes the probed `path`/`w_id` back, which would be + // a cross-workspace existence oracle. Log the detail server-side. + let slack_token = + get_slack_token(&db, path, w_id).await.map_err(|e| { + tracing::warn!( + "Failed to resolve slack token for {w_id}/{path}: {e:#}" + ); + Error::BadRequest("Invalid Slack callback request".to_string()) + })?; let client = Client::new(); let container = payload.container.ok_or_else(|| { Error::BadRequest("No container found.".to_string()) @@ -281,6 +377,7 @@ pub async fn request_slack_approval( send_slack_message( &client, + &db, slack_token.as_str(), channel_id.as_str(), &w_id, @@ -334,12 +431,21 @@ async fn handle_submission( let resource_path = private_metadata.resource_path; let container: Container = private_metadata.container; let hide_cancel = private_metadata.hide_cancel; + let signature = private_metadata.signature; // If hide_cancel is true, we don't need to extract information from the private_metadata if hide_cancel.unwrap_or(false) && action == "cancel" { return Ok(()); } + let w_id = extract_w_id_from_resume_url(&resume_url)?; + // Authorize the submission BEFORE taking any action. `resource_path` comes from the + // (client-held) modal metadata and is not covered by the resume-URL signature, so a + // tampered/unsigned submission must be rejected up front — otherwise it could still drive + // the resume/cancel and reach the decryption below with a swapped path. Require the + // workspace-keyed HMAC minted when the modal was built. + verify_slack_payload(&db, w_id, &[resource_path.as_bytes()], signature.as_deref()).await?; + // Use the common handler to process the resume/cancel action handle_resume_action( authed, @@ -351,8 +457,12 @@ async fn handle_submission( ) .await?; - let w_id = extract_w_id_from_resume_url(&resume_url)?; - let slack_token = get_slack_token(&db, &resource_path, w_id).await?; + let slack_token = get_slack_token(&db, &resource_path, w_id) + .await + .map_err(|e| { + tracing::warn!("Failed to resolve slack token for {w_id}/{resource_path}: {e:#}"); + Error::BadRequest("Invalid Slack callback request".to_string()) + })?; update_original_slack_message(action, slack_token, container).await?; Ok(()) } @@ -780,6 +890,7 @@ async fn get_slack_token(db: &DB, slack_resource_path: &str, w_id: &str) -> anyh // Sends a Slack message with a button that opens a modal async fn send_slack_message( client: &Client, + db: &DB, bot_token: &str, channel_id: &str, w_id: &str, @@ -827,6 +938,18 @@ async fn send_slack_message( value["cancel_button_text"] = serde_json::json!(cancel_button_text); } + // Authenticate the button payload so the unauthenticated callback cannot be driven into + // decrypting an arbitrary variable: bind (w_id, job_id, path) with the workspace key. + // `job_id` is signed over its string form to match how it is parsed back on callback. + let signature = sign_slack_payload( + db, + w_id, + &[job_id.to_string().as_bytes(), resource_path.as_bytes()], + ) + .await + .map_err(|e| Box::new(e) as Box)?; + value["signature"] = serde_json::json!(signature); + let payload = serde_json::json!({ "channel": channel_id, "text": "A flow has been suspended. Please approve or reject the flow.", @@ -893,6 +1016,12 @@ async fn get_modal_blocks( resume_button_text: Option<&str>, cancel_button_text: Option<&str>, ) -> Result, Error> { + // Bind the resource_path embedded in the modal's private_metadata to the workspace key so + // it cannot be tampered with on the way back in `handle_submission`. Computed before `db` + // is moved into `get_approval_form_details`. + let private_metadata_signature = + sign_slack_payload(&db, w_id, &[resource_path.as_bytes()]).await?; + let approval_details = crate::approvals::get_approval_form_details( db, w_id, @@ -947,6 +1076,7 @@ async fn get_modal_blocks( container, resume_button_text, cancel_button_text, + &private_metadata_signature, ))) } @@ -959,6 +1089,7 @@ fn construct_payload( container: Container, resume_button_text: Option<&str>, cancel_button_text: Option<&str>, + signature: &str, ) -> serde_json::Value { let mut view = serde_json::json!({ "type": "modal", @@ -973,7 +1104,7 @@ fn construct_payload( "type": "plain_text", "text": resume_button_text.unwrap_or("Resume Workflow") }, - "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel }).to_string(), + "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature }).to_string(), }); if !hide_cancel { diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 3f0cd3c646..63b9e84863 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -125,10 +125,18 @@ pub fn is_none_or_false(val: &Option) -> bool { } } -/// Returns the keys to strip from trigger/schedule serialization when the -/// source workspace is a fork. Stripping these keys avoids propagating -/// fork-local operational state (enabled flag, runtime listener identifiers) -/// back to the parent workspace through the git-sync round-trip. +/// A fork's git-sync export rewrites each trigger's `mode` (and each schedule's +/// `enabled`) to the *parent* workspace's value, instead of emitting the fork's +/// own (clone-disabled / locally-toggled) state. This keeps the fork's synced +/// file byte-identical to the parent on the operational-state field, so a +/// normal-git PR merge has nothing to resolve — no dropped `mode:` line, no +/// flipped parent trigger. Fork-only paths (absent from the parent) keep the +/// fork's own value: there's no parent state to defer to, so the trigger lands +/// with whatever the fork creator set. The write half of the same rule lives in +/// `windmill-trigger::handler::workspace_is_fork`. +/// +/// Maps trigger `path` → parent `mode` (as the lowercase enum text that matches +/// `TriggerMode`'s serde representation). Empty when not a fork. #[cfg(any( feature = "http_trigger", feature = "websocket", @@ -148,20 +156,78 @@ pub fn is_none_or_false(val: &Option) -> bool { feature = "private" ) ))] -fn fork_trigger_ignore_keys(is_fork: bool) -> Option> { - if is_fork { - Some(vec!["mode", "enabled"]) - } else { - None - } +async fn fork_parent_trigger_modes( + db: &DB, + table_name: &str, + parent_workspace_id: Option<&str>, +) -> Result> { + let Some(parent) = parent_workspace_id else { + return Ok(HashMap::new()); + }; + // Read the parent's rows on the non-RLS pool (like `workspace_is_fork`): the + // substitution must be complete regardless of the exporter's folder perms, + // otherwise a parent path the exporter can't read would fall back to the + // fork's own value and silently re-introduce the divergence we're fixing. + // No leak: only values for paths the fork already has (it's a clone) are used. + // SAFETY: `table_name` is a compile-time `TriggerCrud::TABLE_NAME` constant. + let rows: Vec<(String, String)> = sqlx::query_as(&format!( + "SELECT path, mode::text FROM {} WHERE workspace_id = $1", + table_name + )) + .bind(parent) + .fetch_all(db) + .await?; + Ok(rows.into_iter().collect()) } -fn fork_schedule_ignore_keys(is_fork: bool) -> Option> { - if is_fork { - Some(vec!["enabled"]) - } else { - None - } +/// Build the `{ "mode": }` override for a single trigger, or +/// `None` (keep the fork's own value) when the path is fork-only. +#[cfg(any( + feature = "http_trigger", + feature = "websocket", + feature = "postgres_trigger", + feature = "mqtt_trigger", + feature = "native_trigger", + all( + feature = "enterprise", + any( + feature = "kafka", + feature = "sqs_trigger", + feature = "gcp_trigger", + feature = "azure_trigger", + feature = "nats", + feature = "smtp", + ), + feature = "private" + ) +))] +fn trigger_mode_override( + parent_modes: &HashMap, + path: &str, +) -> Option> { + parent_modes.get(path).map(|mode| { + let mut o = serde_json::Map::new(); + o.insert("mode".to_string(), Value::String(mode.clone())); + o + }) +} + +/// Schedule analog of [`fork_parent_trigger_modes`]: maps schedule `path` → +/// parent `enabled`. Empty when not a fork. +async fn fork_parent_schedule_enabled( + db: &DB, + parent_workspace_id: Option<&str>, +) -> Result> { + let Some(parent) = parent_workspace_id else { + return Ok(HashMap::new()); + }; + // Non-RLS pool, same rationale as `fork_parent_trigger_modes`. + let rows: Vec<(String, bool)> = + sqlx::query_as("SELECT path, enabled FROM schedule WHERE workspace_id = $1") + .bind(parent) + .fetch_all(db) + .await?; + Ok(rows.into_iter().collect()) } enum ArchiveImpl { @@ -260,6 +326,25 @@ pub fn to_string_without_metadata( extra_perms: ExtraPermsBehavior, ignore_keys: Option>, ) -> Result +where + T: ?Sized + Serialize, +{ + to_string_without_metadata_inner(value, extra_perms, ignore_keys, None) +} + +/// Like [`to_string_without_metadata`] but additionally lets the caller +/// override top-level keys after stripping. Used for fork trigger/schedule +/// exports, where `mode`/`enabled` is rewritten to the *parent* workspace's +/// value so the fork's synced file is byte-identical to the parent on those +/// fields — a clean 3-way git merge instead of a dropped line. See the write +/// half of the rule in `windmill-trigger::handler::workspace_is_fork`. +#[inline] +pub fn to_string_without_metadata_inner( + value: &T, + extra_perms: ExtraPermsBehavior, + ignore_keys: Option>, + overrides: Option<&serde_json::Map>, +) -> Result where T: ?Sized + Serialize, { @@ -324,6 +409,12 @@ where obj.remove("default_permissioned_as"); } + if let Some(overrides) = overrides { + for (k, v) in overrides { + obj.insert(k.clone(), v.clone()); + } + } + serde_json::to_string_pretty(&obj).ok() }) .flatten() @@ -504,18 +595,18 @@ pub(crate) async fn tarball_workspace( let mut tx = user_db.begin(&authed).await?; - // Source-of-truth check for fork-ness: the workspace's parent_workspace_id - // column. The wm-fork-* prefix is a creation-time naming convention that - // could in principle drift (rename, manual SQL); the column is the - // contract that matches what the conflict-warning gates read. - let is_fork: bool = sqlx::query_scalar!( - "SELECT parent_workspace_id IS NOT NULL FROM workspace WHERE id = $1", - &w_id + // Source-of-truth for fork-ness: the workspace's parent_workspace_id column. + // The wm-fork-* prefix is a creation-time naming convention that could in + // principle drift (rename, manual SQL); the column is the contract that + // matches what the conflict-warning gates read. The id is also the workspace + // whose trigger `mode` / schedule `enabled` a fork export defers to. + let parent_workspace_id: Option = sqlx::query_scalar::<_, Option>( + "SELECT parent_workspace_id FROM workspace WHERE id = $1", ) + .bind(&w_id) .fetch_optional(&mut *tx) .await? - .flatten() - .unwrap_or(false); + .flatten(); let tmp_dir = TempDir::new_in(&*WINDMILL_DIR)?; @@ -799,12 +890,21 @@ pub(crate) async fn tarball_workspace( .fetch_all(&mut *tx) .await?; - let schedule_ignore_keys = fork_schedule_ignore_keys(is_fork); + // For a fork, defer each schedule's `enabled` to the parent so the + // synced file matches the parent and the merge doesn't flip it. + let parent_enabled = + fork_parent_schedule_enabled(&db, parent_workspace_id.as_deref()).await?; for schedule in schedules { - let app_str = &to_string_without_metadata( + let enabled_override = parent_enabled.get(&schedule.path).map(|enabled| { + let mut o = serde_json::Map::new(); + o.insert("enabled".to_string(), Value::Bool(*enabled)); + o + }); + let app_str = &to_string_without_metadata_inner( &schedule, ExtraPermsBehavior::Drop, - schedule_ignore_keys.clone(), + None, + enabled_override.as_ref(), ) .unwrap(); archive @@ -814,38 +914,25 @@ pub(crate) async fn tarball_workspace( } if include_triggers.unwrap_or(false) { - #[cfg(any( - feature = "http_trigger", - feature = "websocket", - feature = "postgres_trigger", - feature = "mqtt_trigger", - feature = "native_trigger", - all( - feature = "enterprise", - any( - feature = "kafka", - feature = "sqs_trigger", - feature = "gcp_trigger", - feature = "azure_trigger", - feature = "nats", - feature = "smtp", - ), - feature = "private" - ) - ))] - let trigger_ignore_keys = fork_trigger_ignore_keys(is_fork); - #[cfg(feature = "http_trigger")] { use crate::triggers::http::HttpTrigger; let handler = HttpTrigger; let http_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; for trigger in http_triggers { - let trigger_str = &to_string_without_metadata( + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str = &to_string_without_metadata_inner( &trigger, ExtraPermsBehavior::Drop, - trigger_ignore_keys.clone(), + None, + mode_override.as_ref(), ) .unwrap(); archive @@ -862,12 +949,20 @@ pub(crate) async fn tarball_workspace( use crate::triggers::websocket::WebsocketTrigger; let handler = WebsocketTrigger; let websocket_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; for trigger in websocket_triggers { - let trigger_str = &to_string_without_metadata( + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str = &to_string_without_metadata_inner( &trigger, ExtraPermsBehavior::Drop, - trigger_ignore_keys.clone(), + None, + mode_override.as_ref(), ) .unwrap(); archive @@ -884,12 +979,20 @@ pub(crate) async fn tarball_workspace( use crate::triggers::kafka::KafkaTrigger; let handler = KafkaTrigger; let kafka_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; for trigger in kafka_triggers { - let trigger_str = &to_string_without_metadata( + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str = &to_string_without_metadata_inner( &trigger, ExtraPermsBehavior::Drop, - trigger_ignore_keys.clone(), + None, + mode_override.as_ref(), ) .unwrap(); archive @@ -906,12 +1009,20 @@ pub(crate) async fn tarball_workspace( use crate::triggers::sqs::SqsTrigger; let handler = SqsTrigger; let sqs_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; for trigger in sqs_triggers { - let trigger_str = &to_string_without_metadata( + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str = &to_string_without_metadata_inner( &trigger, ExtraPermsBehavior::Drop, - trigger_ignore_keys.clone(), + None, + mode_override.as_ref(), ) .unwrap(); archive @@ -928,12 +1039,20 @@ pub(crate) async fn tarball_workspace( use crate::triggers::gcp::GcpTrigger; let handler = GcpTrigger; let gcp_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; for trigger in gcp_triggers { - let trigger_str = &to_string_without_metadata( + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str = &to_string_without_metadata_inner( &trigger, ExtraPermsBehavior::Drop, - trigger_ignore_keys.clone(), + None, + mode_override.as_ref(), ) .unwrap(); archive @@ -950,12 +1069,20 @@ pub(crate) async fn tarball_workspace( use crate::triggers::azure::AzureTrigger; let handler = AzureTrigger; let azure_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; for trigger in azure_triggers { - let trigger_str = &to_string_without_metadata( + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str = &to_string_without_metadata_inner( &trigger, ExtraPermsBehavior::Drop, - trigger_ignore_keys.clone(), + None, + mode_override.as_ref(), ) .unwrap(); archive @@ -972,12 +1099,20 @@ pub(crate) async fn tarball_workspace( use crate::triggers::nats::NatsTrigger; let handler = NatsTrigger; let nats_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; for trigger in nats_triggers { - let trigger_str: &String = &to_string_without_metadata( + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str: &String = &to_string_without_metadata_inner( &trigger, ExtraPermsBehavior::Drop, - trigger_ignore_keys.clone(), + None, + mode_override.as_ref(), ) .unwrap(); archive @@ -994,12 +1129,20 @@ pub(crate) async fn tarball_workspace( use crate::triggers::postgres::PostgresTrigger; let handler = PostgresTrigger; let postgres_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; for trigger in postgres_triggers { - let trigger_str = &to_string_without_metadata( + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str = &to_string_without_metadata_inner( &trigger, ExtraPermsBehavior::Drop, - trigger_ignore_keys.clone(), + None, + mode_override.as_ref(), ) .unwrap(); archive @@ -1016,12 +1159,20 @@ pub(crate) async fn tarball_workspace( use crate::triggers::mqtt::MqttTrigger; let handler = MqttTrigger; let mqtt_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; for trigger in mqtt_triggers { - let trigger_str = &to_string_without_metadata( + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str = &to_string_without_metadata_inner( &trigger, ExtraPermsBehavior::Drop, - trigger_ignore_keys.clone(), + None, + mode_override.as_ref(), ) .unwrap(); archive @@ -1038,12 +1189,20 @@ pub(crate) async fn tarball_workspace( use crate::triggers::email::EmailTrigger; let handler = EmailTrigger; let email_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; for trigger in email_triggers { - let trigger_str = &to_string_without_metadata( + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str = &to_string_without_metadata_inner( &trigger, ExtraPermsBehavior::Drop, - trigger_ignore_keys.clone(), + None, + mode_override.as_ref(), ) .unwrap(); archive @@ -1065,10 +1224,12 @@ pub(crate) async fn tarball_workspace( list_native_triggers(&mut *tx, &w_id, service_name, None, None, None, None) .await?; - let mut native_ignore_keys = vec!["webhook_token_hash"]; - if let Some(ref extra) = trigger_ignore_keys { - native_ignore_keys.extend_from_slice(extra); - } + // Native triggers (Nextcloud, Google Drive, GitHub) are never + // cloned into a fork — a fork only has one if its owner created + // it there, so it's always "fork-only" and keeps its own mode. + // No parent-value substitution applies; we only strip the + // webhook token hash. + let native_ignore_keys = vec!["webhook_token_hash"]; for trigger in native_triggers { let trigger_str = &to_string_without_metadata( @@ -1359,3 +1520,67 @@ pub(crate) async fn tarball_workspace( ]; Ok((headers, body)) } + +#[cfg(test)] +mod fork_export_tests { + use super::*; + use serde_json::json; + + /// A fork export rewrites `mode` to the parent's value: the serialized file + /// carries the parent's state (`enabled`), not the fork's clone-disabled DB + /// value — so a normal-git merge sees no change on that line. + #[test] + fn override_substitutes_parent_mode() { + let fork_trigger = json!({ + "path": "f/triggers/x", + "script_path": "f/scripts/x", + "mode": "disabled", // fork's local (clone-disabled) state + "is_flow": false, + }); + let mut overrides = serde_json::Map::new(); + overrides.insert("mode".to_string(), Value::String("enabled".to_string())); + + let out = to_string_without_metadata_inner( + &fork_trigger, + ExtraPermsBehavior::Drop, + None, + Some(&overrides), + ) + .unwrap(); + let parsed: Value = serde_json::from_str(&out).unwrap(); + + assert_eq!(parsed["mode"], json!("enabled"), "parent mode substituted"); + // `path` is in the metadata strip list, so it should be removed. + assert!(parsed.get("path").is_none()); + } + + /// A fork-only trigger (no parent counterpart, so no override) keeps the + /// fork creator's chosen state. + #[test] + fn no_override_keeps_fork_value() { + let fork_only = json!({ "mode": "enabled", "script_path": "f/scripts/x" }); + let out = + to_string_without_metadata_inner(&fork_only, ExtraPermsBehavior::Drop, None, None) + .unwrap(); + let parsed: Value = serde_json::from_str(&out).unwrap(); + assert_eq!(parsed["mode"], json!("enabled")); + } + + /// `trigger_mode_override` builds an override only when the parent has the + /// path; fork-only paths return `None` (keep the fork's own value). + #[cfg(feature = "http_trigger")] + #[test] + fn trigger_mode_override_defers_to_parent_or_self() { + let mut parent_modes = HashMap::new(); + parent_modes.insert("f/triggers/shared".to_string(), "enabled".to_string()); + + let shared = trigger_mode_override(&parent_modes, "f/triggers/shared"); + assert_eq!( + shared.as_ref().and_then(|o| o.get("mode")), + Some(&Value::String("enabled".to_string())), + ); + + // Fork-only path: no parent entry → no override → keep fork's own value. + assert!(trigger_mode_override(&parent_modes, "f/triggers/fork_only").is_none()); + } +} diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index 9872950ffb..7fc6a0825a 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -18,6 +18,31 @@ use crate::{ DB, }; +/// Whether `label` denotes a user-created token rather than a system token +/// (`session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`). System-token +/// labels are load-bearing — session cleanup, super_admin propagation, expiry +/// notifications and username overrides all key off them — so they must not be +/// user-editable. `None` (no label) is treated as a user token. +/// +/// This is the canonical copy. When updating it, also update its mirrors: +/// - the `update_token_label` editability guard (SQL `WHERE`) in +/// windmill-api-users/src/users.rs +/// - `isUserToken` in frontend/src/lib/components/settings/TokensTable.svelte +pub fn is_user_token(label: Option<&str>) -> bool { + match label { + None => true, + Some(l) => { + // `ephemeral` is matched case-insensitively to agree exactly with the + // frontend mirror (`label.toLowerCase().startsWith('ephemeral')`) and + // the SQL `lower(label) NOT LIKE 'ephemeral%'` guard. + l != "session" + && !l.to_lowercase().starts_with("ephemeral") + && l != "debugger-token" + && !l.starts_with("mcp-oauth-") + } + } +} + /// Hash a raw token using SHA-256 (hex-encoded, 64 chars). /// Used to store and look up tokens without keeping plaintext in the DB. pub fn hash_token(token: &str) -> String { @@ -641,3 +666,35 @@ pub mod aws { Ok(assume_role_with_web_identity_fluent_builder) } } + +#[cfg(test)] +mod tests { + use super::is_user_token; + + #[test] + fn user_tokens_are_editable() { + assert!(is_user_token(None)); // no label + assert!(is_user_token(Some(""))); + assert!(is_user_token(Some("my-ci-token"))); + assert!(is_user_token(Some("webhook-foo"))); // username-override prefix, not a system kind here + } + + #[test] + fn system_tokens_are_not_editable() { + assert!(!is_user_token(Some("session"))); + assert!(!is_user_token(Some("ephemeral-script"))); + assert!(!is_user_token(Some("ephemeral-webhook-x"))); + assert!(!is_user_token(Some("Ephemeral lsp token"))); + assert!(!is_user_token(Some("debugger-token"))); + assert!(!is_user_token(Some("mcp-oauth-client"))); + } + + #[test] + fn ephemeral_match_is_case_insensitive() { + // Must agree with the frontend mirror (`toLowerCase().startsWith('ephemeral')`) + // so a token can't be relabeled to a casing the backend allows but the UI hides. + assert!(!is_user_token(Some("Ephemeral-test"))); + assert!(!is_user_token(Some("ePhemeral-test"))); + assert!(!is_user_token(Some("EPHEMERAL-test"))); + } +} diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 75f3fdf06b..8192f186d0 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -58,6 +58,11 @@ pub const NSJAIL_TMPFS_SIZE_MB_SETTING: &str = "nsjail_tmpfs_size_mb"; pub const NSJAIL_TMP_BACKING_SETTING: &str = "nsjail_tmp_backing"; pub const NSJAIL_TMP_BACKING_DISK: &str = "disk"; pub const NSJAIL_TMP_BACKING_TMPFS: &str = "tmpfs"; +pub const SANDBOX_IMAGE_MAX_SIZE_MB_SETTING: &str = "sandbox_image_max_size_mb"; +pub const SANDBOX_IMAGE_CACHE_MAX_MB_SETTING: &str = "sandbox_image_cache_max_mb"; +pub const SANDBOX_IMAGE_PULL_POLICY_SETTING: &str = "sandbox_image_pull_policy"; +pub const SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING: &str = "sandbox_image_default_registry"; +pub const SANDBOX_REGISTRY_AUTH_SETTING: &str = "sandbox_registry_auth"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 2239868982..bb56d5d0cc 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -976,6 +976,7 @@ const SENSITIVE_SETTINGS: &[&str] = &[ "ruby_repos", "powershell_repo_pat", "workspace_registries", + "sandbox_registry_auth", ]; /// Object-valued settings that contain sensitive sub-fields. diff --git a/backend/windmill-common/src/log_context.rs b/backend/windmill-common/src/log_context.rs index 4a517c7afb..cdb1e27aea 100644 --- a/backend/windmill-common/src/log_context.rs +++ b/backend/windmill-common/src/log_context.rs @@ -35,6 +35,11 @@ pub struct LogContext { pub uri: Option, pub trace_id: Option, + // Inbound W3C `traceparent` captured at enqueue (reserved `_wm_traceparent` + // arg). Carried here so the worker's OTLP span and the script's injected + // TRACEPARENT env can relocate into the originating distributed trace. + pub inbound_traceparent: Option, + // Auth (windmill-api-auth/src/auth.rs) pub email: Option, pub username: Option, diff --git a/backend/windmill-common/src/ssrf.rs b/backend/windmill-common/src/ssrf.rs index 507a773431..2f100d5ae1 100644 --- a/backend/windmill-common/src/ssrf.rs +++ b/backend/windmill-common/src/ssrf.rs @@ -2,6 +2,8 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use crate::error::Error; +pub const ALLOW_PRIVATE_MCP_SERVER_URLS_ENV: &str = "ALLOW_PRIVATE_MCP_SERVER_URLS"; + /// Why a URL failed SSRF validation. /// /// The distinction matters for callers that gate private endpoints behind a @@ -116,6 +118,49 @@ pub async fn validate_url_for_ssrf(url: &str) -> Result<(), SsrfValidationError> Ok(()) } +pub fn allow_private_mcp_server_urls() -> bool { + std::env::var(ALLOW_PRIVATE_MCP_SERVER_URLS_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1") +} + +pub async fn validate_mcp_server_url(url: &str) -> Result<(), SsrfValidationError> { + let parsed = + url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; + + match parsed.scheme() { + "http" | "https" => {} + scheme => return Err(SsrfValidationError::DisallowedScheme(scheme.to_string())), + } + + parsed.host_str().ok_or(SsrfValidationError::MissingHost)?; + + if allow_private_mcp_server_urls() { + return Ok(()); + } + + validate_url_for_ssrf(url).await +} + +pub async fn validate_mcp_server_url_for_bad_request(url: &str, label: &str) -> Result<(), Error> { + validate_mcp_server_url(url).await.map_err(|e| { + Error::BadRequest(format!( + "{label} is not allowed: {}", + mcp_ssrf_error_message(&e) + )) + }) +} + +pub fn mcp_ssrf_error_message(e: &SsrfValidationError) -> String { + match e { + SsrfValidationError::Private { .. } => format!( + "{e}. If you need to use private/internal MCP server URLs, \ + set the {ALLOW_PRIVATE_MCP_SERVER_URLS_ENV}=true environment variable" + ), + _ => e.to_string(), + } +} + fn is_private_ip(ip: &IpAddr) -> bool { match ip { IpAddr::V4(ipv4) => is_private_ipv4(ipv4), @@ -152,6 +197,32 @@ fn is_private_ipv6(ip: &Ipv6Addr) -> bool { mod tests { use super::*; + static TEST_ENV_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + struct PrivateMcpServerUrlsEnvGuard { + previous: Option, + } + + impl PrivateMcpServerUrlsEnvGuard { + fn set(value: Option<&str>) -> Self { + let previous = std::env::var(ALLOW_PRIVATE_MCP_SERVER_URLS_ENV).ok(); + match value { + Some(value) => std::env::set_var(ALLOW_PRIVATE_MCP_SERVER_URLS_ENV, value), + None => std::env::remove_var(ALLOW_PRIVATE_MCP_SERVER_URLS_ENV), + } + Self { previous } + } + } + + impl Drop for PrivateMcpServerUrlsEnvGuard { + fn drop(&mut self) { + match &self.previous { + Some(value) => std::env::set_var(ALLOW_PRIVATE_MCP_SERVER_URLS_ENV, value), + None => std::env::remove_var(ALLOW_PRIVATE_MCP_SERVER_URLS_ENV), + } + } + } + #[test] fn test_private_ipv4() { assert!(is_private_ipv4(&"127.0.0.1".parse().unwrap())); @@ -227,4 +298,66 @@ mod tests { Err(SsrfValidationError::Private { resolved: false }) )); } + + #[tokio::test] + async fn validate_mcp_server_url_blocks_private_by_default() { + let _lock = TEST_ENV_LOCK.lock().await; + let _guard = PrivateMcpServerUrlsEnvGuard::set(None); + + assert!(matches!( + validate_mcp_server_url("http://127.0.0.1/foo").await, + Err(SsrfValidationError::Private { resolved: false }) + )); + } + + #[tokio::test] + async fn validate_mcp_server_url_allows_private_when_env_is_enabled() { + let _lock = TEST_ENV_LOCK.lock().await; + let _guard = PrivateMcpServerUrlsEnvGuard::set(Some("true")); + + assert!(validate_mcp_server_url("http://127.0.0.1/foo") + .await + .is_ok()); + } + + #[tokio::test] + async fn validate_mcp_server_url_allows_private_when_env_is_one() { + let _lock = TEST_ENV_LOCK.lock().await; + let _guard = PrivateMcpServerUrlsEnvGuard::set(Some("1")); + + assert!(validate_mcp_server_url("http://10.0.0.1/foo").await.is_ok()); + } + + #[tokio::test] + async fn validate_mcp_server_url_keeps_syntax_guards_when_private_urls_are_allowed() { + let _lock = TEST_ENV_LOCK.lock().await; + let _guard = PrivateMcpServerUrlsEnvGuard::set(Some("true")); + + assert!(matches!( + validate_mcp_server_url("localhost:11434/v1").await, + Err(SsrfValidationError::DisallowedScheme(_)) + )); + assert!(matches!( + validate_mcp_server_url("file:///tmp/socket").await, + Err(SsrfValidationError::DisallowedScheme(_)) + )); + } + + #[tokio::test] + async fn private_mcp_error_message_includes_env_hint_only_for_private_urls() { + let _lock = TEST_ENV_LOCK.lock().await; + let _guard = PrivateMcpServerUrlsEnvGuard::set(None); + + let private_error = validate_mcp_server_url("http://127.0.0.1/foo") + .await + .unwrap_err(); + assert!( + mcp_ssrf_error_message(&private_error).contains("ALLOW_PRIVATE_MCP_SERVER_URLS=true") + ); + + let invalid_error = validate_mcp_server_url("localhost:11434/v1") + .await + .unwrap_err(); + assert!(!mcp_ssrf_error_message(&invalid_error).contains(ALLOW_PRIVATE_MCP_SERVER_URLS_ENV)); + } } diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 07ac44b755..9c944cb778 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -174,6 +174,26 @@ pub async fn generate_approval_token( Ok(hex::encode(mac.finalize().into_bytes())) } +/// Stateless read-share signature for a job: `HMAC(workspace_key, job_id || "view_token")`. +/// Mirrors [`generate_approval_token`] but in a distinct domain so an approval token can +/// never be used as a view token (or vice-versa). Used to build a "share read link" that +/// grants an authenticated workspace member read access to a job (and its flow subtree) +/// they otherwise lack ACL on. No expiry/revocation (stateless), like the approval token. +pub async fn generate_view_token( + w_id: &str, + job_id: uuid::Uuid, + db: &DB, +) -> crate::error::Result { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + let key = get_workspace_key(w_id, db).await?; + let mut mac = Hmac::::new_from_slice(key.as_bytes()) + .map_err(|e| crate::Error::internal_err(format!("HMAC key error: {e}")))?; + mac.update(job_id.as_bytes()); + mac.update(b"view_token"); + Ok(hex::encode(mac.finalize().into_bytes())) +} + pub async fn get_secret_value_as_admin( db: &DB, w_id: &str, diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 92ebf08477..c7816d7a70 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -859,6 +859,37 @@ pub struct BashAnnotations { pub sandbox: bool, } +impl BashAnnotations { + /// If the script declares `# sandbox ` (an image ref after the sandbox + /// annotation), returns that image ref. This selects the daemonless, sandboxed + /// container runtime: extract the image's rootfs and run it inside the job's + /// nsjail sandbox. + /// + /// A bare `# sandbox` (no image argument) returns `None` and keeps the plain + /// nsjail-sandboxed-bash behavior (the `sandbox` boolean modifier). `# docker` + /// is unaffected and keeps the legacy v1 (dind/daemon) path. + pub fn sandbox_image(code: &str) -> Option { + for line in code.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + // Mirror the annotation parser: stop at the first non-comment line. + if !line.starts_with('#') { + break; + } + let mut tokens = line[1..].split_whitespace(); + if tokens.next() == Some("sandbox") { + // `# sandbox ` -> container; bare `# sandbox` -> nsjail bash. + if let Some(image) = tokens.next() { + return Some(image.to_string()); + } + } + } + None + } +} + #[derive(Debug, Clone, Copy, PartialEq)] pub enum SqlResultCollectionStrategy { LastStatementAllRows, @@ -2224,6 +2255,34 @@ mod tests { use super::*; use std::collections::HashMap; + #[test] + fn test_bash_sandbox_image_annotation() { + // `# sandbox ` selects the container runtime and returns the image. + assert_eq!( + BashAnnotations::sandbox_image("# sandbox alpine:latest\necho hi"), + Some("alpine:latest".to_string()) + ); + // Extra whitespace and a leading non-spaced `#` still work. + assert_eq!( + BashAnnotations::sandbox_image("#sandbox python:3.12-slim\n"), + Some("python:3.12-slim".to_string()) + ); + // A bare `# sandbox` (no image) keeps the nsjail-bash modifier -> None. + assert_eq!(BashAnnotations::sandbox_image("# sandbox\necho hi"), None); + // `sandbox` must be its own token, not a prefix. + assert_eq!(BashAnnotations::sandbox_image("# sandboxed foo"), None); + // Stops at the first non-comment line (image declared too late is ignored). + assert_eq!( + BashAnnotations::sandbox_image("echo hi\n# sandbox alpine"), + None + ); + // `# docker` is a different annotation -> not a sandbox image. + assert_eq!( + BashAnnotations::sandbox_image("# docker alpine\necho hi"), + None + ); + } + #[test] fn test_mixed_tags() { let input = vec![ diff --git a/backend/windmill-mcp/Cargo.toml b/backend/windmill-mcp/Cargo.toml index 3968ea0836..36e0d03d18 100644 --- a/backend/windmill-mcp/Cargo.toml +++ b/backend/windmill-mcp/Cargo.toml @@ -29,3 +29,6 @@ http = { workspace = true, optional = true } tokio-util = { workspace = true, features = ["rt"], optional = true } tokio = { workspace = true, optional = true } futures.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/backend/windmill-mcp/src/client/mod.rs b/backend/windmill-mcp/src/client/mod.rs index bc6d2c24ac..34dc141209 100644 --- a/backend/windmill-mcp/src/client/mod.rs +++ b/backend/windmill-mcp/src/client/mod.rs @@ -22,8 +22,6 @@ use rmcp::{ }; use serde_json::{json, Value}; use std::str::FromStr; -use windmill_common::variables::get_secret_value_as_admin; -use windmill_common::DB; /// MCP client for communicating with external MCP servers pub struct McpClient { @@ -34,18 +32,34 @@ pub struct McpClient { } impl McpClient { - /// Create a new MCP client from a resource configuration - pub async fn from_resource(resource: McpResource, db: &DB, w_id: &str) -> Result { + /// Create a new MCP client from a resource configuration. + /// + /// `token`, when present, is the already-resolved bearer token sent as an + /// `Authorization` header. It MUST be resolved by the caller through the + /// permissioned (RLS + audit) variable path — `from_resource` never reads + /// secrets itself, so a caller cannot trick it into decrypting a variable + /// they are not allowed to read. + pub async fn from_resource(resource: McpResource, token: Option) -> Result { + // The resource URL is author-controlled and we send a (potentially + // secret) bearer token to it, so it must be validated against SSRF + // before we connect (e.g. cloud metadata endpoints, internal services). + windmill_common::ssrf::validate_mcp_server_url(&resource.url) + .await + .map_err(|e| { + anyhow::anyhow!( + "MCP server URL is not allowed: {}", + windmill_common::ssrf::mcp_ssrf_error_message(&e) + ) + })?; + // Build custom reqwest client with headers if provided let mut headers = HeaderMap::new(); - if let Some(token_path) = &resource.token { - if !token_path.trim().is_empty() { - let value = - get_secret_value_as_admin(db, w_id, token_path.trim_start_matches("$var:")) - .await?; + if let Some(token) = token { + let token = token.trim(); + if !token.is_empty() { headers.insert( HeaderName::from_static("authorization"), - HeaderValue::from_str(format!("Bearer {}", value).as_str())?, + HeaderValue::from_str(format!("Bearer {}", token).as_str())?, ); } } @@ -64,6 +78,12 @@ impl McpClient { let reqwest_client = reqwest::Client::builder() .default_headers(headers) + // Don't follow redirects: the SSRF check above only validates the + // initial (author-controlled) URL, so following a redirect could + // still reach a private/internal address with the bearer token + // attached. The MCP streamable-HTTP endpoint is a direct endpoint + // and does not legitimately rely on redirects. + .redirect(reqwest::redirect::Policy::none()) .build() .context("Failed to build HTTP client")?; @@ -210,3 +230,61 @@ impl McpClient { } } } + +#[cfg(test)] +mod tests { + use super::*; + + struct PrivateMcpServerUrlsEnvGuard { + previous: Option, + } + + impl PrivateMcpServerUrlsEnvGuard { + fn unset() -> Self { + let previous = + std::env::var(windmill_common::ssrf::ALLOW_PRIVATE_MCP_SERVER_URLS_ENV).ok(); + std::env::remove_var(windmill_common::ssrf::ALLOW_PRIVATE_MCP_SERVER_URLS_ENV); + Self { previous } + } + } + + impl Drop for PrivateMcpServerUrlsEnvGuard { + fn drop(&mut self) { + match &self.previous { + Some(value) => std::env::set_var( + windmill_common::ssrf::ALLOW_PRIVATE_MCP_SERVER_URLS_ENV, + value, + ), + None => { + std::env::remove_var(windmill_common::ssrf::ALLOW_PRIVATE_MCP_SERVER_URLS_ENV) + } + } + } + } + + /// Regression test: `from_resource` must refuse to connect to a URL that + /// targets a private/internal address (here the AWS + /// instance-metadata endpoint), so a resource author cannot use the MCP + /// client as an SSRF primitive against internal services. The guard runs + /// before any connection attempt, so this fails fast without network access. + #[tokio::test] + async fn from_resource_rejects_ssrf_url() { + let _guard = PrivateMcpServerUrlsEnvGuard::unset(); + + let resource = McpResource { + name: "evil".to_string(), + url: "http://169.254.169.254".to_string(), + token: None, + headers: None, + }; + + let msg = match McpClient::from_resource(resource, None).await { + Ok(_) => panic!("a link-local metadata URL must be rejected before connecting"), + Err(e) => e.to_string(), + }; + assert!( + msg.contains("not allowed") && msg.contains("private"), + "error should explain the URL was rejected as private/internal, got: {msg}" + ); + } +} diff --git a/backend/windmill-mcp/src/client_registration.rs b/backend/windmill-mcp/src/client_registration.rs index eca4ed650f..b0b07cada1 100644 --- a/backend/windmill-mcp/src/client_registration.rs +++ b/backend/windmill-mcp/src/client_registration.rs @@ -17,7 +17,7 @@ use windmill_common::db::DB; use windmill_common::error; use windmill_common::variables::{build_crypt, decrypt, encrypt}; -use crate::oauth::AuthorizationManager; +use crate::oauth::{no_redirect_http_client, AuthorizationManager}; /// MCP client credentials returned by [`get_or_refresh_mcp_client`]. pub struct McpClientCredentials { @@ -77,7 +77,14 @@ async fn register_client( redirect_uri: &str, client_name: &str, ) -> Result { - let client = reqwest::Client::new(); + windmill_common::ssrf::validate_mcp_server_url_for_bad_request( + registration_endpoint, + "MCP server registration endpoint URL", + ) + .await?; + + let client = no_redirect_http_client() + .map_err(|e| error::Error::BadRequest(format!("Failed to build DCR client: {e}")))?; let request = DcrRequest { client_name: client_name.to_string(), redirect_uris: vec![redirect_uri.to_string()], @@ -121,6 +128,12 @@ pub async fn get_or_refresh_mcp_client( let base_url = (**windmill_common::BASE_URL.load()).clone(); let redirect_uri = format!("{}/api/mcp/oauth/callback", base_url); + windmill_common::ssrf::validate_mcp_server_url_for_bad_request( + mcp_server_url, + "MCP server URL", + ) + .await?; + let cached_client: Option = sqlx::query_as("SELECT mcp_server_url, client_id, client_secret, client_secret_expires_at, token_endpoint FROM mcp_oauth_client WHERE mcp_server_url = $1") .bind(mcp_server_url) @@ -131,6 +144,11 @@ pub async fn get_or_refresh_mcp_client( if let Some(client) = cached_client { if !client.is_expired() { tracing::debug!("Using cached MCP client for {}", mcp_server_url); + windmill_common::ssrf::validate_mcp_server_url_for_bad_request( + &client.token_endpoint, + "MCP server token endpoint URL", + ) + .await?; let decrypted_secret = if let Some(ref encrypted_secret) = client.client_secret { Some(decrypt_client_secret(db, encrypted_secret).await?) } else { @@ -145,17 +163,27 @@ pub async fn get_or_refresh_mcp_client( tracing::debug!("Cached MCP client expired, re-registering"); } - windmill_common::ssrf::validate_url_for_ssrf(mcp_server_url).await?; - - let manager = AuthorizationManager::new(mcp_server_url) + let mut manager = AuthorizationManager::new(mcp_server_url) .await .map_err(|e| error::Error::BadRequest(format!("Failed to create auth manager: {e}")))?; + let discovery_client = no_redirect_http_client().map_err(|e| { + error::Error::BadRequest(format!("Failed to build MCP OAuth discovery client: {e}")) + })?; + manager + .with_client(discovery_client) + .map_err(|e| error::Error::BadRequest(format!("Failed to configure auth manager: {e}")))?; let metadata = manager .discover_metadata() .await .map_err(|e| error::Error::BadRequest(format!("OAuth discovery failed: {e}")))?; + windmill_common::ssrf::validate_mcp_server_url_for_bad_request( + &metadata.token_endpoint, + "MCP server token endpoint URL", + ) + .await?; + let supports_dynamic_registration = metadata.registration_endpoint.is_some(); let (client_id, client_secret, expires_at) = if supports_dynamic_registration { diff --git a/backend/windmill-mcp/src/lib.rs b/backend/windmill-mcp/src/lib.rs index 7df6ee9f39..d9cae883de 100644 --- a/backend/windmill-mcp/src/lib.rs +++ b/backend/windmill-mcp/src/lib.rs @@ -38,11 +38,70 @@ pub mod client_registration; pub mod oauth { //! Re-exports of rmcp auth and oauth2 types for MCP OAuth implementations + use std::time::Duration; + pub use rmcp::transport::auth::AuthorizationManager; + const DEFAULT_OAUTH_HTTP_TIMEOUT: Duration = Duration::from_secs(30); + + pub fn no_redirect_http_client() -> Result { + no_redirect_http_client_with_timeout(DEFAULT_OAUTH_HTTP_TIMEOUT) + } + + pub(crate) fn no_redirect_http_client_with_timeout( + timeout: Duration, + ) -> Result { + reqwest::Client::builder() + .timeout(timeout) + .redirect(reqwest::redirect::Policy::none()) + .build() + } + // Re-export oauth2 types needed for MCP OAuth flow pub use oauth2::{ basic::BasicClient, AuthUrl, ClientId, ClientSecret, CsrfToken, PkceCodeChallenge, RedirectUrl, Scope, TokenUrl, }; + + #[cfg(test)] + mod tests { + use super::*; + use std::{ + io::Read, + net::TcpListener, + thread, + time::{Duration, Instant}, + }; + + #[tokio::test] + async fn no_redirect_http_client_times_out_stalled_responses() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + + let handle = thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); + let mut buffer = [0; 1024]; + let _ = stream.read(&mut buffer); + thread::sleep(Duration::from_millis(300)); + } + }); + + let client = no_redirect_http_client_with_timeout(Duration::from_millis(50)).unwrap(); + let started = Instant::now(); + let err = client + .get(format!("http://{addr}/stall")) + .send() + .await + .expect_err("stalled response should time out"); + + assert!(err.is_timeout(), "expected timeout error, got: {err}"); + assert!( + started.elapsed() < Duration::from_secs(2), + "stalled request should fail promptly" + ); + + handle.join().unwrap(); + } + } } diff --git a/backend/windmill-native-triggers/src/github/routes.rs b/backend/windmill-native-triggers/src/github/routes.rs index 4528c1b612..b2196f2249 100644 --- a/backend/windmill-native-triggers/src/github/routes.rs +++ b/backend/windmill-native-triggers/src/github/routes.rs @@ -2,9 +2,10 @@ use std::sync::Arc; use axum::{extract::Path, routing::get, Extension, Json, Router}; use http::Method; +use windmill_api_auth::ApiAuthed; use windmill_common::{error::JsonResult, DB}; -use crate::{get_workspace_integration, External, ServiceName}; +use crate::{get_workspace_integration, require_native_integration_use, External, ServiceName}; use super::{GitHub, GithubApiRepoResponse, GithubRepoEntry}; @@ -12,10 +13,12 @@ const PER_PAGE: usize = 100; const MAX_PAGES: usize = 10; async fn list_repos( + authed: ApiAuthed, Extension(handler): Extension>, Extension(db): Extension, Path(workspace_id): Path, ) -> JsonResult> { + require_native_integration_use(&authed)?; get_workspace_integration(&db, &workspace_id, ServiceName::Github).await?; let mut all_entries = Vec::new(); diff --git a/backend/windmill-native-triggers/src/google/routes.rs b/backend/windmill-native-triggers/src/google/routes.rs index f49a9f9fe6..c5f7611f10 100644 --- a/backend/windmill-native-triggers/src/google/routes.rs +++ b/backend/windmill-native-triggers/src/google/routes.rs @@ -7,9 +7,10 @@ use axum::{ }; use http::Method; use serde::{Deserialize, Serialize}; +use windmill_api_auth::ApiAuthed; use windmill_common::{error::JsonResult, DB}; -use crate::{get_workspace_integration, External, ServiceName}; +use crate::{get_workspace_integration, require_native_integration_use, External, ServiceName}; use super::Google; @@ -84,10 +85,12 @@ pub struct DriveFilesQuery { } async fn list_calendars( + authed: ApiAuthed, Extension(handler): Extension>, Extension(db): Extension, Path(workspace_id): Path, ) -> JsonResult> { + require_native_integration_use(&authed)?; get_workspace_integration(&db, &workspace_id, ServiceName::Google).await?; let url = format!( @@ -113,11 +116,13 @@ async fn list_calendars( } async fn list_drive_files( + authed: ApiAuthed, Extension(handler): Extension>, Extension(db): Extension, Path(workspace_id): Path, Query(query): Query, ) -> JsonResult { + require_native_integration_use(&authed)?; get_workspace_integration(&db, &workspace_id, ServiceName::Google).await?; let drive_query = if query.shared_with_me { @@ -186,10 +191,12 @@ struct SharedDriveApiEntry { } async fn list_shared_drives( + authed: ApiAuthed, Extension(handler): Extension>, Extension(db): Extension, Path(workspace_id): Path, ) -> JsonResult> { + require_native_integration_use(&authed)?; get_workspace_integration(&db, &workspace_id, ServiceName::Google).await?; let url = format!( diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 743b9f8210..a81217d8d5 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -1226,6 +1226,20 @@ pub async fn store_workspace_integration( Ok(()) } +/// Authorization gate for the integration *use* routes (calendar/drive/repo/event +/// pickers). A workspace admin configures the integration, but any member who can +/// create a native trigger needs the pickers to configure one. Operators are +/// read-only and cannot create triggers, so they must not be able to drive the +/// admin-configured integration's upstream API and enumerate its data. +pub fn require_native_integration_use(authed: &ApiAuthed) -> Result<()> { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot use workspace integrations".to_string(), + )); + } + Ok(()) +} + pub async fn get_workspace_integration<'c, E: sqlx::Executor<'c, Database = Postgres>>( db: E, workspace_id: &str, diff --git a/backend/windmill-native-triggers/src/nextcloud/routes.rs b/backend/windmill-native-triggers/src/nextcloud/routes.rs index 9c40d2403c..2f54124d64 100644 --- a/backend/windmill-native-triggers/src/nextcloud/routes.rs +++ b/backend/windmill-native-triggers/src/nextcloud/routes.rs @@ -7,17 +7,21 @@ use windmill_common::{ DB, }; +use windmill_api_auth::ApiAuthed; + use crate::{ get_workspace_integration, nextcloud::{NextCloudEventType, OcsResponse}, - External, ServiceName, + require_native_integration_use, External, ServiceName, }; async fn list_available_events( + authed: ApiAuthed, Extension(handler): Extension>, Extension(db): Extension, Path(workspace_id): Path, ) -> JsonResult> { + require_native_integration_use(&authed)?; let integration = get_workspace_integration(&db, &workspace_id, ServiceName::Nextcloud).await?; let base_url = integration diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index 874a859200..252f7ace28 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -26,7 +26,6 @@ use windmill_common::error::{self, to_anyhow, Error}; use windmill_common::more_serde::maybe_number_opt; use windmill_common::oauth2::*; use windmill_common::utils::now_from_db; -use windmill_common::variables::{build_crypt, encrypt}; use windmill_common::BASE_URL; pub type DB = sqlx::Pool; @@ -92,6 +91,12 @@ pub struct OAuthConfig { /// entry, `build_oauth_clients` registers a second client under that key. #[serde(skip_serializing_if = "Option::is_none")] pub sandbox: Option, + /// Frontend-only metadata for per-instance OAuth providers (Snowflake, + /// ServiceNow, …) whose authorize/token URLs are derived from an + /// admin-entered instance name. Ignored by the backend, which only ever + /// sees the resulting concrete `connect_config`. + #[serde(skip_serializing_if = "Option::is_none")] + pub connect_config_template: Option, } /// URL overrides for an OAuth provider's sandbox environment. Inherits @@ -106,6 +111,43 @@ pub struct OAuthSandboxOverride { pub userinfo_url: Option, } +/// Frontend metadata for a per-instance OAuth provider. The instance-settings +/// UI renders one generic instance-name input and substitutes `{instance}` into +/// `auth_url`/`token_url` to build the per-client `connect_config`. Adding a new +/// per-instance provider needs only a registry entry carrying this template — +/// no frontend code change. The backend never reads it. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ConnectConfigTemplate { + /// Properly-cased provider name for the settings dropdown (e.g. "ServiceNow"); + /// the UI falls back to a capitalized registry key when absent. + #[serde(skip_serializing_if = "Option::is_none")] + pub display_name: Option, + pub label: String, + pub placeholder: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub help_url: Option, + pub auth_url: String, + pub token_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub req_body_auth: Option, + /// Key under `connect_config.extra_params` where the instance name is + /// stored (defaults to `instance`). Snowflake uses `account_identifier` for + /// backward compatibility with previously-saved configs. + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_params_key: Option, + /// Optional host suffix stripped from the input before substitution (e.g. + /// `.service-now.com`), so the admin can paste a full host or a bare name. + #[serde(skip_serializing_if = "Option::is_none")] + pub strip_suffix: Option, + /// Maps OAuth-connected resource arg fields to value templates substituting + /// `{instance}` (e.g. ServiceNow's `instance_url` -> + /// `https://{instance}.service-now.com`). Applied by the resource-connect + /// flow so the created resource carries the instance-specific fields the + /// scripts need (ServiceNow's token response omits the host). + #[serde(skip_serializing_if = "Option::is_none")] + pub resource_mapping: Option>, +} + impl OAuthConfig { /// Returns a copy of this config with sandbox URL overrides applied and /// the nested `sandbox` field cleared. Returns `None` if no overrides are @@ -482,11 +524,14 @@ pub struct OAuthAccountInfo { pub scopes: Option>, } -/// Refresh an OAuth token and update the database. +/// Refresh an OAuth token and update the `account` row. /// Fetches the account from DB, then delegates to `refresh_token_for_account`. +/// +/// Returns the freshly minted access token. Persisting it to the secret variable +/// backing the resource is the caller's responsibility (it must route through the +/// configured secret backend — see `store_oauth_token_value` in `windmill-store`). pub async fn refresh_token<'c>( mut tx: Transaction<'c, Postgres>, - path: &str, w_id: &str, id: i32, db: &DB, @@ -506,7 +551,6 @@ pub async fn refresh_token<'c>( refresh_token_for_account( tx, - path, w_id, id, db, @@ -519,9 +563,14 @@ pub async fn refresh_token<'c>( } /// Refresh an OAuth token given pre-fetched account info (no additional SELECT). +/// +/// Exchanges the refresh token, updates the `account` row (`refresh_token`, +/// `expires_at`, `refresh_error`) and returns the new access token. It does NOT +/// persist the token to the secret variable — the caller must do that through the +/// configured secret backend (`store_oauth_token_value`), otherwise an external +/// secret backend would keep serving the stale connect-time token. pub async fn refresh_token_for_account<'c>( mut tx: Transaction<'c, Postgres>, - path: &str, w_id: &str, id: i32, db: &DB, @@ -633,17 +682,6 @@ pub async fn refresh_token_for_account<'c>( tx.commit().await?; let token_str = token.access_token.to_string(); - let mc = build_crypt(db, w_id).await?; - let encrypted_token = encrypt(&mc, token_str.as_str()); - - sqlx::query!( - "UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3", - encrypted_token, - w_id, - path - ) - .execute(db) - .await?; tracing::info!( grant_type = %account.grant_type, @@ -817,6 +855,7 @@ mod tests { token_url: Some("https://account-d.example.com/oauth/token".to_string()), userinfo_url: None, }), + connect_config_template: None, } } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 964fd558f1..0a707418ba 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -5058,7 +5058,7 @@ async fn push_inner<'c, 'd>( content, path, hash, - language, + mut language, lock, cache_ttl, cache_ignore_s3_path, @@ -5068,6 +5068,21 @@ async fn push_inner<'c, 'd>( debouncing_settings, modules, }) => { + // Reconcile the preview language with the `//native` annotation, mirroring the + // deploy-time logic in `worker_lockfiles`. The editor sends `bun` for a TypeScript + // script even when it carries `//native`, which would otherwise tag the preview as + // `bun` and route it to a regular bun worker. A native-mode worker neither matches + // the `bun` tag nor accepts a non-native `script_lang`, so previewing a `//native` + // script on a native-only worker setup fails. Normalizing to `bunnative` (tag + // `nativets`) makes the preview run exactly like the deployed script would. + if language == ScriptLang::Bun || language == ScriptLang::Bunnative { + let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); + if anns.native && language == ScriptLang::Bun { + language = ScriptLang::Bunnative; + } else if !anns.native && language == ScriptLang::Bunnative { + language = ScriptLang::Bun; + } + } // Inject modules into job args as _MODULES so the worker can extract them if let Some(ref modules) = modules { match serde_json::to_string(modules).and_then(|s| RawValue::from_string(s)) { @@ -5277,6 +5292,7 @@ async fn push_inner<'c, 'd>( expr: skip_handler.stop_condition, skip_if_stopped: true, error_message: Some(skip_handler.stop_message), + error_include_result: false, }), ..Default::default() }); diff --git a/backend/windmill-store/Cargo.toml b/backend/windmill-store/Cargo.toml index cb1e41ad05..b3aca5e669 100644 --- a/backend/windmill-store/Cargo.toml +++ b/backend/windmill-store/Cargo.toml @@ -45,6 +45,8 @@ tracing.workspace = true uuid.workspace = true quick_cache.workspace = true lazy_static.workspace = true +sha2.workspace = true +hex.workspace = true sql-builder.workspace = true async-recursion.workspace = true futures.workspace = true diff --git a/backend/windmill-store/src/lib.rs b/backend/windmill-store/src/lib.rs index c7e606de43..b14466c79b 100644 --- a/backend/windmill-store/src/lib.rs +++ b/backend/windmill-store/src/lib.rs @@ -14,3 +14,6 @@ pub mod resources; pub mod secret_backend_ext; pub mod var_resource_cache; pub mod variables; + +#[cfg(all(test, feature = "oauth2", feature = "private", feature = "enterprise"))] +mod oauth_refresh_secret_backend_tests; diff --git a/backend/windmill-store/src/oauth_refresh_oss.rs b/backend/windmill-store/src/oauth_refresh_oss.rs index e8edb19ccd..13e24dfd8b 100644 --- a/backend/windmill-store/src/oauth_refresh_oss.rs +++ b/backend/windmill-store/src/oauth_refresh_oss.rs @@ -24,9 +24,8 @@ pub async fn _refresh_token<'c>( id: i32, db: &DB, ) -> error::Result { - windmill_oauth::refresh_token( + let token = windmill_oauth::refresh_token( tx, - path, w_id, id, db, @@ -34,5 +33,12 @@ pub async fn _refresh_token<'c>( &windmill_oauth::OAUTH_HTTP_CLIENT, include_str!("../../oauth_connect.json"), ) - .await + .await?; + + // Persist the refreshed token through the configured secret backend so an + // external backend (Vault / Azure KV / AWS Secrets Manager) is updated too, + // not just the in-DB variable mirror. + crate::secret_backend_ext::store_oauth_token_value(db, w_id, path, &token).await?; + + Ok(token) } diff --git a/backend/windmill-store/src/oauth_refresh_secret_backend_tests.rs b/backend/windmill-store/src/oauth_refresh_secret_backend_tests.rs new file mode 100644 index 0000000000..5a11dea03e --- /dev/null +++ b/backend/windmill-store/src/oauth_refresh_secret_backend_tests.rs @@ -0,0 +1,337 @@ +//! E2E regression tests for OAuth token refresh persistence through the +//! configured secret backend. +//! +//! Regression for windmill#9471 / windmill-ee-private#607: the lazy on-fetch +//! OAuth refresh used to persist the freshly minted token with a raw +//! `UPDATE variable SET value = `, bypassing the secret-backend +//! abstraction. With an external backend (AWS Secrets Manager / Azure Key +//! Vault / Vault) reads resolve through the backend and ignore `variable.value` +//! entirely, so the external store stayed frozen at its connect-time token and +//! every read that did not itself trigger a mint served a stale/expired token. +//! +//! These tests exercise the persistence step (`store_oauth_token_value`) — the +//! exact code path that was fixed — against both the database backend and an +//! external (AWS Secrets Manager via LocalStack) backend, plus the self-healing +//! reset on a failed persist. They are opt-in (they mutate the shared +//! `global_settings.secret_backend` row and workspace/variable/account rows on a +//! real DB) and skip unless `RUN_SECRET_BACKEND_E2E=1` is set. +//! +//! ## Run +//! +//! Database-backend case (needs a migrated DB). Note `RUN_SECRET_BACKEND_E2E=1` +//! is required or every test skips: +//! +//! ```bash +//! RUN_SECRET_BACKEND_E2E=1 \ +//! DATABASE_URL=postgres://postgres:changeme@127.0.0.1:5432/windmill \ +//! cargo test -p windmill-store --features private,enterprise,oauth2 \ +//! oauth_refresh_secret_backend_tests -- --nocapture --test-threads=1 +//! ``` +//! +//! External-backend cases additionally need LocalStack `secretsmanager` and +//! `RUN_AWS_SM_TESTS=1`: +//! +//! ```bash +//! docker run -d -e SERVICES=secretsmanager localstack/localstack:3.8 +//! RUN_SECRET_BACKEND_E2E=1 RUN_AWS_SM_TESTS=1 AWS_SM_ENDPOINT=http://:4566 \ +//! DATABASE_URL=postgres://postgres:changeme@127.0.0.1:5432/windmill \ +//! cargo test -p windmill-store --features private,enterprise,oauth2 \ +//! oauth_refresh_secret_backend_tests -- --nocapture --test-threads=1 +//! ``` + +use crate::secret_backend_ext::{get_secret_value, store_oauth_token_value, store_secret_value}; +use sqlx::postgres::PgPoolOptions; +use sqlx::{Pool, Postgres}; + +// global_settings holds a single `secret_backend` row shared across tests; +// serialize the test bodies so concurrent runs don't clobber each other's +// configured backend. (Also run with --test-threads=1 for good measure.) +static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn env_flag(name: &str) -> bool { + std::env::var(name) + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(false) +} + +// Opt-in gate so the suite never runs (and mutates shared DB state) as part of a +// normal `cargo test` invocation. +fn run_e2e() -> bool { + env_flag("RUN_SECRET_BACKEND_E2E") +} + +fn run_aws_sm() -> bool { + env_flag("RUN_AWS_SM_TESTS") +} + +/// Restore the default (database) backend so we don't leave the instance +/// pointed at a test backend for any concurrently-running suite. +async fn reset_backend(db: &Pool) { + set_backend(db, serde_json::json!({ "type": "Database" })).await; +} + +fn aws_sm_endpoint() -> String { + std::env::var("AWS_SM_ENDPOINT").unwrap_or_else(|_| "http://localhost:4566".to_string()) +} + +async fn db() -> Pool { + let url = std::env::var("DATABASE_URL") + .expect("DATABASE_URL must point at a migrated windmill database"); + PgPoolOptions::new() + .max_connections(5) + .connect(&url) + .await + .expect("connect to DATABASE_URL") +} + +/// Fresh workspace + key + clean variable/account rows for `w_id`. +async fn setup_workspace(db: &Pool, w_id: &str) { + sqlx::query("DELETE FROM variable WHERE workspace_id = $1") + .bind(w_id) + .execute(db) + .await + .unwrap(); + sqlx::query("DELETE FROM account WHERE workspace_id = $1") + .bind(w_id) + .execute(db) + .await + .unwrap(); + sqlx::query("DELETE FROM workspace_key WHERE workspace_id = $1") + .bind(w_id) + .execute(db) + .await + .unwrap(); + sqlx::query("DELETE FROM workspace WHERE id = $1") + .bind(w_id) + .execute(db) + .await + .unwrap(); + + sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'admin@windmill.dev')") + .bind(w_id) + .execute(db) + .await + .unwrap(); + sqlx::query( + "INSERT INTO workspace_key (workspace_id, kind, key) VALUES ($1, 'cloud', 'e2ekey')", + ) + .bind(w_id) + .execute(db) + .await + .unwrap(); +} + +async fn set_backend(db: &Pool, config: serde_json::Value) { + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ('secret_backend', $1) \ + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(config) + .execute(db) + .await + .unwrap(); +} + +fn aws_sm_config(endpoint: &str) -> serde_json::Value { + serde_json::json!({ + "type": "AwsSecretsManager", + "region": "us-east-1", + "access_key_id": "test", + "secret_access_key": "test", + "endpoint_url": endpoint, + "prefix": "windmill-e2e/" + }) +} + +/// Simulate `Connect`: store the initial token through the backend and create +/// the linked secret variable + account (expired, with a refresh token). +async fn simulate_connect(db: &Pool, w_id: &str, path: &str, initial_token: &str) -> i32 { + let stored = store_secret_value(db, w_id, path, initial_token) + .await + .expect("store initial token"); + + let account_id: i32 = sqlx::query_scalar( + "INSERT INTO account (workspace_id, expires_at, refresh_token, client, grant_type) \ + VALUES ($1, now() - interval '1 hour', 'rt_dummy', 'gdrive', 'authorization_code') \ + RETURNING id", + ) + .bind(w_id) + .fetch_one(db) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO variable (workspace_id, path, value, is_secret, is_oauth, account, expires_at) \ + VALUES ($1, $2, $3, true, true, $4, now() - interval '1 hour')", + ) + .bind(w_id) + .bind(path) + .bind(&stored) + .bind(account_id) + .execute(db) + .await + .unwrap(); + + account_id +} + +async fn variable_value(db: &Pool, w_id: &str, path: &str) -> String { + sqlx::query_scalar("SELECT value FROM variable WHERE workspace_id = $1 AND path = $2") + .bind(w_id) + .bind(path) + .fetch_one(db) + .await + .unwrap() +} + +async fn account_expires_in_past(db: &Pool, w_id: &str, account_id: i32) -> bool { + sqlx::query_scalar("SELECT expires_at < now() FROM account WHERE workspace_id = $1 AND id = $2") + .bind(w_id) + .bind(account_id) + .fetch_one(db) + .await + .unwrap() +} + +/// Database backend (the "without external storage" case): refresh must +/// re-encrypt the new token into `variable.value`; reads serve the new token. +#[tokio::test] +async fn database_backend_persists_refreshed_token() { + if !run_e2e() { + println!( + "Skipping database_backend_persists_refreshed_token: set RUN_SECRET_BACKEND_E2E=1" + ); + return; + } + let _guard = SERIAL.lock().await; + let db = db().await; + let w_id = "wm_e2e_db"; + let path = "f/google/gdrive"; + + set_backend(&db, serde_json::json!({ "type": "Database" })).await; + setup_workspace(&db, w_id).await; + let _ = simulate_connect(&db, w_id, path, "OLD_TOKEN").await; + + // Connect-time token is served. + let v = variable_value(&db, w_id, path).await; + assert_eq!( + get_secret_value(&db, w_id, path, &v).await.unwrap(), + "OLD_TOKEN" + ); + + // Refresh persists the new token. + store_oauth_token_value(&db, w_id, path, "NEW_TOKEN") + .await + .unwrap(); + + let v = variable_value(&db, w_id, path).await; + assert_eq!( + get_secret_value(&db, w_id, path, &v).await.unwrap(), + "NEW_TOKEN", + "database backend should serve the refreshed token" + ); + println!(" ✓ database backend serves refreshed token"); + reset_backend(&db).await; +} + +/// External backend (the "with external storage" case): refresh must write +/// the new token to AWS Secrets Manager. Before the fix the external store +/// stayed frozen and reads served the stale connect-time token. +#[tokio::test] +async fn external_backend_persists_refreshed_token() { + if !run_e2e() || !run_aws_sm() { + println!("Skipping external_backend_persists_refreshed_token: set RUN_SECRET_BACKEND_E2E=1 and RUN_AWS_SM_TESTS=1"); + return; + } + let _guard = SERIAL.lock().await; + let db = db().await; + let w_id = "wm_e2e_awssm"; + let path = "f/google/gsheets"; + + set_backend(&db, aws_sm_config(&aws_sm_endpoint())).await; + setup_workspace(&db, w_id).await; + let _ = simulate_connect(&db, w_id, path, "OLD_TOKEN").await; + + // Connect-time token is served from the external store. + let marker = variable_value(&db, w_id, path).await; + assert!( + marker.starts_with("$aws_sm:"), + "external backend should store a marker in variable.value, got {marker}" + ); + assert_eq!( + get_secret_value(&db, w_id, path, &marker).await.unwrap(), + "OLD_TOKEN" + ); + + // Demonstrate the original bug shape: a raw DB write to variable.value is + // futile because reads resolve through the backend and ignore it. + sqlx::query( + "UPDATE variable SET value = 'ignored_db_blob' WHERE workspace_id = $1 AND path = $2", + ) + .bind(w_id) + .bind(path) + .execute(&db) + .await + .unwrap(); + assert_eq!( + get_secret_value(&db, w_id, path, "ignored_db_blob") + .await + .unwrap(), + "OLD_TOKEN", + "reads ignore variable.value for external backends — a raw UPDATE can't refresh the served token" + ); + + // The fix: persist through the backend. + store_oauth_token_value(&db, w_id, path, "NEW_TOKEN") + .await + .unwrap(); + + let marker = variable_value(&db, w_id, path).await; + assert_eq!( + get_secret_value(&db, w_id, path, &marker).await.unwrap(), + "NEW_TOKEN", + "external backend should serve the refreshed token written back to AWS SM" + ); + println!(" ✓ external (AWS SM) backend serves refreshed token written back to the store"); + reset_backend(&db).await; +} + +/// If persisting the refreshed token fails (e.g. transient external-backend +/// error) after the account was committed fresh, the account expiry must be +/// reset to the past so the next fetch retries instead of serving a stale +/// token for the whole token lifetime. +#[tokio::test] +async fn failed_persist_resets_account_expiry() { + if !run_e2e() || !run_aws_sm() { + println!("Skipping failed_persist_resets_account_expiry: set RUN_SECRET_BACKEND_E2E=1 and RUN_AWS_SM_TESTS=1"); + return; + } + let _guard = SERIAL.lock().await; + let db = db().await; + let w_id = "wm_e2e_selfheal"; + let path = "f/google/gdrive"; + + // Working backend first to seed the variable + a *fresh* account. + set_backend(&db, aws_sm_config(&aws_sm_endpoint())).await; + setup_workspace(&db, w_id).await; + let account_id = simulate_connect(&db, w_id, path, "OLD_TOKEN").await; + sqlx::query("UPDATE account SET expires_at = now() + interval '1 hour' WHERE workspace_id = $1 AND id = $2") + .bind(w_id) + .bind(account_id) + .execute(&db) + .await + .unwrap(); + assert!(!account_expires_in_past(&db, w_id, account_id).await); + + // Point the backend at an unreachable endpoint so the persist fails. + set_backend(&db, aws_sm_config("http://127.0.0.1:1")).await; + + let res = store_oauth_token_value(&db, w_id, path, "NEW_TOKEN").await; + assert!(res.is_err(), "persist to unreachable backend should fail"); + assert!( + account_expires_in_past(&db, w_id, account_id).await, + "a failed persist must reset account.expires_at to the past so refresh retries" + ); + println!(" ✓ failed persist reset account expiry (self-healing)"); + reset_backend(&db).await; +} diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 3e5292e805..402628e3ad 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -17,7 +17,7 @@ use windmill_common::db::DB; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use crate::secret_backend_ext::rename_vault_secret; -use crate::var_resource_cache::{cache_resource, get_cached_resource}; +use crate::var_resource_cache::{auth_identity, cache_resource, get_cached_resource}; use windmill_common::utils::{escape_ilike_pattern, BulkDeleteRequest}; use windmill_common::webhook::{WebhookMessage, WebhookShared}; @@ -550,8 +550,18 @@ pub async fn get_resource_value_interpolated_internal<'a>( return Ok(Some(pg_creds)); } - if allow_cache { - if let Some(cached_value) = get_cached_resource(&workspace, &path) { + // Scope the cache to the caller's full authorization identity (not just email): the + // cached value is already decrypted/interpolated under this caller's RLS context, so it + // must never be served to a context that resolves to different permissions. Only + // job-independent values are ever stored (see the write below), so a hit is always safe + // to return regardless of the current `job_id`. + let cache_identity = allow_cache.then(|| match db_with_opt_authed.authed() { + Some(authed) => auth_identity(authed), + None => format!("\0system:{}", db_with_opt_authed.email()), + }); + + if let Some(identity) = cache_identity.as_deref() { + if let Some(cached_value) = get_cached_resource(&workspace, &path, identity) { return Ok(Some(cached_value)); } } @@ -575,17 +585,24 @@ pub async fn get_resource_value_interpolated_internal<'a>( let value = not_found_if_none(value_o, "Resource", path)?; if let Some(value) = value { - let r = transform_json_value( + // Track whether interpolation pulled in a `$WM_*` contextual variable. If it did, the + // result is job-dependent (and may embed `$WM_TOKEN`) and must not be cached; if not, + // it's job-independent and safe to cache and to serve to any job context. + let used_job_context = std::sync::atomic::AtomicBool::new(false); + let r = transform_json_value_tracked( &db_with_opt_authed, workspace, value, &job_id, token_for_context, 0, + &used_job_context, ) .await?; - if allow_cache { - cache_resource(&workspace, &path, r.clone()); + if let Some(identity) = cache_identity.as_deref() { + if !used_job_context.load(std::sync::atomic::Ordering::Relaxed) { + cache_resource(&workspace, &path, identity, r.clone()); + } } Ok(Some(r)) } else { @@ -601,14 +618,41 @@ pub async fn get_resource_value_interpolated_internal<'a>( // access could otherwise use to crash the API process. pub const MAX_RESOURCE_INTERPOLATION_DEPTH: u8 = 50; -#[async_recursion] pub async fn transform_json_value( - db_with_opt_authed: &DbWithOptAuthed, + db_with_opt_authed: &DbWithOptAuthed<'_, ApiAuthed>, workspace: &str, v: Value, job_id: &Option, token: Option<&str>, depth: u8, +) -> Result { + // Discard the job-context flag; callers that need it use `transform_json_value_tracked`. + let used_job_context = std::sync::atomic::AtomicBool::new(false); + transform_json_value_tracked( + db_with_opt_authed, + workspace, + v, + job_id, + token, + depth, + &used_job_context, + ) + .await +} + +/// Like [`transform_json_value`], but records into `used_job_context` whether the value +/// contains a `$WM_*` contextual variable (resolved from `job_id`/`token`). A value that did +/// not is job-independent and safe to cache; one that did must not be cached or shared across +/// jobs. +#[async_recursion] +pub async fn transform_json_value_tracked( + db_with_opt_authed: &DbWithOptAuthed<'_, ApiAuthed>, + workspace: &str, + v: Value, + job_id: &Option, + token: Option<&str>, + depth: u8, + used_job_context: &std::sync::atomic::AtomicBool, ) -> Result { if depth >= MAX_RESOURCE_INTERPOLATION_DEPTH { return Err(Error::internal_err(format!( @@ -652,15 +696,35 @@ pub async fn transform_json_value( tx.commit().await?; let v = not_found_if_none(v, "Resource", path)?; if let Some(v) = v { - transform_json_value(db_with_opt_authed, workspace, v, job_id, token, depth + 1) - .await + transform_json_value_tracked( + db_with_opt_authed, + workspace, + v, + job_id, + token, + depth + 1, + used_job_context, + ) + .await } else { Ok(Value::Null) } } - Value::String(y) if y.starts_with("$") && job_id.is_some() => { + // `$WM_*` is the reserved contextual-variable namespace (`$WM_TOKEN`, `$WM_JOB_ID`, + // ...); its resolved value depends on the job, so a value containing one is + // job-dependent and must never be cached — including on a no-job read, where the + // placeholder is left unresolved (caching it would then serve a stale placeholder to a + // later job read). Any other `$...` string (custom workspace envs, `$5.00`, `$HOME`, jq + // paths) is NOT interpolated here — it resolves to itself regardless of context and so + // stays cacheable (handled by the catch-all below). Note: custom workspace envs are + // intentionally not resolved inside resource values (they remain available to scripts). + Value::String(y) if y.starts_with("$WM_") => { + used_job_context.store(true, std::sync::atomic::Ordering::Relaxed); + let Some(job_id) = *job_id else { + // No job context to resolve against; leave the placeholder unchanged. + return Ok(Value::String(y)); + }; let mut tx = db_with_opt_authed.begin().await?; - let job_id = job_id.unwrap(); let job = sqlx::query!( "SELECT v2_job.permissioned_as_email, @@ -731,13 +795,14 @@ pub async fn transform_json_value( Value::Array(mut arr) if depth <= 2 && arr.len() <= 1000 => { for i in 0..arr.len() { let val = std::mem::take(&mut arr[i]); - arr[i] = transform_json_value( + arr[i] = transform_json_value_tracked( db_with_opt_authed, workspace, val, job_id, token, depth + 1, + used_job_context, ) .await?; } @@ -754,13 +819,14 @@ pub async fn transform_json_value( } Value::Object(mut m) => { for (a, b) in m.clone().into_iter() { - let v = transform_json_value( + let v = transform_json_value_tracked( db_with_opt_authed, workspace, b, job_id, token, depth + 1, + used_job_context, ) .await?; m.insert(a.clone(), v); diff --git a/backend/windmill-store/src/secret_backend_ext.rs b/backend/windmill-store/src/secret_backend_ext.rs index a83cb56b9e..1ae32c80ce 100644 --- a/backend/windmill-store/src/secret_backend_ext.rs +++ b/backend/windmill-store/src/secret_backend_ext.rs @@ -26,7 +26,10 @@ use windmill_common::{ #[cfg(all(feature = "private", feature = "enterprise"))] use windmill_common::{ global_settings::{load_value_from_global_settings, SECRET_BACKEND_SETTING}, - secret_backend::{AwsSecretsManagerBackend, AwsSecretsManagerSettings, AzureKeyVaultBackend, AzureKeyVaultSettings, SecretBackendConfig, VaultBackend, VaultSettings}, + secret_backend::{ + AwsSecretsManagerBackend, AwsSecretsManagerSettings, AzureKeyVaultBackend, + AzureKeyVaultSettings, SecretBackendConfig, VaultBackend, VaultSettings, + }, }; #[cfg(all(feature = "private", feature = "enterprise"))] @@ -225,7 +228,12 @@ pub async fn is_vault_backend_configured(db: &DB) -> Result { None => SecretBackendConfig::default(), }; - Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_) | SecretBackendConfig::AzureKeyVault(_) | SecretBackendConfig::AwsSecretsManager(_))) + Ok(matches!( + config, + SecretBackendConfig::HashiCorpVault(_) + | SecretBackendConfig::AzureKeyVault(_) + | SecretBackendConfig::AwsSecretsManager(_) + )) } /// Get a secret value using the configured backend @@ -252,12 +260,8 @@ pub async fn get_secret_value( // Fetch from Vault directly backend.get_secret(workspace_id, path).await } - "azure_key_vault" => { - backend.get_secret(workspace_id, path).await - } - "aws_secrets_manager" => { - backend.get_secret(workspace_id, path).await - } + "azure_key_vault" => backend.get_secret(workspace_id, path).await, + "aws_secrets_manager" => backend.get_secret(workspace_id, path).await, _ => Err(Error::internal_err(format!( "Unknown backend: {}", backend.backend_name() @@ -303,6 +307,93 @@ pub async fn store_secret_value( } } +/// Persist a freshly minted OAuth access token to the secret variable backing +/// a resource, routing through the configured secret backend. +/// +/// This is the write counterpart of the lazy on-fetch OAuth refresh: it stores +/// the token via [`store_secret_value`] (which writes to the external backend — +/// AWS Secrets Manager / Azure Key Vault / Vault — when one is configured, or +/// encrypts for the database backend) and updates `variable.value` with the +/// returned value (the encrypted blob for the DB backend, or a `$...:` marker +/// for an external backend). Using a raw `UPDATE variable SET value = ` +/// here instead would leave the external store frozen at its connect-time token +/// while reads (which resolve through the backend) keep serving the stale value. +/// +/// The caller has already committed the `account` row as fresh (advanced +/// `expires_at`) by the time we get here. If persisting the token fails — most +/// likely a transient error talking to an external backend — that would leave +/// the account marked fresh while the served secret is stale, so the on-fetch +/// refresh gate (`now() > expires_at`) would skip refresh and keep serving the +/// stale token for the whole token lifetime. To avoid that we reset `expires_at` +/// to the past (and record `refresh_error`) on failure — looking the account up +/// via `variable.account` — so the very next fetch retries the refresh instead. +/// +/// Authorization contract: this performs NO access control. It writes the +/// caller-supplied token into the secret variable at `path` and may mutate the +/// linked `account` row, so callers MUST have already authorized the operation +/// against `workspace_id`/`path` (the OAuth refresh adapters only run after the +/// read path has resolved and gated the variable). It is therefore kept +/// `pub(crate)` and intended solely for the in-crate refresh adapters. +#[cfg(feature = "oauth2")] +pub(crate) async fn store_oauth_token_value( + db: &DB, + workspace_id: &str, + path: &str, + token: &str, +) -> Result<()> { + let persist = async { + let value = store_secret_value(db, workspace_id, path, token).await?; + sqlx::query("UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3") + .bind(value) + .bind(workspace_id) + .bind(path) + .execute(db) + .await?; + Ok::<(), Error>(()) + } + .await; + + if let Err(e) = persist { + // Mark the account expired again so the next fetch re-runs the refresh + // instead of serving the now-stale token until it naturally expires. The + // account id is the one linked from the variable being refreshed. + let account_id: Option = sqlx::query_scalar::<_, Option>( + "SELECT account FROM variable WHERE workspace_id = $1 AND path = $2", + ) + .bind(workspace_id) + .bind(path) + .fetch_optional(db) + .await + .ok() + .flatten() + .flatten(); + + if let Some(account_id) = account_id { + if let Err(reset_err) = sqlx::query( + "UPDATE account SET expires_at = now() - interval '1 minute', refresh_error = $1 \ + WHERE workspace_id = $2 AND id = $3", + ) + .bind(format!( + "OAuth token was refreshed but persisting it to the secret backend failed: {e}" + )) + .bind(workspace_id) + .bind(account_id) + .execute(db) + .await + { + tracing::error!( + workspace_id = %workspace_id, + account_id = %account_id, + "failed to reset account expiry after token persistence error: {reset_err}" + ); + } + } + return Err(e); + } + + Ok(()) +} + /// Delete a secret from the configured backend (if using Vault) /// /// For database backend: no-op (DB delete is handled separately) diff --git a/backend/windmill-store/src/var_resource_cache.rs b/backend/windmill-store/src/var_resource_cache.rs index f7ce2aeecf..3e89f8579e 100644 --- a/backend/windmill-store/src/var_resource_cache.rs +++ b/backend/windmill-store/src/var_resource_cache.rs @@ -8,7 +8,9 @@ use quick_cache::sync::Cache; use serde_json::Value; +use sha2::{Digest, Sha256}; use std::time::{SystemTime, UNIX_EPOCH}; +use windmill_common::db::Authable; /// Cache TTL for variables and resources (30seconds) const CACHE_TTL_SECS: u64 = 30; @@ -40,11 +42,23 @@ impl CacheEntry { } } -lazy_static::lazy_static! { - /// Cache for individual variable values: key = "workspace_id:path" - pub static ref VARIABLE_CACHE: Cache> = Cache::new(1000); +/// A cached variable value plus whether it is a secret. `is_secret` is retained so a +/// cache hit can re-run the per-read side effects of a secret read (the +/// `variables.decrypt_secret` audit and running-job secret registration) that the +/// original miss performed — a hit must be observably equivalent to a miss. +#[derive(Clone, Debug)] +pub struct CachedVariable { + pub value: String, + pub is_secret: bool, +} - /// Cache for resource values: key = "workspace_id:path" +lazy_static::lazy_static! { + /// Cache for individual variable values. Key: [`identity_cache_key`] + /// (`identity:workspace_id:path`) — scoped to the caller's authorization context. + pub static ref VARIABLE_CACHE: Cache> = Cache::new(1000); + + /// Cache for interpolated resource values. Key: [`identity_cache_key`] + /// (`identity:workspace_id:path`) — scoped to the caller's authorization context. pub static ref RESOURCE_CACHE: Cache> = Cache::new(1000); } @@ -53,9 +67,73 @@ pub fn cache_key(workspace_id: &str, path: &str) -> String { format!("{}:{}", workspace_id, path) } -/// Get cached variable if available and not expired -pub fn get_cached_variable(workspace_id: &str, path: &str) -> Option { - let key = cache_key(workspace_id, path); +/// Hash the caller's full authorization context into a stable identity string. +/// +/// Email alone is **not** a sufficient scope: the same email can resolve to different +/// effective permissions (`username`, groups, folders, scopes, admin/operator) through +/// job- or owner-scoped tokens that share an email but carry a narrower `permissioned_as`. +/// Every input that determines what the caller may read is folded in, mirroring +/// `job_read_access_cache_key` in windmill-api, so a lower-privilege context can never +/// reuse a higher-privilege context's cache entry. Variable-length fields are +/// length-prefixed to keep the encoding injective. +pub fn auth_identity(authed: &A) -> String { + let mut hasher = Sha256::new(); + let field = |hasher: &mut Sha256, bytes: &[u8]| { + hasher.update((bytes.len() as u32).to_be_bytes()); + hasher.update(bytes); + }; + hasher.update([authed.is_admin() as u8, authed.is_operator() as u8]); + field(&mut hasher, authed.email().as_bytes()); + field(&mut hasher, authed.username().as_bytes()); + let mut groups: Vec<&str> = authed.groups().iter().map(String::as_str).collect(); + groups.sort_unstable(); + hasher.update((groups.len() as u32).to_be_bytes()); + for g in groups { + field(&mut hasher, g.as_bytes()); + } + let mut folders: Vec<&str> = authed.folders().iter().map(|f| f.0.as_str()).collect(); + folders.sort_unstable(); + hasher.update((folders.len() as u32).to_be_bytes()); + for f in folders { + field(&mut hasher, f.as_bytes()); + } + match authed.scopes() { + // u32::MAX length-prefix marks "no scopes" so it can't collide with an empty list. + None => hasher.update(u32::MAX.to_be_bytes()), + Some(scopes) => { + let mut scopes: Vec<&str> = scopes.iter().map(String::as_str).collect(); + scopes.sort_unstable(); + hasher.update((scopes.len() as u32).to_be_bytes()); + for s in scopes { + field(&mut hasher, s.as_bytes()); + } + } + } + hex::encode(hasher.finalize()) +} + +/// Generate an identity-scoped cache key (`identity:workspace_id:path`). +/// +/// Both the variable and resource caches store *already-decrypted* values that were +/// resolved under the caller's row-level-security context. The cache is consulted before +/// the per-folder RLS query runs, so an unscoped `workspace:path` key would let an entry +/// warmed by one caller (via `allow_cache=true`) be served to a different caller who has +/// no access to the underlying folder, leaking decrypted secrets within the TTL. `identity` +/// is [`auth_identity`] — the hash of the caller's full authorization context — so a hit +/// can only ever be returned to a caller whose authorized read populated it. +fn identity_cache_key(identity: &str, workspace_id: &str, path: &str) -> String { + format!("{}:{}", identity, cache_key(workspace_id, path)) +} + +/// Get cached variable if available and not expired. Scoped to `identity` +/// ([`auth_identity`]); see [`identity_cache_key`]. Returns the value and its `is_secret` +/// flag so the caller can re-run a secret read's side effects on a hit. +pub fn get_cached_variable( + workspace_id: &str, + path: &str, + identity: &str, +) -> Option { + let key = identity_cache_key(identity, workspace_id, path); VARIABLE_CACHE.get(&key).and_then(|entry| { if entry.is_expired() { VARIABLE_CACHE.remove(&key); @@ -67,17 +145,21 @@ pub fn get_cached_variable(workspace_id: &str, path: &str) -> Option { }) } -/// Cache variable data -pub fn cache_variable(workspace_id: &str, path: &str, email: &str, variable: String) { - let key = format!("{}:{}", email, cache_key(workspace_id, path)); +/// Cache variable data, scoped to the caller identity. See [`get_cached_variable`]. +pub fn cache_variable(workspace_id: &str, path: &str, identity: &str, variable: CachedVariable) { + let key = identity_cache_key(identity, workspace_id, path); let entry = CacheEntry::new(variable); VARIABLE_CACHE.insert(key.clone(), entry); tracing::debug!("Cached variable {}", key); } -/// Get cached resource if available and not expired -pub fn get_cached_resource(workspace_id: &str, path: &str) -> Option { - let key = cache_key(workspace_id, path); +/// Get cached resource if available and not expired. +/// +/// Scoped to `identity` ([`auth_identity`]); see [`identity_cache_key`]. The cached value +/// is the *already-interpolated* resource — its `$var:`/`$res:` secrets are resolved and +/// decrypted inline — so it must never cross authorization boundaries. +pub fn get_cached_resource(workspace_id: &str, path: &str, identity: &str) -> Option { + let key = identity_cache_key(identity, workspace_id, path); RESOURCE_CACHE.get(&key).and_then(|entry| { if entry.is_expired() { RESOURCE_CACHE.remove(&key); @@ -89,22 +171,28 @@ pub fn get_cached_resource(workspace_id: &str, path: &str) -> Option { }) } -/// Cache resource data -pub fn cache_resource(workspace_id: &str, path: &str, resource: Value) { - let key = cache_key(workspace_id, path); +/// Cache resource data, scoped to the caller identity. See [`get_cached_resource`]. +pub fn cache_resource(workspace_id: &str, path: &str, identity: &str, resource: Value) { + let key = identity_cache_key(identity, workspace_id, path); let entry = CacheEntry::new(resource); RESOURCE_CACHE.insert(key.clone(), entry); tracing::debug!("Cached resource {}", key); } -/// Invalidate specific variable from cache +/// Invalidate a variable from the cache. +/// +/// NOTE: entries are keyed by [`identity_cache_key`] (`identity:workspace:path`), so this +/// `workspace:path` key cannot target them — it only removes a legacy unscoped entry, if +/// any. Per-identity entries are not enumerable here; rely on the 30s TTL for staleness, +/// or use [`clear_all_caches`] to force a full flush. Currently unused. pub fn invalidate_variable_cache(workspace_id: &str, path: &str) { let key = cache_key(workspace_id, path); VARIABLE_CACHE.remove(&key); tracing::info!("Variable cache invalidated for {}", key); } -/// Invalidate specific resource from cache +/// Invalidate a resource from the cache. Same identity-scoping caveat as +/// [`invalidate_variable_cache`]. Currently unused. pub fn invalidate_resource_cache(workspace_id: &str, path: &str) { let key = cache_key(workspace_id, path); RESOURCE_CACHE.remove(&key); @@ -118,3 +206,106 @@ pub fn clear_all_caches() { RESOURCE_CACHE.clear(); tracing::debug!("All variable/resource caches cleared"); } + +#[cfg(test)] +mod tests { + use super::*; + + /// Minimal [`Authable`] double so we can assert which authorization fields the + /// cache identity is sensitive to, without standing up a full auth stack. + struct FakeAuthed { + email: String, + username: String, + is_admin: bool, + is_operator: bool, + groups: Vec, + folders: Vec<(String, bool, bool)>, + scopes: Option>, + } + + impl FakeAuthed { + fn base() -> Self { + Self { + email: "alice@x.dev".to_string(), + username: "alice".to_string(), + is_admin: false, + is_operator: false, + groups: vec!["all".to_string()], + folders: vec![("shared".to_string(), false, false)], + scopes: None, + } + } + } + + impl Authable for FakeAuthed { + fn email(&self) -> &str { + &self.email + } + fn username(&self) -> &str { + &self.username + } + fn is_admin(&self) -> bool { + self.is_admin + } + fn is_operator(&self) -> bool { + self.is_operator + } + fn groups(&self) -> &[String] { + &self.groups + } + fn folders(&self) -> &[(String, bool, bool)] { + &self.folders + } + fn scopes(&self) -> Option<&[String]> { + self.scopes.as_deref() + } + } + + // Email alone must NOT determine the cache identity: two contexts that share an email + // but resolve to different effective permissions must get distinct identities, so a + // lower-privilege context can never reuse a higher-privilege one's cached secret. + #[test] + fn auth_identity_is_not_just_email() { + let base = auth_identity(&FakeAuthed::base()); + + let mut more_folders = FakeAuthed::base(); + more_folders + .folders + .push(("secret".to_string(), false, false)); + assert_ne!(base, auth_identity(&more_folders), "folders must matter"); + + let mut more_groups = FakeAuthed::base(); + more_groups.groups.push(("devs").to_string()); + assert_ne!(base, auth_identity(&more_groups), "groups must matter"); + + let mut other_user = FakeAuthed::base(); + other_user.username = "bob".to_string(); + assert_ne!(base, auth_identity(&other_user), "username must matter"); + + let mut admin = FakeAuthed::base(); + admin.is_admin = true; + assert_ne!(base, auth_identity(&admin), "is_admin must matter"); + + let mut operator = FakeAuthed::base(); + operator.is_operator = true; + assert_ne!(base, auth_identity(&operator), "is_operator must matter"); + + let mut scoped = FakeAuthed::base(); + scoped.scopes = Some(vec!["resources:read:f/secret/x".to_string()]); + assert_ne!(base, auth_identity(&scoped), "scopes must matter"); + } + + // Identical authorization contexts must produce the same identity (so the same caller + // gets a cache hit), and ordering of groups/folders must not change the identity. + #[test] + fn auth_identity_is_stable_and_order_independent() { + let a = FakeAuthed::base(); + assert_eq!(auth_identity(&a), auth_identity(&FakeAuthed::base())); + + let mut reordered = FakeAuthed::base(); + reordered.groups = vec!["all".to_string(), "devs".to_string()]; + let mut other_order = FakeAuthed::base(); + other_order.groups = vec!["devs".to_string(), "all".to_string()]; + assert_eq!(auth_identity(&reordered), auth_identity(&other_order)); + } +} diff --git a/backend/windmill-store/src/variables.rs b/backend/windmill-store/src/variables.rs index 2d767b393b..24739ce940 100644 --- a/backend/windmill-store/src/variables.rs +++ b/backend/windmill-store/src/variables.rs @@ -42,7 +42,9 @@ use windmill_common::{ worker::CLOUD_HOSTED, }; -use crate::var_resource_cache::{cache_variable, get_cached_variable}; +use crate::var_resource_cache::{ + auth_identity, cache_variable, get_cached_variable, CachedVariable, +}; use lazy_static::lazy_static; use serde::Deserialize; use sqlx::{Acquire, Postgres, Transaction}; @@ -1204,15 +1206,55 @@ fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value { } } +/// Emit the `variables.decrypt_secret` audit event for a secret-variable read. Run on both +/// the cache-miss and cache-hit paths so `allow_cache` never skips secret-access auditing. +async fn audit_decrypt_secret( + db_with_opt_authed: &DbWithOptAuthed<'_, ApiAuthed>, + w_id: &str, + path: &str, +) -> Result<()> { + let mut tx = db_with_opt_authed.db().begin().await?; + audit_log( + &mut *tx, + db_with_opt_authed, + "variables.decrypt_secret", + ActionKind::Execute, + w_id, + Some(path), + None, + ) + .await?; + tx.commit().await?; + Ok(()) +} + pub async fn get_value_internal<'a>( db_with_opt_authed: &'a DbWithOptAuthed<'a, ApiAuthed>, w_id: &str, path: &str, allow_cache: bool, ) -> Result { - if allow_cache { - if let Some(cached_variable) = get_cached_variable(&w_id, &path) { - return Ok(cached_variable); + // Scope the cache to the caller's full authorization identity (not just email): the + // cached value is the decrypted variable, resolved under this caller's RLS context. + let cache_identity = allow_cache.then(|| match db_with_opt_authed.authed() { + Some(authed) => auth_identity(authed), + None => format!("\0system:{}", db_with_opt_authed.email()), + }); + + if let Some(identity) = cache_identity.as_deref() { + if let Some(cached) = get_cached_variable(&w_id, &path, identity) { + // A cache hit must be observably equivalent to a miss: re-run the per-read side + // effects a secret read performs (the `variables.decrypt_secret` audit and + // running-job secret registration) so `allow_cache` never silently skips them. + if cached.is_secret { + audit_decrypt_secret(db_with_opt_authed, &w_id, &path).await?; + if !cached.value.is_empty() { + windmill_common::sensitive_log_masks::register_secret_for_all_running_jobs( + &cached.value, + ); + } + } + return Ok(cached.value); } } @@ -1234,19 +1276,7 @@ pub async fn get_value_internal<'a>( }; let r = if variable.is_secret { - // let audit_author = - let mut tx = db_with_opt_authed.db().begin().await?; - audit_log( - &mut *tx, - db_with_opt_authed, - "variables.decrypt_secret", - ActionKind::Execute, - &w_id, - Some(&variable.path), - None, - ) - .await?; - tx.commit().await?; + audit_decrypt_secret(db_with_opt_authed, &w_id, &variable.path).await?; let value = variable.value; if variable.is_expired.unwrap_or(false) && variable.account.is_some() { @@ -1282,9 +1312,16 @@ pub async fn get_value_internal<'a>( windmill_common::sensitive_log_masks::register_secret_for_all_running_jobs(&r); } - // Cache the result when explicitly allowed and caching appropriate - if allow_cache { - cache_variable(&w_id, &path, db_with_opt_authed.email(), r.clone()); + // Cache the result when explicitly allowed. Secrets are cached too: their per-read side + // effects (audit + running-job registration) are re-run on a hit (see the hit path above), + // and `is_secret` is stored so the hit knows to do so. + if let Some(identity) = cache_identity.as_deref() { + cache_variable( + &w_id, + &path, + identity, + CachedVariable { value: r.clone(), is_secret: variable.is_secret }, + ); } Ok(r) diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 79dcc3bd84..ae2de880a4 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -914,6 +914,26 @@ pub async fn run_deployed_relative_imports( .await .unwrap(); + // Regression guard for the Deno lock-gen import map (generate_deno_lock): + // it must resolve workspace `/f/`/`/u/` imports, otherwise `deno cache --lock` + // fails with "not a dependency and not in import map". We match that + // specific failure rather than asserting lock_error_logs is empty — + // the field also captures benign, non-fatal lock-job output (e.g. Bun's + // "empty dependencies, skipping install"). (Runtime query to avoid + // touching the sqlx offline cache.) + let lock_error: Option = + sqlx::query_scalar("SELECT lock_error_logs FROM script WHERE path = $1") + .bind("f/system/test_import") + .fetch_one(&db2) + .await + .unwrap(); + if let Some(err) = &lock_error { + assert!( + !err.contains("not in import map"), + "lock generation failed to resolve a workspace import: {err}" + ); + } + let job = RunJob::from(JobPayload::ScriptHash { path: "f/system/test_import".to_string(), hash: ScriptHash(script.hash), diff --git a/backend/windmill-trigger-http/src/http_trigger_auth.rs b/backend/windmill-trigger-http/src/http_trigger_auth.rs index 19766cdbc2..10927d1ef9 100644 --- a/backend/windmill-trigger-http/src/http_trigger_auth.rs +++ b/backend/windmill-trigger-http/src/http_trigger_auth.rs @@ -337,6 +337,20 @@ mod zoom { return Ok(None); } + // Prevent this challenge endpoint from being used as a signing oracle. + // Legitimate Zoom validation tokens are short random hex strings that + // never contain colons. The exploit requires crafting a plainToken in the + // `v0:{timestamp}:{body}` webhook-signing format (always containing colons) + // to obtain a valid signature for an arbitrary body. Reject any token that + // does not look like a legitimate Zoom validation token. + if zoom_request_body.payload.plain_token.contains(':') + || zoom_request_body.payload.plain_token.len() > 128 + { + return Err(AuthenticationError::InvalidChallengeResponse( + "Zoom: invalid plainToken format".to_string(), + )); + } + let hmac_signature = calculate_hmac_signature( HmacAlgorithm::Sha256, &signature_config_data.secret_key, @@ -1540,6 +1554,52 @@ mod tests { assert!(response.is_none()); } + #[test] + fn test_zoom_challenge_normal_token_succeeds() { + // A legitimate Zoom validation token is a short random alphanumeric string. + let payload = r#"{"event":"endpoint.url_validation","event_ts":1234567890,"payload":{"plainToken":"qgg8vlvZRS6UYooatFL8Aw"}}"#; + + let handler = WebhookType::Zoom.get_webhook_handler().unwrap(); + let config_data = SignatureConfigData { secret_key: "zoom_secret" }; + let response = handler + .handle_challenge_request(&HeaderMap::new(), &config_data, payload) + .unwrap(); + assert!(response.is_some()); + } + + #[test] + fn test_zoom_challenge_token_with_colons_rejected() { + // Exploit attempt: a plainToken crafted in the `v0:{ts}:{body}` signing format + // would let an attacker obtain a valid webhook signature for an arbitrary body. + let payload = r#"{"event":"endpoint.url_validation","event_ts":1234567890,"payload":{"plainToken":"v0:1234567890:{\"forged\":\"body\"}"}}"#; + + let handler = WebhookType::Zoom.get_webhook_handler().unwrap(); + let config_data = SignatureConfigData { secret_key: "zoom_secret" }; + let result = handler.handle_challenge_request(&HeaderMap::new(), &config_data, payload); + assert!(matches!( + result, + Err(AuthenticationError::InvalidChallengeResponse(_)) + )); + } + + #[test] + fn test_zoom_challenge_token_too_long_rejected() { + // A plainToken exceeding 128 chars cannot be a legitimate Zoom validation token. + let long_token = "a".repeat(129); + let payload = format!( + r#"{{"event":"endpoint.url_validation","event_ts":1234567890,"payload":{{"plainToken":"{}"}}}}"#, + long_token + ); + + let handler = WebhookType::Zoom.get_webhook_handler().unwrap(); + let config_data = SignatureConfigData { secret_key: "zoom_secret" }; + let result = handler.handle_challenge_request(&HeaderMap::new(), &config_data, &payload); + assert!(matches!( + result, + Err(AuthenticationError::InvalidChallengeResponse(_)) + )); + } + // --- Custom webhook end-to-end --- #[test] diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index f425f97b53..3591c88931 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -32,6 +32,27 @@ use std::sync::Arc; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_git_sync::handle_deployment_metadata; +/// True when the workspace is a fork (`parent_workspace_id IS NOT NULL`). +/// +/// Operational state (`mode`) belongs to the parent workspace: a git-sync / +/// merge / clone / UI-create write into a fork must never set it. On create we +/// force `disabled` so a fork trigger can't compete with the parent's listener; +/// on update we preserve the fork's existing value. `setmode` is the intended +/// explicit mutator of a fork's mode (and carries its own conflict warning) — +/// runtime error handling may still auto-disable an errored trigger, which is +/// orthogonal to this rule. This is the write half whose read half lives in +/// `workspaces_export.rs` (parent-value substitution on fork export), and it is +/// the single authority shared by both the git-sync round-trip and the in-app +/// compare-workspaces merge. +async fn workspace_is_fork(db: &DB, workspace_id: &str) -> Result { + let is_fork: Option = + sqlx::query_scalar("SELECT parent_workspace_id IS NOT NULL FROM workspace WHERE id = $1") + .bind(workspace_id) + .fetch_optional(db) + .await?; + Ok(is_fork.unwrap_or(false)) +} + #[async_trait] pub trait TriggerCrud: Send + Sync + 'static { type Trigger: Serialize @@ -440,6 +461,13 @@ async fn create_trigger( let mut tx = user_db.begin(&authed).await?; + // Writing into a fork never sets operational state: force `disabled` so a + // cloned / synced / merged / UI-created trigger can't compete with the + // parent's listener. The fork owner re-enables locally via `setmode`. + if workspace_is_fork(&db, &workspace_id).await? { + new_trigger.base.set_mode(TriggerMode::Disabled); + } + let new_path = new_trigger.base.path.clone(); let labels = new_trigger.base.labels.clone(); @@ -591,11 +619,16 @@ async fn update_trigger( let mut tx = user_db.begin(&authed).await?; - // When the request omits `mode`/`enabled`, preserve the existing DB value - // instead of falling back to the BaseTriggerData default (Enabled). This - // keeps fork→parent git-sync round-trips from flipping the parent's - // operational state — see fork_trigger_ignore_keys in workspaces_export.rs. - if edit_trigger.base.is_mode_unspecified() { + // Preserve the existing DB `mode` instead of writing the incoming value + // when either: + // * the target is a fork — a fork's operational state is fork-local and + // is never set through a git-sync/merge write (only via `setmode`); or + // * the request omits `mode`/`enabled` (legacy clients / YAML round-trip), + // where falling back to the BaseTriggerData default (Enabled) would flip + // the parent on a fork→parent merge. + // Read half of the rule: parent-value substitution on fork export in + // workspaces_export.rs. + if workspace_is_fork(&db, &workspace_id).await? || edit_trigger.base.is_mode_unspecified() { let existing_mode: Option = sqlx::query_scalar(&format!( "SELECT mode FROM {} WHERE workspace_id = $1 AND path = $2", T::TABLE_NAME diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs index 9335f60d4f..c7d291d6a1 100644 --- a/backend/windmill-types/src/flows.rs +++ b/backend/windmill-types/src/flows.rs @@ -315,6 +315,11 @@ pub struct StopAfterIf { pub expr: String, pub skip_if_stopped: bool, pub error_message: Option, + /// When stopping with an error (`error_message` set), embed the stopping + /// step's own result inside the raised error object (as `error.result`) + /// instead of discarding it. The top-level result stays `{ "error": .. }`. + #[serde(default, skip_serializing_if = "is_false")] + pub error_include_result: bool, } #[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index a3b571a6ad..ac15430759 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -561,6 +561,13 @@ pub struct OnBehalfOf { pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; +/// Reserved job-arg key holding the inbound W3C `traceparent` captured from the +/// request that enqueued the job (run endpoints). It rides the `args` jsonb like +/// [`ENTRYPOINT_OVERRIDE`]; normal scripts never see it because args are bound by +/// declared parameter name. Read back at root-job completion to link the job's +/// OTLP span to the originating distributed trace (EE/OTel only). +pub const WM_TRACEPARENT: &str = "_wm_traceparent"; + /// The entrypoint override (`_ENTRYPOINT_OVERRIDE` job arg -> /// `v2_job.script_entrypoint_override`) is interpolated verbatim into /// generated worker wrappers in a code position (e.g. the NativeTS diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index e56ef66de0..18957bb4a5 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -5,10 +5,21 @@ hostname: "python" log_level: ERROR time_limit: 900 -rlimit_as: 2048 +# uv's --compile-bytecode spawns a bytecode-compile thread pool sized to the +# host's CPU count. Each thread reserves virtual address space for its stack, so +# on high-core machines the aggregate overruns a low rlimit_as and installs fail +# intermittently with "OS can't spawn worker thread: Resource temporarily +# unavailable (os error 11)" / "memory allocation failed". A low cap (was 2048) +# is the address-space companion to the fd exhaustion fixed below; raised well +# above the run sandbox's 4096 to give the compile pool headroom on large nodes. +rlimit_as: 8192 rlimit_cpu: 1000 rlimit_fsize: 1024 -rlimit_nofile: 64 +# uv's --compile-bytecode spawns a Python interpreter that compiles .py files +# with parallelism scaling to the host's CPU count, opening many fds at once. +# A low cap (was 64) is exhausted on high-core machines -> "Too many open files". +# Matches the runtime configs (run.python3/run.ansible) which already use 10000. +rlimit_nofile: 10000 envar: "HOME=/user" envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" diff --git a/backend/windmill-worker/nsjail/run.docker.config.proto b/backend/windmill-worker/nsjail/run.docker.config.proto new file mode 100644 index 0000000000..a2da459fbe --- /dev/null +++ b/backend/windmill-worker/nsjail/run.docker.config.proto @@ -0,0 +1,103 @@ +name: "docker v2 run" + +mode: ONCE +hostname: "container" +log_level: ERROR +time_limit: {TIMEOUT} + +disable_rl: true + +cwd: {WORKDIR} + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +# keep_env forwards nsjail's OWN process env (only windmill-trusted keys: reserved +# vars + proxy) to the child. The image's attacker-controlled Env is delivered via +# the envar directives below — NEVER nsjail's process env, so a hostile image cannot +# set LD_PRELOAD/LD_LIBRARY_PATH/LD_AUDIT on the nsjail binary itself. +keep_env: true +mount_proc: true + +# Image Env (+ PATH/HOME fallbacks), proto-escaped. Applied to the child only. +{ENVARS} + +# Map uid/gid 0 inside the jail to the (single) worker user outside. The image's +# rootfs is extracted as the worker user, so a root process inside the container +# owns the rootfs and runs like a normal "root in container" — without any subuid +# range. Multi-uid images are a later enhancement (newuidmap range). +uidmap { + inside_id: "0" + outside_id: "" + count: 1 +} +gidmap { + inside_id: "0" + outside_id: "" + count: 1 +} + +# The image's root filesystem, bound one top-level entry at a time. Binding the +# whole rootfs at "/" trips nsjail's read-only remount of its base root in a +# rootless userns ("mount(... MS_REMOUNT|MS_BIND|MS_RDONLY): Operation not +# permitted"); per-entry binds sit as rw submounts under nsjail's own tmpfs root +# and avoid it. Generated from the extracted rootfs. +{ROOTFS_MOUNTS} + +# Pseudo-filesystems the image expects. /tmp honors the same instance settings as +# every other nsjail job (nsjail_tmp_backing tmpfs/disk, nsjail_tmpfs_size_mb); +# /dev gets the standard nodes; /proc comes from mount_proc (the jail's own pid ns). +{TMP_MOUNT_BLOCK} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/zero" + dst: "/dev/zero" + is_bind: true + rw: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +# Host DNS config layered over the image's /etc so name resolution works on the +# job's network (mandatory:false: some minimal images have no /etc files to shadow). +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +# `# volume` mounts (and the same-worker /tmp/shared folder). Placed after the +# rootfs binds and the tmpfs /tmp so a volume target overrides any colliding image +# path and isn't shadowed by the tmpfs. Empty when there are no volumes. +{SHARED_MOUNT} + +iface_no_lo: true + +#{DEV} diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index 74e75ef0a5..353bab17a0 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -7,6 +7,8 @@ use std::{ }; use uuid::Uuid; use windmill_ai::types::*; +#[cfg(feature = "mcp")] +use windmill_common::client::AuthedClient; use windmill_common::flows::FlowModuleValue; use windmill_common::{ db::DB, @@ -546,7 +548,7 @@ pub async fn load_mcp_tools( db: &DB, workspace_id: &str, mcp_configs: Vec, - auth_token: &str, + client: &AuthedClient, ) -> Result<(HashMap>, Vec), Error> { let mut all_mcp_tools = Vec::new(); let mut mcp_clients = HashMap::new(); @@ -573,27 +575,47 @@ pub async fn load_mcp_tools( let resource_name = mcp_resource.name.clone(); - // Check if token needs refresh before creating MCP client - if let Some(ref token_path) = mcp_resource.token { + // Resolve the token through the job's permissioned (RLS + audit) path so + // the AI agent cannot exfiltrate a secret its identity is not allowed to + // read by pointing an MCP resource's token at it. + let token = if let Some(ref token_path) = mcp_resource.token { let token_var_path = token_path.trim_start_matches("$var:"); - if let Err(e) = - refresh_token_if_expired(db, workspace_id, token_var_path, auth_token).await - { - tracing::warn!( - "Failed to refresh token for MCP resource {}: {}. Proceeding with possibly expired token.", - resource_name, e - ); + if token_var_path.trim().is_empty() { + None + } else { + // Refresh first (best-effort) so the value we read is current. + if let Err(e) = + refresh_token_if_expired(db, workspace_id, token_var_path, &client.token).await + { + tracing::warn!( + "Failed to refresh token for MCP resource {}: {}. Proceeding with possibly expired token.", + resource_name, e + ); + } + Some( + client + .get_variable_value(token_var_path) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to resolve token variable {} for MCP resource {}: {}", + token_var_path, resource_name, e + )) + })?, + ) } - } + } else { + None + }; // Create new MCP client for this execution tracing::debug!("Creating fresh MCP client for {}", resource_name); - let client = McpClient::from_resource(mcp_resource, db, workspace_id) + let mcp_conn = McpClient::from_resource(mcp_resource, token) .await .context("Failed to create MCP client")?; // Get raw MCP tools from client - let raw_mcp_tools = client.available_tools(); + let raw_mcp_tools = mcp_conn.available_tools(); // Convert to Windmill Tool format let converted_tools = @@ -616,7 +638,7 @@ pub async fn load_mcp_tools( all_mcp_tools.extend(filtered_tools); // Store client for later use and cleanup - let mcp_client = Arc::new(client); + let mcp_client = Arc::new(mcp_conn); mcp_clients.insert(resource_name, mcp_client); } @@ -663,7 +685,7 @@ pub async fn load_mcp_tools( _db: &DB, _workspace_id: &str, _mcp_configs: Vec, - _auth_token: &str, + _client: &windmill_common::client::AuthedClient, ) -> Result<(HashMap>, Vec), Error> { Ok((HashMap::new(), Vec::new())) } diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index b2478e3be1..2591cc6495 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -432,7 +432,7 @@ pub async fn handle_ai_agent_job( let mcp_clients = if !mcp_configs.is_empty() { let (clients, mcp_tools) = - load_mcp_tools(db, &job.workspace_id, mcp_configs, &client.token).await?; + load_mcp_tools(db, &job.workspace_id, mcp_configs, client).await?; tools.extend(mcp_tools); clients } else { diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 516c75fdba..4c470b8dfc 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -40,9 +40,9 @@ use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{ common::{ - build_args_map, build_command_with_isolation, get_reserved_variables, read_file, - read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, - OccupancyMetrics, DEV_CONF_NSJAIL, + build_args_map, build_command_with_isolation, get_reserved_variables, raw_to_string, + read_file, read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, + start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, @@ -57,14 +57,6 @@ lazy_static::lazy_static! { pub static ref ANSI_ESCAPE_RE: Regex = Regex::new(r"\x1b\[[0-9;]*m").unwrap(); } -fn raw_to_string(x: &str) -> String { - match serde_json::from_str::(x) { - Ok(serde_json::Value::String(x)) => x, - Ok(x) => serde_json::to_string(&x).unwrap_or_else(|_| String::new()), - _ => String::new(), - } -} - #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_bash_job( mem_peak: &mut i32, @@ -84,6 +76,28 @@ pub async fn handle_bash_job( ) -> Result, Error> { let annotation = windmill_common::worker::BashAnnotations::parse(&content); + // `# sandbox ` selects the daemonless, nsjail-sandboxed container runtime + // (extract the image's rootfs + run it inside the job's sandbox). A bare + // `# sandbox` keeps the plain nsjail-bash modifier; `# docker` keeps v1 (dind). + if let Some(image) = windmill_common::worker::BashAnnotations::sandbox_image(content) { + return crate::docker_v2::handle_docker_v2_job( + &image, + mem_peak, + canceled_by, + job, + conn, + client, + parent_runnable_path, + content, + job_dir, + shared_mount, + base_internal_url, + worker_name, + occupancy_metrics, + ) + .await; + } + // Check if sandbox annotation is used but nsjail is not available if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { return Err(Error::ExecutionErr( diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 9864093cd0..1bef9e4c6d 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -68,6 +68,16 @@ mount { #[cfg(not(debug_assertions))] pub const DEV_CONF_NSJAIL: &str = ""; +/// Turn a JSON value into the string a shell/CLI arg should receive: a JSON string +/// becomes its inner value, anything else is re-serialized compactly. +pub(crate) fn raw_to_string(x: &str) -> String { + match serde_json::from_str::(x) { + Ok(serde_json::Value::String(x)) => x, + Ok(x) => serde_json::to_string(&x).unwrap_or_else(|_| String::new()), + _ => String::new(), + } +} + pub async fn build_args_map<'a>( job: &'a MiniPulledJob, client: &AuthedClient, diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 70df9fb4e0..944236cf2b 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -56,9 +56,13 @@ const DOTNET_ROOT_DEFAULT: &str = "C:\\Program Files\\dotnet"; #[cfg(unix)] const DOTNET_ROOT_DEFAULT: &str = "/usr/share/dotnet"; +#[cfg(feature = "csharp")] +const DOTNET_TARGET_FRAMEWORK_DEFAULT: &str = "net9.0"; + #[cfg(feature = "csharp")] lazy_static::lazy_static! { static ref DOTNET_ROOT: String = std::env::var("DOTNET_ROOT").unwrap_or_else(|_| DOTNET_ROOT_DEFAULT.to_string()); + static ref DOTNET_TARGET_FRAMEWORK: String = std::env::var("DOTNET_TARGET_FRAMEWORK").unwrap_or_else(|_| DOTNET_TARGET_FRAMEWORK_DEFAULT.to_string()); } #[cfg(feature = "csharp")] @@ -212,6 +216,7 @@ fn gen_cs_proj( ) }; + let target_framework = DOTNET_TARGET_FRAMEWORK.as_str(); write_file( job_dir, "Main.csproj", @@ -219,7 +224,7 @@ fn gen_cs_proj( r#" Exe - net9.0 + {target_framework} enable WindmillScriptCSharpInternal.Wrapper true @@ -510,9 +515,10 @@ pub async fn handle_csharp_job( let ws_suffix = crate::workspace_registry_cache_suffix(&job.workspace_id).await; let mut hash = calculate_hash(&format!( - "{}{}", + "{}{}{}", inner_content, - requirements_o.unwrap_or(&String::new()) + requirements_o.unwrap_or(&String::new()), + DOTNET_TARGET_FRAMEWORK.as_str() )); hash.push_str(&ws_suffix); let bin_path = format!("{}/{hash}", *CSHARP_CACHE_DIR); diff --git a/backend/windmill-worker/src/docker_v2.rs b/backend/windmill-worker/src/docker_v2.rs new file mode 100644 index 0000000000..3a4936e85d --- /dev/null +++ b/backend/windmill-worker/src/docker_v2.rs @@ -0,0 +1,892 @@ +//! Sandboxed container runtime: run a container as a sandboxed subprogram of the job. +//! +//! Unlike the legacy `# docker` (dind/daemon) path, this has no daemon and no Docker +//! API. It splits *pull* from *run*: +//! +//! 1. **pull/extract** (`crane`, no daemon/store/root): materialize the image's root +//! filesystem into `{job_dir}/rootfs` and read its OCI config +//! (Env/Cmd/Entrypoint/WorkingDir), via a digest-keyed rootfs cache. +//! 2. **run** (the job's own nsjail sandbox): execute the image command with the +//! extracted rootfs bound in as the new root, so the container inherits exactly +//! the job's confinement (filesystem mask, pid namespace, network, uid) and can't +//! escape past what the job itself can reach. +//! +//! Selected by `# sandbox ` (a bare `# sandbox` keeps plain nsjail-bash; +//! `# docker` keeps the v1 daemon path). The script body runs inside the image via +//! `/bin/sh`; an empty body runs the image's ENTRYPOINT/CMD. + +use std::process::Stdio; + +use serde::Deserialize; +use serde_json::{json, value::RawValue}; +use sqlx::types::Json; +use tokio::process::Command; + +use windmill_common::{client::AuthedClient, scripts::ScriptLang}; +use windmill_common::{ + error::Error, + worker::{to_raw_value, write_file, Connection}, +}; + +use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; + +use crate::{ + common::{ + build_args_map, get_reserved_variables, raw_to_string, resolve_nsjail_timeout, + resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, + }, + get_proxy_envs_for_lang, + handle_child::handle_child, + DISABLE_NUSER, NSJAIL_AVAILABLE, NSJAIL_PATH, SANDBOX_IMAGE_CACHE_MAX_MB, + SANDBOX_IMAGE_DEFAULT_REGISTRY, SANDBOX_IMAGE_MAX_SIZE_MB, SANDBOX_IMAGE_PULL_POLICY, + SANDBOX_REGISTRY_AUTH, +}; + +const NSJAIL_CONFIG_RUN_DOCKER_CONTENT: &str = include_str!("../nsjail/run.docker.config.proto"); + +const DEFAULT_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +lazy_static::lazy_static! { + /// `crane` (google/go-containerregistry) — pulls + flattens an image to a rootfs + /// without a daemon, store, root, or privileged container. We never *run* the + /// image via crane (nsjail does the run), so a full container engine is overkill. + pub static ref CRANE_PATH: String = + std::env::var("CRANE_PATH").unwrap_or_else(|_| "crane".to_string()); + + /// `linux/` for the worker, pinned on every crane call so multi-arch images + /// resolve deterministically (and `crane manifest` returns a real manifest, not an + /// index). + static ref CRANE_PLATFORM: String = format!("linux/{}", match std::env::consts::ARCH { + "x86_64" => "amd64", + "aarch64" => "arm64", + other => other, + }); + + /// Content-addressed cache of flattened rootfs tars, keyed by image digest. crane + /// has no persistent store, so this is what gives cross-job dedup (and, since it's + /// digest-keyed, automatic freshness when a moving tag changes). + static ref ROOTFS_CACHE_DIR: String = + format!("{}sandbox_rootfs", *windmill_common::worker::ROOT_CACHE_DIR); +} + +/// Guards against overlapping cache-eviction passes across concurrent jobs. +static EVICTION_RUNNING: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// `sandbox_image_pull_policy` instance setting. With the digest-keyed cache, `newer` +/// (default) re-resolves the digest each job (cheap manifest fetch) so moving tags +/// like `:latest` stay fresh while unchanged digests reuse the cache. `missing` skips +/// the registry when a digest is already cached for the ref; `never` only uses the +/// cache (errors if absent); `always` == `newer` here. +async fn pull_policy() -> String { + let p = SANDBOX_IMAGE_PULL_POLICY.read().await.clone(); + match p.as_deref() { + Some(p @ ("missing" | "newer" | "always" | "never")) => p.to_string(), + _ => "newer".to_string(), + } +} + +/// `sandbox_image_max_size_mb` instance setting; 0 (or unset/non-positive) = no limit. +async fn max_image_size_mb() -> u64 { + SANDBOX_IMAGE_MAX_SIZE_MB.read().await.unwrap_or(0).max(0) as u64 +} + +/// `sandbox_image_cache_max_mb` instance setting; 0 (or unset/non-positive) = unbounded. +async fn image_cache_max_mb() -> u64 { + SANDBOX_IMAGE_CACHE_MAX_MB.read().await.unwrap_or(0).max(0) as u64 +} + +/// A ref is registry-qualified if the component before the first `/` looks like a +/// host (contains `.` or `:`, or is `localhost`). Bare repos (`alpine`, +/// `alpine:latest`, `myorg/img`) are unqualified and resolve against docker.io — +/// or the configured default registry. +fn registry_qualified(image: &str) -> bool { + match image.split_once('/') { + None => false, + Some((first, _)) => first.contains('.') || first.contains(':') || first == "localhost", + } +} + +/// Prepend the `sandbox_image_default_registry` instance setting to unqualified image +/// refs (fully-qualified refs are left untouched). +async fn resolve_image_ref(image: &str) -> String { + let registry = SANDBOX_IMAGE_DEFAULT_REGISTRY.read().await.clone(); + match registry { + Some(registry) if !registry.trim().is_empty() && !registry_qualified(image) => { + format!("{}/{}", registry.trim().trim_end_matches('/'), image) + } + _ => image.to_string(), + } +} + +/// If the `sandbox_registry_auth` instance setting holds a docker `auth.json` blob, +/// write it to a per-job `DOCKER_CONFIG` dir (`{job_dir}/.docker/config.json`, 0600, +/// removed with the job) and return the dir to pass to crane via `DOCKER_CONFIG`. +/// Returns `None` when unset. (docker `config.json` and podman `auth.json` share the +/// `{"auths": {...}}` schema, so the same blob works.) +async fn write_auth_dir(job_dir: &str) -> Result, Error> { + let auth = SANDBOX_REGISTRY_AUTH.read().await.clone(); + let Some(auth) = auth.filter(|a| !a.trim().is_empty()) else { + return Ok(None); + }; + let dir = format!("{job_dir}/.docker"); + tokio::fs::create_dir_all(&dir).await?; + let path = format!("{dir}/config.json"); + // Create 0600 from the start (registry credentials) — no world-readable window. + #[cfg(unix)] + { + use tokio::io::AsyncWriteExt; + let mut f = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(&path) + .await?; + f.write_all(auth.as_bytes()).await?; + } + #[cfg(not(unix))] + tokio::fs::write(&path, auth).await?; + Ok(Some(dir)) +} + +/// The subset of an image's OCI config we apply to the run. +#[derive(Deserialize, Default, Debug)] +struct OciConfig { + #[serde(default, rename = "Env")] + env: Option>, + #[serde(default, rename = "Cmd")] + cmd: Option>, + #[serde(default, rename = "Entrypoint")] + entrypoint: Option>, + #[serde(default, rename = "WorkingDir")] + working_dir: Option, +} + +/// Quote a string as a protobuf-text-format string literal for safe inclusion in +/// the nsjail config. Image-controlled values (mount srcs/dsts, symlink targets, +/// WorkingDir) flow into the config, so they MUST be escaped — an unescaped `"` or +/// newline would otherwise let a hostile image config inject arbitrary nsjail +/// directives and break out of the sandbox. Every byte is emitted as a printable +/// ASCII char or a valid protobuf escape (`\"`, `\\`, `\n`/`\r`/`\t`, or 3-digit +/// octal `\NNN` for control/non-ASCII bytes), so the result always parses. +fn proto_str(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + out.push('"'); + for &b in s.as_bytes() { + match b { + b'"' => out.push_str("\\\""), + b'\\' => out.push_str("\\\\"), + b'\n' => out.push_str("\\n"), + b'\r' => out.push_str("\\r"), + b'\t' => out.push_str("\\t"), + 0x20..=0x7e => out.push(b as char), + _ => out.push_str(&format!("\\{b:03o}")), + } + } + out.push('"'); + out +} + +/// Render container env vars as nsjail `envar:` directives (one per line). Each +/// `KEY=VALUE` is proto-escaped, so image-controlled keys/values can neither break +/// the config nor reach nsjail's own process environment. +fn render_envars(env: &[(String, String)]) -> String { + env.iter() + .map(|(k, v)| format!("envar: {}", proto_str(&format!("{k}={v}")))) + .collect::>() + .join("\n") +} + +/// Run `crane` with the optional per-job `DOCKER_CONFIG` auth dir. +async fn crane(args: &[&str], auth_dir: Option<&str>) -> Result { + let mut cmd = Command::new(CRANE_PATH.as_str()); + cmd.args(args); + if let Some(dir) = auth_dir { + cmd.env("DOCKER_CONFIG", dir); + } + cmd.output() + .await + .map_err(|e| Error::ExecutionErr(format!("failed to run crane {}: {e}", args.join(" ")))) +} + +/// `crane config` output: the image config (Env/Cmd/Entrypoint/WorkingDir) is nested +/// under the top-level `config` key. +#[derive(Deserialize, Default)] +struct CraneConfig { + #[serde(default)] + config: OciConfig, +} + +/// Filesystem-safe cache key for a digest (`sha256:ab..` -> `sha256_ab..`). +fn digest_key(digest: &str) -> String { + digest.replace([':', '/'], "_") +} + +/// Filesystem-safe, collision-resistant key for an image ref (the ref->digest file). +fn ref_key(image: &str) -> String { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + image.hash(&mut h); + let safe: String = image + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') { + c + } else { + '_' + } + }) + .collect(); + let safe = &safe[safe.len().saturating_sub(80)..]; + format!("{safe}_{:016x}", h.finish()) +} + +/// Resolve the image ref to a content digest, honoring the pull policy + a ref->digest +/// cache. `missing`/`never` reuse a cached digest without hitting the registry (`never` +/// errors if absent); `newer`/`always` always re-resolve via `crane digest`. +async fn resolve_digest( + image: &str, + policy: &str, + auth_dir: Option<&str>, +) -> Result { + let refs_dir = format!("{}/refs", *ROOTFS_CACHE_DIR); + let ref_file = format!("{refs_dir}/{}", ref_key(image)); + + if matches!(policy, "missing" | "never") { + if let Ok(d) = tokio::fs::read_to_string(&ref_file).await { + let d = d.trim().to_string(); + if !d.is_empty() + && tokio::fs::metadata(format!("{}/{}.tar", *ROOTFS_CACHE_DIR, digest_key(&d))) + .await + .is_ok() + { + return Ok(d); + } + } + if policy == "never" { + return Err(Error::ExecutionErr(format!( + "image {image} is not in the sandbox cache and SANDBOX_IMAGE_PULL_POLICY=never" + ))); + } + } + + let out = crane(&["digest", "--platform", &CRANE_PLATFORM, image], auth_dir).await?; + if !out.status.success() { + return Err(Error::ExecutionErr(format!( + "failed to resolve image {image}: {}", + String::from_utf8_lossy(&out.stderr) + ))); + } + let digest = String::from_utf8_lossy(&out.stdout).trim().to_string(); + let _ = tokio::fs::create_dir_all(&refs_dir).await; + // tmp+rename so a concurrent `missing`/`never` reader never sees a torn ref file. + let ref_tmp = format!("{ref_file}.tmp.{}", digest_key(&digest)); + if tokio::fs::write(&ref_tmp, &digest).await.is_ok() { + let _ = tokio::fs::rename(&ref_tmp, &ref_file).await; + } + Ok(digest) +} + +/// Pull (if not cached) and unpack `image` into `{job_dir}/rootfs`, returning its OCI +/// config. Uses `crane export`/`config` (no daemon/store/root) with a content-addressed +/// rootfs+config cache keyed by digest for cross-job dedup. +async fn extract_image(image: &str, job_dir: &str) -> Result { + let rootfs = format!("{job_dir}/rootfs"); + tokio::fs::create_dir_all(&rootfs).await?; + tokio::fs::create_dir_all(&*ROOTFS_CACHE_DIR).await?; + + let auth_dir = write_auth_dir(job_dir).await?; + let auth = auth_dir.as_deref(); + let digest = resolve_digest(image, &pull_policy().await, auth).await?; + // Pin every subsequent fetch to the resolved digest, not the (mutable) tag, so the + // content can't diverge from the digest we cache under if the tag moves mid-fetch. + let pinned = format!("{}@{digest}", image.split('@').next().unwrap_or(image)); + let key = digest_key(&digest); + let tar = format!("{}/{key}.tar", *ROOTFS_CACHE_DIR); + let cfg = format!("{}/{key}.json", *ROOTFS_CACHE_DIR); + let size_file = format!("{}/{key}.size", *ROOTFS_CACHE_DIR); + let token = std::path::Path::new(job_dir) + .file_name() + .map(|x| x.to_string_lossy().into_owned()) + .unwrap_or_default(); + + // Enforce the size cap on EVERY job (not just cache misses), using a cached size so + // a cache reuse needs no registry call — lowering the limit rejects cached images too. + enforce_image_size_limit(&pinned, &size_file, auth).await?; + + // Materialize the flattened rootfs. The cache tar can be evicted concurrently, so up + // to two attempts: hardlink the cache tar into the job dir (pins the inode against + // eviction) before extracting; if it vanished first, re-fetch. + let job_tar = format!("{job_dir}/rootfs.tar"); + for attempt in 0..2 { + if tokio::fs::metadata(&tar).await.is_err() { + fetch_into_cache(&pinned, &tar, &cfg, &token, auth).await?; + } + let config = read_oci_config(&cfg).await; + let _ = tokio::fs::remove_file(&job_tar).await; + // Stage the cache tar into the job dir so concurrent eviction can't unlink it out + // from under `tar -xf`. Prefer a hardlink (free), but the cache volume and the job + // dir are usually on *different* filesystems in the shipped deployments (the cache + // is its own volume/PVC) — there `hard_link` returns EXDEV, so fall back to a copy. + // `copy` reads through the source inode, so an eviction mid-copy still completes. + let staged = match tokio::fs::hard_link(&tar, &job_tar).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(e), // vanished — re-fetch + Err(_) => tokio::fs::copy(&tar, &job_tar).await.map(|_| ()), + }; + match staged { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound && attempt == 0 => { + continue; // evicted between the check and the staging — re-fetch + } + Err(e) => return Err(Error::ExecutionErr(format!("failed to stage rootfs: {e}"))), + } + // Extract as the worker user (rootfs is worker-owned → uid 0 inside the jail). + let untar = Command::new("tar") + .args(["-xf", &job_tar, "-C", &rootfs]) + .output() + .await + .map_err(|e| Error::ExecutionErr(format!("failed to run tar: {e}")))?; + let _ = tokio::fs::remove_file(&job_tar).await; + if !untar.status.success() { + return Err(Error::ExecutionErr(format!( + "failed to unpack image {image}: {}", + String::from_utf8_lossy(&untar.stderr) + ))); + } + return Ok(config); + } + Err(Error::ExecutionErr(format!( + "failed to materialize rootfs for {image} (cache evicted twice)" + ))) +} + +/// Fetch + flatten `pinned` (a `name@digest` ref) into the cache: export the rootfs tar +/// and write the OCI config sidecar, both via tmp+rename so concurrent readers never see +/// a torn file. The tar is published last (a present tar implies a present config). +async fn fetch_into_cache( + pinned: &str, + tar: &str, + cfg: &str, + token: &str, + auth: Option<&str>, +) -> Result<(), Error> { + let tar_tmp = format!("{tar}.tmp.{token}"); + let cfg_tmp = format!("{cfg}.tmp.{token}"); + let exported = crane( + &["export", "--platform", &CRANE_PLATFORM, pinned, &tar_tmp], + auth, + ) + .await?; + if !exported.status.success() { + let _ = tokio::fs::remove_file(&tar_tmp).await; + return Err(Error::ExecutionErr(format!( + "failed to export image {pinned}: {}", + String::from_utf8_lossy(&exported.stderr) + ))); + } + let config = crane(&["config", "--platform", &CRANE_PLATFORM, pinned], auth).await?; + if !config.status.success() { + let _ = tokio::fs::remove_file(&tar_tmp).await; + return Err(Error::ExecutionErr(format!( + "failed to read image {pinned} config: {}", + String::from_utf8_lossy(&config.stderr) + ))); + } + let _ = tokio::fs::write(&cfg_tmp, &config.stdout).await; + let _ = tokio::fs::rename(&cfg_tmp, cfg).await; + tokio::fs::rename(&tar_tmp, tar).await?; + Ok(()) +} + +/// Read the cached OCI config (Env/Cmd/Entrypoint/WorkingDir); tolerate a missing or torn +/// sidecar by falling back to defaults (the run still works off the body + image FS). +async fn read_oci_config(cfg: &str) -> OciConfig { + match tokio::fs::read(cfg).await { + Ok(bytes) => serde_json::from_slice::(&bytes) + .map(|c| c.config) + .unwrap_or_default(), + Err(_) => OciConfig::default(), + } +} + +/// Manifest descriptor (`crane manifest`), for the pre-download size guard. +#[derive(Deserialize, Default)] +struct CraneDescriptor { + #[serde(default)] + size: u64, +} +#[derive(Deserialize, Default)] +struct CraneManifest { + #[serde(default)] + layers: Vec, + #[serde(default)] + config: CraneDescriptor, +} + +/// Reject the image if its compressed download size exceeds `SANDBOX_IMAGE_MAX_SIZE_MB`. +/// Runs on EVERY job (so lowering the limit rejects already-cached images too); the size +/// is read from a `{digest}.size` sidecar when present (no registry call on cache reuse) +/// and otherwise fetched once via `crane manifest` (before any layer download) and cached. +/// No-op when the limit is 0 (unset). +async fn enforce_image_size_limit( + pinned: &str, + size_file: &str, + auth_dir: Option<&str>, +) -> Result<(), Error> { + let max = max_image_size_mb().await; + if max == 0 { + return Ok(()); + } + let bytes = match tokio::fs::read_to_string(size_file) + .await + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + Some(b) => b, + None => { + let out = crane( + &["manifest", "--platform", &CRANE_PLATFORM, pinned], + auth_dir, + ) + .await?; + if !out.status.success() { + // Don't silently bypass the guard — surface it so an operator can see the + // size limit isn't being enforced for this image. + tracing::warn!( + "sandbox image size guard: `crane manifest {pinned}` failed, not enforcing \ + SANDBOX_IMAGE_MAX_SIZE_MB: {}", + String::from_utf8_lossy(&out.stderr) + ); + return Ok(()); + } + let manifest: CraneManifest = match serde_json::from_slice(&out.stdout) { + Ok(m) => m, + Err(e) => { + tracing::warn!( + "sandbox image size guard: cannot parse `crane manifest` json: {e}" + ); + return Ok(()); + } + }; + let b = manifest.config.size + manifest.layers.iter().map(|l| l.size).sum::(); + let _ = tokio::fs::write(size_file, b.to_string()).await; + b + } + }; + let mb = bytes / 1_000_000; + if mb > max { + return Err(Error::ExecutionErr(format!( + "image {pinned} is {mb} MB (compressed), over the SANDBOX_IMAGE_MAX_SIZE_MB limit of {max} MB" + ))); + } + Ok(()) +} + +/// Best-effort eviction: while the cached rootfs tars exceed `SANDBOX_IMAGE_CACHE_MAX_MB`, +/// remove the oldest by mtime (creation order — tars are write-once, cache hits don't +/// touch mtime). No-op when the limit is 0 (unset). Skipped if another pass is already +/// running. The per-job extracted rootfs lives in the job dir (cleaned with the job), so +/// only the content-addressed tar+config+size cache is pruned. Also sweeps orphaned +/// `*.tmp.*` files left by a crashed mid-export. +async fn enforce_image_cache_limit() { + use std::sync::atomic::Ordering; + let max_mb = image_cache_max_mb().await; + if max_mb == 0 { + return; + } + if EVICTION_RUNNING + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return; + } + // Reset the guard on every exit path (incl. an early `break` or a panic), so a + // stuck flag can never permanently disable eviction until a worker restart. + struct ResetOnDrop; + impl Drop for ResetOnDrop { + fn drop(&mut self) { + EVICTION_RUNNING.store(false, std::sync::atomic::Ordering::SeqCst); + } + } + let _reset = ResetOnDrop; + let max_bytes = max_mb.saturating_mul(1_000_000); + + // (path, size, mtime) for every cached rootfs tar; also sweep orphaned tmp files. + async fn list_tars() -> Vec<(std::path::PathBuf, u64, std::time::SystemTime)> { + let mut out = Vec::new(); + let Ok(mut rd) = tokio::fs::read_dir(&*ROOTFS_CACHE_DIR).await else { + return out; + }; + while let Ok(Some(e)) = rd.next_entry().await { + let p = e.path(); + let name = e.file_name(); + let name = name.to_string_lossy(); + // Reclaim leftover `*.tmp.` files from a crashed mid-export. + if name.contains(".tmp.") { + let _ = tokio::fs::remove_file(&p).await; + continue; + } + if p.extension().and_then(|x| x.to_str()) != Some("tar") { + continue; + } + if let Ok(m) = e.metadata().await { + let mtime = m.modified().unwrap_or(std::time::UNIX_EPOCH); + out.push((p, m.len(), mtime)); + } + } + out + } + + loop { + let mut tars = list_tars().await; + let total: u64 = tars.iter().map(|(_, s, _)| *s).sum(); + if total <= max_bytes || tars.is_empty() { + break; + } + tars.sort_by_key(|(_, _, mtime)| *mtime); + let victim = tars[0].0.clone(); + if tokio::fs::remove_file(&victim).await.is_err() { + break; // can't reclaim — stop rather than spin on the same victim + } + // Drop the sibling config + size sidecars too. + let _ = tokio::fs::remove_file(victim.with_extension("json")).await; + let _ = tokio::fs::remove_file(victim.with_extension("size")).await; + tracing::info!("sandbox image cache eviction: removed {}", victim.display()); + } + // `_reset` drops here and clears EVICTION_RUNNING. +} + +/// Build the nsjail mount block that binds each top-level entry of the rootfs in +/// place. Binding the whole rootfs at `/` trips nsjail's read-only remount of its +/// base root in a rootless userns; per-entry binds avoid it. `proc`, `dev`, `tmp` +/// and `sys` are skipped — the profile provides them. +async fn generate_rootfs_mounts(rootfs: &str) -> Result { + let mut block = String::new(); + let mut entries = tokio::fs::read_dir(rootfs).await?; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if matches!(name.as_ref(), "proc" | "dev" | "tmp" | "sys") { + continue; + } + let src = proto_str(&format!("{rootfs}/{name}")); + let dst = proto_str(&format!("/{name}")); + let file_type = entry.file_type().await?; + if file_type.is_symlink() { + // Recreate top-level symlinks (e.g. usr-merged /bin -> usr/bin) as + // symlinks in the jail. The target is image-controlled but only ever + // *resolved inside the jail* (against the bound rootfs dirs / jail + // pseudo-fs) — there is no host `/` in the jail for it to point at — and + // it is escaped via proto_str, so it can neither escape nor inject config. + let target = tokio::fs::read_link(entry.path()) + .await + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(); + block.push_str(&format!( + "mount {{\n src: {}\n dst: {dst}\n is_symlink: true\n mandatory: false\n}}\n", + proto_str(&target), + )); + } else { + block.push_str(&format!( + "mount {{\n src: {src}\n dst: {dst}\n is_bind: true\n rw: true\n mandatory: false\n}}\n", + )); + } + } + Ok(block) +} + +#[tracing::instrument(level = "trace", skip_all)] +pub async fn handle_docker_v2_job( + image: &str, + mem_peak: &mut i32, + canceled_by: &mut Option, + job: &MiniPulledJob, + conn: &Connection, + client: &AuthedClient, + parent_runnable_path: Option, + content: &str, + job_dir: &str, + shared_mount: &str, + base_internal_url: &str, + worker_name: &str, + occupancy_metrics: &mut OccupancyMetrics, +) -> Result, Error> { + // The sandboxed container runtime *is* nsjail, so it requires nsjail. (`# docker` + // keeps the v1 dind path for non-sandboxed workers.) + if NSJAIL_AVAILABLE.is_none() { + return Err(Error::ExecutionErr(format!( + "`# sandbox {image}` runs the image inside nsjail, which is not available on \ + this worker. Install nsjail, or use a bare `# docker` (dind) instead." + ))); + } + + // Apply the default-registry instance setting to unqualified refs. + let resolved_image = resolve_image_ref(image).await; + let image = resolved_image.as_str(); + + append_logs( + &job.id, + &job.workspace_id, + format!("\n\n--- SANDBOXED CONTAINER (nsjail) ---\nextracting image {image}...\n"), + conn, + ) + .await; + + let config = extract_image(image, job_dir).await?; + let rootfs = format!("{job_dir}/rootfs"); + + // Best-effort: keep the cached rootfs tars under their size cap (overlaps the run). + tokio::spawn(enforce_image_cache_limit()); + + // Resolve the script args from the bash signature, like the bash executor. + let args = build_args_map(job, client, conn).await?.map(Json); + let job_args = if args.is_some() { + args.as_ref() + } else { + job.args.as_ref() + }; + let args_owned = windmill_parser_bash::parse_bash_sig(content)? + .args + .iter() + .map(|arg| { + job_args + .and_then(|x| x.get(&arg.name).map(|x| raw_to_string(x.get()))) + .unwrap_or_else(String::new) + }) + .collect::>(); + + // The body is everything that isn't a leading `#` annotation/comment line. With + // a body we run it via the image's `/bin/sh`; without one we run the image's + // ENTRYPOINT + CMD. + let has_body = content + .lines() + .any(|l| !l.trim().is_empty() && !l.trim_start().starts_with('#')); + + let cmd_args: Vec = if has_body { + // Pass the body straight to `sh -c` rather than writing a script file into + // the image-controlled rootfs: a malicious image could plant that path as a + // symlink to a host file and capture the worker's write before nsjail starts + // (sandbox-boundary bypass). `sh -c sh ` binds args as $1.. . + let mut v = vec![ + "/bin/sh".to_string(), + "-c".to_string(), + format!("set -e\n{content}"), + "sh".to_string(), + ]; + v.extend(args_owned.iter().cloned()); + v + } else { + let mut v = config.entrypoint.clone().unwrap_or_default(); + v.extend(config.cmd.clone().unwrap_or_default()); + if v.is_empty() { + return Err(Error::ExecutionErr(format!( + "image {image} has no ENTRYPOINT/CMD and the script body is empty — \ + nothing to run" + ))); + } + v.extend(args_owned.iter().cloned()); + v + }; + + let working_dir = config + .working_dir + .as_deref() + .filter(|w| !w.is_empty()) + .unwrap_or("/"); + + // The image's OCI Env is attacker-controlled (BOTH keys and values), so it must + // NOT enter the nsjail launcher's own process env: a hostile image could set + // LD_PRELOAD / LD_LIBRARY_PATH / LD_AUDIT and have the dynamic loader run code in + // the nsjail binary as the worker — outside the jail — before it sandboxes. + // Deliver it to the *child only* via proto-escaped `envar:` directives. + let mut container_env: Vec<(String, String)> = Vec::new(); + for kv in config.env.unwrap_or_default() { + if let Some((k, v)) = kv.split_once('=') { + container_env.push((k.to_string(), v.to_string())); + } + } + if !container_env.iter().any(|(k, _)| k == "PATH") { + container_env.push(("PATH".to_string(), DEFAULT_PATH.to_string())); + } + if !container_env.iter().any(|(k, _)| k == "HOME") { + container_env.push(("HOME".to_string(), "/root".to_string())); + } + let envars = render_envars(&container_env); + + // Render the nsjail profile: dynamic per-entry rootfs binds + image WorkingDir. + let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; + let rootfs_mounts = generate_rootfs_mounts(&rootfs).await?; + write_file( + job_dir, + "run.docker.config.proto", + &NSJAIL_CONFIG_RUN_DOCKER_CONTENT + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + // proto_str-quoted: WorkingDir is image-controlled, must not break out + // of the `cwd:` string and inject nsjail directives. + .replace("{WORKDIR}", &proto_str(working_dir)) + .replace("{ROOTFS_MOUNTS}", &rootfs_mounts) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + // `# volume` mounts + same-worker shared folder (empty if none). + .replace("{SHARED_MOUNT}", shared_mount) + // Image env as `envar:` directives (child-only), so it never touches + // nsjail's process env. + .replace("{ENVARS}", &envars) + .replace("#{DEV}", DEV_CONF_NSJAIL), + )?; + + // nsjail's OWN process env: only windmill-trusted keys (reserved vars so + // `wmill`/API calls work, + proxy). `keep_env: true` forwards these to the + // child. The image env is NOT here — see container_env above. + let mut reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; + reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); + reserved_variables.insert( + "BASE_INTERNAL_URL".to_string(), + base_internal_url.to_string(), + ); + + let proxy_envs = get_proxy_envs_for_lang( + &ScriptLang::Bash, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?; + + let mut nsjail_run_args = vec!["--config", "run.docker.config.proto", "--"]; + nsjail_run_args.extend(cmd_args.iter().map(|s| s.as_str())); + + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .envs(proxy_envs) + .args(nsjail_run_args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child = start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await?; + + handle_child( + &job.id, + conn, + mem_peak, + canceled_by, + child, + true, + worker_name, + &job.workspace_id, + "sandboxed container run", + job.timeout, + true, + &mut Some(occupancy_metrics), + None, + None, + ) + .await?; + + Ok(to_raw_value(&json!(format!( + "sandboxed container ({image}) completed successfully" + )))) +} + +#[cfg(test)] +mod tests { + use super::{digest_key, proto_str, ref_key, registry_qualified, render_envars}; + + #[test] + fn digest_key_is_filesystem_safe() { + assert_eq!(digest_key("sha256:4d889c14e7d5"), "sha256_4d889c14e7d5"); + // No `:` or `/` survives (both would break the cache filename). + let k = digest_key("sha256:ab/cd:ef"); + assert!(!k.contains(':') && !k.contains('/')); + } + + #[test] + fn ref_key_is_safe_and_stable() { + // Deterministic for a given ref... + assert_eq!(ref_key("ghcr.io/o/i:tag"), ref_key("ghcr.io/o/i:tag")); + // ...distinguishes different refs... + assert_ne!(ref_key("alpine:latest"), ref_key("alpine:edge")); + // ...and is filesystem-safe (no `/` or `:`), incl. for multibyte refs (no panic + // on the trailing-80 byte slice since every char maps to single-byte ASCII). + for r in [ + "alpine", + "ghcr.io/o/i:tag", + "localhost:5000/r@sha256:ab", + "rég/imagé:tag", + ] { + let k = ref_key(r); + assert!(!k.contains('/') && !k.contains(':')); + } + } + + #[test] + fn render_envars_emits_proto_directives() { + // Image-controlled env (incl. loader vars) is rendered as `envar:` directives + // — i.e. delivered to the child via the config, NOT nsjail's process env, so + // it can never set LD_PRELOAD/etc. on the nsjail binary itself. + let env = vec![ + ("PATH".to_string(), "/usr/bin".to_string()), + ("LD_PRELOAD".to_string(), "rootfs/evil.so".to_string()), + ]; + let out = render_envars(&env); + assert_eq!( + out, + "envar: \"PATH=/usr/bin\"\nenvar: \"LD_PRELOAD=rootfs/evil.so\"" + ); + // A value trying to inject extra directives is escaped, not interpreted. + let evil = vec![("X".to_string(), "v\"\nclone_newuser: false".to_string())]; + let line = render_envars(&evil); + assert!(line.starts_with("envar: \"")); + assert!(!line.contains("\nclone_newuser")); + assert!(line.contains("\\n")); + } + + #[test] + fn proto_str_escapes_injection() { + // Normal paths are just wrapped in quotes. + assert_eq!(proto_str("/app"), "\"/app\""); + // A `"` is escaped so it cannot close the surrounding string and inject + // subsequent nsjail directives — this is what the WorkingDir / mount-src + // sandboxing fixes depend on. + let malicious = "/x\"\nmount { src: \"/\" dst: \"/host\" is_bind: true }\n#"; + let escaped = proto_str(malicious); + assert!(escaped.starts_with('"') && escaped.ends_with('"')); + // No raw quote or newline survives inside the rendered literal. + let inner = &escaped[1..escaped.len() - 1]; + assert!(!inner.contains('\n')); + assert!(!inner.contains("\"") || inner.contains("\\\"")); + assert!(escaped.contains("\\\"")); // the inner quote is backslash-escaped + assert!(escaped.contains("\\n")); // the newline is escaped + // Control and non-ASCII bytes render as valid 3-digit octal escapes (never + // a raw byte or an invalid `\u{..}` that nsjail's parser would reject). + assert_eq!(proto_str("a\u{1b}b"), "\"a\\033b\""); // ESC (0x1b) + assert_eq!(proto_str("é"), "\"\\303\\251\""); // UTF-8 bytes 0xc3 0xa9 + } + + #[test] + fn registry_qualified_classifies_refs() { + // Unqualified: bare repos (with/without tag) and docker.io org/repo. + for img in ["alpine", "alpine:latest", "myorg/img", "myorg/img:1.2"] { + assert!(!registry_qualified(img), "{img} should be unqualified"); + } + // Qualified: the first path component is a host (has `.`/`:`) or localhost. + for img in [ + "ghcr.io/org/img", + "registry.example.com/img:tag", + "localhost:5000/img", + "localhost/img", + "host:5000/a/b", + ] { + assert!(registry_qualified(img), "{img} should be qualified"); + } + } +} diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 08f9381205..7727982cf8 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -31,6 +31,7 @@ mod csharp_executor; mod dedicated_worker_ee; mod dedicated_worker_oss; mod deno_executor; +mod docker_v2; #[cfg(feature = "duckdb")] mod duckdb_executor; mod global_cache; diff --git a/backend/windmill-worker/src/otel_oss.rs b/backend/windmill-worker/src/otel_oss.rs index 2f65c534b4..913fee6003 100644 --- a/backend/windmill-worker/src/otel_oss.rs +++ b/backend/windmill-worker/src/otel_oss.rs @@ -7,3 +7,6 @@ use windmill_queue::MiniPulledJob; #[cfg(not(feature = "private"))] pub fn add_root_flow_job_to_otlp(_queued_job: &MiniPulledJob, _success: bool) {} + +#[cfg(not(feature = "private"))] +pub fn set_job_span_parent(_span: &tracing::Span, _job: &MiniPulledJob, _rj: &uuid::Uuid) {} diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 3895c823b7..f74a523072 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -597,11 +597,57 @@ async fn postinstall( Ok(()) } +/// Python hard keywords cannot be used as a bare name in `import ` / +/// `from import `. A flow inline step whose id (or a folder on its +/// path) is such a keyword — e.g. a step id `in` — otherwise generates +/// `from pkg import in as inner_script`, a SyntaxError. Prefix these with `_`, +/// mirroring the existing digit-leading guard. +fn is_python_keyword(s: &str) -> bool { + matches!( + s, + "False" + | "None" + | "True" + | "and" + | "as" + | "assert" + | "async" + | "await" + | "break" + | "class" + | "continue" + | "def" + | "del" + | "elif" + | "else" + | "except" + | "finally" + | "for" + | "from" + | "global" + | "if" + | "import" + | "in" + | "is" + | "lambda" + | "nonlocal" + | "not" + | "or" + | "pass" + | "raise" + | "return" + | "try" + | "while" + | "with" + | "yield" + ) +} + /// Compute the directory (relative to job_dir) where Python writes the main script. /// Module files must be placed in this same directory for relative imports to work. pub fn compute_python_module_dir(script_path: &str) -> String { let script_path_splitted = script_path.split("/").map(|x| { - if x.starts_with(|x: char| x.is_ascii_digit()) { + if x.starts_with(|x: char| x.is_ascii_digit()) || is_python_keyword(x) { format!("_{}", x) } else { x.to_string() @@ -1236,6 +1282,12 @@ pub fn compute_py_codegen(content: &str, script_path: &str) -> PyScriptCodegen { .replace("-", "_") .replace(" ", "_") .to_lowercase(); + // `last` is lowercased above, so this catches a keyword id in any case. + let last = if is_python_keyword(&last) { + format!("_{last}") + } else { + last + }; let sig = windmill_parser_py::parse_python_signature(content, None, false).unwrap_or_default(); let pre_sig = windmill_parser_py::parse_python_signature( @@ -1605,6 +1657,12 @@ async fn prepare_wrapper( .replace("-", "_") .replace(" ", "_") .to_lowercase(); + // `last` is lowercased above, so this catches a keyword id in any case. + let last = if is_python_keyword(&last) { + format!("_{last}") + } else { + last + }; let module_dir = format!("{}/{}", job_dir, dirs); tokio::fs::create_dir_all(format!("{module_dir}/")).await?; @@ -3286,6 +3344,13 @@ mod tests { assert_eq!(compute_python_module_dir("u/@admin/script"), "u/.admin"); } + #[test] + fn test_compute_python_module_dir_keyword_segment() { + // A folder whose name is a Python keyword would otherwise produce an + // invalid `from f.in.x import ...`; it is underscore-prefixed. + assert_eq!(compute_python_module_dir("f/in/script"), "f/_in"); + } + #[test] fn test_compute_py_codegen_basic_args() { let code = "def main(x: str, y: int):\n return x\n"; @@ -3297,6 +3362,22 @@ mod tests { assert_eq!(cg.module_name, "script"); } + #[test] + fn test_compute_py_codegen_keyword_step_id() { + // Regression for a flow inline step whose auto-assigned id is a Python + // keyword (e.g. `in`): the generated wrapper must not emit + // `from pkg import in as inner_script` (SyntaxError). The module name is + // underscore-prefixed, matching the digit-leading guard. + let code = "def main():\n return 1\n"; + let cg = compute_py_codegen(code, "u/admin/myflow/in"); + assert_eq!(cg.module_name, "_in"); + assert_eq!(cg.module_dir_dot, "u.admin.myflow"); + + // Non-keyword ids are unaffected. + let cg2 = compute_py_codegen(code, "u/admin/myflow/step"); + assert_eq!(cg2.module_name, "step"); + } + #[test] fn test_compute_py_codegen_with_datetime_and_bytes() { let code = "import datetime\n\ndef main(name: str, created_at: datetime.datetime, file: bytes):\n return name\n"; diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 5d06d3e673..0ae6b3462a 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -537,6 +537,11 @@ impl PyV { &v, "--python-preference=only-managed", "--no-bin", + // Compile the runtime's stdlib to bytecode at install time. The + // runtime is mounted read-only into the job nsjail, so without + // precompiled .pyc Python would recompile ~stdlib from source on + // every job (and can never persist it). Requires uv >= 0.9.25. + "--compile-bytecode", ]) // TODO: Do we need these? .envs([ diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 0a591bc36d..fdef059c26 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -185,6 +185,9 @@ async fn process_jc( span.record("labels", labels.join(",")); } } + // The secondary `job_postprocessing` span stays on the UUID-derived context + // (MiniCompletedJob carries no args, so the inbound traceparent isn't + // available here); the primary job span is relocated in `create_span_with_name`. windmill_common::otel_oss::set_span_parent(&span, &rj); if let Some(lg) = jc.job.script_lang.as_ref() { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 9029de5572..8e15e053ea 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -694,6 +694,27 @@ lazy_static::lazy_static! { /// RAM-backed tmpfs sized by `nsjail_tmpfs_size_mb`. pub static ref NSJAIL_TMP_BACKING: Arc>> = Arc::new(RwLock::new(None)); + /// Reject a `# sandbox ` whose compressed download size exceeds this many + /// MB, before download. `None`/non-positive = no limit. (`sandbox_image_max_size_mb`.) + pub static ref SANDBOX_IMAGE_MAX_SIZE_MB: Arc>> = Arc::new(RwLock::new(None)); + + /// Best-effort cap (MB) on the worker's cached rootfs tars; oldest evicted after a + /// run when exceeded. `None`/non-positive = unbounded. (`sandbox_image_cache_max_mb`.) + pub static ref SANDBOX_IMAGE_CACHE_MAX_MB: Arc>> = Arc::new(RwLock::new(None)); + + /// Sandbox image pull policy (`missing`/`newer`/`always`/`never`). `None`/unrecognized + /// falls back to `newer`. (`sandbox_image_pull_policy`.) + pub static ref SANDBOX_IMAGE_PULL_POLICY: Arc>> = Arc::new(RwLock::new(None)); + + /// If set, unqualified sandbox image refs (e.g. `alpine`) are pulled from this + /// registry instead of docker.io. Fully-qualified refs are unaffected. + /// (`sandbox_image_default_registry`.) + pub static ref SANDBOX_IMAGE_DEFAULT_REGISTRY: Arc>> = Arc::new(RwLock::new(None)); + + /// Optional docker `auth.json` blob for private registries, written to a per-job + /// `DOCKER_CONFIG` dir for crane. (`sandbox_registry_auth`.) + pub static ref SANDBOX_REGISTRY_AUTH: Arc>> = Arc::new(RwLock::new(None)); + /// Optional mirror URL for `uv python install`. Wires to the `UV_PYTHON_INSTALL_MIRROR` /// env var when forwarded to uv. Can be set via the `UV_PYTHON_INSTALL_MIRROR` env var /// or the `uv_python_install_mirror` instance setting. @@ -919,14 +940,50 @@ pub async fn is_otel_tracing_proxy_enabled_for_lang(lang: &ScriptLang) -> bool { } } +/// Strict check that a string is a well-formed W3C `traceparent` +/// (`version-traceid-spanid-flags`, lowercase hex, non-zero ids, version != ff). +/// Used before forwarding an inbound header value verbatim to a job subprocess, +/// so we don't hand downstream OTel parsers something they'll reject. +#[cfg(all(feature = "private", feature = "enterprise"))] +fn valid_w3c_traceparent(tp: &str) -> bool { + let p: Vec<&str> = tp.split('-').collect(); + p.len() == 4 + && p[0].len() == 2 + && p[1].len() == 32 + && p[2].len() == 16 + && p[3].len() == 2 + // version "ff" is reserved/invalid per the W3C spec + && p[0] != "ff" + && p[1] != "00000000000000000000000000000000" + && p[2] != "0000000000000000" + // W3C mandates lowercase hex + && p + .iter() + .all(|s| s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))) +} + /// Get OTEL trace context environment variables for a job (TRACEPARENT, OTEL_TRACE_ID, OTEL_SPAN_ID). /// Returns an empty vec when OTEL tracing is not enabled or on non-enterprise builds. +/// +/// When the request that enqueued the job carried a valid inbound `traceparent` +/// (propagated via the job's [`LogContext`](windmill_common::log_context::LogContext)), +/// it is forwarded verbatim so the script's spans join the originating +/// distributed trace. Otherwise the trace context is derived from the job UUID. pub fn get_otel_context_envs(job_id: &uuid::Uuid) -> Vec<(&'static str, String)> { #[cfg(all(feature = "private", feature = "enterprise"))] if windmill_common::OTEL_TRACING_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { - let trace_id = format!("{:032x}", job_id.as_u128()); - let span_id = format!("{:016x}", job_id.as_u64_pair().1); - let traceparent = format!("00-{}-{}-01", trace_id, span_id); + let inbound = windmill_common::log_context::current_log_context() + .and_then(|c| c.inbound_traceparent.clone()) + .filter(|tp| valid_w3c_traceparent(tp)); + let (traceparent, trace_id, span_id) = if let Some(tp) = inbound { + let trace_id = tp[3..35].to_string(); + let span_id = tp[36..52].to_string(); + (tp, trace_id, span_id) + } else { + let trace_id = format!("{:032x}", job_id.as_u128()); + let span_id = format!("{:016x}", job_id.as_u64_pair().1); + (format!("00-{}-{}-01", trace_id, span_id), trace_id, span_id) + }; return vec![ ("TRACEPARENT", traceparent), ("OTEL_TRACE_ID", trace_id), @@ -1466,7 +1523,10 @@ pub fn create_span_with_name( span.record("script_hash", script_hash.to_string().as_str()); } - windmill_common::otel_oss::set_span_parent(&span, &rj); + // Parent the job span on the inbound distributed trace when the request that + // enqueued it (or its flow root) carried a W3C `traceparent`; otherwise on + // the UUID-derived context. See `otel_ee::set_job_span_parent`. + crate::otel_oss::set_job_span_parent(&span, arc_job, &rj); span } @@ -1567,10 +1627,21 @@ pub fn log_context_for_job( trigger_kind: arc_job.trigger_kind.as_ref().map(|k| k.to_string()), trigger: arc_job.trigger.clone(), hostname: hostname.map(|h| h.to_string()), + inbound_traceparent: job_inbound_traceparent(arc_job), ..existing } } +/// Extract the inbound W3C `traceparent` captured at enqueue from a job's args +/// (reserved `_wm_traceparent` key). Present only on directly-triggered jobs +/// (and flow steps that inherited it). +pub(crate) fn job_inbound_traceparent(job: &MiniPulledJob) -> Option { + job.args + .as_ref() + .and_then(|a| a.get(windmill_common::jobs::WM_TRACEPARENT)) + .and_then(|raw| serde_json::from_str::(raw.get()).ok()) +} + pub async fn handle_all_job_kind_error( conn: &Connection, authed_client: &AuthedClient, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 281a9785c2..c8d109c465 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -311,13 +311,14 @@ struct RecoveryObject { recover: Option, } -fn get_stop_after_if_data(stop_after_if: Option<&StopAfterIf>) -> (bool, Option) { +/// Returns `(skip_if_stopped, error_message, include_step_result)`. +fn get_stop_after_if_data(stop_after_if: Option<&StopAfterIf>) -> (bool, Option, bool) { if let Some(stop_after_if) = stop_after_if { // skip_if_stopped and error_message are mutually exclusive: // skip_if_stopped=true means clean stop (mark remaining as skipped), // error_message means stop with error. skip_if_stopped takes precedence. if stop_after_if.skip_if_stopped { - return (true, None); + return (true, None, false); } let err_msg = stop_after_if.error_message.as_ref().and_then(|message| { if message.is_empty() { @@ -326,9 +327,9 @@ fn get_stop_after_if_data(stop_after_if: Option<&StopAfterIf>) -> (bool, Option< Some(message.clone()) } }); - return (false, err_msg); + return (false, err_msg, stop_after_if.error_include_result); } - return (false, None); + return (false, None, false); } async fn get_id_ctx_for_expr( @@ -358,6 +359,7 @@ async fn evaluate_stop_after_all_iters_if( stop_early: &mut bool, skip_if_stop_early: &mut bool, stop_early_err_msg: &mut Option, + stop_early_include_result: &mut bool, nresult: &mut Option>>, args: HashMap>, flow_env: Option<&HashMap>>, @@ -394,8 +396,11 @@ async fn evaluate_stop_after_all_iters_if( if stop_early_after_all_iters { *stop_early = true; - (*skip_if_stop_early, *stop_early_err_msg) = - get_stop_after_if_data(Some(stop_after_all_iters_if)); + ( + *skip_if_stop_early, + *stop_early_err_msg, + *stop_early_include_result, + ) = get_stop_after_if_data(Some(stop_after_all_iters_if)); } Ok(()) } @@ -655,19 +660,24 @@ pub async fn update_flow_status_after_job_completion_internal( false }; - let (mut stop_early, mut stop_early_err_msg, mut skip_if_stop_early, continue_on_error) = - if stop_early_override.is_some() - && !is_flow_stop_early_override - && !parallel_loop - && !parallel_branchall - { - // we ignore stop_early_override (stop_early in children) if module is parallel or is a flow step - let se = stop_early_override.as_ref().unwrap(); - (true, None, *se, false) - } else if is_failure_step || module_step.is_preprocessor_step() { - (false, None, false, false) - } else if let Some(current_module) = current_module { - let stop_early = success + let ( + mut stop_early, + mut stop_early_err_msg, + mut skip_if_stop_early, + mut stop_early_include_result, + continue_on_error, + ) = if stop_early_override.is_some() + && !is_flow_stop_early_override + && !parallel_loop + && !parallel_branchall + { + // we ignore stop_early_override (stop_early in children) if module is parallel or is a flow step + let se = stop_early_override.as_ref().unwrap(); + (true, None, *se, false, false) + } else if is_failure_step || module_step.is_preprocessor_step() { + (false, None, false, false, false) + } else if let Some(current_module) = current_module { + let stop_early = success && !is_branch_all // we don't support stop_early per branch && !parallel_loop // we don't support anymore stop_early per iteration when parallel for loop (removed from frontend) && !is_identity_job // don't evaluate stop_after_if for skipped (identity) steps @@ -717,22 +727,23 @@ pub async fn update_flow_status_after_job_completion_internal( } else { false }; - let (skip_if_stopped, stop_early_err_msg) = if stop_early { - get_stop_after_if_data(current_module.stop_after_if.as_ref()) - } else { - (false, None) - }; - - ( - stop_early, - stop_early_err_msg, - skip_if_stopped, - current_module.continue_on_error.unwrap_or(false), - ) + let (skip_if_stopped, stop_early_err_msg, include_result) = if stop_early { + get_stop_after_if_data(current_module.stop_after_if.as_ref()) } else { - (false, None, false, false) + (false, None, false) }; + ( + stop_early, + stop_early_err_msg, + skip_if_stopped, + include_result, + current_module.continue_on_error.unwrap_or(false), + ) + } else { + (false, None, false, false, false) + }; + let skip_seq_branch_failure = match module_status { FlowStatusModule::InProgress { branchall: Some(BranchAllStatus { branch, .. }), @@ -974,6 +985,7 @@ pub async fn update_flow_status_after_job_completion_internal( &mut stop_early, &mut skip_if_stop_early, &mut stop_early_err_msg, + &mut stop_early_include_result, &mut nresult, args, resolved_flow_env.as_deref(), @@ -1173,6 +1185,7 @@ pub async fn update_flow_status_after_job_completion_internal( stop_early = false; stop_early_err_msg = None; skip_if_stop_early = false; + stop_early_include_result = false; } if is_loop || (is_branch_all && !stop_early) { @@ -1194,6 +1207,7 @@ pub async fn update_flow_status_after_job_completion_internal( &mut stop_early, &mut skip_if_stop_early, &mut stop_early_err_msg, + &mut stop_early_include_result, &mut nresult, args, resolved_flow_env.as_deref(), @@ -1310,12 +1324,22 @@ pub async fn update_flow_status_after_job_completion_internal( }; if stop_early && stop_early_err_msg.is_some() { - nresult = Some(Arc::new(to_raw_value(&serde_json::json! ({ - "error": { - "name": "EarlyStopError", - "message": stop_early_err_msg.as_ref().unwrap(), - } - })))); + let mut error = serde_json::json!({ + "name": "EarlyStopError", + "message": stop_early_err_msg.as_ref().unwrap(), + }); + if stop_early_include_result { + // Embed the stopping step's own result inside the error object instead + // of discarding it, keeping the top-level result shape `{ "error": .. }` + // unchanged. `nresult` is already set for loops/branchall (aggregated + // iteration results), otherwise fall back to the step result. + let step_result = nresult.clone().unwrap_or_else(|| result.clone()); + error["result"] = + serde_json::to_value(&step_result).unwrap_or(serde_json::Value::Null); + } + nresult = Some(Arc::new(to_raw_value( + &serde_json::json!({ "error": error }), + ))); } let step_counter = if inc_step_counter { @@ -4174,6 +4198,21 @@ async fn push_next_flow_job( } } + // Propagate the inbound W3C traceparent captured at enqueue to each step + // so the whole flow shares the originating distributed trace (the trace + // identity is otherwise derived from the root job UUID). Observability + // only — no security impact — so unlike _TEMP_SCRIPT_REFS it is not + // gated to previews. + if let Some(traceparent) = arc_flow_job_args + .as_ref() + .get(windmill_common::jobs::WM_TRACEPARENT) + { + push_args.extra.get_or_insert_with(HashMap::new).insert( + windmill_common::jobs::WM_TRACEPARENT.to_string(), + traceparent.clone(), + ); + } + tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed args for job {i} of {len}"); let value_with_parallel = module.get_value_with_parallel()?; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 38d55b5319..1634b4dc88 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -2070,6 +2070,32 @@ pub async fn handle_app_dependency_job( .and_then(|x| x.get("temp_script_refs")) .and_then(|v| serde_json::from_str(v.get()).ok()); + // The version captured at job creation can be stale (the app may have been + // redeployed since). Relock the current latest instead, mirroring the flow + // dependency handler. + let id = if triggered_by_relative_import { + let latest_version = sqlx::query_scalar!( + "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", + job_path, + job.workspace_id + ) + .fetch_optional(db) + .await?; + match latest_version { + Some(latest_version) if latest_version != id => { + tracing::info!( + "App version changed since dependency job was queued ({} -> {}), using latest", + id, + latest_version + ); + latest_version + } + _ => id, + } + } else { + id + }; + sqlx::query!( "DELETE FROM workspace_runnable_dependencies WHERE app_path = $1 AND workspace_id = $2", job_path, @@ -2163,13 +2189,14 @@ pub async fn handle_app_dependency_job( .execute(db) .await?; - // NOTE: Temporary solution. - // Ideally we do this for every job regardless whether it was triggered by relative import or by creation/update of the app. - // NOTE: For now is not solving any problem but at some point we will introduce latest version caching - // and when we do this will be last operation that will make new version appear as the latest and will trigger cache invalidation for all worker. + // Re-publish the relocked version as latest for cache invalidation, but + // only if it is still the latest: the guard makes this a single atomic, + // never-demoting statement. Without it, a concurrent deploy that landed a + // newer version (e.g. same git-sync push) would be reverted, pointing a + // raw app's bundle_secret at a version with no bundle (404 / white screen). if triggered_by_relative_import { sqlx::query!( - "UPDATE app SET versions = array_append(versions, $1::bigint) WHERE path = $2 AND workspace_id = $3", + "UPDATE app SET versions = array_append(versions, $1::bigint) WHERE path = $2 AND workspace_id = $3 AND versions[array_upper(versions, 1)] = $1::bigint", id, &job_path, &job.workspace_id diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index dff2f0fa12..3cd0c3effe 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.714.0"; +export const VERSION = "v1.719.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/package-lock.json b/cli/package-lock.json index f14fd32f8d..c502b3b25e 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -17,6 +17,7 @@ "jszip": "3.8.0", "minimatch": "^10.0.0", "open": "^10.0.0", + "pg-gateway": "0.3.0-beta.4", "svelte": "^5.45.2", "tar-stream": "^3.1.7", "windmill-parser-wasm-csharp": "1.510.1", @@ -1236,6 +1237,12 @@ "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", "license": "(MIT AND Zlib)" }, + "node_modules/pg-gateway": { + "version": "0.3.0-beta.4", + "resolved": "https://registry.npmjs.org/pg-gateway/-/pg-gateway-0.3.0-beta.4.tgz", + "integrity": "sha512-CTjsM7Z+0Nx2/dyZ6r8zRsc3f9FScoD5UAOlfUx1Fdv/JOIWvRbF7gou6l6vP+uypXQVoYPgw8xZDXgMGvBa4Q==", + "license": "MIT" + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", diff --git a/cli/src/commands/init/init.ts b/cli/src/commands/init/init.ts index fd6b93380d..f1a10d7cc9 100644 --- a/cli/src/commands/init/init.ts +++ b/cli/src/commands/init/init.ts @@ -18,6 +18,7 @@ import { import { generateRTNamespace } from "../resource-type/resource-type.ts"; import { generateCommentedTemplate } from "./template.ts"; import { refreshPrompts } from "../refresh/prompts.ts"; +import { refreshTsconfig } from "../refresh/tsconfig.ts"; export interface InitOptions { useDefault?: boolean; @@ -238,6 +239,18 @@ async function initAction(opts: InitOptions) { await refreshPrompts({ yes: opts.useDefault === true }); + // Generate the IDE tsconfig (managed tsconfig.wmill.json + user tsconfig.json + // that extends it). Independent of any workspace binding — it's purely local. + try { + await refreshTsconfig({ yes: opts.useDefault === true }); + } catch (error) { + log.warn( + `Could not generate tsconfig: ${ + error instanceof Error ? error.message : error + }` + ); + } + // Generate resource type namespace (only if a workspace was bound) if (didBindWorkspace && boundProfile) { try { diff --git a/cli/src/commands/refresh/prompts.ts b/cli/src/commands/refresh/prompts.ts index 3f933d3de2..863ab4f9ff 100644 --- a/cli/src/commands/refresh/prompts.ts +++ b/cli/src/commands/refresh/prompts.ts @@ -32,7 +32,8 @@ export async function refreshPrompts(opts: { // If config can't be read, use the conservative default above. } - const interactive = process.stdin.isTTY && !opts.yes; + const assumeYes = opts.yes === true; + const interactive = process.stdin.isTTY && !assumeYes; try { const result = await writeAiGuidanceFiles({ @@ -42,8 +43,13 @@ export async function refreshPrompts(opts: { agentsSourcePath: process.env[WMILL_INIT_AI_AGENTS_SOURCE_ENV], claudeSourcePath: process.env[WMILL_INIT_AI_CLAUDE_SOURCE_ENV], resolveAgentsMdMigration: async () => { - if (!interactive) return "append"; - return await promptMigration(); + // Consent model (matches `wmill refresh tsconfig`): we only touch an + // existing user-owned file that we don't recognize when the user opts + // in. `--yes` (and `wmill init --default`) appends without asking; an + // interactive run prompts; a plain non-interactive run leaves it alone. + if (assumeYes) return "append"; + if (interactive) return await promptMigration(); + return "skip"; }, }); @@ -175,7 +181,7 @@ const command = new Command() .description("Refresh AGENTS.cli.md and managed skills. User-owned AGENTS.md and CLAUDE.md are never overwritten unless you opt in.") .option( "--yes", - "Non-interactive: skip the migration prompt for existing AGENTS.md / CLAUDE.md without the expected include; defaults to appending the include." + "Non-interactive: append the @AGENTS.cli.md include to an existing AGENTS.md / CLAUDE.md without prompting. Without it, a non-interactive run leaves an unlinked file untouched." ) .action(promptsAction as any); diff --git a/cli/src/commands/refresh/refresh.ts b/cli/src/commands/refresh/refresh.ts index 882284c428..d1fea23af2 100644 --- a/cli/src/commands/refresh/refresh.ts +++ b/cli/src/commands/refresh/refresh.ts @@ -1,8 +1,12 @@ import { Command } from "@cliffy/command"; import promptsCommand from "./prompts.ts"; +import tsconfigCommand from "./tsconfig.ts"; const command = new Command() - .description("Refresh wmill-managed project files (AGENTS.cli.md and skills)") - .command("prompts", promptsCommand); + .description( + "Refresh wmill-managed project files (AGENTS.cli.md, skills, tsconfig.wmill.json)" + ) + .command("prompts", promptsCommand) + .command("tsconfig", tsconfigCommand); export default command; diff --git a/cli/src/commands/refresh/tsconfig.ts b/cli/src/commands/refresh/tsconfig.ts new file mode 100644 index 0000000000..10d9993d50 --- /dev/null +++ b/cli/src/commands/refresh/tsconfig.ts @@ -0,0 +1,496 @@ +import { execSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import process from "node:process"; + +import { colors } from "@cliffy/ansi/colors"; +import { Command } from "@cliffy/command"; +import { Confirm } from "@cliffy/prompt/confirm"; +import * as log from "../../core/log.ts"; +import { readConfigFile } from "../../core/conf.ts"; + +/** + * The on-disk folders (`f/`, `u/`) that the absolute workspace import paths + * `/f/...` and `/u/...` map to. tsconfig `paths` and Deno import maps remap the + * `/f/`,`/u/` prefixes to these local folders, so the same workspace import + * resolves both on the Windmill worker and in a local editor (tsc/Bun/Deno all + * honor the `/`-prefixed key). + */ +const WORKSPACE_IMPORT_DIRS = ["f", "u"]; + +// wmill-managed files holding the recommended config. They are always +// (re)written so we can ship updated recommendations over time; users keep +// their own overrides in tsconfig.json / deno.json, which reference these +// managed files and are never overwritten. This mirrors how AGENTS.cli.md +// (managed) and AGENTS.md (user-owned) work for AI prompts. +const MANAGED_TSCONFIG = "tsconfig.wmill.json"; +const MANAGED_IMPORT_MAP = "import_map.wmill.json"; + +const MANAGED_NOTICE = + "// Managed by wmill — regenerated by `wmill init` / `wmill refresh tsconfig`.\n" + + "// Do not edit; put your overrides in tsconfig.json (which extends this file).\n"; + +// Embedded in tsconfig.wmill.json so any command can detect a stale managed file +// (the recommended config changed) and nudge the user to `wmill refresh tsconfig` +// — mirroring the prompts freshness marker in AGENTS.cli.md. +const TSCONFIG_HASH_PREFIX = "// wmill-tsconfig-hash: "; +const TSCONFIG_HASH_REGEX = /^\/\/ wmill-tsconfig-hash: ([0-9a-f]{12})/m; + +/** + * The recommended managed tsconfig, minus environment-dependent bits (`types` + * depends on whether bun-types is installed locally). This is both the source + * of the written file and the input to the freshness hash, so the hash only + * changes when wmill's *recommended* config changes — not when bun-types + * appears/disappears on a given machine. + */ +function buildManagedTsconfig(): { + compilerOptions: Record; + include: string[]; +} { + // Map "/f/*" -> ["./f/*"], "/u/*" -> ["./u/*"] so the editor resolves + // workspace imports against the local script folders. + // + // Known limitation: this resolves imports written with a plain `.ts` extension + // (the canonical form). Scripts stored with a flavor-specific extension — + // `.bun.ts`/`.deno.ts`/`.fetch.ts` for languages other than the project default + // (see filePathExtensionFromContentType) — won't resolve via these `paths` in a + // local editor. The worker (extension-agnostic API) and the in-app editor (ATA + // normalizes to `.ts`) handle those fine; only local tsc / VS Code is affected. + const paths: Record = {}; + for (const dir of WORKSPACE_IMPORT_DIRS) { + paths[`/${dir}/*`] = [`./${dir}/*`]; + } + return { + compilerOptions: { + target: "ESNext", + module: "ESNext", + moduleResolution: "bundler", + // Workspace imports carry an explicit `.ts` extension (e.g. "/f/foo/bar.ts"); + // allow it so the editor doesn't flag every cross-script import. + allowImportingTsExtensions: true, + noEmit: true, + strict: false, + // No `baseUrl`: with moduleResolution "bundler" the `paths` patterns resolve + // relative to this file, and `baseUrl` is deprecated in TypeScript 7+. + paths, + }, + include: ["**/*.ts", "rt.d.ts"], + }; +} + +function currentTsconfigHash(): string { + return createHash("sha256") + .update(JSON.stringify(buildManagedTsconfig())) + .digest("hex") + .slice(0, 12); +} + +// Exact `tsconfig.json` shapes the *previous* CLI generated (single-file, before +// the managed/user split — see the now-deleted resource-type/tsconfig.ts). When +// an existing tsconfig.json matches one of these verbatim, we know it's ours (not +// a user customization), so we can safely replace it wholesale with the thin stub +// that extends tsconfig.wmill.json. Anything else is treated as user-authored. +const LEGACY_GENERATED_TSCONFIGS: Record[] = [ + { + compilerOptions: { + target: "ESNext", + module: "ESNext", + moduleResolution: "bundler", + noEmit: true, + strict: false, + }, + include: ["**/*.ts", "rt.d.ts"], + }, + { + compilerOptions: { + target: "ESNext", + module: "ESNext", + moduleResolution: "bundler", + noEmit: true, + strict: false, + types: ["bun-types"], + }, + include: ["**/*.ts", "rt.d.ts"], + }, +]; + +// Order-sensitive deep equality (arrays compared positionally, objects by key +// set). Used only on small parsed JSON config objects. +function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (Array.isArray(a) || Array.isArray(b)) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) { + return false; + } + return a.every((x, i) => deepEqual(x, b[i])); + } + if (a && b && typeof a === "object" && typeof b === "object") { + const ka = Object.keys(a as object); + const kb = Object.keys(b as object); + if (ka.length !== kb.length) return false; + return ka.every((k) => + deepEqual((a as Record)[k], (b as Record)[k]) + ); + } + return false; +} + +// How to handle an existing, user-authored config that doesn't yet reference the +// managed file. `assumeYes` (from `--yes` / `wmill init --default`) wires it +// without asking; otherwise we only wire it after an interactive confirmation — +// a non-interactive run with neither leaves the file untouched. +type WireMode = { interactive: boolean; assumeYes: boolean }; + +/** + * (Re)generate the wmill-managed TypeScript/Deno IDE config so the editor + * resolves `/f/`/`/u/` workspace imports against the local script folders. + * + * Split into a managed base file (always refreshed) and a user-owned file that + * references it. We only ever touch files that are ours: the managed file is + * regenerated, a tsconfig.json still in the previously-generated shape is + * migrated to the new split, and a genuinely custom config is wired only with + * the user's consent (interactive prompt, or `--yes`). + * + * Programmatic entry point reused by `wmill init`; also exposed as + * `wmill refresh tsconfig`. + */ +export async function refreshTsconfig(opts?: { yes?: boolean }): Promise { + let defaultTs: "bun" | "deno" = "bun"; + try { + const conf = await readConfigFile({ warnIfMissing: false }); + if (conf?.defaultTs === "deno") { + defaultTs = "deno"; + } + } catch { + // fall back to bun if wmill.yaml is missing or unreadable + } + + const assumeYes = opts?.yes === true; + const mode: WireMode = { + assumeYes, + interactive: !!process.stdin.isTTY && !assumeYes, + }; + + // tsconfig.json is useful for Bun and general TS tooling regardless of the + // default; the Deno import map is only relevant for Deno-default projects + // (the Deno LSP ignores tsconfig.json). + await refreshManagedTsconfig(defaultTs, mode); + if (defaultTs === "deno") { + await refreshManagedDenoImportMap(mode); + } +} + +async function refreshManagedTsconfig(defaultTs: "bun" | "deno", mode: WireMode) { + const managed = buildManagedTsconfig(); + + // Only reference bun-types if it's actually available; otherwise the IDE + // would flag the missing type definitions. (Excluded from the freshness hash + // since it's environment-, not recommendation-, dependent.) + const bunTypesAvailable = + defaultTs === "bun" ? ensureBunTypesAvailable() : false; + if (bunTypesAvailable) { + managed.compilerOptions.types = ["bun-types"]; + } + + const header = MANAGED_NOTICE + TSCONFIG_HASH_PREFIX + currentTsconfigHash() + "\n"; + writeFileSync( + path.join(process.cwd(), MANAGED_TSCONFIG), + header + JSON.stringify(managed, null, 2) + "\n" + ); + log.info(colors.green(`Refreshed ${MANAGED_TSCONFIG}`)); + + await ensureUserReferencesManaged({ + file: "tsconfig.json", + create: { extends: `./${MANAGED_TSCONFIG}` }, + legacyFormats: LEGACY_GENERATED_TSCONFIGS, + mode, + wire: (parsed) => { + // TypeScript does NOT merge `compilerOptions.paths` across `extends` — the + // nearest config that defines `paths` wins wholesale. So a config with its + // own `paths` would shadow the managed `/f/`/`/u/` mappings and they would + // silently fail to resolve. We don't touch the user's paths — warn + // and leave it for them to wire manually. + const co = parsed.compilerOptions; + const paths = + co && typeof co === "object" && !Array.isArray(co) + ? (co as Record).paths + : undefined; + if ( + paths && + typeof paths === "object" && + !Array.isArray(paths) && + Object.keys(paths).length > 0 + ) { + return ( + "defines its own `compilerOptions.paths` (TS won't merge ours in via " + + '`extends`); add "/f/*": ["./f/*"] and "/u/*": ["./u/*"] to it' + ); + } + const ext = parsed.extends; + const managed = `./${MANAGED_TSCONFIG}`; + // Insert the managed config FIRST in `extends` so the user's own base config + // keeps precedence on overlapping compilerOptions (strict/target/module) + // rather than being overridden by our defaults. + if (ext === undefined) { + parsed.extends = managed; + } else if (typeof ext === "string") { + parsed.extends = [managed, ext]; + } else if (Array.isArray(ext)) { + ext.unshift(managed); + } else { + return "unexpected `extends` value"; + } + return true; + }, + token: MANAGED_TSCONFIG, + hint: `"extends": "./${MANAGED_TSCONFIG}"`, + }); +} + +async function refreshManagedDenoImportMap(mode: WireMode) { + // Import-map prefix keys must end with "/": "/f/" -> "./f/", "/u/" -> "./u/". + const imports: Record = {}; + for (const dir of WORKSPACE_IMPORT_DIRS) { + imports[`/${dir}/`] = `./${dir}/`; + } + + // Deno import maps only allow `imports`/`scopes`, so no comment header here. + writeFileSync( + path.join(process.cwd(), MANAGED_IMPORT_MAP), + JSON.stringify({ imports }, null, 2) + "\n" + ); + log.info(colors.green(`Refreshed ${MANAGED_IMPORT_MAP}`)); + + await ensureUserReferencesManaged({ + file: "deno.json", + // Don't write deno.json if the project already uses deno.jsonc — a new + // deno.json would take precedence and shadow the existing config. + altFiles: ["deno.jsonc"], + create: { importMap: `./${MANAGED_IMPORT_MAP}` }, + mode, + wire: (parsed) => { + // Deno rejects `imports` + `importMap` together, so we can't auto-wire a + // deno.json that already defines its own imports. + if (parsed.imports !== undefined) { + return "deno.json already defines `imports` (can't also use importMap)"; + } + if ( + parsed.importMap !== undefined && + parsed.importMap !== `./${MANAGED_IMPORT_MAP}` + ) { + return "deno.json already sets a different `importMap`"; + } + parsed.importMap = `./${MANAGED_IMPORT_MAP}`; + return true; + }, + token: MANAGED_IMPORT_MAP, + hint: `"importMap": "./${MANAGED_IMPORT_MAP}"`, + }); +} + +/** + * Ensure a user-owned config file references the wmill-managed file. Mirrors how + * `wmill refresh prompts` wires `@AGENTS.cli.md` into AGENTS.md: + * - missing → create the minimal file (already linked); + * - exists & linked → leave it alone; + * - exists & unlinked → auto-wire it (parse JSON, apply `wire`, write back). + * Falls back to a one-line warning when the file can't be auto-edited safely + * (JSONC comments fail JSON.parse, or the structure already conflicts) — we + * never corrupt a file we can't round-trip. + */ +async function ensureUserReferencesManaged(opts: { + file: string; + // Sibling configs that, if already present, must not be shadowed by writing + // `opts.file` next to them (e.g. an existing deno.jsonc vs a new deno.json). + altFiles?: string[]; + create: Record; + // Mutate the parsed user config to reference the managed file. Returns true + // when wired, or a short reason string when it can't be wired cleanly (→ warn). + wire: (parsed: Record) => true | string; + // Verbatim shapes a previous CLI generated for this file. A match means the + // file is ours, so it's replaced wholesale (no prompt); anything else is + // treated as user-authored and only wired with consent. + legacyFormats?: Record[]; + mode: WireMode; + token: string; + hint: string; +}) { + // Prefer any existing config (including alternates) over creating a fresh one, + // so we never shadow a config the user already has. + const existing = [opts.file, ...(opts.altFiles ?? [])] + .map((f) => path.join(process.cwd(), f)) + .find((p) => existsSync(p)); + + if (!existing) { + const userPath = path.join(process.cwd(), opts.file); + writeFileSync(userPath, JSON.stringify(opts.create, null, 2) + "\n"); + log.info(colors.green(`Created ${opts.file} (references ${opts.token})`)); + return; + } + + const existingName = path.basename(existing); + let text = ""; + try { + text = readFileSync(existing, "utf-8"); + } catch { + return; + } + if (text.includes(opts.token)) { + log.info( + colors.gray(`${existingName} already references ${opts.token}, leaving it untouched`) + ); + return; + } + + // We only ever rewrite a config we can round-trip as JSON. Files with comments + // (JSONC) fail JSON.parse, so we warn instead of corrupting them. + let parsed: Record; + try { + parsed = JSON.parse(text); + } catch { + log.warn( + `${existingName} couldn't be auto-edited (it may contain comments). Add ${opts.hint} ` + + `to pick up wmill's recommended settings (incl. workspace /f/, /u/ import resolution).` + ); + return; + } + + // The file is still exactly what a previous CLI generated → it's ours, so + // migrate it to the new split (replace with the thin stub that extends the + // managed file). No prompt: we're not touching user-authored content. + if (opts.legacyFormats?.some((fmt) => deepEqual(parsed, fmt))) { + writeFileSync(existing, JSON.stringify(opts.create, null, 2) + "\n"); + log.info( + colors.green( + `Migrated previously-generated ${existingName} to reference ${opts.token}` + ) + ); + return; + } + + // Custom config. Try wiring a clone so we can report un-wireable cases without + // mutating, then only persist with the user's consent. + const next = JSON.parse(JSON.stringify(parsed)) as Record; + const wired = opts.wire(next); + if (wired !== true) { + log.warn( + `${existingName}: ${wired}. Add ${opts.hint} manually to pick up wmill's ` + + `recommended settings (incl. workspace /f/, /u/ import resolution).` + ); + return; + } + + const consent = opts.mode.assumeYes + ? true + : opts.mode.interactive + ? await Confirm.prompt({ + message: + `${existingName} isn't linked to wmill's ${opts.token}. Add ${opts.hint}? ` + + `Your settings are preserved (it's inserted first, so your config wins).`, + default: true, + }) + : false; + + if (!consent) { + log.info( + colors.gray( + `Left ${existingName} unchanged — add ${opts.hint} when ready to enable ` + + `workspace /f/, /u/ import resolution (or re-run \`wmill refresh tsconfig\`).` + ) + ); + return; + } + + writeFileSync(existing, JSON.stringify(next, null, 2) + "\n"); + log.info(colors.green(`Linked ${existingName} → ${opts.token}`)); +} + +function ensureBunTypesAvailable(): boolean { + const cwd = process.cwd(); + if (existsSync(path.join(cwd, "node_modules", "bun-types"))) { + return true; + } + + try { + execSync("bun --version", { stdio: "ignore" }); + } catch { + log.info( + "Install bun (https://bun.sh), run 'bun add -d bun-types', then re-run 'wmill refresh tsconfig' for Bun API autocompletion." + ); + return false; + } + + try { + log.info( + colors.yellow("Installing bun-types with 'bun add -d bun-types'...") + ); + execSync("bun add -d bun-types", { stdio: "inherit" }); + log.info(colors.green("Installed bun-types.")); + return true; + } catch (e) { + log.warn( + `Failed to install bun-types automatically: ${ + e instanceof Error ? e.message : e + }` + ); + log.info( + "Run 'bun add -d bun-types' manually, then 'wmill refresh tsconfig', for Bun API autocompletion." + ); + return false; + } +} + +/** + * One-line, non-blocking warning (to stderr) when the managed config is out of + * date. Mirrors the prompts freshness check (`warnIfPromptsStale`) exactly: + * gated on the *managed* file existing (= the project opted in by running + * init/refresh), and only ever warns that it's **stale** — never about a + * missing or unlinked user tsconfig.json. So a deliberately-unlinked / custom + * setup is never nagged, and a not-yet-initialized project stays silent. + * Gated identically in main.ts so it never fires during `wmill init`/`refresh`. + */ +export async function warnIfTsconfigStale(opts?: { cwd?: string }): Promise { + const cwd = opts?.cwd ?? process.cwd(); + const managedPath = path.join(cwd, MANAGED_TSCONFIG); + if (!existsSync(managedPath)) return; + + let managedText: string; + try { + managedText = readFileSync(managedPath, "utf-8"); + } catch { + return; + } + const match = managedText.match(TSCONFIG_HASH_REGEX); + if (!match) { + emitTsconfigWarning( + `${MANAGED_TSCONFIG} predates versioning. Run \`wmill refresh tsconfig\` to refresh it.` + ); + return; + } + if (match[1] !== currentTsconfigHash()) { + emitTsconfigWarning( + `${MANAGED_TSCONFIG} is out of date. Run \`wmill refresh tsconfig\` to refresh.` + ); + } +} + +// Send to stderr (not log.warn → stdout) so it never contaminates a piped +// command's output, matching the prompts freshness warning. +function emitTsconfigWarning(message: string): void { + process.stderr.write(`${colors.yellow(message)}\n`); +} + +const command = new Command() + .description( + "Refresh the wmill-managed tsconfig.wmill.json (and Deno import map for Deno projects)" + ) + .option( + "--yes", + "Non-interactive: wire an existing custom tsconfig.json/deno.json to the managed file without prompting (a previously-generated config is always migrated automatically)." + ) + .action((async (opts: { yes?: boolean }) => { + await refreshTsconfig({ yes: opts.yes === true }); + }) as any); + +export default command; diff --git a/cli/src/commands/resource-type/resource-type.ts b/cli/src/commands/resource-type/resource-type.ts index c51e4bff3a..96c8429eb4 100644 --- a/cli/src/commands/resource-type/resource-type.ts +++ b/cli/src/commands/resource-type/resource-type.ts @@ -12,7 +12,6 @@ import { } from "../../types.ts"; import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace } from "../../core/context.ts"; -import { generateTsconfigForIde } from "./tsconfig.ts"; import { colors } from "@cliffy/ansi/colors"; import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; @@ -190,8 +189,6 @@ export async function generateRTNamespace(opts: GlobalOptions) { "Created rt.d.ts with resource types namespace (RT) for TypeScript." ) ); - - await generateTsconfigForIde(); } const command = new Command() diff --git a/cli/src/commands/resource-type/tsconfig.ts b/cli/src/commands/resource-type/tsconfig.ts deleted file mode 100644 index 2aa2c5335b..0000000000 --- a/cli/src/commands/resource-type/tsconfig.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { execSync } from "node:child_process"; -import { existsSync, writeFileSync } from "node:fs"; -import path from "node:path"; -import process from "node:process"; - -import { colors } from "@cliffy/ansi/colors"; -import * as log from "../../core/log.ts"; -import { readConfigFile } from "../../core/conf.ts"; - -export async function generateTsconfigForIde() { - const tsconfigPath = path.join(process.cwd(), "tsconfig.json"); - if (existsSync(tsconfigPath)) { - log.info(colors.gray("tsconfig.json already exists, skipping")); - return; - } - - let defaultTs: "bun" | "deno" = "bun"; - try { - const conf = await readConfigFile({ warnIfMissing: false }); - if (conf?.defaultTs === "deno") { - defaultTs = "deno"; - } - } catch { - // fall back to bun if wmill.yaml is missing or unreadable - } - - // Only reference bun-types in tsconfig if it's actually available; otherwise - // the IDE will flag the missing type definitions. - const bunTypesAvailable = - defaultTs === "bun" ? ensureBunTypesAvailable() : false; - - const tsconfig: { - compilerOptions: Record; - include: string[]; - } = { - compilerOptions: { - target: "ESNext", - module: "ESNext", - moduleResolution: "bundler", - noEmit: true, - strict: false, - }, - include: ["**/*.ts", "rt.d.ts"], - }; - - if (bunTypesAvailable) { - tsconfig.compilerOptions.types = ["bun-types"]; - } - - writeFileSync(tsconfigPath, JSON.stringify(tsconfig, null, 2) + "\n"); - log.info(colors.green("Created tsconfig.json for IDE type support.")); -} - -function ensureBunTypesAvailable(): boolean { - const cwd = process.cwd(); - if (existsSync(path.join(cwd, "node_modules", "bun-types"))) { - return true; - } - - try { - execSync("bun --version", { stdio: "ignore" }); - } catch { - log.info( - "Install bun (https://bun.sh) then run 'bun add -d bun-types' and add \"types\": [\"bun-types\"] to tsconfig.json for Bun API autocompletion." - ); - return false; - } - - try { - log.info( - colors.yellow("Installing bun-types with 'bun add -d bun-types'...") - ); - execSync("bun add -d bun-types", { stdio: "inherit" }); - log.info(colors.green("Installed bun-types.")); - return true; - } catch (e) { - log.warn( - `Failed to install bun-types automatically: ${ - e instanceof Error ? e.message : e - }` - ); - log.info( - "Run 'bun add -d bun-types' manually and add \"types\": [\"bun-types\"] to tsconfig.json for Bun API autocompletion." - ); - return false; - } -} diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 0299744056..717e8acf46 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -297,7 +297,9 @@ export async function handleFile( if ( !isAppInlineScriptPath(path) && !isFlowInlineScriptPath(path) && - !isRawAppBackendPath(path) && + // Raw-app files (frontend included) belong to the app bundle, never + // standalone scripts — pushed via pushRawApp, not here. + !isRawAppPath(path) && (!isScriptModulePath(path) || moduleEntryPoint) && exts.some((exts) => path.endsWith(exts)) ) { diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 2070ceb590..ed3b7828f1 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -4066,10 +4066,11 @@ export async function push( continue; } if ( - change.path.endsWith(".script.json") || - change.path.endsWith(".script.yaml") || - change.path.endsWith(".lock") || - isFileResource(change.path) + !isRawAppFile(change.path) && + (change.path.endsWith(".script.json") || + change.path.endsWith(".script.yaml") || + change.path.endsWith(".lock") || + isFileResource(change.path)) ) { continue; } else if ( diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index fc3ee0a773..31df17af6b 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -4,3 +4,10 @@ */ export const WM_FORK_PREFIX = "wm-fork"; + +// CLI version — source of truth. Release tooling (.github/change-versions*.sh) +// rewrites this line. Kept here, rather than in main.ts, so low-level modules +// (e.g. utils.ts) can read it without importing main.ts and creating a circular +// dependency (main → workspace → utils → main) that triggers a TDZ. +// Re-exported from main.ts for backwards compatibility. +export const VERSION = "1.719.0"; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 8b6f21053d..61d92755fb 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -4831,7 +4831,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -6455,12 +6455,14 @@ List all queues with their metrics ### refresh -Refresh wmill-managed project files (AGENTS.cli.md and skills) +Refresh wmill-managed project files (AGENTS.cli.md, skills, tsconfig.wmill.json) **Subcommands:** - \`refresh prompts\` - Refresh AGENTS.cli.md and managed skills. User-owned AGENTS.md and CLAUDE.md are never overwritten unless you opt in. - - \`--yes\` - Non-interactive: skip the migration prompt for existing AGENTS.md / CLAUDE.md without the expected include; defaults to appending the include. + - \`--yes\` - Non-interactive: append the @AGENTS.cli.md include to an existing AGENTS.md / CLAUDE.md without prompting. Without it, a non-interactive run leaves an unlinked file untouched. +- \`refresh tsconfig\` - Refresh the wmill-managed tsconfig.wmill.json (and Deno import map for Deno projects) + - \`--yes\` - Non-interactive: wire an existing custom tsconfig.json/deno.json to the managed file without prompting (a previously-generated config is always migrated automatically). ### resource diff --git a/cli/src/main.ts b/cli/src/main.ts index da4a46da71..c9e8d557c4 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -89,10 +89,14 @@ export { token, }; -export const VERSION = "1.714.0"; - -// Re-exported from constants.ts to maintain backwards compatibility -export { WM_FORK_PREFIX } from "./core/constants.ts"; +// VERSION and WM_FORK_PREFIX are defined in constants.ts (which keeps its +// imports minimal) and re-exported here for backwards compatibility. VERSION is +// also imported below for internal use. Defining VERSION in constants.ts rather +// than here lets utils.ts read it without importing main.ts, which previously +// created a circular dependency (main → workspace → utils → main) and a TDZ +// crash ("Cannot access 'workspace' before initialization") on some load orders. +import { VERSION } from "./core/constants.ts"; +export { VERSION, WM_FORK_PREFIX } from "./core/constants.ts"; // Re-implementation of cliffy's internal `checkVersion` so the help path // can wrap it in try/catch. `_check_version` is not in cliffy's package @@ -304,6 +308,10 @@ async function main() { if (shouldRunFreshnessCheck(process.argv)) { const { warnIfPromptsStale } = await import("./guidance/freshness.ts"); await warnIfPromptsStale({ argv: process.argv }).catch(() => {}); + const { warnIfTsconfigStale } = await import( + "./commands/refresh/tsconfig.ts" + ); + await warnIfTsconfigStale().catch(() => {}); } await command.parse(args); diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index ff9fe9b78c..4af76517fa 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -11,6 +11,7 @@ import { readdir, readFile } from "node:fs/promises"; import { fetchVersion } from "../core/context.ts"; import { updateGlobalVersions } from "../commands/sync/global.ts"; import { isRawAppPath } from "./resource_folders.ts"; +import { VERSION } from "../core/constants.ts"; export function deepEqual(a: T, b: T): boolean { if (a === b) return true; @@ -320,6 +321,7 @@ export async function fetchRemoteVersion( if (version) { updateGlobalVersions(version); } + log.info(colors.gray("CLI version: " + VERSION)); log.info(colors.gray("Remote version: " + version)); } diff --git a/cli/test/raw_app_sync.test.ts b/cli/test/raw_app_sync.test.ts index ee20bc1947..3cf1172daa 100644 --- a/cli/test/raw_app_sync.test.ts +++ b/cli/test/raw_app_sync.test.ts @@ -382,6 +382,81 @@ excludes: []`, "utf-8"); }); }); +test("Raw App: frontend .ts file sorting first does not short-circuit the app push", async () => { + // Regression: in the push apply loop, raw-app changes are collapsed to a + // single representative change (changes[0]). Because every file inside a + // raw_app folder shares the same sort order, changes[0] is just the + // alphabetically-first changed path. When that path was a frontend file + // with a script extension (e.g. "Api.ts", which sorts before "App.tsx"), + // handleFile() mistook it for a standalone script: it pushed a bogus script + // at the truncated path (f/test/) AND returned true, so the loop + // `continue`d and pushRawApp() never ran. Result: the whole raw app silently + // failed to deploy while the CLI still reported success. + await withTestBackend(async (backend, tempDir) => { + const testWorkspace = { + remote: backend.baseUrl, + workspaceId: backend.workspace, + name: "raw_app_ts_first_test", + token: backend.token + }; + await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir }); + + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: + - "**" +excludes: []`, "utf-8"); + + const appDir = path.join(tempDir, "f", "test", "ts_first_app.raw_app"); + await mkdir(path.join(tempDir, "f", "test"), { recursive: true }); + await createRawAppOnDisk(appDir); + + // A frontend .ts file whose name sorts before "App.tsx". + const apiTsPath = path.join(appDir, "Api.ts"); + await writeFile(apiTsPath, "export const API = '/api/v1'\n", "utf-8"); + + // Initial push: create the raw app on the backend. + const pushResult1 = await backend.runCLICommand( + ['sync', 'push', '--yes'], + tempDir, "raw_app_ts_first_test" + ); + expect(pushResult1.code).toEqual(0); + + // Edit App.tsx (and the .ts file that sorts first) and push again. + const appTsxPath = path.join(appDir, "App.tsx"); + const appTsxContent = await readFileContent(appTsxPath); + await writeFile( + appTsxPath, + appTsxContent.replace("hello world", "REGRESSION MARKER"), + "utf-8" + ); + await writeFile(apiTsPath, "export const API = '/api/v2'\n", "utf-8"); + + const pushResult2 = await backend.runCLICommand( + ['sync', 'push', '--yes'], + tempDir, "raw_app_ts_first_test" + ); + expect(pushResult2.code).toEqual(0); + + // The App.tsx edit must have landed on the remote app's bundled files. + const appResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/apps/get/p/f/test/ts_first_app` + ); + expect(appResp.status).toEqual(200); + const appJson = await appResp.json(); + const files = appJson?.value?.files ?? {}; + expect(files["/App.tsx"]).toContain("REGRESSION MARKER"); + // The first-sorting .ts file is part of the app bundle, with fresh content. + expect(files["/Api.ts"]).toContain("/api/v2"); + + // And no bogus standalone script was created at the truncated path + // (f/test/ts_first_app.raw_app/Api.ts -> f/test/ts_first_app). + const scriptResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/get/p/f/test/ts_first_app` + ); + expect(scriptResp.status).toEqual(404); + }); +}); + test("Raw App: delete file and push", async () => { await withTestBackend(async (backend, tempDir) => { // Set up workspace diff --git a/docker-compose.yml b/docker-compose.yml index 8b636c7702..8b30479481 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,25 +49,6 @@ services: logging: *default-logging - # Docker-in-Docker sidecar: provides an isolated Docker daemon so user scripts - # can run containers without accessing the host Docker socket. - dind: - image: docker:dind - privileged: true - restart: unless-stopped - environment: - DOCKER_TLS_CERTDIR: "" - volumes: - - dind-data:/var/lib/docker - expose: - - 2375 - healthcheck: - test: ["CMD", "docker", "info"] - interval: 10s - timeout: 5s - retries: 5 - logging: *default-logging - windmill_worker: image: ${WM_IMAGE} pull_policy: always @@ -89,22 +70,19 @@ services: # If running with non-root/non-windmill UID (e.g., user: "1001:1001"), # add: - HOME=/tmp - FAVOR_UNSHARE_PID=true - # Connect to the dind sidecar instead of the host Docker socket - - DOCKER_HOST=tcp://dind:2375 depends_on: db: condition: service_healthy - dind: - condition: service_healthy # to mount the worker folder to debug, KEEP_JOB_DIR=true and mount /tmp/windmill volumes: - worker_dependency_cache:/tmp/windmill/cache - worker_logs:/tmp/windmill/logs - ## WARNING: mounting the host Docker socket grants user scripts full access to - ## the host Docker daemon, enabling host filesystem access and privilege escalation. - ## Only use this if you fully trust all users who can run scripts. - ## To use it, remove the DOCKER_HOST env var and dind depends_on above, - ## and uncomment the line below: + ## Sandboxed containers (`# sandbox `) run daemonless via crane + nsjail + ## inside the worker itself — no Docker socket or dind sidecar required. + ## For the legacy full-compat docker (a bare `# docker`, trusted users only), + ## mount the host Docker socket by uncommenting the line below. WARNING: this + ## grants user scripts full access to the host Docker daemon (host filesystem + ## access and privilege escalation) — only use it if you fully trust all users. # - /var/run/docker.sock:/var/run/docker.sock logging: *default-logging @@ -237,4 +215,3 @@ volumes: windmill_index: null lsp_cache: null caddy_data: null - dind-data: null diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 3301d73506..aca18aea22 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -54,14 +54,17 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmo ENV TZ=Etc/UTC # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtime to temp location (will copy with world-writable perms later) -RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY +# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run +# under the read-only nsjail runtime mount (uv >= 0.9.25). The copy below MUST preserve +# timestamps or Python's mtime-based .pyc invalidation discards these compiled files. +RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode # Copy to final location with world-writable permissions for arbitrary UID support RUN mkdir -p /tmp/windmill/cache && \ - cp -r /tmp/build_cache/* /tmp/windmill/cache/ && \ + cp -r --preserve=timestamps /tmp/build_cache/* /tmp/windmill/cache/ && \ chmod -R a+rw /tmp/windmill/cache && \ rm -rf /tmp/build_cache && \ mkdir -p -m 777 /tmp/windmill/cache/uv @@ -83,10 +86,20 @@ COPY --from=docker:29-dind /usr/local/bin/docker /usr/local/bin/ # nsjail runtime deps and binary RUN apt-get update \ - && apt-get install -y --no-install-recommends libprotobuf-dev libnl-route-3-dev \ + && apt-get install -y --no-install-recommends libprotobuf32 libnl-route-3-200 libnl-3-200 \ && apt-get clean && rm -rf /var/lib/apt/lists/* COPY --from=nsjail /nsjail/nsjail /bin/nsjail +# crane: pulls + flattens images for the sandboxed container runtime (`# sandbox `). +# Single static binary — no daemon/store/root needed. See docs/docker-v2-runtime.md. +ARG CRANE_VERSION=v0.20.6 +RUN arch="$(dpkg --print-architecture)"; \ + case "$arch" in amd64) crane_arch=x86_64 ;; arm64) crane_arch=arm64 ;; *) echo >&2 "error: unsupported arch '$arch' for crane"; exit 1 ;; esac; \ + wget -O /tmp/crane.tgz "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${crane_arch}.tar.gz" \ + && tar -xzf /tmp/crane.tgz -C /usr/local/bin crane \ + && rm /tmp/crane.tgz \ + && chmod +x /usr/local/bin/crane + WORKDIR ${APP} COPY --from=ghcr.io/windmill-labs/windmill:dev --chmod=755 ${APP}/windmill ${APP}/windmill diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 48cd3698a1..b8e5c06a01 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -54,14 +54,17 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmo ENV TZ=Etc/UTC # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtime to temp location (will copy with world-writable perms later) -RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY +# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run +# under the read-only nsjail runtime mount (uv >= 0.9.25). The copy below MUST preserve +# timestamps or Python's mtime-based .pyc invalidation discards these compiled files. +RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode # Copy to final location with world-writable permissions for arbitrary UID support RUN mkdir -p /tmp/windmill/cache && \ - cp -r /tmp/build_cache/* /tmp/windmill/cache/ && \ + cp -r --preserve=timestamps /tmp/build_cache/* /tmp/windmill/cache/ && \ chmod -R a+rw /tmp/windmill/cache && \ rm -rf /tmp/build_cache && \ mkdir -p -m 777 /tmp/windmill/cache/uv @@ -83,10 +86,20 @@ COPY --from=docker:29-dind /usr/local/bin/docker /usr/local/bin/ # nsjail runtime deps and binary RUN apt-get update \ - && apt-get install -y --no-install-recommends libprotobuf-dev libnl-route-3-dev \ + && apt-get install -y --no-install-recommends libprotobuf32 libnl-route-3-200 libnl-3-200 \ && apt-get clean && rm -rf /var/lib/apt/lists/* COPY --from=nsjail /nsjail/nsjail /bin/nsjail +# crane: pulls + flattens images for the sandboxed container runtime (`# sandbox `). +# Single static binary — no daemon/store/root needed. See docs/docker-v2-runtime.md. +ARG CRANE_VERSION=v0.20.6 +RUN arch="$(dpkg --print-architecture)"; \ + case "$arch" in amd64) crane_arch=x86_64 ;; arm64) crane_arch=arm64 ;; *) echo >&2 "error: unsupported arch '$arch' for crane"; exit 1 ;; esac; \ + wget -O /tmp/crane.tgz "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${crane_arch}.tar.gz" \ + && tar -xzf /tmp/crane.tgz -C /usr/local/bin crane \ + && rm /tmp/crane.tgz \ + && chmod +x /usr/local/bin/crane + WORKDIR ${APP} COPY --from=ghcr.io/windmill-labs/windmill-ee:dev --chmod=755 ${APP}/windmill ${APP}/windmill diff --git a/docs/docker-v2-runtime.md b/docs/docker-v2-runtime.md new file mode 100644 index 0000000000..f38d17d5e6 --- /dev/null +++ b/docs/docker-v2-runtime.md @@ -0,0 +1,110 @@ +# Sandboxed container runtime (daemonless docker) + +Windmill bash scripts can run a container image. There are **two** runtimes: + +| | legacy `# docker` | sandboxed `# sandbox ` | +|---|---|---| +| selected by | bare `# docker` | `# sandbox ` | +| runtime | dind / Docker daemon (bollard, `dind` feature) | daemonless: extract rootfs + nsjail-run | +| boundary | separate (daemon outside the jail) | the job's own nsjail sandbox | +| nsjail | not provided (trusted-tenant) | **required** — this *is* the sandbox | +| safety | trusted-tenant | sandboxed (untrusted-capable) | +| compat | full `docker run`/`-d`/API | run-a-command subset | + +The three bash annotations are distinct and don't overload each other: + +- `# docker` → legacy daemon docker (unchanged). +- `# sandbox` → run the bash script under nsjail. +- `# sandbox ` → run that image's command under nsjail (this runtime). + +## Using it + +Put the image ref on a `# sandbox` annotation line; the rest of the script runs +**inside** that image: + +```bash +# sandbox python:3.12-slim +name="$1" # windmill args bind positionally, like any bash script +python3 -c "import sys; print('hello', sys.argv[1])" "$name" +``` + +- The body runs via the image's `/bin/sh -c` (so the image needs a shell). +- An **empty** body runs the image's `ENTRYPOINT` + `CMD`. +- Windmill args (declared `x="$1"`, …) are appended to the command. +- The image's `Env`, `WorkingDir` are applied; the windmill reserved variables + (`WM_TOKEN`, `BASE_INTERNAL_URL`, …) are injected so `wmill`/API calls work. + +## How it works + +1. **Pull/extract** ([`crane`](https://github.com/google/go-containerregistry), no + daemon/store/root): `crane export ` streams the image's flattened root + filesystem to a tar (layers + whiteouts applied, like `docker export`) and + `crane config` reads its OCI config. The tar + config are cached + content-addressed by digest (`crane digest`) so unchanged digests reuse the + cache; `tar -x` materializes the per-job `{job_dir}/rootfs`. crane is a single + ~25 MB static binary — we never *run* the image with it (nsjail does), so a full + container engine like podman isn't needed. +2. **Run** (the job's nsjail sandbox): nsjail binds each top-level entry of the + rootfs in place (binding the whole rootfs at `/` trips nsjail's read-only + remount of its base root in a rootless userns), mounts the standard + pseudo-filesystems (`/proc` from the jail's pid namespace, a tmpfs `/tmp`, + `/dev` nodes), maps uid/gid 0 inside → the worker user outside, and runs the + command. The container *is* the jail. + +``` +# sandbox ─▶ crane export → digest-keyed rootfs cache → tar -x → {job_dir}/rootfs ─▶ nsjail (chroot rootfs) + crane config (OCI config) ───────────────────────────────────────────▶ Env / Cmd / WorkingDir +``` + +Because the run is just the job's own nsjail with the image's filesystem as root, +the container inherits exactly the job's confinement: + +- **Filesystem**: only the rootfs + the job's mounts are visible — no host `/`, + no other job dirs, no dep cache. There is nothing to bind-mount escape to. +- **/proc**: the jail's own pid namespace — the worker and other jobs aren't + visible. +- **uid**: a single-uid jail — an escape lands as the unprivileged worker user. +- **network**: the job's network (same as any bash job). + +## Image storage, freshness & limits + +- **Where pulls live:** a content-addressed cache of flattened rootfs tars (+ OCI + config sidecars) keyed by image digest, under `{ROOT_CACHE_DIR}/sandbox_rootfs` + (persistent, dedups pulls across jobs). The per-job extracted rootfs lives in + `{job_dir}/rootfs` and is removed with the job. +- **Freshness (`SANDBOX_IMAGE_PULL_POLICY`, default `newer`):** the cache is keyed + by digest, so a moving tag whose digest changed re-pulls automatically. `newer` + (default) / `always` re-resolve the digest each job (one cheap `crane digest` + manifest fetch); `missing` reuses a cached digest for the ref without hitting the + registry; `never` only uses the cache (errors if absent). Pinning a digest + (`img@sha256:…`) is immutable and never stale. +- **Per-image size cap (`SANDBOX_IMAGE_MAX_SIZE_MB`, default 0 = off):** images + whose *compressed download* size (`crane manifest`) exceeds the cap are rejected + **before any layer is downloaded**. +- **Cache size cap (`SANDBOX_IMAGE_CACHE_MAX_MB`, default 0 = off):** best-effort + eviction — after a run, the oldest cached rootfs tars (by creation time) are + removed until the cache is back under the cap. + +## Requirements + +- [`crane`](https://github.com/google/go-containerregistry) and `tar` on the worker + for image pull/extract (a single static binary — no daemon, root, or privileged). +- `nsjail` on the worker — **required**. If nsjail is absent, a `# sandbox ` + job errors clearly (use a bare `# docker` + a daemon instead). + +## Limitations (by design — daemonless, run-to-completion) + +- No `docker run -d` + later `exec`/`attach`/`logs -f`, no `docker build`, + `compose`, swarm, healthchecks. +- No arbitrary `-v` host bind mounts, `--privileged`, `--cap-add`, `--device`, + host namespace sharing. +- Images that drop to a non-root uid or chown to arbitrary uids inside need a + subuid **range** in the jail (single-uid only today — follow-up: `newuidmap` + range mapping). +- The script result is a completion message; capture output via stdout/logs. + +## Follow-ups + +- Subuid-range nsjail variant for multi-uid images. +- Per-container isolated networking (slirp/pasta). +- Support under the non-nsjail `unshare` isolation mode. diff --git a/docs/fork-triggers.md b/docs/fork-triggers.md index c2b5bf2231..308d21f1be 100644 --- a/docs/fork-triggers.md +++ b/docs/fork-triggers.md @@ -51,27 +51,53 @@ parent and incoming mail would be delivered arbitrarily. The clone filter copies email triggers only when `workspaced_local_part IS TRUE` (or `CLOUD_HOSTED`, since cloud scopes email lookup by `workspace_id` natively). -## Merge-direction filter (always on) +## Operational state is owned by the parent -Whenever the source workspace has `parent_workspace_id IS NOT NULL` (i.e. -it's a fork), the tarball export at `/api/w/{workspace}/workspaces/tarball` -strips fork-local fields: +The rule that makes both the normal-git PR merge and the in-app merge behave: -- `mode` from every `*_trigger` row -- `enabled` from every `schedule` row +> **A trigger's `mode` (and a schedule's `enabled`) belongs to the parent +> workspace. Git-sync *reads* the parent's value into a fork's synced file and +> *never writes* a fork's value back. No git-sync / merge / create / update +> write sets a fork's operational state; the `setmode` / `setenabled` endpoint +> is the intended explicit mutator.** -The fork-detection key is the column, not the `wm-fork-*` naming convention, -so it stays consistent with the conflict-warning gates in `set_trigger_mode` -and `set_schedule_enabled` and survives any future ID rename. +(Runtime error handling can still auto-disable an errored trigger or schedule — +that's orthogonal to this rule, which governs git-sync/merge/create/update +writes.) -The trigger update handler complements this: when an incoming `update_trigger` -request omits both `mode` and `enabled`, the existing DB value is preserved -instead of falling back to the BaseTriggerData default of `Enabled`. This -means the fork→parent merge cannot flip the parent's operational state, even -if the fork has an explicit (locally-disabled) state for that path. +This is enforced in two halves, keyed off `parent_workspace_id IS NOT NULL` +(the column, not the `wm-fork-*` naming convention — it stays consistent with +the conflict-warning gates and survives any future ID rename): -The schedule `EditSchedule` payload already lacks an `enabled` field, so its -update path is naturally safe. +**Read half — parent-value substitution on export.** When the source workspace +is a fork, the tarball export at `/api/w/{workspace}/workspaces/tarball` +rewrites each trigger's `mode` (and each schedule's `enabled`) to the +*parent's* value for the same path, looked up at export time. A fork-only path +(absent from the parent) keeps the fork's own value — there's no parent state +to defer to, so it lands with whatever the fork creator set. + +The earlier design *stripped* these fields instead. That broke a normal-git PR +merge: the parent branch (and the merge base) carries the line, the fork branch +dropped it, so the 3-way merge either silently deleted `mode`/`enabled` from +the parent — corrupting the source of truth — or conflicted outright when the +parent had also edited it. Substituting the parent's value makes the fork's +file byte-identical to the parent on that field, so the merge has nothing to +resolve. + +**Write half — fork writes never set operational state.** A write into a fork +(git-sync push, merge deploy, clone, or a plain UI create) must not set the +state, otherwise pulling the substituted parent value straight back into the +fork would re-enable it. So `create_trigger`/`create_schedule` force `disabled` +for a fork target, and `update_trigger` preserves the fork's existing `mode` +(`workspace_is_fork` in `windmill-trigger/src/handler.rs`; schedule `enabled` is +naturally preserved because `EditSchedule` has no `enabled` field). The same +handlers serve both merge paths, so the two can't diverge. The fork owner +re-enables locally via `setmode`/`setenabled` (which carry the conflict +warning below). + +For a non-fork target the incoming value is applied as given — so a fork→parent +merge of an existing trigger writes the parent's own (substituted) value (a +no-op), and a fork-only trigger lands with the fork creator's chosen state. ## Conflict warning on enable @@ -161,22 +187,23 @@ but `tally_deployed_object_changes` still records mutations against them; the deploy will fail at the workspace-collision check if the user tries to deploy a non-workspaced row to a fork. -Operational state (`mode` for triggers, `enabled` for schedules) is handled -asymmetrically between update and create: +Operational state (`mode` for triggers, `enabled` for schedules) follows the +same "owned by the parent" rule as the git round-trip (see *Operational state +is owned by the parent* above) — the two paths share the backend `create`/ +`update` handlers, so they can't diverge. -- **Update**: the merge deploy strips `mode`/`enabled` so the target's existing - state is preserved. Triggers rely on the backend's `is_mode_unspecified()` - safeguard in `update_trigger`; schedules rely on `EditSchedule` lacking the - `enabled` field. Both deploy paths perform the strip - (`stripOperationalState` in `utils_deployable.ts`, `preparePayload` in - `merge.ts`). This matches the YAML round-trip's `fork_trigger_ignore_keys` - / `fork_schedule_ignore_keys`. -- **Create**: the source's `mode`/`enabled` is passed through. There's no - target row to preserve, so a fork-only trigger or schedule lands with the - state the fork creator chose. When the source omits the flag entirely (e.g. - legacy clients), the backend defaults to `enabled` for both kinds — - `BaseTriggerData::mode()` returns `Enabled` and the schedule insert defaults - to `true` to match. +- **Update**: the merge deploy strips `mode`/`enabled` + (`stripOperationalStateOnUpdate` in the shared `windmill-utils-internal` + package, `cli/windmill-utils-internal/src/deploy.ts`), so the target's + existing value is preserved — equivalent to substituting the target's value. + For a fork target the backend preserves it regardless (`workspace_is_fork`); + for a parent target the `is_mode_unspecified()` safeguard does. Schedules also + rely on `EditSchedule` lacking the `enabled` field. +- **Create**: the source's `mode`/`enabled` is passed through. Into a **parent** + there's no row to preserve, so a fork-only trigger/schedule lands with the + state the fork creator chose (omitting the flag defaults to `enabled`: + `BaseTriggerData::mode()` → `Enabled`, schedule insert → `true`). Into a + **fork** the backend forces `disabled` — a fork write never enables anything. ## Future work — runtime listener suffix diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a6fd9ba5e5..a00584ef83 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.714.0", + "version": "1.719.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.714.0", + "version": "1.719.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 1a89f6209e..583d39886a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.714.0", + "version": "1.719.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 8abc303e18..f2bddb8874 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -21,6 +21,7 @@ export const copilotInfo = writable<{ enabled: boolean codeCompletionModel?: AIProviderModel defaultModel?: AIProviderModel + metadataModel?: AIProviderModel aiModels: AIProviderModel[] customPrompts?: Record maxTokensPerModel?: Record @@ -28,6 +29,7 @@ export const copilotInfo = writable<{ enabled: false, codeCompletionModel: undefined, defaultModel: undefined, + metadataModel: undefined, aiModels: [], customPrompts: {}, maxTokensPerModel: {} @@ -65,6 +67,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { enabled: true, codeCompletionModel: aiConfig.code_completion_model, defaultModel: aiConfig.default_model, + metadataModel: aiConfig.metadata_model, aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {} @@ -76,6 +79,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { enabled: false, codeCompletionModel: undefined, defaultModel: undefined, + metadataModel: undefined, aiModels: [], customPrompts: {}, maxTokensPerModel: {} @@ -92,6 +96,15 @@ export function getCurrentModel(): AIProviderModel { return model } +export function getMetadataModel(): AIProviderModel { + const info = get(copilotInfo) + const model = info.metadataModel ?? info.defaultModel ?? info.aiModels[0] + if (!model) { + throw new Error('No model selected') + } + return model +} + export function tryGetCurrentModel(): AIProviderModel | undefined { return get(copilotSessionModel) ?? get(copilotInfo).defaultModel ?? get(copilotInfo).aiModels[0] } diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 7b6ed37ecc..4872629cc1 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -13,6 +13,7 @@ type ResourceType } from '$lib/gen' import { emptyString, truncateRev, urlize } from '$lib/utils' + import oauthConnectRegistry from '$oauth_connect_registry' import { createEventDispatcher, onDestroy } from 'svelte' import Path from './Path.svelte' import { Button, Skeleton } from './common' @@ -489,12 +490,25 @@ throw Error(`Resource at path ${path} already exists. Delete it or pick another path`) } - if (resourceType == 'snowflake_oauth') { - const account_identifier = extra_params.find(([key, _]) => key == 'account_identifier') - if (account_identifier) { - args['account_identifier'] = account_identifier[1] + // Per-instance OAuth providers (Snowflake, ServiceNow, …): copy the + // admin-configured instance from the OAuth client's extra_params into the + // resource args, per the registry template's resource_mapping (e.g. + // ServiceNow -> instance_url: https://{instance}.service-now.com). Generic + // so a new per-instance provider needs only a registry entry. + const connectTemplate = (oauthConnectRegistry as Record)[resourceType] + ?.connect_config_template + if (connectTemplate?.resource_mapping) { + const instanceKey = connectTemplate.extra_params_key ?? 'instance' + const found = extra_params.find(([key, _]) => key === instanceKey) + if (found) { + for (const [argField, valueTemplate] of Object.entries( + connectTemplate.resource_mapping as Record + )) { + args[argField] = valueTemplate.replaceAll('{instance}', found[1]) + } } - } else if (resourceType === 'quickbooks' && responseExtra['realmId']) { + } + if (resourceType === 'quickbooks' && responseExtra['realmId']) { args['realmId'] = responseExtra['realmId'] } diff --git a/frontend/src/lib/components/AuthSettings.svelte b/frontend/src/lib/components/AuthSettings.svelte index 734c6a1fe3..31706433f6 100644 --- a/frontend/src/lib/components/AuthSettings.svelte +++ b/frontend/src/lib/components/AuthSettings.svelte @@ -29,7 +29,10 @@ import oauthConnectRegistry from '$oauth_connect_registry' interface Props { - snowflakeAccountIdentifier?: string + // Per-instance OAuth providers (Snowflake, ServiceNow, …): instance name + // keyed by provider, used to build their per-instance connect_config URLs. + // Required (and always bound by InstanceSettings) so it is never undefined. + instanceInputs: Record oauths?: Record requirePreexistingUserForOauth?: boolean baseUrl?: string @@ -39,7 +42,7 @@ } let { - snowflakeAccountIdentifier = $bindable(), + instanceInputs = $bindable(), oauths = $bindable(), requirePreexistingUserForOauth = $bindable(), baseUrl, @@ -49,9 +52,6 @@ }: Props = $props() $effect(() => { - if (snowflakeAccountIdentifier == undefined) { - snowflakeAccountIdentifier = '' - } if (oauths == undefined) { oauths = {} } @@ -79,7 +79,6 @@ 'visma', 'sage_intacct', 'spotify', - 'snowflake_oauth', 'teams', 'zoho', 'xero', @@ -96,9 +95,20 @@ const windmillBuiltinsWithSandbox = Object.entries(oauthConnectRegistry) .filter(([, cfg]) => cfg && typeof cfg === 'object' && 'sandbox' in cfg) .map(([name]) => name) + // Per-instance providers (Snowflake, ServiceNow, …): registry entries that + // carry a `connect_config_template`. Derived from the registry so adding a + // new one needs only a JSON entry — they get a builtin tile + the generic + // instance-name input below, with no frontend change. + const connectConfigTemplates: Record = Object.fromEntries( + Object.entries(oauthConnectRegistry) + .filter(([, cfg]) => cfg && typeof cfg === 'object' && 'connect_config_template' in cfg) + .map(([name, cfg]) => [name, (cfg as any).connect_config_template]) + ) + const windmillBuiltinsTemplated = Object.keys(connectConfigTemplates) const windmillBuiltins = [ ...windmillBuiltinsBase, - ...windmillBuiltinsWithSandbox.map((n) => `${n}_sandbox`) + ...windmillBuiltinsWithSandbox.map((n) => `${n}_sandbox`), + ...windmillBuiltinsTemplated ] let showCustomOAuthForm = $state(false) @@ -238,6 +248,20 @@ } }) + // Add per-instance providers (registry entries with a connect_config_template) + windmillBuiltinsTemplated.forEach((name) => { + if (!oauths || !oauths[name]) { + const icon = getOAuthProviderIcon(name) + items.push({ + // Prefer the template's display_name (properly cased, e.g. "ServiceNow") + // over capitalize(name) which yields "Servicenow"/"Snowflake_oauth". + displayName: connectConfigTemplates[name]?.display_name ?? capitalize(name), + action: () => createOAuthClient(name), + icon: icon + }) + } + }) + // Add custom option items.push({ displayName: `Custom OAuth client ${!$enterpriseLicense ? '(requires ee)' : ''}`, @@ -486,19 +510,23 @@ {:else if !windmillBuiltins.includes(k) && k != 'slack'} {/if} - {#if k == 'snowflake_oauth'} + {#if connectConfigTemplates[k]} + {@const tmpl = connectConfigTemplates[k]} {/if} diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 8ac06662d8..707cda66e1 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -6,6 +6,7 @@ import { copyToClipboard, parseS3Object, roughSizeOfObject } from '$lib/utils' import { base } from '$lib/base' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' + import { appendViewToken } from '$lib/viewToken' import { Button, Drawer, DrawerContent } from './common' import { ClipboardCopy, @@ -176,9 +177,11 @@ let resultApiPath = $derived( workspaceId && jobId - ? nodeId - ? `/w/${workspaceId}/jobs/result_by_id/${jobId}/${nodeId}` - : `/w/${workspaceId}/jobs_u/completed/get_result/${jobId}` + ? appendViewToken( + nodeId + ? `/w/${workspaceId}/jobs/result_by_id/${jobId}/${nodeId}` + : `/w/${workspaceId}/jobs_u/completed/get_result/${jobId}` + ) : undefined ) let resultDownloadHref = $derived( @@ -407,6 +410,23 @@ return json } + // The explicit column order from a leading header row (see + // handleArrayOfObjectsHeaders). Returned as an array so the order survives: + // baking it into object keys loses integer-like names like "1234", which JS + // enumerates first in ascending numeric order. + function getForcedColumnOrder(json: any): string[] | undefined { + if ( + Array.isArray(json) && + json.length > 0 && + Array.isArray(json[0]) && + json[0].length > 0 && + json[0].every((item) => typeof item === 'string') + ) { + return json[0] + } + return undefined + } + type InputObject = { [key: string]: number[] } function objectOfArraysToObjects(input: InputObject): any[] { @@ -640,6 +660,7 @@ ? 'absolute inset-0 [&>div]:h-full [&>div]:min-h-[10rem]' : ''} objects={handleArrayOfObjectsHeaders(data)} + headerOrder={getForcedColumnOrder(data)} /> {:else if !forceJson && resultKind === 'html'}
@@ -1016,9 +1037,7 @@ {#if largeObject}
{#if resultApiPath && shouldDownloadViaClient()} - {:else} diff --git a/frontend/src/lib/components/DisplayResultControlBar.svelte b/frontend/src/lib/components/DisplayResultControlBar.svelte index d45ca3f92d..45011d2db1 100644 --- a/frontend/src/lib/components/DisplayResultControlBar.svelte +++ b/frontend/src/lib/components/DisplayResultControlBar.svelte @@ -4,6 +4,7 @@ import Popover from './Popover.svelte' import { copyToClipboard } from '$lib/utils' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' + import { appendViewToken } from '$lib/viewToken' import type { DisplayResultUi } from './custom_ui' import { createEventDispatcher } from 'svelte' @@ -41,9 +42,11 @@ let resultApiPath = $derived( workspaceId && jobId - ? nodeId - ? `/w/${workspaceId}/jobs/result_by_id/${jobId}/${nodeId}` - : `/w/${workspaceId}/jobs_u/completed/get_result/${jobId}` + ? appendViewToken( + nodeId + ? `/w/${workspaceId}/jobs/result_by_id/${jobId}/${nodeId}` + : `/w/${workspaceId}/jobs_u/completed/get_result/${jobId}` + ) : undefined ) let downloadName = $derived(`${filename ?? 'result'}.json`) diff --git a/frontend/src/lib/components/DrillPicker.svelte b/frontend/src/lib/components/DrillPicker.svelte new file mode 100644 index 0000000000..1b6a161d02 --- /dev/null +++ b/frontend/src/lib/components/DrillPicker.svelte @@ -0,0 +1,532 @@ + + + + leafHaystack(x.leaf)} + opts={{}} +/> + +{#snippet defaultLeafIcon(leaf: DrillLeaf)} + {#if leafIcon} + {@render leafIcon(leaf)} + {:else if leaf.icon} + {@const Icon = leaf.icon} + + {/if} +{/snippet} + +{#snippet defaultBranchIcon(branch: DrillBranch)} + {#if branchIcon} + {@render branchIcon(branch)} + {:else if branch.icon} + {@const Icon = branch.icon} + + {/if} +{/snippet} + +{#snippet leafRow(leaf: DrillLeaf, secondary: string | undefined, baseClass: string)} + {@const key = leaf.key} + {@const isHl = key === highlightedKey} + {@const isCur = !!leaf.current} + +{/snippet} + + +
(mouseActive = true)} +> + {#if externalFilter === undefined} +
+ +
+ {/if} + + {#if scope.length > 0 && !isSearching} + + {/if} + +
+ {#if isSearching} + {@const total = (searchedItems ?? []).length} + {#if !searchedItems} +
+ Searching… +
+ {:else if total === 0} +
No matches
+ {:else} + {#each searchResultsByGroup as { group, items } (group?.key ?? '__none')} + {#if group} +
+ {group.label} +
+ {/if} +
    + {#each items as r (r.leaf.key)} +
  • {@render leafRow(r.leaf, r.leaf.secondary ?? r.leaf.label, 'py-1.5')}
    • + {/each} +
+ {/each} + {/if} + {:else if branchLoading && entryList.length === 0} +
+ Loading… +
+ {:else if entryList.length === 0} +
Empty
+ {:else} +
+ {#each entryList as entry (entry.key)} + {@const isHl = entry.key === highlightedKey} + {#if entry.type === 'leaf'} + {@render leafRow( + entry.node, + leafSecondary?.(entry.node, scope) ?? entry.node.secondary, + 'py-1.5' + )} + {:else} + + {/if} + {/each} +
+ {/if} +
+
+ + diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 8c36de8b11..24c0071a5e 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -133,7 +133,8 @@ onSaveDraftError, onSaveDraftOnlyAtNewPath, onHistoryRestore, - onNavigate + onNavigate, + onTestJob }: FlowBuilderProps = $props() let initialPathStore = writable(initialPath) @@ -735,7 +736,18 @@ flowStore.val = redo(history) } + let flowBuilderRoot: HTMLDivElement | undefined = $state() + function onKeyDown(event: KeyboardEvent) { + // Defer to anything that has explicitly grabbed focus — menus, modals, + // drawers etc. live outside the flow root. Flow nodes aren't focusable, + // so the unfocused default (activeElement === body) means "flow is the + // canvas" and we should react. + const active = document.activeElement + if (active && active !== document.body && !flowBuilderRoot?.contains(active)) { + return + } + let classes = event.target?.['className'] if ( (typeof classes === 'string' && classes.includes('inputarea')) || @@ -1175,7 +1187,7 @@ -
+
{ + onRunPreview={(jobId) => { stepsInputArgs.resetManuallyEditedArgs() modulesTestStates.hideJobsInGraph() localModuleStates = {} showJobStatus = true + if (jobId) { + onTestJob?.({ jobId }) + } }} /> {/snippet} diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 49f195cd0d..eb9bbea4e7 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -59,7 +59,7 @@ scrollTop?: number localModuleStates?: Record localDurationStatuses?: Record - onRunPreview?: () => void + onRunPreview?: (jobId?: string) => void render?: boolean onJobDone?: () => void upToId?: string | undefined @@ -200,7 +200,7 @@ savedArgs = $state.snapshot(previewArgs.val) inputSelected = undefined } - onRunPreview?.() + onRunPreview?.(newJobId) } catch (e) { sendUserToast('Could not run preview', true, undefined, e.toString()) isRunning = false diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 809eff58d9..d0bf66d61b 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -28,6 +28,7 @@ import ModuleStatus from './ModuleStatus.svelte' import { clone, isScriptPreview, msToSec, readFieldsRecursively, truncateRev } from '$lib/utils' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' + import { appendViewToken } from '$lib/viewToken' import JobArgs from './JobArgs.svelte' import { ChevronDown, Download, ExternalLink, Hourglass } from 'lucide-svelte' import { deepEqual } from 'fast-equals' @@ -1839,7 +1840,9 @@ style="min-height: {minTabHeight}px" > {#if !hideDownloadLogs && !isReplay && job?.id} - {@const logsApiPath = `/w/${workspace}/jobs_u/get_flow_all_logs/${job.id}`} + {@const logsApiPath = appendViewToken( + `/w/${workspace}/jobs_u/get_flow_all_logs/${job.id}` + )} {@const logsName = `windmill_flow_logs_${job.id}.txt`}
{#if shouldDownloadViaClient()} diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 7342f1d3fa..9581357e84 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -13,6 +13,7 @@ import { createEventDispatcher } from 'svelte' import { setLicense } from '$lib/enterpriseUtils' import AuthSettings from './AuthSettings.svelte' + import oauthConnectRegistry from '$oauth_connect_registry' import InstanceSetting from './InstanceSetting.svelte' import { writable, type Writable } from 'svelte/store' import { ExternalLink, Loader2 } from 'lucide-svelte' @@ -54,7 +55,9 @@ let initialValues: Record = $state({}) let baseUrlIsFallback = $state(false) - let snowflakeAccountIdentifier = $state('') + // Per-instance OAuth providers (Snowflake, ServiceNow, …): instance name + // keyed by provider, used to build their per-instance connect_config URLs. + let instanceInputs: Record = $state({}) let version: string = $state('') let loading = $state(true) @@ -147,12 +150,8 @@ $values = nvalues loading = false - // populate snowflake account identifier from db - const account_identifier = - oauths?.snowflake_oauth?.connect_config?.extra_params?.account_identifier - if (account_identifier) { - snowflakeAccountIdentifier = account_identifier - } + // populate per-instance OAuth provider inputs (snowflake, servicenow, …) from db + loadInstanceInputs(oauths) } export async function saveSettings() { @@ -162,13 +161,7 @@ } } - if ( - oauths?.snowflake_oauth && - oauths?.snowflake_oauth?.connect_config?.extra_params?.account_identifier !== - snowflakeAccountIdentifier - ) { - setupSnowflakeUrls() - } + setupTemplatedOauthUrls() // Remove empty or invalid entries for critical error channels $values.critical_error_channels = $values.critical_error_channels.filter((entry: any) => { @@ -283,19 +276,54 @@ } } - function setupSnowflakeUrls() { - // strip all whitespaces from account identifier - snowflakeAccountIdentifier = snowflakeAccountIdentifier.replace(/\s/g, '') + // Per-instance OAuth providers (Snowflake, ServiceNow, …) keyed by name -> + // their registry connect_config_template. Adding a new one needs only a + // registry entry — no code here. + const connectConfigTemplates: Record = Object.fromEntries( + Object.entries(oauthConnectRegistry) + .filter(([, cfg]) => cfg && typeof cfg === 'object' && 'connect_config_template' in cfg) + .map(([name, cfg]) => [name, (cfg as any).connect_config_template]) + ) - const connect_config = { - scopes: [], - auth_url: `https://${snowflakeAccountIdentifier}.snowflakecomputing.com/oauth/authorize`, - token_url: `https://${snowflakeAccountIdentifier}.snowflakecomputing.com/oauth/token-request`, - req_body_auth: false, - extra_params: { account_identifier: snowflakeAccountIdentifier }, - extra_params_callback: {} + function normalizeInstanceInput(tmpl: any, raw: string): string { + let v = (raw ?? '').replace(/\s/g, '') + if (tmpl.strip_suffix) { + // accept a full host/URL or a bare name -> reduce to the bare instance + v = v.replace(/^https?:\/\//, '').replace(/\/.*$/, '') + if (v.endsWith(tmpl.strip_suffix)) { + v = v.slice(0, -tmpl.strip_suffix.length) + } + } + return v + } + + // Build each per-instance provider's connect_config from the admin-entered + // instance name + its registry template (substituting {instance} into the + // URLs). Replaces the old per-provider setup functions. + function setupTemplatedOauthUrls() { + for (const [name, tmpl] of Object.entries(connectConfigTemplates)) { + if (!oauths?.[name]) continue + const key = tmpl.extra_params_key ?? 'instance' + const v = normalizeInstanceInput(tmpl, instanceInputs[name] ?? '') + instanceInputs[name] = v + if (oauths[name].connect_config?.extra_params?.[key] === v) continue + oauths[name].connect_config = { + scopes: [], + auth_url: tmpl.auth_url.replaceAll('{instance}', v), + token_url: tmpl.token_url.replaceAll('{instance}', v), + req_body_auth: tmpl.req_body_auth ?? false, + extra_params: { [key]: v }, + extra_params_callback: {} + } + } + } + + // Recover the instance-name inputs from a saved oauths config (for load/discard). + function loadInstanceInputs(savedOauths: Record) { + for (const [name, tmpl] of Object.entries(connectConfigTemplates)) { + const key = tmpl.extra_params_key ?? 'instance' + instanceInputs[name] = savedOauths?.[name]?.connect_config?.extra_params?.[key] ?? '' } - oauths['snowflake_oauth'].connect_config = connect_config } let sendingStats = $state(false) @@ -510,9 +538,7 @@ if (category === 'Auth/OAuth/SAML') { oauths = JSON.parse(JSON.stringify(initialOauths)) requirePreexistingUserForOauth = initialRequirePreexistingUserForOauth - const account_identifier = - initialOauths?.snowflake_oauth?.connect_config?.extra_params?.account_identifier - snowflakeAccountIdentifier = account_identifier ?? '' + loadInstanceInputs(initialOauths) } else if (category === 'Registries') { const v = initialValues['workspace_registries'] $values['workspace_registries'] = v !== undefined ? JSON.parse(JSON.stringify(v)) : undefined @@ -524,9 +550,7 @@ $values = JSON.parse(JSON.stringify(initialValues)) oauths = JSON.parse(JSON.stringify(initialOauths)) requirePreexistingUserForOauth = initialRequirePreexistingUserForOauth - const account_identifier = - initialOauths?.snowflake_oauth?.connect_config?.extra_params?.account_identifier - snowflakeAccountIdentifier = account_identifier ?? '' + loadInstanceInputs(initialOauths) if (yamlMode) { syncFormToYaml() } @@ -535,13 +559,7 @@ export async function saveCategorySettings(category: string) { // Category-specific pre-processing if (category === 'Auth/OAuth/SAML') { - if ( - oauths?.snowflake_oauth && - oauths?.snowflake_oauth?.connect_config?.extra_params?.account_identifier !== - snowflakeAccountIdentifier - ) { - setupSnowflakeUrls() - } + setupTemplatedOauthUrls() } if (category === 'Alerts' && $values?.critical_error_channels) { @@ -1116,7 +1134,7 @@ {:else if category == 'Auth/OAuth/SAML'} request a share link). + if (status === 403 || status === 404 || errorIteration == 5) { notfound = true job = undefined clearCurrentId() @@ -754,6 +764,13 @@ params.set('token', token.token) } + // Share read link: SSE/EventSource can't set the X-View-Token header, + // so carry the token as a query param instead. + const viewToken = getViewToken() + if (viewToken) { + params.set('view_token', viewToken) + } + const sseUrl = `/api/w/${workspace}/jobs_u/getupdate_sse/${id}?${params.toString()}` currentEventSource = new EventSource(sseUrl) diff --git a/frontend/src/lib/components/LocalDraftBanner.svelte b/frontend/src/lib/components/LocalDraftBanner.svelte index 357744c2b2..2fb2de4c11 100644 --- a/frontend/src/lib/components/LocalDraftBanner.svelte +++ b/frontend/src/lib/components/LocalDraftBanner.svelte @@ -3,7 +3,7 @@ import DiffDrawer from '$lib/components/DiffDrawer.svelte' import { classes } from '$lib/components/common/alert/model' import { type Value } from '$lib/utils' - import { AlertCircle } from 'lucide-svelte' + import { AlertCircle, Diff } from 'lucide-svelte' import { twMerge } from 'tailwind-merge' import { slide } from 'svelte/transition' @@ -84,7 +84,8 @@
diff --git a/frontend/src/lib/components/LogViewer.svelte b/frontend/src/lib/components/LogViewer.svelte index ca90e3968f..b5a31265be 100644 --- a/frontend/src/lib/components/LogViewer.svelte +++ b/frontend/src/lib/components/LogViewer.svelte @@ -17,6 +17,7 @@ import { base } from '$lib/base' import { withExternalDomain } from '$lib/externalDomain' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' + import { appendViewToken } from '$lib/viewToken' import { workspaceStore } from '$lib/stores' import { AnsiUp } from 'ansi_up' import NoWorkerWithTagWarning from './runs/NoWorkerWithTagWarning.svelte' @@ -241,7 +242,7 @@ fetchedSkippedJobId = undefined } }) - let logsApiPath = $derived(`/w/${$workspaceStore}/jobs_u/get_logs/${jobId}`) + let logsApiPath = $derived(appendViewToken(`/w/${$workspaceStore}/jobs_u/get_logs/${jobId}`)) let downloadHref = $derived(withExternalDomain(`${base}/api${logsApiPath}`)) let downloadName = $derived(`windmill_logs_${jobId}.txt`) let truncatedContent = $derived( diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 360eb9276d..b520faec7a 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -259,12 +259,11 @@ createdBy: filters.val.user || undefined, scriptPathStart: filters.val.folder ? `f/${filters.val.folder}/` : undefined, jobKinds: jobKinds == '' ? undefined : jobKinds, - success: - filters.val.status == 'success' - ? true - : filters.val.status == 'failure' - ? false - : undefined, + success: filters.val.status == 'success' ? true : undefined, + status: + filters.val.status == 'failure' || filters.val.status == 'canceled' + ? filters.val.status + : undefined, running: filters.val.status == 'running' || filters.val.status == 'suspended' ? true @@ -677,6 +676,14 @@ }} {item} /> + {#if filters.val.status == 'waiting'} void }[] = initialPath != '' && customUi?.topBar?.extraDeployOptions != false @@ -841,35 +863,6 @@ : []) ] : []), - ...(customUi?.topBar?.diff !== false && savedScript && diffDrawer - ? [ - { - label: 'Show diff', - onClick: async () => { - if (!savedScript) { - return - } - await syncWithDeployed() - - const currentDraftTriggers = structuredClone( - triggersState.getDraftTriggersSnapshot() - ) - - const deployed = deployedValue ?? savedScript - const current = { ...script, draft_triggers: currentDraftTriggers } - if (current.assets && !current.assets.length) delete current.assets - - diffDrawer?.openDrawer() - diffDrawer?.setDiff({ - mode: 'normal', - deployed, - draft: savedScript['draft'], - current - }) - } - } - ] - : []), ...(!inSessionPane && !script.draft_only && script.kind === 'script' && @@ -1015,21 +1008,6 @@ function onScriptLanguageTrigger(lang: 'docker' | 'bunnative' | ScriptLang) { if (lang == 'docker') { - if (isCloudHosted()) { - sendUserToast( - 'You cannot use Docker scripts on the multi-tenant platform. Use a dedicated instance or self-host windmill instead.', - true, - [ - { - label: 'Learn more', - callback: () => { - window.open('https://www.windmill.dev/docs/advanced/docker', '_blank') - } - } - ] - ) - return - } template = 'docker' } else if (lang == 'bunnative') { template = 'bunnative' @@ -1588,7 +1566,7 @@ if (script.timeout && script.timeout != undefined) { script.timeout = undefined } else { - script.timeout = 300 + script.timeout = customUi?.defaultTimeout ?? 300 } }} options={{ @@ -2035,6 +2013,21 @@ {/if} {/snippet} + {#snippet diffButton()} + {#if customUi?.topBar?.diff != false} + + {/if} + {/snippet} {#if compactTopbar} {#snippet buttonReplacement()} @@ -2048,8 +2041,10 @@ /> {/snippet} + {@render diffButton()} {@render settingsButton()} {:else} + {@render diffButton()} {#if customUi?.topBar?.tagEdit != false} {#if $workerTags} {#if $workerTags?.length ?? 0 > 0} @@ -2080,7 +2075,7 @@ handleEditScript(false, detail)} />
@@ -2090,6 +2085,7 @@ {disableAi} bind:selectedTab={selectedInputTab} {customUi} + {onTestJob} collabMode edit={initialPath != ''} on:format={() => { diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 9a7b4aaf83..5567a20d57 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -177,6 +177,9 @@ * to clicking a run button on the graph. */ onTestStateChange?: (running: boolean) => void + // Fired whenever a test run is started from this editor, with the + // preview job id. Used by whitelabel embedders to track test jobs. + onTestJob?: (e: { jobId: string }) => void // When true the right-hand test/run pane mounts collapsed. The user // can still expand it via `toggleTestPanel`. Defaults to false so the // regular /scripts/edit route keeps its current open-by-default UX; @@ -218,6 +221,7 @@ enablePreprocessorSnippet = false, previewLayout = 'right', onTestStateChange, + onTestJob, initialTestPanelCollapsed = false }: Props = $props() @@ -774,6 +778,9 @@ undefined, activeModuleTab !== null ? undefined : modules ) + if (job) { + onTestJob?.({ jobId: job }) + } logPanel?.setFocusToLogs() return job } diff --git a/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte b/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte index dda620dad5..9df993e4f3 100644 --- a/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte +++ b/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte @@ -1,44 +1,30 @@ - (x.summary ? `${x.summary} (${x.path})` : x.path)} - opts={{}} -/> - -{#snippet leafRow(it: Item, secondary: string, baseClass: string)} - {@const key = leafKey(it)} - pick(it)} - onmouseenter={() => setHoverHighlight(key)} - /> +{#snippet leafIcon(leaf: DrillLeaf)} + {/snippet} - -
(mouseActive = true)} -> -
- -
- - {#if scope} - {@const s = scope} - +{#snippet branchIcon(branch: DrillBranch)} + {#if branch.key === 'kind:flow' || branch.key === 'kind:script' || branch.key === 'kind:app'} + {@const k = branch.key.slice(5) as Kind} + + {:else if branch.icon} + {@const Icon = branch.icon} + {/if} +{/snippet} -
- {#if isSearching} - {@const total = (searchedItems ?? []).length} - {@const anyKindLoading = kinds.some((k) => loadingKind[k])} - {#if !searchedItems || anyKindLoading} - -
- Searching… -
- {:else if total === 0} -
No matches
- {:else} - {#each kinds as k (k)} - {@const results = searchResultsByKind[k]} - {#if results.length > 0} -
- {KIND_LABEL[k]} -
-
    - {#each results as it (leafKey(it))} -
  • {@render leafRow(it, it.path, 'py-1.5')}
    • - {/each} -
- {/if} - {/each} - {/if} - {:else if scopeLoading && entries.length === 0} -
- Loading… -
- {:else if entries.length === 0} -
Empty
- {:else} -
- {#each entries as entry (entry.key)} - {@const isHl = entry.key === highlightedKey} - {#if entry.type === 'leaf'} - {@render leafRow( - entry.item, - scope?.dir ? entry.item.path.slice(scope.dir.length + 1) : entry.item.path, - 'py-1.5' - )} - {:else} - - {/if} - {/each} -
- {/if} -
-
- - + onPick(leaf.data)} + initialScope={computedInitialScope} + {initialHighlight} + {externalFilter} + {autoFocus} + {flush} + {leafIcon} + {branchIcon} + leafSecondary={(leaf, scope) => relativizeWorkspacePath(leaf.data.path, scope)} + onScopeChange={(scope) => { + if (scope.length > 0) loader.ensureForScopeSegment(scope[0]) + // Single-kind layout has no kind branch at root — `buildWorkspaceTree` + // collapses to the kind's children. The picker mounts with scope=[], + // so without this fallback nothing fires until the user searches. + else if (kinds.length === 1) loader.ensureLoaded(kinds[0]) + }} + onFilterChange={loader.onFilterChange} +/> diff --git a/frontend/src/lib/components/copilot/CronGen.svelte b/frontend/src/lib/components/copilot/CronGen.svelte index 02a45aa767..6a8dc6892e 100644 --- a/frontend/src/lib/components/copilot/CronGen.svelte +++ b/frontend/src/lib/components/copilot/CronGen.svelte @@ -1,7 +1,7 @@ - -
{ - // avoids triggering onblur on the textinput and closing the tooltip - // but allow input elements to receive focus for the search input - if (!(e.target instanceof HTMLInputElement)) { - e.preventDefault() - } - }} - role="listbox" - tabindex={0} -> - {#if stringSearch.length > 0} - - {#each filteredAvailableContext as element, i (element.type + '-' + element.title)} - {@const Icon = ContextIconMap[element.type]} - - {/each} - {#if filteredAvailableContext.length === 0} -
No matching context
- {/if} - {:else if currentView === 'categories'} - - {#each availableCategories as category, i (category.id)} - {@const Icon = category.icon} - - {/each} - {#if availableCategories.length === 0} -
No available context
- {/if} - {:else if isSearchableView} - - - - - - {#if workspaceSearchLoading} -
- - Searching... -
- {:else if workspaceSearchResults.length === 0} -
- No results found -
- {:else} - {#each workspaceSearchResults as item, i (currentView + '-' + item.path)} - {@const isAlreadySelected = selectedContext.some( - (c) => - ((c.type === 'workspace_script' && currentView === 'scripts') || - (c.type === 'workspace_flow' && currentView === 'flows')) && - c.title === item.path - )} - - {/each} - {/if} - {:else} - - - - {#if currentCategoryItems.length === 0} -
No items in this category
- {:else} - {#each currentCategoryItems as element, i (element.type + '-' + element.title)} - {@const Icon = ContextIconMap[element.type]} - - {/each} - {/if} - {/if} -
diff --git a/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte b/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte new file mode 100644 index 0000000000..def6ea737b --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte @@ -0,0 +1,273 @@ + + + +{#snippet leafIcon(leaf: DrillLeaf)} + {@const d = leaf.data} + {#if 'kind' in d} + + {:else if d.type === 'flow_module'} + + {:else} + {@const Icon = ContextIconMap[d.type]} + {#if Icon}{/if} + {/if} +{/snippet} + +{#snippet branchIcon(branch: DrillBranch)} + {#if branch.key === 'kind:flow' || branch.key === 'kind:script' || branch.key === 'kind:app'} + {@const k = branch.key.slice(5) as WorkspaceItemKind} + + {:else if branch.icon} + {@const Icon = branch.icon} + + {/if} +{/snippet} + + + 'kind' in leaf.data ? relativizeWorkspacePath(leaf.data.path, scope) : undefined} + onScopeChange={handleScopeChange} + onFilterChange={loader.onFilterChange} +/> diff --git a/frontend/src/lib/components/copilot/chat/ContextElementBadge.svelte b/frontend/src/lib/components/copilot/chat/ContextElementBadge.svelte index ae58552055..b42eae4835 100644 --- a/frontend/src/lib/components/copilot/chat/ContextElementBadge.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextElementBadge.svelte @@ -30,9 +30,13 @@ {#snippet trigger()} + {@const label = + contextElement.type === 'diff' + ? contextElement.title.replace(/_/g, ' ') + : contextElement.title}
(showDelete = true)} onmouseleave={() => (showDelete = false)} @@ -50,11 +54,7 @@ {/if} - - {contextElement.type === 'diff' - ? contextElement.title.replace(/_/g, ' ') - : contextElement.title} - + {label}
{/snippet} {#snippet content()} @@ -127,11 +127,7 @@
{contextElement.source} (L{contextElement.startLine}-L{contextElement.endLine})
- +
{:else if contextElement.type === 'app_datatable'}
diff --git a/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts b/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts index be993f312e..a69d6670b4 100644 --- a/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts @@ -149,9 +149,17 @@ export default class ContextManager { let newSelectedContext: ContextElement[] = [...currentlySelectedContext] - // Filter selected context to only include available items + // Filter selected context to only include available items. Workspace + // references (workspace_script / workspace_flow) are user-picked via + // the @-mention picker and intentionally aren't in availableContext — + // preserve them unconditionally so the badge survives editor refreshes. newSelectedContext = newSelectedContext - .filter((c) => newAvailableContext.some((ac) => ac.type === c.type && ac.title === c.title)) + .filter( + (c) => + c.type === 'workspace_script' || + c.type === 'workspace_flow' || + newAvailableContext.some((ac) => ac.type === c.type && ac.title === c.title) + ) .map((c) => c.type === 'db' && dbSchemas[c.title] ? { @@ -232,16 +240,22 @@ export default class ContextManager { ] } - let newSelectedContext: ContextElement[] = [...currentlySelectedContext] - - newSelectedContext = [ + // Seed with the (refreshed) code block + everything else previously + // selected. The filter further down validates each entry against + // newAvailableContext (and the per-type allowlist for code_piece / + // workspace_*); types that are auto-derived (diff/error/db) survive + // when they're still in availableContext, user-picked workspace refs + // survive unconditionally, and `code` is excluded from the carryover + // because we just rebuilt it. + let newSelectedContext: ContextElement[] = [ { type: 'code', title: this.getContextCodePath(scriptOptions) ?? '', content: scriptOptions.code, lang: scriptOptions.lang, deletable: false - } + }, + ...currentlySelectedContext.filter((c) => c.type !== 'code') ] const db = this.getSelectedDBSchema(scriptOptions, dbSchemas) @@ -265,22 +279,33 @@ export default class ContextManager { (c) => (c.type === 'code_piece' && scriptOptions.code.includes(c.content)) || c.type === 'code' || + // Workspace references are user-picked via @-mention and not in + // availableContext; preserve so badges survive editor refreshes. + c.type === 'workspace_script' || + c.type === 'workspace_flow' || newAvailableContext.some((ac) => ac.type === c.type && ac.title === c.title) ) - .map((c) => - c.type === 'code' - ? { - ...c, - content: scriptOptions.code, - title: this.getContextCodePath(scriptOptions) - } - : c.type === 'db' && dbSchemas[c.title] - ? { - ...c, - schema: dbSchemas[c.title] - } - : c - ) + .map((c) => { + if (c.type === 'code') { + return { + ...c, + content: scriptOptions.code, + title: this.getContextCodePath(scriptOptions) + } + } + if (c.type === 'db' && dbSchemas[c.title]) { + return { ...c, schema: dbSchemas[c.title] } + } + // For other auto-derived types (diff, error), rehydrate from the + // freshly-built newAvailableContext so the carryover doesn't keep + // stale `content` / `diff` payloads — preserve the user-set + // `deletable` flag on top of the fresh entry. + const fresh = newAvailableContext.find((ac) => ac.type === c.type && ac.title === c.title) + if (fresh && 'deletable' in c) { + return { ...fresh, deletable: c.deletable } as ContextElement + } + return fresh ?? c + }) this.availableContext = newAvailableContext this.selectedContext = newSelectedContext diff --git a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte index 6f924de457..b89e7954cb 100644 --- a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte @@ -1,22 +1,26 @@
@@ -343,10 +398,12 @@
- { @@ -356,14 +413,10 @@ onAddContext(element) updateInstructionsWithContext(element) showContextTooltip = false - // Refocus the textarea since focus may have been on the search input setTimeout(() => textarea?.focus(), 0) }} - showAllAvailable={true} - stringSearch={contextTooltipWord.slice(1)} - onViewChange={(newNumber) => { - tooltipCurrentViewNumber = newNumber - }} + externalFilter={contextTooltipWord.slice(1)} + autoFocus={false} setShowing={(showing) => { showContextTooltip = showing }} diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index 97af435dda..37db9a90ba 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -196,14 +196,15 @@ return { errorCount: 0, warningCount: 0, errors: [], warnings: [] } }, - setFlowJson: async ({ modules, schema, preprocessorModule, failureModule, groups }) => { + setFlowJson: async ({ modules, schema, preprocessorModule, failureModule, groups, notes }) => { try { if ( modules !== undefined || schema !== undefined || preprocessorModule !== undefined || failureModule !== undefined || - groups !== undefined + groups !== undefined || + notes !== undefined ) { // Take snapshot of current flowStore and set as beforeFlow if (!diffManager?.hasPendingChanges) { @@ -218,7 +219,8 @@ schema, preprocessorModule, failureModule, - groups + groups, + notes }) // Refresh the state store to update UI diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 77fc955d56..af616cca84 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -32,7 +32,13 @@ import type { ContextElement } from '../context' import type { ExtendedOpenFlow } from '$lib/components/flows/types' import { findModuleInFlow, findModuleInModules } from '$lib/components/flows/flowTree' import { createInlineScriptSession, type InlineScriptSession } from './inlineScriptsUtils' -import { validateFlowGroups, type FlowGroup, type FlowJsonUpdateResult } from './helperUtils' +import { + validateFlowGroups, + validateFlowNotes, + type FlowGroup, + type FlowNote, + type FlowJsonUpdateResult +} from './helperUtils' import { flowModuleSchema } from './openFlowZod.gen' import { collectAllFlowModuleIdsFromModules } from '$lib/components/flows/flowTree' import { @@ -50,6 +56,7 @@ type FlowJsonUpdate = { preprocessorModule?: FlowModule | null failureModule?: FlowModule | null groups?: FlowGroup[] | null + notes?: FlowNote[] | null } function formatEmptyInlineScriptWarning({ @@ -179,6 +186,13 @@ const setFlowJsonToolSchema = z.object({ .nullable() .describe( 'JSON string containing the optional array of semantic flow groups. Each group has summary, note, autocollapse, start_id, end_id, color. color MUST be one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray — never hex codes or other strings. Pass null to clear groups.' + ), + notes: z + .string() + .optional() + .nullable() + .describe( + 'JSON string containing the optional array of free-floating sticky notes attached to the flow. Use notes to surface important flow-wide information (what the flow does, key assumptions, warnings, TODOs). Each note has id (unique string), text (markdown), color (one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray — never hex codes), and optional position {x,y} and size {width,height} — omit both and the editor places and sizes the note automatically. Always use type "free". The "group" note type is DEPRECATED — to segment a complex flow into labelled, colored sections use the `groups` field instead (each group carries its own note and color). Pass null to clear notes.' ) }) @@ -548,7 +562,8 @@ export const flowTools: Tool[] = [ schema: parsedFlow.schema, preprocessorModule: parsedFlow.preprocessor_module, failureModule: parsedFlow.failure_module, - groups: parsedFlow.groups + groups: parsedFlow.groups, + notes: parsedFlow.notes }) const warning = formatEmptyInlineScriptWarning(updateResult) @@ -651,13 +666,14 @@ export const flowTools: Tool[] = [ showDetails: true, showFade: true, fn: async ({ args, helpers, toolId, toolCallbacks }) => { - const { modules, schema, preprocessor_module, failure_module, groups } = args + const { modules, schema, preprocessor_module, failure_module, groups, notes } = args let parsedModules: FlowModule[] | null | undefined let parsedSchema: Record | null | undefined let parsedPreprocessorModule: FlowModule | null | undefined let parsedFailureModule: FlowModule | null | undefined let parsedGroups: FlowGroup[] | null | undefined + let parsedNotes: FlowNote[] | null | undefined // Parse JSON strings parsedModules = parseOptionalJsonArg(modules, 'modules') as FlowModule[] | null | undefined @@ -674,6 +690,7 @@ export const flowTools: Tool[] = [ | null | undefined parsedGroups = parseOptionalJsonArg(groups, 'groups') as FlowGroup[] | null | undefined + parsedNotes = parseOptionalJsonArg(notes, 'notes') as FlowNote[] | null | undefined if (parsedModules === null) { parsedModules = undefined } @@ -702,11 +719,16 @@ export const flowTools: Tool[] = [ ) parsedFailureModule = validateSpecialFlowModule(parsedFailureModule, 'failure_module') - if (parsedGroups !== undefined) { + if (parsedGroups !== undefined || parsedNotes !== undefined) { const effectiveModules = parsedModules ?? helpers.getFlowAndSelectedId().flow.value.modules ?? [] const moduleIdsForGroups = new Set(collectAllFlowModuleIdsFromModules(effectiveModules)) - parsedGroups = validateFlowGroups(parsedGroups, moduleIdsForGroups) + if (parsedGroups !== undefined) { + parsedGroups = validateFlowGroups(parsedGroups, moduleIdsForGroups) + } + if (parsedNotes !== undefined) { + parsedNotes = validateFlowNotes(parsedNotes, moduleIdsForGroups) + } } const ids = [ @@ -729,7 +751,8 @@ export const flowTools: Tool[] = [ ? { preprocessorModule: parsedPreprocessorModule } : {}), ...(parsedFailureModule !== undefined ? { failureModule: parsedFailureModule } : {}), - ...(parsedGroups !== undefined ? { groups: parsedGroups } : {}) + ...(parsedGroups !== undefined ? { groups: parsedGroups } : {}), + ...(parsedNotes !== undefined ? { notes: parsedNotes } : {}) }) const warning = formatEmptyInlineScriptWarning(updateResult) @@ -831,7 +854,7 @@ export function prepareFlowSystemMessage(customPrompt?: string): ChatCompletionS Use \`patch_flow_json\` for small, localized changes when you can target an exact snippet from the \`CURRENT FLOW JSON COMPACT\` block below. Always copy the exact search text from the \`CURRENT FLOW JSON COMPACT\` block below. -The compact JSON is a single object with \`modules\`, \`schema\`, \`preprocessor_module\`, \`failure_module\`, and \`groups\` keys. +The compact JSON is a single object with \`modules\`, \`schema\`, \`preprocessor_module\`, \`failure_module\`, \`groups\`, and \`notes\` keys. **Parameters:** - \`old_string\`: Exact JSON text to find @@ -859,7 +882,16 @@ Use the \`set_flow_json\` tool to set the entire flow structure at once. Provide - \`schema\`: Flow input schema in JSON Schema format (optional) - \`preprocessor_module\`: Special module that runs before \`modules\` (optional, separate from \`modules\`) - \`failure_module\`: Special module that runs on failure (optional, separate from \`modules\`) -- \`groups\`: Array of semantic groups for organizing modules in the editor (optional). Each group has \`summary\` (display name), \`note\` (markdown description shown below the group header — attached directly to the group, not a separate sticky note), \`autocollapse\`, \`start_id\`, \`end_id\`, and \`color\`. \`start_id\` and \`end_id\` must reference existing module IDs in the flow (not \`preprocessor\` or \`failure\`). \`color\` MUST be one of these exact names: \`yellow\`, \`blue\`, \`green\`, \`purple\`, \`pink\`, \`orange\`, \`red\`, \`cyan\`, \`lime\`, \`gray\` — do NOT use hex codes, CSS colors, or any other strings. Omit \`color\` entirely if no preference and the editor will assign one automatically. Groups do not affect execution — they provide naming and collapsibility in the editor. Pass \`null\` to clear existing groups. +- \`groups\`: Array of semantic groups for organizing modules in the editor (optional, but **strongly recommended** — proactively segment any non-trivial flow into groups so it reads clearly; don't wait to be asked). Each group has \`summary\` (display name), \`note\` (markdown description shown below the group header — attached directly to the group, not a separate sticky note), \`autocollapse\`, \`start_id\`, \`end_id\`, and \`color\`. \`start_id\` and \`end_id\` must reference existing module IDs in the flow (not \`preprocessor\` or \`failure\`). \`color\` MUST be one of these exact names: \`yellow\`, \`blue\`, \`green\`, \`purple\`, \`pink\`, \`orange\`, \`red\`, \`cyan\`, \`lime\`, \`gray\` — do NOT use hex codes, CSS colors, or any other strings. Omit \`color\` entirely if no preference and the editor will assign one automatically. Groups do not affect execution — they provide naming and collapsibility in the editor. Pass \`null\` to clear existing groups. +- \`notes\`: Array of free-floating sticky notes shown in the editor (optional). Each note has \`id\` (unique string), \`text\` (markdown content), \`color\` (same palette as groups: \`yellow\`, \`blue\`, \`green\`, \`purple\`, \`pink\`, \`orange\`, \`red\`, \`cyan\`, \`lime\`, \`gray\` — never hex codes), and optional \`position\` {x, y} / \`size\` {width, height} (omit both — the editor auto-places and sizes the note). Always set \`type\` to \`free\`. The \`group\` note type is **deprecated** — do not create group notes; use the \`groups\` field to segment a flow instead. Notes are documentation only and do not affect execution. Pass \`null\` to clear existing notes. + +### When to use notes vs groups + +**Strongly prefer \`groups\` to organize flows.** Groups are the primary way to make a flow readable: whenever a flow has more than a couple of steps, or any time consecutive steps form a logical stage (e.g. "fetch", "transform", "notify"), segment them into \`groups\`. Each group spans a range of steps (\`start_id\`..\`end_id\`), carries its own \`summary\`, \`note\` (markdown under the group header), and \`color\`, and can be collapsed. Proactively add or update groups when building or restructuring a flow — do not wait to be asked. Aim for every meaningful step to belong to a semantic group. + +- **\`groups\` (default, use liberally):** segment a flow into labelled semantic sections. This is the main organizational tool — reach for it on essentially any non-trivial flow, not just "complex" ones. +- **\`notes\` (free sticky notes, use sparingly):** reserve for important flow-wide information that does not belong to a specific span of steps — overall purpose, key assumptions, warnings, or TODOs. Usually a single note is enough; do not use notes to label sequences of steps (that is what \`groups\` are for). +- Do **not** use \`group\`-type notes (deprecated) — \`groups\` is the supported way to group steps. **Example - Simple flow:** \`\`\`javascript diff --git a/frontend/src/lib/components/copilot/chat/flow/editableFlowJson.ts b/frontend/src/lib/components/copilot/chat/flow/editableFlowJson.ts index a4b0ea72d5..2f1cb5a699 100644 --- a/frontend/src/lib/components/copilot/chat/flow/editableFlowJson.ts +++ b/frontend/src/lib/components/copilot/chat/flow/editableFlowJson.ts @@ -3,7 +3,7 @@ import type { FlowModule, FlowValue } from '$lib/gen' import { collectAllFlowModuleIdsFromModules } from '$lib/components/flows/flowTree' import { SPECIAL_MODULE_IDS } from '../shared' import type { InlineScriptSession } from './inlineScriptsUtils' -import { validateFlowGroups, type FlowGroup } from './helperUtils' +import { validateFlowGroups, validateFlowNotes, type FlowGroup, type FlowNote } from './helperUtils' import { flowModuleSchema, flowModulesSchema } from './openFlowZod.gen' /** @@ -19,6 +19,7 @@ export type EditableFlowJson = { preprocessor_module: FlowModule | null failure_module: FlowModule | null groups: FlowGroup[] | null + notes: FlowNote[] | null } /** Optional input to the rich-error path of `validateEditableFlowJson`. */ @@ -145,8 +146,10 @@ function formatJsonSchemaForError(jsonSchema: any): string { if (jsonSchema.enum) { return `one of: ${jsonSchema.enum.map((v: any) => JSON.stringify(v)).join(', ')}` } - if (jsonSchema.oneOf) return jsonSchema.oneOf.map((s: any) => formatJsonSchemaForError(s)).join(' | ') - if (jsonSchema.anyOf) return jsonSchema.anyOf.map((s: any) => formatJsonSchemaForError(s)).join(' | ') + if (jsonSchema.oneOf) + return jsonSchema.oneOf.map((s: any) => formatJsonSchemaForError(s)).join(' | ') + if (jsonSchema.anyOf) + return jsonSchema.anyOf.map((s: any) => formatJsonSchemaForError(s)).join(' | ') if (jsonSchema.description) return jsonSchema.description return jsonSchema.type || JSON.stringify(jsonSchema) } @@ -177,7 +180,10 @@ function getExpectedFormat(schema: z.ZodType): string | null { return null } -export function validateFlowModules(rawModules: unknown, ctx: SchemaErrorContext = {}): FlowModule[] { +export function validateFlowModules( + rawModules: unknown, + ctx: SchemaErrorContext = {} +): FlowModule[] { if (!Array.isArray(rawModules)) { throw new Error('Flow modules must be an array') } @@ -275,6 +281,7 @@ export function validateEditableFlowJson( const failureModule = validateOptionalFlowModule(flow.failure_module, 'failure_module') const groupModuleIds = new Set(collectAllFlowModuleIdsFromModules(modules)) const groups = validateFlowGroups(flow.groups, groupModuleIds) + const notes = validateFlowNotes(flow.notes, groupModuleIds) if (preprocessorModule) { if (preprocessorModule.id !== SPECIAL_MODULE_IDS.PREPROCESSOR) { @@ -316,7 +323,8 @@ export function validateEditableFlowJson( schema, preprocessor_module: preprocessorModule, failure_module: failureModule, - groups + groups, + notes } } @@ -371,7 +379,8 @@ export function buildEditableFlowJson( schema: flow.schema ?? null, preprocessor_module: preprocessorModule ?? null, failure_module: failureModule ?? null, - groups: flow.value.groups ?? null + groups: flow.value.groups ?? null, + notes: flow.value.notes ?? null } } @@ -407,6 +416,7 @@ export function applyEditableFlowJsonToFlow( preprocessor_module: restoreSpecialRawscriptModule(editable.preprocessor_module, session) ?? undefined, failure_module: restoreSpecialRawscriptModule(editable.failure_module, session) ?? undefined, - groups: editable.groups ?? undefined + groups: editable.groups ?? undefined, + notes: editable.notes ?? undefined } } diff --git a/frontend/src/lib/components/copilot/chat/flow/helperUtils.test.ts b/frontend/src/lib/components/copilot/chat/flow/helperUtils.test.ts index 75749013a8..6dd0e1f791 100644 --- a/frontend/src/lib/components/copilot/chat/flow/helperUtils.test.ts +++ b/frontend/src/lib/components/copilot/chat/flow/helperUtils.test.ts @@ -3,7 +3,8 @@ import type { FlowModule } from '$lib/gen' import { applyFlowJsonUpdate, updateRawScriptModuleContent, - validateFlowGroups + validateFlowGroups, + validateFlowNotes } from './helperUtils' import { createInlineScriptSession } from './inlineScriptsUtils' @@ -222,6 +223,70 @@ describe('applyFlowJsonUpdate', () => { expect((flow.value as any).groups).toEqual(existingGroups) }) + it('persists notes passed in the flow json update', () => { + const flow = { + value: { + modules: [makeRawScriptModule('fetch_data', 'existing code')] + } + } + const inlineScriptSession = createInlineScriptSession() + inlineScriptSession.set('fetch_data', 'existing code') + + applyFlowJsonUpdate(flow as any, inlineScriptSession, { + notes: [ + { + id: 'note_1', + text: 'Remember to set the API key', + color: 'yellow', + type: 'free' + } as any + ] + }) + + expect((flow.value as any).notes).toEqual([ + { + id: 'note_1', + text: 'Remember to set the API key', + color: 'yellow', + type: 'free' + } + ]) + }) + + it('clears notes when an empty array or null is passed', () => { + const flow = { + value: { + modules: [], + notes: [{ id: 'n', text: 't', color: 'yellow', type: 'free' }] + } + } + const inlineScriptSession = createInlineScriptSession() + + applyFlowJsonUpdate(flow as any, inlineScriptSession, { notes: [] }) + expect((flow.value as any).notes).toBeUndefined() + ;(flow.value as any).notes = [{ id: 'n', text: 't', color: 'yellow', type: 'free' }] + applyFlowJsonUpdate(flow as any, inlineScriptSession, { notes: null }) + expect((flow.value as any).notes).toBeUndefined() + }) + + it('leaves notes untouched when not provided in the update', () => { + const existingNotes = [{ id: 'n', text: 't', color: 'yellow', type: 'free' }] + const flow = { + value: { + modules: [makeRawScriptModule('a', 'existing code')], + notes: existingNotes + } + } + const inlineScriptSession = createInlineScriptSession() + inlineScriptSession.set('a', 'existing code') + + applyFlowJsonUpdate(flow as any, inlineScriptSession, { + modules: [makeRawScriptModule('a', 'inline_script.a')] + }) + + expect((flow.value as any).notes).toEqual(existingNotes) + }) + it('updates ai agent rawscript tools in place when changing module code', () => { const flow = { value: { @@ -319,3 +384,162 @@ describe('validateFlowGroups', () => { expect(result).toEqual([{ start_id: 'a', end_id: 'b' }]) }) }) + +describe('validateFlowNotes', () => { + it('returns null for null/undefined input', () => { + expect(validateFlowNotes(null)).toBeNull() + expect(validateFlowNotes(undefined)).toBeNull() + }) + + it('rejects non-array input', () => { + expect(() => validateFlowNotes({})).toThrow('Flow notes must be an array') + }) + + it('rejects a note that is not an object', () => { + expect(() => validateFlowNotes(['nope'])).toThrow('Invalid note at index 0: must be an object') + }) + + it('rejects a note with a missing or non-string id', () => { + expect(() => validateFlowNotes([{ text: 't' }])).toThrow( + 'Invalid note at index 0: id must be a non-empty string' + ) + expect(() => validateFlowNotes([{ id: '', text: 't' }])).toThrow( + 'Invalid note at index 0: id must be a non-empty string' + ) + }) + + it('rejects duplicate note ids', () => { + expect(() => + validateFlowNotes([ + { id: 'n', text: 'a' }, + { id: 'n', text: 'b' } + ]) + ).toThrow('Invalid note at index 1: duplicate note id "n"') + }) + + it('rejects a non-string text', () => { + expect(() => validateFlowNotes([{ id: 'n', text: 42 }])).toThrow( + 'Invalid note at index 0: text must be a string' + ) + }) + + it('rejects an invalid type', () => { + expect(() => validateFlowNotes([{ id: 'n', text: 't', type: 'sticky' }])).toThrow( + 'Invalid note at index 0: type must be "free" or "group"' + ) + }) + + it('rejects an unknown color name', () => { + expect(() => validateFlowNotes([{ id: 'n', text: 't', color: '#ffff00' }])).toThrow( + /color must be one of/ + ) + }) + + it('defaults type to free and color to the default note color', () => { + const [note] = validateFlowNotes([{ id: 'n', text: 't' }])! + expect(note).toMatchObject({ id: 'n', text: 't', type: 'free', color: 'green' }) + }) + + it('preserves a provided palette color rather than overriding it', () => { + const [note] = validateFlowNotes([{ id: 'n', text: 't', color: 'purple' }])! + expect(note).toMatchObject({ id: 'n', text: 't', type: 'free', color: 'purple' }) + }) + + it('gives a free note missing geometry a concrete position and size', () => { + const [note] = validateFlowNotes([{ id: 'n', text: 't' }])! + expect(note.position).toEqual({ x: expect.any(Number), y: expect.any(Number) }) + expect(note.size).toEqual({ width: expect.any(Number), height: expect.any(Number) }) + expect(note.size!.width).toBeGreaterThan(0) + expect(note.size!.height).toBeGreaterThan(0) + }) + + it('staggers the default y position of multiple geometry-less free notes', () => { + const notes = validateFlowNotes([ + { id: 'a', text: 't' }, + { id: 'b', text: 't' } + ])! + expect(notes[0].position!.y).not.toEqual(notes[1].position!.y) + }) + + it('does not override a free note that already has geometry', () => { + const [note] = validateFlowNotes([ + { id: 'n', text: 't', position: { x: 5, y: 6 }, size: { width: 400, height: 90 } } + ])! + expect(note.position).toEqual({ x: 5, y: 6 }) + expect(note.size).toEqual({ width: 400, height: 90 }) + }) + + it('does not add geometry to group notes', () => { + const [note] = validateFlowNotes([ + { id: 'n', text: 't', type: 'group', contained_node_ids: [] } + ])! + expect(note.position).toBeUndefined() + expect(note.size).toBeUndefined() + }) + + it('rejects a malformed position', () => { + expect(() => validateFlowNotes([{ id: 'n', text: 't', position: { x: 1 } }])).toThrow( + 'Invalid note at index 0: position must be an object with numeric x and y' + ) + expect(() => validateFlowNotes([{ id: 'n', text: 't', position: [1, 2] }])).toThrow( + 'Invalid note at index 0: position must be an object with numeric x and y' + ) + }) + + it('rejects a malformed size', () => { + expect(() => validateFlowNotes([{ id: 'n', text: 't', size: { width: '10' } }])).toThrow( + 'Invalid note at index 0: size must be an object with numeric width and height' + ) + }) + + it('accepts a free note with valid position and size', () => { + const result = validateFlowNotes([ + { + id: 'n', + text: 't', + color: 'blue', + position: { x: 10, y: 20 }, + size: { width: 300, height: 80 } + } + ]) + expect(result).toEqual([ + { + id: 'n', + text: 't', + type: 'free', + color: 'blue', + position: { x: 10, y: 20 }, + size: { width: 300, height: 80 } + } + ]) + }) + + it('rejects group note contained_node_ids that are not strings', () => { + expect(() => + validateFlowNotes([{ id: 'n', text: 't', type: 'group', contained_node_ids: [1] }]) + ).toThrow('Invalid note at index 0: contained_node_ids must be an array of strings') + }) + + it('rejects group note contained_node_ids that do not match a module', () => { + const moduleIds = new Set(['a', 'b']) + expect(() => + validateFlowNotes( + [{ id: 'n', text: 't', type: 'group', contained_node_ids: ['missing'] }], + moduleIds + ) + ).toThrow( + 'Invalid note at index 0: contained_node_ids "missing" does not match any flow module' + ) + }) + + it('accepts a valid group note whose contained ids are all modules', () => { + const moduleIds = new Set(['a', 'b']) + const result = validateFlowNotes( + [{ id: 'n', text: 't', color: 'blue', type: 'group', contained_node_ids: ['a', 'b'] }], + moduleIds + ) + expect(result).toEqual([ + { id: 'n', text: 't', color: 'blue', type: 'group', contained_node_ids: ['a', 'b'] } + ]) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/flow/helperUtils.ts b/frontend/src/lib/components/copilot/chat/flow/helperUtils.ts index f424490d24..35b6aae909 100644 --- a/frontend/src/lib/components/copilot/chat/flow/helperUtils.ts +++ b/frontend/src/lib/components/copilot/chat/flow/helperUtils.ts @@ -1,19 +1,26 @@ -import type { FlowModule, FlowValue, OpenFlow, RawScript } from '$lib/gen' +import type { FlowModule, FlowNote, FlowValue, OpenFlow, RawScript } from '$lib/gen' import { forEachFlowModule } from '$lib/components/flows/dfs' import { findModuleInFlow } from '$lib/components/flows/flowTree' -import { NoteColor } from '$lib/components/graph/noteColors' +import { + DEFAULT_NOTE_COLOR, + MIN_NOTE_HEIGHT, + MIN_NOTE_WIDTH, + NoteColor +} from '$lib/components/graph/noteColors' import type { InlineScriptSession } from './inlineScriptsUtils' -/** Allowed group color names — matches the NoteColor palette the group - * editor uses. Other strings would render with default-blue styling at best - * and break the color picker UI at worst. */ -const ALLOWED_GROUP_COLORS = new Set(Object.values(NoteColor)) +/** Allowed note/group color names — matches the NoteColor palette the note and + * group editors use. The note renderer keys `NOTE_COLORS` by these exact names, + * so other strings (hex codes, CSS colors) render with no styling at best and + * break the color picker UI at worst. */ +const ALLOWED_NOTE_COLORS = new Set(Object.values(NoteColor)) type FlowLike = Pick & { schema?: Record } export type FlowGroup = NonNullable[number] +export type { FlowNote } export interface FlowJsonUpdate { modules?: FlowModule[] @@ -21,6 +28,7 @@ export interface FlowJsonUpdate { preprocessorModule?: FlowModule | null failureModule?: FlowModule | null groups?: FlowGroup[] | null + notes?: FlowNote[] | null } export interface FlowJsonUpdateResult { @@ -78,9 +86,9 @@ export function validateFlowGroups( } } if (g.color !== undefined && g.color !== null) { - if (typeof g.color !== 'string' || !ALLOWED_GROUP_COLORS.has(g.color)) { + if (typeof g.color !== 'string' || !ALLOWED_NOTE_COLORS.has(g.color)) { throw new Error( - `Invalid group at index ${index}: color must be one of ${[...ALLOWED_GROUP_COLORS].join(', ')}` + `Invalid group at index ${index}: color must be one of ${[...ALLOWED_NOTE_COLORS].join(', ')}` ) } } @@ -88,10 +96,138 @@ export function validateFlowGroups( }) } +/** + * Validate the optional array of sticky notes the agent attached to the flow. + * Notes are editor-only annotations and do not affect execution. + * + * `free` notes are the supported kind (standalone canvas annotations). The + * `group` note type is deprecated for creation — the chat prompt steers the + * agent toward `groups` instead — but it is still ACCEPTED here so flows that + * already contain group notes round-trip cleanly through `patch_flow_json` / + * `set_flow_json` rather than being rejected. When `moduleIds` is provided, + * every `contained_node_ids` entry of a `group` note must reference an existing + * module. + * + * A provided palette `color` is always preserved as-is; the default is only + * filled in when a note omits `color` entirely (FlowNote.color is required). + * + * Free notes are also given a concrete `position` and `size` when missing. A + * free note without geometry is not draggable/resizable in the editor (you'd + * have to resize it first to give it a size) — UI-created notes always set both, + * so agent-created notes must too. Provided geometry is preserved untouched. + */ +export function validateFlowNotes(rawNotes: unknown, moduleIds?: Set): FlowNote[] | null { + if (rawNotes == null) { + return null + } + + if (!Array.isArray(rawNotes)) { + throw new Error('Flow notes must be an array') + } + + const seenIds = new Set() + return rawNotes.map((note, index) => { + if (!note || typeof note !== 'object' || Array.isArray(note)) { + throw new Error(`Invalid note at index ${index}: must be an object`) + } + const n = note as Record + if (typeof n.id !== 'string' || !n.id) { + throw new Error(`Invalid note at index ${index}: id must be a non-empty string`) + } + if (seenIds.has(n.id)) { + throw new Error(`Invalid note at index ${index}: duplicate note id "${n.id}"`) + } + seenIds.add(n.id) + if (typeof n.text !== 'string') { + throw new Error(`Invalid note at index ${index}: text must be a string`) + } + const type = n.type ?? 'free' + if (type !== 'free' && type !== 'group') { + throw new Error(`Invalid note at index ${index}: type must be "free" or "group"`) + } + if (n.color !== undefined && n.color !== null) { + if (typeof n.color !== 'string' || !ALLOWED_NOTE_COLORS.has(n.color)) { + throw new Error( + `Invalid note at index ${index}: color must be one of ${[...ALLOWED_NOTE_COLORS].join(', ')}` + ) + } + } + if (n.position !== undefined && n.position !== null) { + const p = n.position as Record + if ( + typeof p !== 'object' || + Array.isArray(n.position) || + typeof p.x !== 'number' || + typeof p.y !== 'number' + ) { + throw new Error( + `Invalid note at index ${index}: position must be an object with numeric x and y` + ) + } + } + if (n.size !== undefined && n.size !== null) { + const s = n.size as Record + if ( + typeof s !== 'object' || + Array.isArray(n.size) || + typeof s.width !== 'number' || + typeof s.height !== 'number' + ) { + throw new Error( + `Invalid note at index ${index}: size must be an object with numeric width and height` + ) + } + } + if (type === 'group' && n.contained_node_ids !== undefined) { + if ( + !Array.isArray(n.contained_node_ids) || + n.contained_node_ids.some((id) => typeof id !== 'string') + ) { + throw new Error( + `Invalid note at index ${index}: contained_node_ids must be an array of strings` + ) + } + if (moduleIds) { + for (const id of n.contained_node_ids as string[]) { + if (!moduleIds.has(id)) { + throw new Error( + `Invalid note at index ${index}: contained_node_ids "${id}" does not match any flow module` + ) + } + } + } + } + const normalized = { + ...(n as FlowNote), + type, + // Preserve a provided color; only seed the default when omitted. + color: typeof n.color === 'string' ? n.color : DEFAULT_NOTE_COLOR + } as FlowNote + + // Free notes need explicit geometry to be draggable/resizable. Place + // missing ones to the left of the flow column, staggered by index so + // several new notes don't land exactly on top of each other. Group notes + // derive their layout from contained nodes, so they are left alone. + if (type === 'free') { + if (normalized.position == null) { + normalized.position = { + x: -(MIN_NOTE_WIDTH + 100), + y: index * (MIN_NOTE_HEIGHT + 24) + } + } + if (normalized.size == null) { + normalized.size = { width: MIN_NOTE_WIDTH, height: MIN_NOTE_HEIGHT } + } + } + + return normalized + }) +} + export function applyFlowJsonUpdate( flow: FlowLike, inlineScriptSession: InlineScriptSession, - { modules, schema, preprocessorModule, failureModule, groups }: FlowJsonUpdate + { modules, schema, preprocessorModule, failureModule, groups, notes }: FlowJsonUpdate ): FlowJsonUpdateResult { const emptyInlineScriptModuleIds = new Set() @@ -125,6 +261,10 @@ export function applyFlowJsonUpdate( flow.value.groups = groups == null || groups.length === 0 ? undefined : groups } + if (notes !== undefined) { + flow.value.notes = notes == null || notes.length === 0 ? undefined : notes + } + return { emptyInlineScriptModuleIds: Array.from(emptyInlineScriptModuleIds) } diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlow.json b/frontend/src/lib/components/copilot/chat/flow/openFlow.json index dc95ea41b0..0309aeaf91 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlow.json +++ b/frontend/src/lib/components/copilot/chat/flow/openFlow.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"version":"1.692.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file +{"openapi":"3.0.3","info":{"version":"1.716.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts index 1214dc2369..21e6735ab1 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts @@ -1,6 +1,6 @@ import { z } from "zod" -export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration. Use stop_after_if to control when the loop ends"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')") }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined"), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("Short description of what this tool does (shown to the AI)").optional(), "value": z.any().superRefine((x, ctx) => { +export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration. Use stop_after_if to control when the loop ends"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')") }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined"), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("Short description of what this tool does (shown to the AI)").optional(), "value": z.any().superRefine((x, ctx) => { const schemas = [z.intersection(z.object({ "tool_type": z.literal("flowmodule") }), z.lazy(() => flowModuleValueSchema)).describe("A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module"), z.object({ "tool_type": z.literal("mcp"), "resource_path": z.string().describe("Path to the MCP resource/server configuration"), "include_tools": z.array(z.string()).describe("Whitelist of specific tools to include from this MCP server").optional(), "exclude_tools": z.array(z.string()).describe("Blacklist of tools to exclude from this MCP server").optional() }).describe("Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers"), z.object({ "tool_type": z.literal("websearch") }).describe("A tool implemented as a websearch tool. The AI can call this like any other websearch tool")]; const errors = schemas.reduce( (errors, schema) => @@ -20,7 +20,7 @@ export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "i }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task"), "type": z.literal("aiagent"), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type") -export const flowModuleSchema = z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in each iteration. Use stop_after_if to control when the loop ends"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')") }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined"), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("Short description of what this tool does (shown to the AI)").optional(), "value": z.any().superRefine((x, ctx) => { +export const flowModuleSchema = z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in each iteration. Use stop_after_if to control when the loop ends"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')") }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined"), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("Short description of what this tool does (shown to the AI)").optional(), "value": z.any().superRefine((x, ctx) => { const schemas = [z.intersection(z.object({ "tool_type": z.literal("flowmodule") }), z.lazy(() => flowModuleValueSchema)).describe("A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module"), z.object({ "tool_type": z.literal("mcp"), "resource_path": z.string().describe("Path to the MCP resource/server configuration"), "include_tools": z.array(z.string()).describe("Whitelist of specific tools to include from this MCP server").optional(), "exclude_tools": z.array(z.string()).describe("Blacklist of tools to exclude from this MCP server").optional() }).describe("Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers"), z.object({ "tool_type": z.literal("websearch") }).describe("A tool implemented as a websearch tool. The AI can call this like any other websearch tool")]; const errors = schemas.reduce( (errors, schema) => @@ -37,7 +37,7 @@ export const flowModuleSchema = z.object({ "id": z.string().describe("Unique ide message: "Invalid input: Should pass single schema", }); } - }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task"), "type": z.literal("aiagent"), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type"), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch") + }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task"), "type": z.literal("aiagent"), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type"), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch") export const flowModulesSchema = z.array(flowModuleSchema) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index db8ecb871f..960cd7557c 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -647,7 +647,7 @@ function itemMatches( ) } -function scriptToItem(script: Script, includeValue: boolean): WorkspaceItem { +function scriptToItem(script: Script | NewScript, includeValue: boolean): WorkspaceItem { return { type: 'script', path: script.path, @@ -1153,10 +1153,16 @@ async function readWorkspaceItem( triggerKind?: TriggerKind ): Promise { switch (type) { - case 'script': - return scriptToItem(await ScriptService.getScriptByPath({ workspace, path }), true) - case 'flow': - return flowToItem(await FlowService.getFlowByPath({ workspace, path }), true) + case 'script': { + // Prefer the DB draft (newer than the deployed version) when one exists. + const script = await ScriptService.getScriptByPathWithDraft({ workspace, path }) + return scriptToItem(script.draft ?? script, true) + } + case 'flow': { + // Prefer the DB draft (newer than the deployed version) when one exists. + const flow = await FlowService.getFlowByPathWithDraft({ workspace, path }) + return flowToItem(flow.draft ?? flow, true) + } case 'schedule': return scheduleToItem(await ScheduleService.getSchedule({ workspace, path }), true) case 'trigger': diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 378e1073f7..c26abe3717 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -14,10 +14,7 @@ import { import { processToolCall, type Tool, type ToolCallbacks } from './shared' import type { ResponseStream } from 'openai/lib/responses/ResponseStream.mjs' import type { AIProviderModel } from '$lib/gen' -import { - openAIResponsesUsageToChatTokenUsage, - type ChatTokenUsage -} from './tokenUsage' +import { openAIResponsesUsageToChatTokenUsage, type ChatTokenUsage } from './tokenUsage' interface ParsedCompletionResult { shouldContinue: boolean @@ -172,13 +169,22 @@ export async function getOpenAIResponsesCompletion( export async function* getOpenAIResponsesCompletionStream( messages: ChatCompletionMessageParam[], abortController: AbortController, - tools?: OpenAI.Chat.Completions.ChatCompletionTool[] + tools?: OpenAI.Chat.Completions.ChatCompletionTool[], + options?: { + forceModelProvider?: AIProviderModel + openaiClient?: OpenAI + } ): AsyncGenerator { - const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, tools }) + const { provider, config } = getProviderAndCompletionConfig({ + messages, + stream: true, + tools, + forceModelProvider: options?.forceModelProvider + }) const { instructions, input } = convertMessagesToResponsesInput(messages) const responsesConfig = convertCompletionConfigToResponsesConfig(config) - const openaiClient = workspaceAIClients.getOpenaiClient() + const openaiClient = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() const runner = openaiClient.responses.stream( { @@ -380,17 +386,17 @@ export async function parseOpenAIResponsesCompletion( export async function getNonStreamingOpenAIResponsesCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, - testOptions?: { + options?: { apiKey?: string workspace?: string resourcePath?: string - forceModelProvider: AIProviderModel + forceModelProvider?: AIProviderModel } ): Promise { const { provider, config } = getProviderAndCompletionConfig({ messages, stream: false, - forceModelProvider: testOptions?.forceModelProvider + forceModelProvider: options?.forceModelProvider }) const { instructions, input } = convertMessagesToResponsesInput(messages) @@ -406,22 +412,22 @@ export async function getNonStreamingOpenAIResponsesCompletion( } } - if (testOptions?.resourcePath) { + if (options?.resourcePath) { fetchOptions.headers = { ...fetchOptions.headers, - 'X-Resource-Path': testOptions.resourcePath + 'X-Resource-Path': options.resourcePath } - } else if (testOptions?.apiKey) { + } else if (options?.apiKey) { fetchOptions.headers = { ...fetchOptions.headers, - 'X-API-Key': testOptions.apiKey + 'X-API-Key': options.apiKey } } - const openaiClient = testOptions?.apiKey + const openaiClient = options?.apiKey ? createOpenAIProxyClient(getAiProxyBaseURL()) - : testOptions?.workspace - ? workspaceAIClients.createOpenaiClient(testOptions.workspace) + : options?.workspace + ? workspaceAIClients.createOpenaiClient(options.workspace) : workspaceAIClients.getOpenaiClient() const response = await openaiClient.responses.create( diff --git a/frontend/src/lib/components/copilot/lib.test.ts b/frontend/src/lib/components/copilot/lib.test.ts index 369987d53f..541bc60b3d 100644 --- a/frontend/src/lib/components/copilot/lib.test.ts +++ b/frontend/src/lib/components/copilot/lib.test.ts @@ -28,6 +28,13 @@ describe('modelConfig', () => { expect(modelDisallowsSamplingParams('anthropic/claude-opus-4-7')).toBe(true) }) + it('flags Opus 4.8 model IDs via includes matching', () => { + expect(modelDisallowsSamplingParams('claude-opus-4-8')).toBe(true) + expect(modelDisallowsSamplingParams('claude-opus-4-8@20260416')).toBe(true) + expect(modelDisallowsSamplingParams('claude-opus-4-8/thinking')).toBe(true) + expect(modelDisallowsSamplingParams('anthropic/claude-opus-4-8')).toBe(true) + }) + it('omits deterministic temperature for Anthropic Opus 4.7 chat requests', () => { expect( getDefaultChatTemperature({ provider: 'anthropic', model: 'claude-opus-4-7' }) @@ -43,6 +50,50 @@ describe('modelConfig', () => { it('keeps deterministic temperature for older Anthropic models', () => { expect(getDefaultChatTemperature({ provider: 'anthropic', model: 'claude-sonnet-4-6' })).toBe(0) }) + + it('flags gpt-5+ and o-series reasoning models via prefix matching', () => { + expect(modelDisallowsSamplingParams('gpt-5')).toBe(true) + expect(modelDisallowsSamplingParams('gpt-5.5')).toBe(true) + expect(modelDisallowsSamplingParams('gpt-5-mini')).toBe(true) + expect(modelDisallowsSamplingParams('o1')).toBe(true) + expect(modelDisallowsSamplingParams('o3')).toBe(true) + expect(modelDisallowsSamplingParams('o4-mini')).toBe(true) + // provider-prefixed identifiers (e.g. OpenRouter) match on the bare model id + expect(modelDisallowsSamplingParams('openai/gpt-5')).toBe(true) + expect(modelDisallowsSamplingParams('openai/o3')).toBe(true) + }) + + it('keeps sampling params for non-reasoning models that merely share a prefix', () => { + // gpt-4o starts with "gpt-" but not "gpt-5"; the "o" is mid-string, not a prefix + expect(modelDisallowsSamplingParams('gpt-4o')).toBe(false) + expect(modelDisallowsSamplingParams('gpt-4o-mini')).toBe(false) + // the provider prefix "openai/" must not be mistaken for an o-series model + expect(modelDisallowsSamplingParams('openai/gpt-4o')).toBe(false) + // the o-series match requires a digit after "o", so non-OpenAI ids that + // start with "o" (Mistral open-* family, OpenRouter optimus-*/openchat-*) + // keep their deterministic temperature + expect(modelDisallowsSamplingParams('open-mistral-7b')).toBe(false) + expect(modelDisallowsSamplingParams('open-mixtral-8x7b')).toBe(false) + expect(modelDisallowsSamplingParams('open-mistral-nemo-2407')).toBe(false) + expect(modelDisallowsSamplingParams('optimus-alpha')).toBe(false) + expect(modelDisallowsSamplingParams('openchat/openchat-7b')).toBe(false) + }) + + it('keeps deterministic temperature for Mistral open-* models', () => { + expect(getDefaultChatTemperature({ provider: 'mistral', model: 'open-mixtral-8x7b' })).toBe(0) + }) + + it('omits deterministic temperature for gpt-5.5 routed through the customai gateway', () => { + expect(getDefaultChatTemperature({ provider: 'customai', model: 'gpt-5.5' })).toBeUndefined() + }) + + it('omits deterministic temperature for o-series models on the customai gateway', () => { + expect(getDefaultChatTemperature({ provider: 'customai', model: 'o3' })).toBeUndefined() + }) + + it('keeps deterministic temperature for gpt-4o on the customai gateway', () => { + expect(getDefaultChatTemperature({ provider: 'customai', model: 'gpt-4o' })).toBe(0) + }) }) describe('fim autocomplete', () => { diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 00d2334acc..9c56fa4dd5 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -14,7 +14,7 @@ import Anthropic from '@anthropic-ai/sdk' import { get, type Writable } from 'svelte/store' import { OpenAPI, ResourceService, type Script } from '../../gen' import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' -import { getDefaultChatTemperature } from './modelConfig' +import { getDefaultChatTemperature, modelDisallowsSamplingParams } from './modelConfig' import { formatResourceTypes } from './utils' import { processToolCall, type Tool, type ToolCallbacks } from './chat/shared' import { @@ -24,7 +24,7 @@ import { import { convertOpenAIToAnthropicMessages } from './chat/anthropic' import type { Stream } from 'openai/core/streaming.mjs' import { generateRandomString } from '$lib/utils' -import { copilotInfo, getCurrentModel } from '$lib/aiStore' +import { copilotInfo, getCurrentModel, getMetadataModel } from '$lib/aiStore' import { emptyChatTokenUsage, openAICompletionsUsageToChatTokenUsage, @@ -317,7 +317,7 @@ function getModelSpecificConfig( const defaultTemperature = getDefaultChatTemperature(modelProvider) if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai') && - (modelProvider.model.startsWith('o') || modelProvider.model.startsWith('gpt-5')) + modelDisallowsSamplingParams(modelProvider.model) ) { return { model: modelProvider.model, @@ -753,18 +753,18 @@ export function getProviderAndCompletionConfig({ export async function getNonStreamingCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, - testOptions?: { + options?: { apiKey?: string // testing API KEY using the global ai proxy resourcePath?: string // testing resource path passed as a header to the backend proxy workspace?: string // use a specific workspace proxy when testing a workspace resource - forceModelProvider: AIProviderModel + forceModelProvider?: AIProviderModel } ) { let response: string | undefined = '' const { provider, config } = getProviderAndCompletionConfig({ messages, stream: false, - forceModelProvider: testOptions?.forceModelProvider + forceModelProvider: options?.forceModelProvider }) // Use Responses API for OpenAI and Azure OpenAI @@ -773,7 +773,7 @@ export async function getNonStreamingCompletion( const response = await getNonStreamingOpenAIResponsesCompletion( messages, abortController, - testOptions + options ) return response } catch (error) { @@ -790,25 +790,25 @@ export async function getNonStreamingCompletion( 'X-Provider': provider } } - if (testOptions?.resourcePath) { + if (options?.resourcePath) { fetchOptions.headers = { ...fetchOptions.headers, - 'X-Resource-Path': testOptions.resourcePath + 'X-Resource-Path': options.resourcePath } - } else if (testOptions?.apiKey) { + } else if (options?.apiKey) { if (provider === 'customai') { throw new Error('Cannot test API key for Custom AI, only resource path is supported') } fetchOptions.headers = { ...fetchOptions.headers, - 'X-API-Key': testOptions.apiKey + 'X-API-Key': options.apiKey } } - const openaiClient = testOptions?.apiKey + const openaiClient = options?.apiKey ? createOpenAIProxyClient(getAiProxyBaseURL()) - : testOptions?.workspace - ? workspaceAIClients.createOpenaiClient(testOptions.workspace) + : options?.workspace + ? workspaceAIClients.createOpenaiClient(options.workspace) : workspaceAIClients.getOpenaiClient() const completion = await openaiClient.chat.completions.create(config, fetchOptions) @@ -816,6 +816,15 @@ export async function getNonStreamingCompletion( return response } +export async function getNonStreamingMetadataCompletion( + messages: ChatCompletionMessageParam[], + abortController: AbortController +) { + return getNonStreamingCompletion(messages, abortController, { + forceModelProvider: getMetadataModel() + }) +} + export const FIM_MAX_TOKENS = 256 const FIM_MAX_LINES = 8 export async function getFimCompletion( @@ -895,7 +904,10 @@ export async function getCompletion( // Use Responses API for OpenAI and Azure OpenAI if ((provider === 'openai' || provider === 'azure_openai') && !options?.forceCompletions) { try { - const stream = getOpenAIResponsesCompletionStream(messages, abortController, tools) as any + const stream = getOpenAIResponsesCompletionStream(messages, abortController, tools, { + forceModelProvider: options?.forceModelProvider, + openaiClient: options?.openaiClient + }) as any return stream } catch (error) { console.error('Error using Responses API:', error) diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index 40361024c2..e80b7ce7a5 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -2,7 +2,20 @@ import type { AIProviderModel } from '$lib/gen' export function modelDisallowsSamplingParams(model: string) { const normalizedModel = model.toLowerCase() - return normalizedModel.includes('claude-opus-4-7') + // Strip any provider prefix (e.g. OpenRouter's "openai/o3") so the + // reasoning-model check matches the bare model id rather than the prefix. + const baseModel = normalizedModel.split('/').pop() ?? normalizedModel + // gpt-5+ and o-series reasoning models reject sampling params such as + // temperature (only the default value is supported), regardless of which + // provider/gateway routes the request — so this must stay provider-agnostic. + // The o-series match requires a digit after the "o" (o1/o3/o4-mini) so it + // does not catch unrelated ids like Mistral's "open-mistral-*" or "optimus-*". + return ( + normalizedModel.includes('claude-opus-4-7') || + normalizedModel.includes('claude-opus-4-8') || + baseModel.startsWith('gpt-5') || + /^o\d/.test(baseModel) + ) } export function getDefaultChatTemperature(modelProvider: AIProviderModel): number | undefined { diff --git a/frontend/src/lib/components/custom_ui.ts b/frontend/src/lib/components/custom_ui.ts index 4fe6833406..68ecf43a57 100644 --- a/frontend/src/lib/components/custom_ui.ts +++ b/frontend/src/lib/components/custom_ui.ts @@ -44,6 +44,9 @@ export type FlowBuilderWhitelabelCustomUi = { aiSandbox?: boolean suggestIntegration?: boolean suggestScript?: boolean + // Default timeout (in seconds) prefilled when enabling a custom step timeout. + // Defaults to 300 (5 minutes) when unset. + defaultTimeout?: number } export type DisplayResultUi = { @@ -161,4 +164,7 @@ export type ScriptBuilderWhitelabelCustomUi = { editorBar?: EditorBarUi previewPanel?: PreviewPanelUi tagSelectPlaceholder?: string + // Default timeout (in seconds) prefilled when enabling a custom script timeout. + // Defaults to 300 (5 minutes) when unset. + defaultTimeout?: number } diff --git a/frontend/src/lib/components/drillPicker.test.ts b/frontend/src/lib/components/drillPicker.test.ts new file mode 100644 index 0000000000..41abab1281 --- /dev/null +++ b/frontend/src/lib/components/drillPicker.test.ts @@ -0,0 +1,183 @@ +import { describe, it, expect } from 'vitest' +import { + collectLeavesGrouped, + leafHaystack, + resolveScope, + scopeChain, + type DrillBranch, + type DrillLeaf, + type DrillNode +} from './drillPicker' + +const leaf = (key: string, label = key, secondary?: string): DrillLeaf => ({ + type: 'leaf', + key, + label, + secondary, + data: key +}) + +const branch = ( + key: string, + children: DrillNode[], + opts: { label?: string; omitFromSearch?: boolean; searchGroup?: boolean } = {} +): DrillBranch => ({ + type: 'branch', + key, + label: opts.label ?? key, + children, + omitFromSearch: opts.omitFromSearch, + searchGroup: opts.searchGroup +}) + +describe('resolveScope', () => { + const tree: DrillNode[] = [ + branch('a', [branch('a.x', [leaf('a.x.1')]), leaf('a.2')]), + branch('b', [leaf('b.1')]), + leaf('top') + ] + + it('returns null at the root (empty scope)', () => { + expect(resolveScope(tree, [])).toBeNull() + }) + + it('returns the branch at a one-level scope', () => { + expect(resolveScope(tree, ['a'])?.key).toBe('a') + }) + + it('returns the branch at a nested scope', () => { + expect(resolveScope(tree, ['a', 'a.x'])?.key).toBe('a.x') + }) + + it('returns null when any segment is missing', () => { + expect(resolveScope(tree, ['a', 'missing'])).toBeNull() + expect(resolveScope(tree, ['nope'])).toBeNull() + }) + + it('returns null when a segment resolves to a leaf (not a branch)', () => { + expect(resolveScope(tree, ['top'])).toBeNull() + expect(resolveScope(tree, ['a', 'a.2'])).toBeNull() + }) +}) + +describe('scopeChain', () => { + const tree: DrillNode[] = [ + branch('a', [branch('a.x', [leaf('a.x.1')]), leaf('a.2')]), + branch('b', [leaf('b.1')]) + ] + + it('returns [] at the root', () => { + expect(scopeChain(tree, [])).toEqual([]) + }) + + it('returns one branch for a one-level scope', () => { + const chain = scopeChain(tree, ['a']) + expect(chain.map((b) => b.key)).toEqual(['a']) + }) + + it('returns each branch along the path for a nested scope', () => { + const chain = scopeChain(tree, ['a', 'a.x']) + expect(chain.map((b) => b.key)).toEqual(['a', 'a.x']) + }) + + it('stops at the first missing/non-branch segment', () => { + const chain = scopeChain(tree, ['a', 'a.2', 'never-reached']) + expect(chain.map((b) => b.key)).toEqual(['a']) + }) +}) + +describe('collectLeavesGrouped', () => { + it('flattens all leaves with null group when no branch has searchGroup', () => { + const tree: DrillNode[] = [branch('a', [leaf('a.1')]), leaf('top')] + const result = collectLeavesGrouped(tree) + expect(result.map((r) => [r.leaf.key, r.group?.key])).toEqual([ + ['a.1', undefined], + ['top', undefined] + ]) + }) + + it('groups leaves under their nearest searchGroup ancestor', () => { + const tree: DrillNode[] = [ + branch('flows', [branch('flows-folder', [leaf('flows-folder.1')]), leaf('flows.root')], { + searchGroup: true + }) + ] + const result = collectLeavesGrouped(tree) + expect(result.map((r) => [r.leaf.key, r.group?.key])).toEqual([ + ['flows-folder.1', 'flows'], + ['flows.root', 'flows'] + ]) + }) + + it('the DEEPEST searchGroup wins when nested', () => { + const tree: DrillNode[] = [ + branch('outer', [branch('inner', [leaf('deep')], { searchGroup: true })], { + searchGroup: true + }) + ] + const result = collectLeavesGrouped(tree) + expect(result[0].group?.key).toBe('inner') + }) + + it('skips branches marked omitFromSearch entirely', () => { + const tree: DrillNode[] = [ + branch('all', [leaf('shared')], { omitFromSearch: true }), + branch('flows', [leaf('shared'), leaf('uniq')], { searchGroup: true }) + ] + const result = collectLeavesGrouped(tree) + // `all` branch is skipped, so `shared` is only seen once and grouped under `flows`. + expect(result.map((r) => [r.leaf.key, r.group?.key])).toEqual([ + ['shared', 'flows'], + ['uniq', 'flows'] + ]) + }) + + it('deduplicates leaves by key (first occurrence wins)', () => { + // Simulate the workspace 'All' branch (omitFromSearch=true) plus per-kind + // branches having the same leaf — even without omitFromSearch the dedup + // would still guarantee no double-counting if the search tree changes. + const tree: DrillNode[] = [ + branch('flows', [leaf('a')], { searchGroup: true }), + branch('scripts', [leaf('a')], { searchGroup: true }) + ] + const result = collectLeavesGrouped(tree) + expect(result.length).toBe(1) + expect(result[0].group?.key).toBe('flows') + }) + + it('handles a mix of top-level leaves and branches', () => { + const tree: DrillNode[] = [ + leaf('root-leaf'), + branch('b', [leaf('b.1')], { searchGroup: true }) + ] + const result = collectLeavesGrouped(tree) + expect(result.map((r) => [r.leaf.key, r.group?.key])).toEqual([ + ['root-leaf', undefined], + ['b.1', 'b'] + ]) + }) +}) + +describe('leafHaystack', () => { + it('uses searchableText when present (overrides label/secondary)', () => { + expect(leafHaystack({ ...leaf('k', 'Label'), searchableText: 'custom' })).toBe('custom') + }) + + it('joins label and secondary with parens when both are present', () => { + expect(leafHaystack(leaf('k', 'My Flow', 'f/demo/my_flow'))).toBe('My Flow (f/demo/my_flow)') + }) + + it('uses just label when secondary is absent', () => { + expect(leafHaystack(leaf('k', 'just label'))).toBe('just label') + }) + + it('falls back to secondary when label is empty', () => { + expect(leafHaystack({ type: 'leaf', key: 'k', label: '', secondary: 'sec', data: 'd' })).toBe( + 'sec' + ) + }) + + it('returns the empty string when nothing is set', () => { + expect(leafHaystack({ type: 'leaf', key: 'k', label: '', data: 'd' })).toBe('') + }) +}) diff --git a/frontend/src/lib/components/drillPicker.ts b/frontend/src/lib/components/drillPicker.ts new file mode 100644 index 0000000000..5f3b5d6a25 --- /dev/null +++ b/frontend/src/lib/components/drillPicker.ts @@ -0,0 +1,116 @@ +import type { Component, ComponentType } from 'svelte' + +/** Icon constructor accepted by the picker — covers Svelte-5 `Component` and + * legacy `ComponentType` (lucide icons resolve to the former, but other + * callers in the repo still hand in the latter, see `TriggersBadge.svelte`). */ +export type DrillIcon = ComponentType | Component + +/** Leaf node — terminal entry the user picks. The picker emits the leaf + * back via `onPick` so callers can react with the original `data` payload. */ +export type DrillLeaf = { + type: 'leaf' + key: string + /** Primary line. */ + label: string + /** Optional secondary line (e.g. full path). */ + secondary?: string + /** Lucide-style component rendered with `size={12}`. The picker also + * accepts a `leafIcon` snippet override that gets the whole leaf. */ + icon?: DrillIcon + data: L + /** Optional override for the fuzzy-search haystack. Defaults to + * `label` (or `secondary` when label is empty). */ + searchableText?: string + /** Marks this leaf as the user's current location — gets `aria-current` + * and a styled, no-op click. */ + current?: boolean + /** When true, leaf is rendered but disabled (greyed + no-op click). */ + disabled?: boolean +} + +/** Branch node — interior entry the user drills into. */ +export type DrillBranch = { + type: 'branch' + key: string + label: string + icon?: DrillIcon + children: DrillNode[] + /** Show a spinner alongside the branch (async loading in progress). */ + loading?: boolean + /** Hide from search index traversal. Used by the workspace adapter to + * keep the cross-kind 'all' branch out of search (its leaves are + * duplicates of the per-kind branches' leaves). */ + omitFromSearch?: boolean + /** When true, leaves under this branch are grouped under its label in + * the search-results display. The DEEPEST such ancestor wins. Used to + * collapse folder hierarchies into kind/section headers — e.g. a leaf + * at `Workspace > Flows > f/demo > foo` groups under "Flows" (not + * "f/demo"). */ + searchGroup?: boolean +} + +export type DrillNode = DrillBranch | DrillLeaf + +/** Walk the tree to the branch at the given scope path. Returns null at + * root (empty scope) or when any segment doesn't resolve to a branch. */ +export function resolveScope(tree: DrillNode[], scope: string[]): DrillBranch | null { + if (scope.length === 0) return null + let level: DrillNode[] = tree + let current: DrillBranch | null = null + for (const key of scope) { + const node = level.find((n) => n.key === key) + if (!node || node.type !== 'branch') return null + current = node + level = node.children + } + return current +} + +/** Walk the tree to the branch at scope, returning ALL branches along the + * path (for breadcrumb rendering). The root is implicit and not returned. */ +export function scopeChain(tree: DrillNode[], scope: string[]): DrillBranch[] { + const chain: DrillBranch[] = [] + let level: DrillNode[] = tree + for (const key of scope) { + const node = level.find((n) => n.key === key) + if (!node || node.type !== 'branch') break + chain.push(node) + level = node.children + } + return chain +} + +/** Flatten the tree into a leaf list with each leaf's deepest + * `searchGroup`-anchor ancestor (or null if none). Skips branches marked + * `omitFromSearch`. Deduplicates leaves by `key` (first occurrence wins). */ +export function collectLeavesGrouped( + tree: DrillNode[] +): { leaf: DrillLeaf; group: DrillBranch | null }[] { + const out: { leaf: DrillLeaf; group: DrillBranch | null }[] = [] + const seen = new Set() + + function walk(nodes: DrillNode[], group: DrillBranch | null) { + for (const n of nodes) { + if (n.type === 'leaf') { + if (!seen.has(n.key)) { + seen.add(n.key) + out.push({ leaf: n, group }) + } + } else { + if (n.omitFromSearch) continue + // Deeper `searchGroup` anchors override shallower ones. + const nextGroup = n.searchGroup ? n : group + walk(n.children, nextGroup) + } + } + } + walk(tree, null) + return out +} + +/** Fuzzy-search haystack string for a leaf. */ +export function leafHaystack(leaf: DrillLeaf): string { + if (leaf.searchableText) return leaf.searchableText + if (leaf.label && leaf.secondary) return `${leaf.label} (${leaf.secondary})` + return leaf.label || leaf.secondary || '' +} diff --git a/frontend/src/lib/components/flow_builder.ts b/frontend/src/lib/components/flow_builder.ts index e91a06e430..5d2493b930 100644 --- a/frontend/src/lib/components/flow_builder.ts +++ b/frontend/src/lib/components/flow_builder.ts @@ -51,4 +51,7 @@ export type FlowBuilderProps = { onDetails?: ({ path }: { path: string }) => void onHistoryRestore?: () => void onNavigate?: (item: WorkspaceItem) => void + // Fired whenever a test run is started from the flow editor, with the + // preview job id. Used by whitelabel embedders to track test jobs. + onTestJob?: (e: { jobId: string }) => void } diff --git a/frontend/src/lib/components/flows/content/FlowInputs.svelte b/frontend/src/lib/components/flows/content/FlowInputs.svelte index 353dbb3a2f..335a37b0c7 100644 --- a/frontend/src/lib/components/flows/content/FlowInputs.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputs.svelte @@ -7,8 +7,6 @@ import FlowScriptPicker from '../pickers/FlowScriptPicker.svelte' import PickHubScript from '../pickers/PickHubScript.svelte' import WorkspaceScriptPicker from '../pickers/WorkspaceScriptPicker.svelte' - import { isCloudHosted } from '$lib/cloud' - import { sendUserToast } from '$lib/toast' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import { Check, Code, Zap } from 'lucide-svelte' @@ -259,23 +257,6 @@ {label} lang={lang == 'docker' ? 'bash' : lang} on:click={() => { - if (lang == 'docker') { - if (isCloudHosted()) { - sendUserToast( - 'You cannot use Docker scripts on the multi-tenant platform. Use a dedicated instance or self-host windmill instead.', - true, - [ - { - label: 'Learn more', - callback: () => { - window.open('https://www.windmill.dev/docs/advanced/docker', '_blank') - } - } - ] - ) - return - } - } dispatch('new', { language: lang == 'docker' ? 'bash' : lang, kind, diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index db199c745a..3f070dd74e 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -4,7 +4,6 @@ + + + + + + diff --git a/frontend/src/lib/components/icons/Auth0Icon.svelte b/frontend/src/lib/components/icons/Auth0Icon.svelte index 2c9919f037..fbf419e3cf 100644 --- a/frontend/src/lib/components/icons/Auth0Icon.svelte +++ b/frontend/src/lib/components/icons/Auth0Icon.svelte @@ -1,16 +1,36 @@ + auth0-svg diff --git a/frontend/src/lib/components/icons/DatabricksIcon.svelte b/frontend/src/lib/components/icons/DatabricksIcon.svelte new file mode 100644 index 0000000000..e767337442 --- /dev/null +++ b/frontend/src/lib/components/icons/DatabricksIcon.svelte @@ -0,0 +1,21 @@ + + + + + diff --git a/frontend/src/lib/components/icons/OktaIcon.svelte b/frontend/src/lib/components/icons/OktaIcon.svelte index c001e63f20..01c5d0715c 100644 --- a/frontend/src/lib/components/icons/OktaIcon.svelte +++ b/frontend/src/lib/components/icons/OktaIcon.svelte @@ -1,7 +1,37 @@ - - oktaddd-svg - - - \ No newline at end of file + + + + okta-svg + + diff --git a/frontend/src/lib/components/icons/WizIcon.svelte b/frontend/src/lib/components/icons/WizIcon.svelte new file mode 100644 index 0000000000..f3ee8e842e --- /dev/null +++ b/frontend/src/lib/components/icons/WizIcon.svelte @@ -0,0 +1,12 @@ + + + + + diff --git a/frontend/src/lib/components/icons/brands/Auth0.svelte b/frontend/src/lib/components/icons/brands/Auth0.svelte index 8d6696411c..d3d102fc02 100644 --- a/frontend/src/lib/components/icons/brands/Auth0.svelte +++ b/frontend/src/lib/components/icons/brands/Auth0.svelte @@ -1,30 +1,24 @@ - auth0ddd-svg - + auth0-svg diff --git a/frontend/src/lib/components/icons/index.ts b/frontend/src/lib/components/icons/index.ts index 78767697a1..56d77c810c 100644 --- a/frontend/src/lib/components/icons/index.ts +++ b/frontend/src/lib/components/icons/index.ts @@ -27,6 +27,8 @@ import QRCodeIcon from './QRCodeIcon.svelte' import LinkedinIcon from './LinkedinIcon.svelte' import HubspotIcon from './HubspotIcon.svelte' import DatadogIcon from './DatadogIcon.svelte' +import DatabricksIcon from './DatabricksIcon.svelte' +import AdobeAcrobatSignIcon from './AdobeAcrobatSignIcon.svelte' import StripeIcon from './StripeIcon.svelte' import TelegramIcon from './TelegramIcon.svelte' import FunkwhaleIcon from './FunkwhaleIcon.svelte' @@ -214,6 +216,7 @@ import YnabIcon from './YnabIcon.svelte' import SpeechifyIcon from './SpeechifyIcon.svelte' import ConvertKitIcon from './ConvertKitIcon.svelte' import BrowserlessIcon from './BrowserlessIcon.svelte' +import WizIcon from './WizIcon.svelte' import type { Component } from 'svelte' export const APP_TO_ICON_COMPONENT = { postgresql: PostgresIcon, @@ -244,6 +247,8 @@ export const APP_TO_ICON_COMPONENT = { linkedin: LinkedinIcon, hubspot: HubspotIcon, datadog: DatadogIcon, + databricks: DatabricksIcon, + adobe_acrobat_sign: AdobeAcrobatSignIcon, stripe: StripeIcon, telegram: TelegramIcon, funkwhale: FunkwhaleIcon, @@ -436,7 +441,8 @@ export const APP_TO_ICON_COMPONENT = { ynab: YnabIcon, speechify: SpeechifyIcon, convertkit: ConvertKitIcon, - browserless: BrowserlessIcon + browserless: BrowserlessIcon, + wiz: WizIcon } as unknown as Record // to generate correct svelte package types export { @@ -647,5 +653,6 @@ export { YnabIcon, SpeechifyIcon, ConvertKitIcon, - BrowserlessIcon + BrowserlessIcon, + WizIcon } diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index a5829bb2a2..10a0519d7e 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -268,6 +268,60 @@ export const settings: Record = { placeholder: '800', storage: 'setting' }, + { + label: 'Sandbox image max size (MB)', + key: 'sandbox_image_max_size_mb', + description: + 'Reject a # sandbox <image> whose compressed download size exceeds this many MB, before any layer is downloaded. Leave empty for no limit.', + fieldType: 'number', + placeholder: 'no limit', + storage: 'setting' + }, + { + label: 'Sandbox image cache cap (MB)', + key: 'sandbox_image_cache_max_mb', + description: + "Best-effort cap on the worker's cached sandbox rootfs tars. When exceeded, the oldest (by creation time) are evicted after a run. Leave empty for unbounded.", + fieldType: 'number', + placeholder: 'unbounded', + storage: 'setting' + }, + { + label: 'Sandbox image pull policy', + key: 'sandbox_image_pull_policy', + description: + 'When to re-pull a # sandbox image. newer (default) re-pulls only when the registry digest changed, so moving tags like :latest stay fresh without re-downloading unchanged layers. missing pulls only if absent (fastest, tags can go stale). always re-checks every job.', + fieldType: 'select', + storage: 'setting', + placeholder: 'newer', + defaultValue: () => 'newer', + select_items: [ + { label: 'Newer (default)', value: 'newer' }, + { label: 'Missing', value: 'missing' }, + { label: 'Always', value: 'always' }, + { label: 'Never', value: 'never' } + ] + }, + { + label: 'Sandbox image default registry', + key: 'sandbox_image_default_registry', + description: + 'If set, unqualified # sandbox images (e.g. alpine) are pulled from this registry instead of docker.io. Fully-qualified refs (e.g. ghcr.io/org/img) are unaffected. Example: myregistry.example.com.', + fieldType: 'text', + placeholder: 'docker.io', + storage: 'setting' + }, + { + label: 'Sandbox registry auth', + key: 'sandbox_registry_auth', + description: + 'Credentials for private registries used by # sandbox images, in docker config.json / auth.json format. Written to a per-job DOCKER_CONFIG dir (removed with the job) and used by crane for the pull.', + fieldType: 'codearea', + codeAreaLang: 'json', + placeholder: + '{\n "auths": {\n "myregistry.example.com": {\n "auth": "BASE64(username:password)"\n }\n }\n}', + storage: 'setting' + }, { label: 'Default timeout', key: 'job_default_timeout', diff --git a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte index 7f9e8e9807..dedb7b5d11 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte @@ -364,6 +364,29 @@ }) } + async function openDiffDrawer() { + if (!savedApp) { + return + } + + // deployedValue should be syncronized when we open Diff + await syncWithDeployed() + + diffDrawer?.openDrawer() + diffDrawer?.setDiff({ + mode: 'normal', + deployed: deployedValue ?? savedApp, + draft: savedApp.draft, + current: { + summary: summary, + value: app, + path: newEditedPath || savedApp.draft?.path || savedApp.path, + policy, + custom_path: customPath + } + }) + } + async function updateApp(npath: string) { if (!app) { sendUserToast(`App hasn't been loaded yet`, true) @@ -682,33 +705,6 @@ action: () => { publishToHubDrawerOpen = true } - }, - { - displayName: 'Diff', - icon: DiffIcon, - action: async () => { - if (!savedApp) { - return - } - - // deployedValue should be syncronized when we open Diff - await syncWithDeployed() - - diffDrawer?.openDrawer() - diffDrawer?.setDiff({ - mode: 'normal', - deployed: deployedValue ?? savedApp, - draft: savedApp.draft, - current: { - summary: summary, - value: app, - path: newEditedPath || savedApp.draft?.path || savedApp.path, - policy, - custom_path: customPath - } - }) - }, - disabled: !savedApp } ]) @@ -965,6 +961,18 @@ {/snippet} + +
- {#snippet text()} - - {#if job && job.job_kind} - {getJobKindDisplayLabel(job.job_kind, job.script_path)} - {/if} - {#if job && job.is_flow_step && job.parent_job} + {#snippet text()} + + {#if job && job.job_kind} + {getJobKindDisplayLabel(job.job_kind, job.script_path)} + {/if} + {#if job && job.is_flow_step && job.parent_job}
Step of flow {truncateRev(job.parent_job, 10)} diff --git a/frontend/src/lib/components/runs/runsFilter.ts b/frontend/src/lib/components/runs/runsFilter.ts index 9d0b85dd75..c070b383c9 100644 --- a/frontend/src/lib/components/runs/runsFilter.ts +++ b/frontend/src/lib/components/runs/runsFilter.ts @@ -163,6 +163,7 @@ export function buildRunsFilterSearchbarSchema({ { label: 'Running', value: 'running' as const }, { label: 'Success', value: 'success' as const }, { label: 'Failure', value: 'failure' as const }, + { label: 'Canceled', value: 'canceled' as const }, { label: 'Waiting', value: 'waiting' as const }, { label: 'Suspended', value: 'suspended' as const } ], diff --git a/frontend/src/lib/components/runs/useJobsLoader.svelte.ts b/frontend/src/lib/components/runs/useJobsLoader.svelte.ts index e7d8337708..b4c4b2e2c0 100644 --- a/frontend/src/lib/components/runs/useJobsLoader.svelte.ts +++ b/frontend/src/lib/components/runs/useJobsLoader.svelte.ts @@ -230,7 +230,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { let scriptPathStart = folder == null || folder === '' ? undefined : `f/${folder}/` let scriptPathExact = path == null || path === '' ? undefined : path let isQueueOnly = success == 'running' || success == 'suspended' || success == 'waiting' - let isCompletedOnly = success == 'success' || success == 'failure' + let isCompletedOnly = success == 'success' || success == 'failure' || success == 'canceled' let promise = JobService.listJobs({ workspace: currentWorkspace, createdBefore: createdBefore ?? undefined, @@ -247,7 +247,8 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { createdBy: user == null || user === '' ? undefined : user, scriptPathStart: scriptPathStart, jobKinds: jobKindsCat == 'all' || jobKinds == '' ? undefined : jobKinds, - success: success == 'success' ? true : success == 'failure' ? false : undefined, + success: success == 'success' ? true : undefined, + status: success == 'failure' || success == 'canceled' ? success : undefined, running: success == 'running' || success == 'suspended' ? true @@ -313,7 +314,8 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { createdBy: user == null || user === '' ? undefined : user, scriptPathStart: folder == null || folder === '' ? undefined : `f/${folder}/`, jobKinds: jobKindsCat == 'all' || jobKinds == '' ? undefined : jobKinds, - success: success == 'success' ? true : success == 'failure' ? false : undefined, + success: success == 'success' ? true : undefined, + status: success == 'failure' || success == 'canceled' ? success : undefined, running: success == 'running' ? true : undefined, isSkipped: showSkipped ? undefined : false, isFlowStep: jobKindsCat != 'all' ? false : undefined, diff --git a/frontend/src/lib/components/script_builder.ts b/frontend/src/lib/components/script_builder.ts index e561600181..cac47d4eef 100644 --- a/frontend/src/lib/components/script_builder.ts +++ b/frontend/src/lib/components/script_builder.ts @@ -48,6 +48,9 @@ export interface ScriptBuilderProps { onSeeDetails?: (e: { path: string }) => void onSaveDraftError?: (e: { path: string; error: any }) => void onNavigate?: (item: WorkspaceItem) => void + // Fired whenever a test run is started from the script editor, with the + // preview job id. Used by whitelabel embedders to track test jobs. + onTestJob?: (e: { jobId: string }) => void // Forwarded to the underlying ScriptEditor. When true, the right-hand // test/run pane opens collapsed. Used by the session preview. initialTestPanelCollapsed?: boolean diff --git a/frontend/src/lib/components/sessions/FlowEditorView.svelte b/frontend/src/lib/components/sessions/FlowEditorView.svelte index 0ea66146ac..dfa928f08c 100644 --- a/frontend/src/lib/components/sessions/FlowEditorView.svelte +++ b/frontend/src/lib/components/sessions/FlowEditorView.svelte @@ -2,13 +2,8 @@ import FlowBuilder from '$lib/components/FlowBuilder.svelte' import DiffDrawer from '$lib/components/DiffDrawer.svelte' import type { WorkspaceItem } from '$lib/components/workspacePicker' - import { untrack } from 'svelte' import type { SessionRuntime } from './sessionRuntime.svelte' - import type { Flow } from '$lib/gen' - import { UserDraft } from '$lib/userDraft.svelte' - import { flowDraftSig } from './flowDraftSig' - import { initFlowState } from '$lib/components/flows/flowState' - import SessionItemNotFound from './SessionItemNotFound.svelte' + import SessionEditorTarget from './SessionEditorTarget.svelte' import { sendUserToast } from '$lib/toast' let { @@ -22,25 +17,14 @@ path: string workspaceId: string onNavigate?: (item: WorkspaceItem) => void - /** - * Only the visible session should claim the workspace's live-editor - * slot — without this, a hidden warm-mounted session can overwrite the - * active session's UserDraft live-editor target (one slot per - * (workspace, kind)), so chat actions like discard / "the open editor" - * resolve to the wrong session. - */ + /** Forwarded to SessionEditorTarget — only the visible session claims the + * workspace's single live-editor slot. */ isActiveSession?: boolean } = $props() let selectedId = $state('settings-metadata') let diffDrawer: DiffDrawer | undefined = $state() - $effect(() => { - if (workspaceId && path) { - untrack(() => runtime.loadFlow(workspaceId, path)) - } - }) - // In a session pane, "restore" just reloads from the current state — the // session target stays put. The Diff drawer's primary use here is viewing // the diff; restore is best-effort. @@ -48,100 +32,6 @@ diffDrawer?.closeDrawer() await runtime.loadFlow(workspaceId, path) } - - // Mark this editor as the "live editor" for the session's workspace so - // the chat's `isLiveDraft` hint and `discard_local_draft` tool resolve to - // this path. Same registration the regular /flows/edit page does on - // mount, scoped to the session's (forked) workspace. - // Gated on `isActiveSession`: warm-but-hidden session editors must not - // claim the workspace's single live-editor slot, else chat actions on the - // visible session resolve to the hidden one's path. - $effect(() => { - if (!workspaceId || !path) return - if (!isActiveSession) return - UserDraft.setLiveEditorDraft({ - workspace: workspaceId, - itemKind: 'flow', - storagePath: path, - effectivePath: runtime.flowStore.val?.path ?? path - }) - return () => - UserDraft.clearLiveEditorDraft('flow', { workspace: workspaceId, storagePath: path }) - }) - - // Bidirectional sync between this preview and `UserDraft`. - // We hold a *live* handle (useMany) rather than reading via the static - // `UserDraft.get`. The handle materializes UserDraft's shared reactive - // `$state` cell for (workspace, 'flow', path), and that cell is what lets - // the chat's writes (UserDraft.save, from write_flow / patch_flow_json / - // set_flow_module_code) reach this preview. Without a live entry those - // writes only touch localStorage and the inbound effect below never - // re-fires. A reactive getter is used (not `use()`) because switching - // open_preview to another flow swaps `path` without remounting this view, - // so the handle must re-acquire. - // - // One-way-reactive discipline: inbound tracks only the handle's draft, - // outbound tracks only `flowStore.val`; the read on the "other side" - // inside each effect goes through `untrack()`. Without that asymmetry, a - // user keystroke would re-fire the inbound effect with the pre-keystroke - // stored value and revert the edit. - const draftHandles = UserDraft.useMany(() => [ - { itemKind: 'flow', path, workspace: workspaceId } - ]) - let lastInboundSig: string | undefined = $state(undefined) - - // Store → editor. Re-runs when the handle's draft changes (AI write from - // this session's chat or another session). flowStore reads are untracked - // so the editor's own mutations don't refire this effect. - $effect(() => { - if (!workspaceId || !path) return - const incoming = draftHandles[0]?.draft - if (!incoming) return - const sig = flowDraftSig(incoming) - untrack(() => { - if (runtime.loadedPath !== path) return - if (sig === lastInboundSig) return - const current = runtime.flowStore.val - if (!current) return - lastInboundSig = sig - runtime.flowStore.val = { - ...current, - value: incoming.value, - schema: incoming.schema ?? current.schema, - summary: incoming.summary ?? current.summary - } - // flowStateStore is keyed by module_id; after an AI write the set - // of module ids may differ, so rebuild the UI state. This wipes - // per-module test args / preview output for the new flow — a - // known v1 trade-off, see the plan's caveats. - void initFlowState(runtime.flowStore.val, runtime.flowStateStore) - }) - }) - - // Editor → store. Re-runs on any deep mutation of flowStore.val - // (modules, schema, module bodies). The store read is untracked. - // Debounced 150ms so a typing burst inside an inline rawscript editor - // results in one serialise-and-write instead of one per keystroke. - let outboundTimer: ReturnType | undefined - $effect(() => { - if (!workspaceId || !path) return - if (runtime.loadedPath !== path) return - const flow = runtime.flowStore.val - if (!flow) return - const sig = flowDraftSig(flow) - if (sig === lastInboundSig) return - if (outboundTimer) clearTimeout(outboundTimer) - outboundTimer = setTimeout(() => { - untrack(() => { - const current = UserDraft.get('flow', path, { workspace: workspaceId }) - if (current && flowDraftSig(current) === sig) return - UserDraft.save('flow', path, flow, { workspace: workspaceId }) - }) - }, 150) - return () => { - if (outboundTimer) clearTimeout(outboundTimer) - } - }) {#if runtime.savedFlow.val} @@ -152,30 +42,36 @@ isFlow /> {/if} -{#if runtime.loadingFlow && !runtime.loadedPath} -
Loading flow {path}…
-{:else if runtime.notFound && !runtime.loadedPath} - -{:else} - - runtime.scheduleForkComparisonRefresh()} - onDeploy={() => { - // FlowBuilder has no deploy toast and the session stays put, so toast - // here, then sync the preview to deployed (pulls the new locks + version_id). - sendUserToast('Deployed') - runtime.syncPreviewWithDeployed(workspaceId, 'flow', path) - }} - /> -{/if} + runtime.flowStore.val?.path ?? path} +> + {#snippet editor()} + + runtime.scheduleForkComparisonRefresh()} + onDeploy={() => { + // FlowBuilder has no deploy toast and the session stays put, so toast + // here, then sync the preview to deployed (pulls the new locks + version_id). + sendUserToast('Deployed') + runtime.syncPreviewWithDeployed(workspaceId, 'flow', path) + }} + /> + {/snippet} + diff --git a/frontend/src/lib/components/sessions/RawAppEditorView.svelte b/frontend/src/lib/components/sessions/RawAppEditorView.svelte index af77298a87..9ed75f9702 100644 --- a/frontend/src/lib/components/sessions/RawAppEditorView.svelte +++ b/frontend/src/lib/components/sessions/RawAppEditorView.svelte @@ -2,12 +2,8 @@ import RawAppEditor from '$lib/components/raw_apps/RawAppEditor.svelte' import DiffDrawer from '$lib/components/DiffDrawer.svelte' import type { WorkspaceItem } from '$lib/components/workspacePicker' - import { untrack } from 'svelte' import type { SessionRuntime } from './sessionRuntime.svelte' - import { UserDraft } from '$lib/userDraft.svelte' - import type { RawAppDraft } from './appDraftCodec' - import { applyDraftToRuntimeRawApp, runtimeRawAppToDraft } from './appDraftCodec' - import SessionItemNotFound from './SessionItemNotFound.svelte' + import SessionEditorTarget from './SessionEditorTarget.svelte' let { runtime, @@ -20,107 +16,17 @@ path: string workspaceId: string onNavigate?: (item: WorkspaceItem) => void - /** - * Only the visible session should claim the workspace's live-editor - * slot — without this, a hidden warm-mounted session can overwrite the - * active session's UserDraft live-editor target (one slot per - * (workspace, kind)), so chat actions like discard / "the open editor" - * resolve to the wrong session. - */ + /** Forwarded to SessionEditorTarget — only the visible session claims the + * workspace's single live-editor slot. */ isActiveSession?: boolean } = $props() let diffDrawer: DiffDrawer | undefined = $state() - $effect(() => { - if (workspaceId && path) { - untrack(() => runtime.loadRawApp(workspaceId, path)) - } - }) - async function restoreFromCurrentTarget() { diffDrawer?.closeDrawer() await runtime.loadRawApp(workspaceId, path) } - - // Mark this editor as the live editor draft for the session's workspace - // so the chat's `isLiveDraft` hint / `discard_local_draft` tool resolve - // to this path — same registration the regular /apps_raw/edit page does. - // Gated on `isActiveSession`: warm-but-hidden session editors must not - // claim the workspace's single live-editor slot, else chat actions on the - // visible session resolve to the hidden one's path. - $effect(() => { - if (!workspaceId || !path) return - if (!isActiveSession) return - UserDraft.setLiveEditorDraft({ - workspace: workspaceId, - itemKind: 'raw_app', - storagePath: path, - effectivePath: runtime.rawApp.val?.path ?? path - }) - return () => - UserDraft.clearLiveEditorDraft('raw_app', { workspace: workspaceId, storagePath: path }) - }) - - // Bidirectional sync between this preview and `UserDraft`. - // We hold a *live* handle (useMany) rather than reading via the static - // `UserDraft.get`: the handle materializes UserDraft's shared reactive - // `$state` cell for (workspace, 'raw_app', path), and that cell is what - // lets the chat's writes (UserDraft.save / setDraftAndMeta, from - // write_app_file / patch_app_file / write_app_runnable) reach this preview. - // Without a live entry those writes only touch localStorage and the inbound - // effect below never re-fires. A reactive getter is used (not `use()`) - // because switching open_preview to another app swaps `path` without - // remounting this view, so the handle must re-acquire. - // - // Same one-way-reactive discipline as ScriptEditorView: inbound tracks only - // the handle's draft, outbound tracks only rawApp.val; each side's read of - // the other goes through untrack() to break the keystroke-revert race. - const draftHandles = UserDraft.useMany(() => [ - { itemKind: 'raw_app', path, workspace: workspaceId } - ]) - let lastInboundSig: string | undefined = $state(undefined) - - // Store → editor. Re-runs when the handle's draft changes (chat write, - // other session edit). - $effect(() => { - if (!workspaceId || !path) return - const incoming = draftHandles[0]?.draft - if (!incoming) return - const sig = JSON.stringify(incoming) - untrack(() => { - if (runtime.loadedRawAppPath !== path) return - if (sig === lastInboundSig) return - const current = runtime.rawApp.val - if (!current) return - lastInboundSig = sig - runtime.rawApp.val = applyDraftToRuntimeRawApp(current, incoming) - }) - }) - - // Editor → store. Debounced 150ms so a typing burst inside a frontend - // file's Monaco editor coalesces into one store write. - let outboundTimer: ReturnType | undefined - $effect(() => { - if (!workspaceId || !path) return - if (runtime.loadedRawAppPath !== path) return - const raw = runtime.rawApp.val - if (!raw) return - const draft = runtimeRawAppToDraft(raw) - const sig = JSON.stringify(draft) - if (sig === lastInboundSig) return - if (outboundTimer) clearTimeout(outboundTimer) - outboundTimer = setTimeout(() => { - untrack(() => { - const current = UserDraft.get('raw_app', path, { workspace: workspaceId }) - if (current && JSON.stringify(current) === sig) return - UserDraft.save('raw_app', path, draft, { workspace: workspaceId }) - }) - }, 150) - return () => { - if (outboundTimer) clearTimeout(outboundTimer) - } - }) {#if runtime.savedRawApp.val} @@ -130,29 +36,37 @@ restoreDraft={restoreFromCurrentTarget} /> {/if} -{#if runtime.loadingRawApp && !runtime.loadedRawAppPath} -
Loading raw app {path}…
-{:else if runtime.notFoundRawApp && !runtime.loadedRawAppPath} - -{:else if runtime.rawApp.val} - { - // Sync the preview to deployed (raw apps deploy only from this editor). - runtime.syncPreviewWithDeployed(workspaceId, 'raw_app', e.path) - }} - defaultSidebarCollapsed - sidebarStorageKey="raw-app-sidebar-collapsed-preview" - defaultSplitWithPreview={false} - /> -{/if} + runtime.rawApp.val?.path ?? path} +> + {#snippet editor()} + {#if runtime.rawApp.val} + { + // Sync the preview to deployed (raw apps deploy only from this editor). + runtime.syncPreviewWithDeployed(workspaceId, 'raw_app', e.path) + }} + defaultSidebarCollapsed + sidebarStorageKey="raw-app-sidebar-collapsed-preview" + defaultSplitWithPreview={false} + /> + {/if} + {/snippet} + diff --git a/frontend/src/lib/components/sessions/ScriptEditorView.svelte b/frontend/src/lib/components/sessions/ScriptEditorView.svelte index da7b7fcb53..bebd6d3cce 100644 --- a/frontend/src/lib/components/sessions/ScriptEditorView.svelte +++ b/frontend/src/lib/components/sessions/ScriptEditorView.svelte @@ -2,11 +2,10 @@ import ScriptBuilder from '$lib/components/ScriptBuilder.svelte' import DiffDrawer from '$lib/components/DiffDrawer.svelte' import type { WorkspaceItem } from '$lib/components/workspacePicker' - import { untrack } from 'svelte' import type { SessionRuntime } from './sessionRuntime.svelte' import { DraftService, ScriptService, type NewScript } from '$lib/gen' import { UserDraft } from '$lib/userDraft.svelte' - import SessionItemNotFound from './SessionItemNotFound.svelte' + import SessionEditorTarget from './SessionEditorTarget.svelte' import { sendUserToast } from '$lib/toast' let { @@ -22,24 +21,13 @@ workspaceId: string onNavigate?: (item: WorkspaceItem) => void initialTestPanelCollapsed?: boolean - /** - * Only the visible session should claim the workspace's live-editor - * slot — without this, a hidden warm-mounted session can overwrite the - * active session's UserDraft live-editor target (one slot per - * (workspace, kind)), so chat actions like discard / "the open editor" - * resolve to the wrong session. - */ + /** Forwarded to SessionEditorTarget — only the visible session claims the + * workspace's single live-editor slot. */ isActiveSession?: boolean } = $props() let diffDrawer: DiffDrawer | undefined = $state() - $effect(() => { - if (workspaceId && path) { - untrack(() => runtime.loadScript(workspaceId, path)) - } - }) - // Restore actions for the diff drawer. The previous shared // `loadScript`-based handler was a no-op: loadScript early-returns on the // already-loaded path (and would re-read the local draft anyway). Instead @@ -78,147 +66,65 @@ workspace: workspaceId }) } - - // Mark this editor as the live editor draft for the session's workspace - // so the chat's `isLiveDraft` hint / `discard_local_draft` tool resolve - // to this path — same registration the regular /scripts/edit page does. - // Gated on `isActiveSession`: warm-but-hidden session editors must not - // claim the workspace's single live-editor slot, else chat actions on the - // visible session resolve to the hidden one's path. - $effect(() => { - if (!workspaceId || !path) return - if (!isActiveSession) return - UserDraft.setLiveEditorDraft({ - workspace: workspaceId, - itemKind: 'script', - storagePath: path, - effectivePath: runtime.scriptStore.val?.path ?? path - }) - return () => - UserDraft.clearLiveEditorDraft('script', { workspace: workspaceId, storagePath: path }) - }) - - // Bidirectional sync between this preview and `UserDraft`. - // The same path under the same workspace is shared with the session's - // chat (read_workspace_item / write_script / edit_script) and any other - // open editor on the same workspace. - // - // We hold a *live* handle (useMany) instead of reading via the static - // `UserDraft.get`. The handle materializes UserDraft's shared reactive - // `$state` cell for (workspace, 'script', path) — and that cell is what - // lets the chat's writes (UserDraft.save, from write_script / edit_script) - // reach this preview. Without a live entry those writes only touch - // localStorage and the inbound effect below never re-fires. A reactive - // getter is used (not `use()`) because switching open_preview to another - // script swaps `path` without remounting this view, so the handle must - // re-acquire. - // - // One-way-reactive discipline: inbound tracks ONLY the handle's `draft` - // (and reads `script.content` via untrack); outbound tracks ONLY - // `script.content` (and reads UserDraft via untrack). Without that - // asymmetry, a user keystroke would re-fire the inbound effect with the - // pre-keystroke stored value and revert the edit. - const draftHandles = UserDraft.useMany(() => [ - { itemKind: 'script', path, workspace: workspaceId } - ]) - let lastInboundContent: string | undefined = $state(undefined) - - // Store → editor. Re-runs when the handle's draft changes (chat write, - // other session edit, …). `script.content` is read inside untrack so user - // keystrokes don't refire this effect. - $effect(() => { - if (!workspaceId || !path) return - const draft = draftHandles[0]?.draft - if (!draft || typeof draft.content !== 'string') return - const incoming = draft.content - untrack(() => { - if (runtime.loadedScriptPath !== path) return - const script = runtime.scriptStore.val - if (!script) return - if (incoming === script.content) return - lastInboundContent = incoming - script.content = incoming - if (draft.language) script.language = draft.language - if (draft.summary !== undefined) script.summary = draft.summary - }) - }) - - // Editor → store. Re-runs on `script.content` mutation (user typing - // or inbound write). UserDraft is read inside untrack so writing here - // doesn't ping-pong the inbound effect. `UserDraft.save` persists - // immediately and, now that the entry is live, updates the same cell the - // inbound effect reads (the content guard there makes it a no-op). - $effect(() => { - if (!workspaceId || !path) return - if (runtime.loadedScriptPath !== path) return - const script = runtime.scriptStore.val - if (!script) return - const content = script.content - if (content === lastInboundContent) return - untrack(() => { - const current = UserDraft.get('script', path, { workspace: workspaceId }) - if (current && current.content === content) return - UserDraft.save( - 'script', - path, - { ...(current ?? script), ...script }, - { - workspace: workspaceId - } - ) - }) - }) {#if runtime.savedScript.val} {/if} -{#if runtime.loadingScript && !runtime.loadedScriptPath} -
Loading script {path}…
-{:else if runtime.notFoundScript && !runtime.loadedScriptPath} - -{:else if runtime.scriptStore.val} - - { - runtime.scheduleForkComparisonRefresh() - // Re-pin parent_hash to the latest version so the next Deploy's conflict - // check (which runs before deploy, while the session stays mounted) - // doesn't misfire. - try { - const latest = await ScriptService.getScriptLatestVersion({ - workspace: workspaceId, - path: e.path - }) - const cur = runtime.scriptStore.val - if (latest?.script_hash && cur) cur.parent_hash = latest.script_hash - } catch (err) { - console.error('Failed to sync parent_hash after save draft', err) - } - }} - onDeploy={(e) => { - // Fires on every deploy (primary, "Deploy & Stay here", and lib — we - // ignore e.stay since the session always stays). Toast, then sync the - // preview to the deployed version. - sendUserToast('Deployed') - runtime.syncPreviewWithDeployed(workspaceId, 'script', e.path) - }} - /> -{/if} + runtime.scriptStore.val?.path ?? path} +> + {#snippet editor()} + {#if runtime.scriptStore.val} + + { + runtime.scheduleForkComparisonRefresh() + // Re-pin parent_hash to the latest version so the next Deploy's conflict + // check (which runs before deploy, while the session stays mounted) + // doesn't misfire. + try { + const latest = await ScriptService.getScriptLatestVersion({ + workspace: workspaceId, + path: e.path + }) + const cur = runtime.scriptStore.val + if (latest?.script_hash && cur) cur.parent_hash = latest.script_hash + } catch (err) { + console.error('Failed to sync parent_hash after save draft', err) + } + }} + onDeploy={(e) => { + // Fires on every deploy (primary, "Deploy & Stay here", and lib — we + // ignore e.stay since the session always stays). Toast, then sync the + // preview to the deployed version. + sendUserToast('Deployed') + runtime.syncPreviewWithDeployed(workspaceId, 'script', e.path) + }} + /> + {/if} + {/snippet} + diff --git a/frontend/src/lib/components/sessions/SessionEditorTarget.svelte b/frontend/src/lib/components/sessions/SessionEditorTarget.svelte new file mode 100644 index 0000000000..55bd697bb9 --- /dev/null +++ b/frontend/src/lib/components/sessions/SessionEditorTarget.svelte @@ -0,0 +1,128 @@ + + +{#snippet loadingOverlay(asOverlay: boolean)} +
+ +
+{/snippet} + +{#if slot.notFound && slot.loadedPath !== path} + + +{:else if slot.loadedPath === undefined} + + {@render loadingOverlay(false)} +{:else} + + {#key slot.loadedPath} + {@render editor()} + {/key} + {#if showOverlay} + {@render loadingOverlay(true)} + {/if} +{/if} diff --git a/frontend/src/lib/components/sessions/SessionForkBar.svelte b/frontend/src/lib/components/sessions/SessionForkBar.svelte index 9205c4c087..22b41a7a00 100644 --- a/frontend/src/lib/components/sessions/SessionForkBar.svelte +++ b/frontend/src/lib/components/sessions/SessionForkBar.svelte @@ -2,6 +2,7 @@ import { Archive, ArrowRight, + Diff, GitCompareArrows, GitFork, GitMerge, @@ -189,9 +190,9 @@
{#snippet actions()} diff --git a/frontend/src/lib/components/settings/TokensTable.svelte b/frontend/src/lib/components/settings/TokensTable.svelte index 5907c808a6..f2e2ca235d 100644 --- a/frontend/src/lib/components/settings/TokensTable.svelte +++ b/frontend/src/lib/components/settings/TokensTable.svelte @@ -35,7 +35,13 @@ let tokenPage = $state(1) let newTokenLabel = $state(untrack(() => defaultNewTokenLabel)) let editingToken = $state< - { prefix: string; scopes: string[] | undefined; workspaceId: string | undefined } | undefined + | { + prefix: string + label: string | undefined + scopes: string[] | undefined + workspaceId: string | undefined + } + | undefined >(undefined) let editModalOpen = $state(false) @@ -43,9 +49,9 @@ listTokens() }) - // When updating this filter, also update: - // - `is_user_token` in backend/src/monitor.rs - // - `register_token_expiry_notification` in backend/windmill-api-auth/src/lib.rs + // Mirror of the canonical `is_user_token` in backend/windmill-common/src/auth.rs. + // When updating this filter, also update that function and the SQL `WHERE` + // mirror in `update_token_label` (backend/windmill-api-users/src/users.rs). function isUserToken(label: string | undefined): boolean { if (!label) return true return ( @@ -104,11 +110,13 @@ function handleEditClick( tokenPrefix: string, + tokenLabel: string | undefined, tokenScopes: string[] | undefined, tokenWorkspaceId: string | undefined ) { editingToken = { prefix: tokenPrefix, + label: tokenLabel, scopes: tokenScopes, workspaceId: tokenWorkspaceId } @@ -198,9 +206,11 @@