mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-10-12 16:02:56 +00:00
Merge remote-tracking branch 'origin/main' into hugo/resource-type-guidance
This commit is contained in:
hugocasa
@@ -28,7 +28,7 @@ ENV PATH="${PATH}:/usr/local/go/bin"
|
||||
ENV GO_PATH=/usr/local/go/bin/go
|
||||
|
||||
# UV
|
||||
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
|
||||
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
|
||||
|
||||
ENV TZ=Etc/UTC
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
|
||||
- uses: denoland/setup-deno@v2
|
||||
with:
|
||||
# Pin to the Deno version shipped in the runtime image (Dockerfile) so CI
|
||||
# Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI
|
||||
# tests what production runs, instead of floating on the latest v2.x.
|
||||
deno-version: 2.2.1
|
||||
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
version: "0.11.24"
|
||||
version: "0.13.0"
|
||||
|
||||
- uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
dotnet-version: "9.0.x"
|
||||
- uses: denoland/setup-deno@v2
|
||||
with:
|
||||
# Pin to the Deno version shipped in the runtime image (Dockerfile) so CI
|
||||
# Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI
|
||||
# tests what production runs, instead of floating on the latest v2.x.
|
||||
deno-version: 2.2.1
|
||||
- uses: actions/setup-go@v7
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
node-version: "24"
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
version: "0.11.24"
|
||||
version: "0.13.0"
|
||||
- uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: "8.3"
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always --health-interval 10s --health-timeout 5s
|
||||
--health-retries 5 --health-cmd "curl
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always --health-interval 10s --health-timeout 5s
|
||||
--health-retries 5 --health-cmd "curl
|
||||
@@ -125,7 +125,7 @@ jobs:
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
MODE: worker
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always
|
||||
|
||||
@@ -136,7 +136,7 @@ jobs:
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
MODE: worker
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always
|
||||
|
||||
@@ -147,7 +147,7 @@ jobs:
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
MODE: worker
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always
|
||||
|
||||
@@ -188,7 +188,7 @@ jobs:
|
||||
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always --health-interval 10s --health-timeout 5s
|
||||
--health-retries 5 --health-cmd "curl
|
||||
@@ -202,7 +202,7 @@ jobs:
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
MODE: worker
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always
|
||||
|
||||
@@ -213,7 +213,7 @@ jobs:
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
MODE: worker
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always
|
||||
|
||||
@@ -224,7 +224,7 @@ jobs:
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
MODE: worker
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always
|
||||
|
||||
@@ -235,7 +235,7 @@ jobs:
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
MODE: worker
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always
|
||||
|
||||
@@ -246,7 +246,7 @@ jobs:
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
MODE: worker
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always
|
||||
|
||||
@@ -257,7 +257,7 @@ jobs:
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
MODE: worker
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always
|
||||
|
||||
@@ -268,7 +268,7 @@ jobs:
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
MODE: worker
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always
|
||||
steps:
|
||||
@@ -309,7 +309,7 @@ jobs:
|
||||
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
|
||||
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
|
||||
options: >-
|
||||
--pull always --health-interval 10s --health-timeout 5s
|
||||
--health-retries 5 --health-cmd "curl
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
{
|
||||
".": "1.829.0"
|
||||
".": "1.830.0"
|
||||
}
|
||||
|
||||
@@ -37,8 +37,9 @@ Open-source platform for internal tools, workflows, API integrations, background
|
||||
- **Operator write rights**: `docs/operator-write-rights.md` — which `operator_settings` flags are
|
||||
enforced rather than cosmetic, and why a right that is granted-unless-withdrawn needs `Option`
|
||||
fields and a jsonb merge rather than a serde default
|
||||
- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace setting that lets
|
||||
operators compose flows, what the composition check must cover, and why it costs a full seat
|
||||
- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace settings that let
|
||||
operators compose flows and full-code apps, what the composition check must cover, which
|
||||
raw-app endpoint stays closed on purpose, and why either right costs a full seat
|
||||
- **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation
|
||||
scope, which token labels email their owner at expiry, how OAuth login matches `login_type`, and
|
||||
that every superadmin route refuses `$WM_TOKEN`. Read before designing anything that creates
|
||||
|
||||
@@ -1,5 +1,25 @@
|
||||
# Changelog
|
||||
|
||||
## [1.830.0](https://github.com/windmill-labs/windmill/compare/v1.829.0...v1.830.0) (2026-10-09)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add a jobs:run:dependencies token scope for lock generation ([#11643](https://github.com/windmill-labs/windmill/issues/11643)) ([1dc4ed5](https://github.com/windmill-labs/windmill/commit/1dc4ed5e7fb86c9658f1f877e7b92cd9dab5e283))
|
||||
* let operators compose full-code apps when the workspace grants it ([#11229](https://github.com/windmill-labs/windmill/issues/11229)) ([02e4697](https://github.com/windmill-labs/windmill/commit/02e4697157112581e9e2a9303180077126db714b))
|
||||
* operators get AI sessions ([#11443](https://github.com/windmill-labs/windmill/issues/11443)) ([338c4e9](https://github.com/windmill-labs/windmill/commit/338c4e921dc3185d886c8b040f253285adf20efb))
|
||||
* replace the hub_sync script with an opt-in daily server sync ([#11624](https://github.com/windmill-labs/windmill/issues/11624)) ([2b55a8a](https://github.com/windmill-labs/windmill/commit/2b55a8afbc8a86c6cebe97b1cab02c313b986f22))
|
||||
* retry transient AI provider failures before output reaches readers ([#11623](https://github.com/windmill-labs/windmill/issues/11623)) ([d3ddef0](https://github.com/windmill-labs/windmill/commit/d3ddef0b7ec4705aa578c4eca24b96e9152485bd))
|
||||
* stop offering deno for new scripts, ship it only on full images ([#11648](https://github.com/windmill-labs/windmill/issues/11648)) ([cda7883](https://github.com/windmill-labs/windmill/commit/cda78830fde1b96bd3de2674e23c245b50e3c051))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** count superadmins as able to preserve ownership on push ([#11653](https://github.com/windmill-labs/windmill/issues/11653)) ([70db567](https://github.com/windmill-labs/windmill/commit/70db567a78b1e1bc9e2dcc9ff2139951d778117e))
|
||||
* harden app secret validation ([#11655](https://github.com/windmill-labs/windmill/issues/11655)) ([8556a7b](https://github.com/windmill-labs/windmill/commit/8556a7bebe093e0bd61c33d67415fd553f0070ad))
|
||||
* open the chat + menu without waiting on the skills and mcp fetches ([#11645](https://github.com/windmill-labs/windmill/issues/11645)) ([1fea1aa](https://github.com/windmill-labs/windmill/commit/1fea1aa6609d644e02bac8d16186f9a2dfcb63cd))
|
||||
* say Running on a chat run card only once its job exists ([#11627](https://github.com/windmill-labs/windmill/issues/11627)) ([d046187](https://github.com/windmill-labs/windmill/commit/d046187e3edd01535240c6080b483afe154ae189))
|
||||
|
||||
## [1.829.0](https://github.com/windmill-labs/windmill/compare/v1.828.0...v1.829.0) (2026-10-09)
|
||||
|
||||
|
||||
|
||||
+1
-4
@@ -83,7 +83,6 @@ COPY /python-client/docs/ /frontend/static/pydocs/
|
||||
|
||||
RUN npm run generate-backend-client
|
||||
ENV NODE_OPTIONS "--max-old-space-size=8192"
|
||||
ARG VITE_BASE_URL ""
|
||||
# Must be declared for the build-arg to reach the bundle. See frontend/svelte.config.js.
|
||||
ARG WM_BUILD_VERSION=""
|
||||
# Read more about macro in docker/dev.nu
|
||||
@@ -242,7 +241,7 @@ ENV PATH="${PATH}:/usr/local/go/bin"
|
||||
ENV GO_PATH=/usr/local/go/bin/go
|
||||
|
||||
# Install UV
|
||||
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
|
||||
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
|
||||
|
||||
# Preinstall python runtimes to temp build location (will copy with world-writable perms later)
|
||||
# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run
|
||||
@@ -287,8 +286,6 @@ COPY --from=builder /frontend/build /static_frontend
|
||||
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
|
||||
COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/target/release/libwindmill_duckdb_ffi_internal.so ${APP}/libwindmill_duckdb_ffi_internal.so
|
||||
|
||||
COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno
|
||||
|
||||
COPY --from=oven/bun:1.4.0 /usr/local/bin/bun /usr/bin/bun
|
||||
|
||||
# Install windmill CLI
|
||||
|
||||
@@ -150,7 +150,7 @@ You can run scripts locally by passing the right environment variables for the `
|
||||
- **Frontend**: Svelte 5
|
||||
- **Sandboxing**: [nsjail](https://github.com/google/nsjail) and PID namespace isolation
|
||||
- **Runtimes**:
|
||||
- TypeScript/JavaScript: Bun (default) and Deno
|
||||
- TypeScript/JavaScript: Bun (default); Deno only on the `windmill-full` images
|
||||
- Python: python3 with uv for dependency management
|
||||
- Go, Bash, PowerShell, PHP, Rust, C#, Java, Ansible
|
||||
|
||||
@@ -318,7 +318,7 @@ running options.
|
||||
This will also avoid compile time issue with sqlx's `query!` macro.
|
||||
2. (optional, linux only) Install [nsjail](https://github.com/google/nsjail) and have it accessible in
|
||||
your PATH
|
||||
3. Install bun, deno and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun`,`/usr/bin/deno`, and
|
||||
3. Install bun and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun` and
|
||||
`/usr/local/bin/python3` or set the corresponding environment variables.
|
||||
4. (optional) Install the [lld linker](https://lld.llvm.org/)
|
||||
5. Go to `frontend/`:
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)\n VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error(\"fail\"); }', 'deno', 'script', 'test-user', '{}', '', '', '', true)\n ",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)\n VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error(\"fail\"); }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>', true)\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -10,5 +10,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde"
|
||||
"hash": "009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')\n ",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\\n//bun.lock\\n<empty>')\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -11,5 +11,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730"
|
||||
"hash": "1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e"
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\",\n COALESCE((operator_settings->>'builder_apps')::boolean, false) AS \"apps!\",\n COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\",\n COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\"\n FROM workspace_settings WHERE workspace_id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "flows!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "apps!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "schedules!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "triggers!",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>')\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -8,5 +8,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877"
|
||||
"hash": "344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')\n ",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\\n//bun.lock\\n<empty>')\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -10,5 +10,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95"
|
||||
"hash": "42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '')\n ",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\\n//bun.lock\\n<empty>')\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -10,5 +10,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4"
|
||||
"hash": "69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>')\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -10,5 +10,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e"
|
||||
"hash": "8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2",
|
||||
"query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('dependencies', 'flowdependencies') THEN 'dependencies'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -31,5 +31,5 @@
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec"
|
||||
"hash": "8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '')\n ",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\\n//bun.lock\\n<empty>')\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -11,5 +11,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c"
|
||||
"hash": "b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "exists",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ",
|
||||
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>')\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -8,5 +8,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20"
|
||||
"hash": "cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT COALESCE((ws.operator_settings->>'builder_flows')::boolean, false)) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg",
|
||||
"query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT (COALESCE((ws.operator_settings->>'builder_flows')::boolean, false) OR COALESCE((ws.operator_settings->>'builder_apps')::boolean, false))) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -34,5 +34,5 @@
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b"
|
||||
"hash": "efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b"
|
||||
}
|
||||
+1
-1
@@ -214,7 +214,7 @@ Whole classes of defect compile and unit-test clean:
|
||||
- **Stack overflow from a large buffer in an async block.** An array declared across an
|
||||
`.await` is baked into the future's state; once that future is boxed a few layers deep
|
||||
by the job poller, two 16 KB arrays abort the worker *process* (`thread
|
||||
'tokio-runtime-worker' has overflowed its stack`). Heap-allocate read buffers
|
||||
'tokio-rt-worker' has overflowed its stack`). Heap-allocate read buffers
|
||||
(`vec![0u8; N]`, not `[0u8; N]`).
|
||||
- Deadlocks from draining only one of a child's pipes, missed cancellation or timeout
|
||||
propagation, and anything depending on the real engine's output format.
|
||||
|
||||
Generated
+1507
-827
File diff suppressed because it is too large
Load Diff
+31
-33
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "windmill"
|
||||
version = "1.829.0"
|
||||
version = "1.830.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
@@ -88,7 +88,7 @@ members = [
|
||||
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.829.0"
|
||||
version = "1.830.0"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
edition = "2021"
|
||||
|
||||
@@ -451,7 +451,7 @@ headers = "^0"
|
||||
hyper = { version = "^1", features = ["full"] }
|
||||
hyper-tls = "^0.6"
|
||||
hyper-util = { version = "^0.1", features = ["client-legacy", "http1", "tokio"] }
|
||||
tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] }
|
||||
tokio = { version = "=1.53.2", features = ["full", "tracing", "time"] }
|
||||
tokio-stream = { version = "0.1.17" }
|
||||
tower = "^0"
|
||||
tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] }
|
||||
@@ -488,9 +488,11 @@ arc-swap = "1"
|
||||
ed25519-dalek = { version = "2", features = ["rand_core"] }
|
||||
magic-crypt = "^3"
|
||||
git-version = "^0"
|
||||
malachite = "=0.4.18"
|
||||
malachite-bigint = "=0.2.0"
|
||||
rustpython-parser = "^0"
|
||||
# Ruff's parser crates are internal to ruff and carry no semver guarantee: pin them exactly,
|
||||
# and to the same release.
|
||||
ruff_python_parser = "=0.0.17"
|
||||
ruff_python_ast = "=0.0.17"
|
||||
ruff_text_size = "=0.0.17"
|
||||
pep440_rs = "0.7.3"
|
||||
php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" }
|
||||
cron = "^0"
|
||||
@@ -519,33 +521,29 @@ aws-sdk-rds = "^1"
|
||||
async-trait = "0.1.88"
|
||||
|
||||
|
||||
v8 = "=137.1.0" # Exact version NOTE: Do not forget to update version and hash in flake.nix
|
||||
# deno_* pin set: deno v2.4.0 base, with deno_ast force-overridden to =0.51.0.
|
||||
# Rationale: deno_ast 0.51.0 is the first version pulling swc_common =14.0.4,
|
||||
# the first swc_common patch that dropped `pub use serde::__private as serde;`
|
||||
# (the line that capped our workspace serde pin at =1.0.220). v2.4.0's other
|
||||
# pins keep deno_tls at 0.196.0 which uses permissive `rustls ^0.23.11`,
|
||||
# compatible with aws-sdk-bedrockruntime's `^0.23.31` requirement. deno_tls
|
||||
# 0.198+ tightened that to exact `=0.23.28`, which would have made any
|
||||
# meaningful deno bump resolver-impossible against aws-sdk.
|
||||
deno_fetch = "0.233.0"
|
||||
deno_tls = "0.196.0"
|
||||
deno_console = "0.209.0"
|
||||
deno_url = "0.209.0"
|
||||
deno_webidl = "0.209.0"
|
||||
deno_web = "0.240.0"
|
||||
# Sibling release of the pinned deno_core 0.352 / deno_web 0.240 stack
|
||||
# (its deps are deno_core ^0.352, deno_web ^0.240, deno_error =0.6.1). A newer
|
||||
# deno_crypto would force bumping the whole deno stack.
|
||||
deno_crypto = "0.223.0"
|
||||
deno_io = "0.119.0"
|
||||
deno_fs = "0.119.0"
|
||||
deno_net = "0.201.0"
|
||||
deno_core = "0.352.0"
|
||||
# deno_* pin set: the crates deno published together on 2026-09-16 (deno_core
|
||||
# 0.412).
|
||||
# V8 is pinned exactly because flake.nix fetches the prebuilt archive of that
|
||||
# version: update its version and hash there with any bump. No default features,
|
||||
# so the pin leaves the feature set deno_core picks untouched.
|
||||
v8 = { version = "=150.4.0", default-features = false }
|
||||
# These release as a matched set: each pins its siblings, deno_error exactly,
|
||||
# and deno_tls pins rustls exactly (=0.23.40, the workspace `rustls` pin below
|
||||
# must equal it). A bump moves all of them at once.
|
||||
# deno_ast is independent of that set and stays at 0.51.0: a newer one pulls
|
||||
# swc_common 17, which would force bumping every swc_* pin the parsers use.
|
||||
deno_fetch = "0.283.0"
|
||||
deno_tls = "0.246.0"
|
||||
deno_webidl = "0.259.0"
|
||||
deno_web = "0.290.0"
|
||||
deno_crypto = "0.273.0"
|
||||
deno_net = "0.251.0"
|
||||
deno_core = "0.412.0"
|
||||
deno_ast = { version = "=0.51.0", features = ["transpiling"] }
|
||||
deno_permissions = "0.68.0"
|
||||
deno_telemetry = "0.31.0"
|
||||
deno_error = "=0.6.1"
|
||||
deno_permissions = "0.118.0"
|
||||
deno_telemetry = "0.81.0"
|
||||
deno_error = "=0.7.1"
|
||||
sys_traits = { version = "0.1.28", features = ["real", "libc", "winapi"] }
|
||||
rustls-pemfile = "2.2.0"
|
||||
|
||||
# only used with special deno_core_mac feature to prevent ffi issue on macos, requires libffi to be installed
|
||||
@@ -682,7 +680,7 @@ aws-sdk-sqs = "^1.89.0"
|
||||
aws-sdk-sts = "^1.91.0"
|
||||
aws-sdk-sso = "=1.77.0"
|
||||
aws-sdk-ssooidc = "=1.78.0"
|
||||
rustls = "=0.23.35"
|
||||
rustls = "=0.23.40"
|
||||
async-once-cell = "0.5.4"
|
||||
size = "0.5.0"
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
5f5d03a47e58d1ea1e3970ad141ec24bd6c5f068
|
||||
06610ecc950ed64a2709d7f1aa8c7b49a95fdbae
|
||||
|
||||
@@ -11,6 +11,6 @@ path = "./src/lib.rs"
|
||||
[dependencies]
|
||||
windmill-parser.workspace = true
|
||||
windmill-parser-sql-asset.workspace = true
|
||||
rustpython-parser.workspace = true
|
||||
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
|
||||
ruff_python_parser.workspace = true
|
||||
ruff_python_ast.workspace = true
|
||||
anyhow.workspace = true
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use rustpython_ast::{Constant, Expr, ExprConstant, Visitor};
|
||||
use rustpython_parser::{ast::Suite, Parse};
|
||||
use ruff_python_ast::{
|
||||
visitor::{walk_expr, walk_stmt, Visitor},
|
||||
Expr, ExprCall, ExprDict, FStringPartRef, InterpolatedStringElement, Keyword, Stmt, StmtAssign,
|
||||
};
|
||||
use ruff_python_parser::parse_module;
|
||||
use std::collections::HashMap;
|
||||
use windmill_parser::asset_parser::{
|
||||
asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind,
|
||||
@@ -8,12 +11,12 @@ use windmill_parser::asset_parser::{
|
||||
use AssetUsageAccessType::*;
|
||||
|
||||
pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
|
||||
let ast = Suite::parse(input, "main.py")
|
||||
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
|
||||
let ast = parse_module(input)
|
||||
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
|
||||
.into_suite();
|
||||
|
||||
let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() };
|
||||
ast.into_iter()
|
||||
.for_each(|stmt| assets_finder.visit_stmt(stmt));
|
||||
assets_finder.visit_body(&ast);
|
||||
|
||||
for (kind, path, _) in assets_finder.var_identifiers.into_values() {
|
||||
// if a db = wmill.datatable() was never used (e.g db.query(...)),
|
||||
@@ -43,13 +46,72 @@ struct AssetsFinder {
|
||||
var_identifiers: HashMap<String, (AssetKind, VarAssetName, VarAssetSchema)>,
|
||||
}
|
||||
|
||||
impl Visitor for AssetsFinder {
|
||||
impl<'a> Visitor<'a> for AssetsFinder {
|
||||
fn visit_stmt(&mut self, stmt: &'a Stmt) {
|
||||
if let Stmt::Assign(node) = stmt {
|
||||
self.track_assignment(node);
|
||||
}
|
||||
// Continue with generic visit to catch any other assets in the statement
|
||||
walk_stmt(self, stmt);
|
||||
}
|
||||
|
||||
fn visit_expr(&mut self, expr: &'a Expr) {
|
||||
match expr {
|
||||
// A detected asset call does not recurse,
|
||||
// so this is only reached when no further context was found
|
||||
Expr::StringLiteral(s) => self.visit_str(s.value.to_str()),
|
||||
Expr::FString(f) => {
|
||||
// Literal text between two interpolations reads as one string, across the
|
||||
// pieces of an implicitly concatenated f-string too.
|
||||
let mut literal = String::new();
|
||||
for part in &f.value {
|
||||
match part {
|
||||
FStringPartRef::Literal(s) => literal.push_str(&s.value),
|
||||
FStringPartRef::FString(f) => {
|
||||
for element in &f.elements {
|
||||
match element {
|
||||
InterpolatedStringElement::Literal(l) => {
|
||||
literal.push_str(&l.value)
|
||||
}
|
||||
InterpolatedStringElement::Interpolation(_) => {
|
||||
self.visit_str(&std::mem::take(&mut literal))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.visit_str(&literal);
|
||||
walk_expr(self, expr);
|
||||
}
|
||||
Expr::Call(node) => {
|
||||
if self.visit_expr_call_inner(node).is_err() {
|
||||
walk_expr(self, expr);
|
||||
}
|
||||
}
|
||||
_ => walk_expr(self, expr),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AssetsFinder {
|
||||
fn visit_str(&mut self, s: &str) {
|
||||
if let Some((kind, path)) = parse_asset_syntax(s, false) {
|
||||
self.assets.push(ParseAssetsResult {
|
||||
kind,
|
||||
path: path.to_string(),
|
||||
access_type: None,
|
||||
columns: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Handle assignment statements like: x = wmill.datatable('name')
|
||||
fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) {
|
||||
fn track_assignment(&mut self, node: &StmtAssign) {
|
||||
// Check if the value is a call to a tracked function
|
||||
if let Some(Expr::Name(expr_name)) = node.targets.first() {
|
||||
// Remove any tracked variables with that name in case of reassignment
|
||||
let Ok(var_name) = expr_name.id.parse::<String>();
|
||||
let var_name = expr_name.id.to_string();
|
||||
let removed = self.var_identifiers.remove(&var_name);
|
||||
// if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)),
|
||||
// we still want to register the asset as unknown access type
|
||||
@@ -69,59 +131,12 @@ impl Visitor for AssetsFinder {
|
||||
|
||||
if let Some((kind, name, schema)) = self.extract_asset_from_call(&node.value) {
|
||||
// Track target variable
|
||||
let Ok(var_name) = expr_name.id.parse::<String>();
|
||||
self.var_identifiers
|
||||
.insert(var_name, (kind.clone(), name.clone(), schema.clone()));
|
||||
}
|
||||
}
|
||||
// Continue with generic visit to catch any other assets in the expression
|
||||
self.generic_visit_stmt_assign(node);
|
||||
}
|
||||
|
||||
// visit_call_expr will not recurse if it detects an asset,
|
||||
// so this will only be called when no further context was found
|
||||
fn visit_expr_constant(&mut self, node: ExprConstant) {
|
||||
match node.value {
|
||||
Constant::Str(s) => {
|
||||
if let Some((kind, path)) = parse_asset_syntax(&s, false) {
|
||||
self.assets.push(ParseAssetsResult {
|
||||
kind,
|
||||
path: path.to_string(),
|
||||
access_type: None,
|
||||
columns: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
_ => self.generic_visit_expr_constant(node),
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_expr_call(&mut self, node: rustpython_ast::ExprCall) {
|
||||
match self.visit_expr_call_inner(&node) {
|
||||
Ok(_) => {}
|
||||
Err(_) => {
|
||||
// Check keyword arguments for assets before falling back to generic visit
|
||||
for keyword in &node.keywords {
|
||||
if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) =
|
||||
&keyword.value
|
||||
{
|
||||
if let Some((kind, path)) = parse_asset_syntax(s, false) {
|
||||
self.assets.push(ParseAssetsResult {
|
||||
kind,
|
||||
path: path.to_string(),
|
||||
access_type: None,
|
||||
columns: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
self.generic_visit_expr_call(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AssetsFinder {
|
||||
/// Extract asset info from calls like wmill.datatable('name'), wmill.ducklake('name'), etc.
|
||||
fn extract_asset_from_call(
|
||||
&self,
|
||||
@@ -147,18 +162,20 @@ impl AssetsFinder {
|
||||
|
||||
// Get the first argument (the asset name)
|
||||
let name = call
|
||||
.arguments
|
||||
.args
|
||||
.first()
|
||||
.or_else(|| {
|
||||
// Try keyword argument 'name' or 'path'
|
||||
call.keywords
|
||||
call.arguments
|
||||
.keywords
|
||||
.iter()
|
||||
.find(|kw| matches!(kw.arg.as_deref(), Some("name") | Some("path")))
|
||||
.map(|kw| &kw.value)
|
||||
})
|
||||
.and_then(|first_arg| {
|
||||
if let Expr::Constant(ExprConstant { value: Constant::Str(name), .. }) = first_arg {
|
||||
Some(name.clone())
|
||||
if let Expr::StringLiteral(name) = first_arg {
|
||||
Some(name.value.to_str().to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -179,44 +196,39 @@ impl AssetsFinder {
|
||||
Some((kind, name, schema))
|
||||
}
|
||||
|
||||
fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> {
|
||||
fn visit_expr_call_inner(&mut self, node: &ExprCall) -> Result<(), ()> {
|
||||
let ident: String = node
|
||||
.func
|
||||
.as_name_expr()
|
||||
.and_then(|o| o.id.parse().ok())
|
||||
.map(|o| o.id.to_string())
|
||||
.or_else(|| {
|
||||
node.func
|
||||
.as_attribute_expr()
|
||||
.and_then(|attr| attr.attr.parse().ok())
|
||||
.map(|attr| attr.attr.to_string())
|
||||
})
|
||||
.ok_or(())?;
|
||||
|
||||
// `obj_name` is the object (receiver) - i.e., `obj` in `obj.method()`
|
||||
let obj_name: String = if let Expr::Attribute(rustpython_ast::ExprAttribute {
|
||||
value, ..
|
||||
}) = node.func.as_ref()
|
||||
{
|
||||
if let Expr::Name(expr_name) = value.as_ref() {
|
||||
expr_name.id.parse().map_err(|_| ())?
|
||||
} else {
|
||||
return Err(());
|
||||
}
|
||||
} else {
|
||||
return Err(());
|
||||
};
|
||||
let obj_name: String = node
|
||||
.func
|
||||
.as_attribute_expr()
|
||||
.and_then(|attr| attr.value.as_name_expr())
|
||||
.map(|name| name.id.to_string())
|
||||
.ok_or(())?;
|
||||
|
||||
if obj_name == "wmill" {
|
||||
// Continue
|
||||
} else if let Some((kind, ref path, ref schema)) = self.var_identifiers.get(&obj_name) {
|
||||
if ident == "query" {
|
||||
let expr_name = node.args.get(0).or_else(|| {
|
||||
node.keywords
|
||||
let expr_name = node.arguments.args.get(0).or_else(|| {
|
||||
node.arguments
|
||||
.keywords
|
||||
.iter()
|
||||
.find(|kw| kw.arg.as_deref() == Some("name"))
|
||||
.map(|kw| &kw.value)
|
||||
});
|
||||
let sql = match expr_name {
|
||||
Some(Expr::Constant(ExprConstant { value: Constant::Str(sql), .. })) => sql,
|
||||
Some(Expr::StringLiteral(sql)) => sql.value.to_str(),
|
||||
_ => return Err(()),
|
||||
};
|
||||
// We use the SQL parser to detect RW, specific tables, etc.
|
||||
@@ -224,7 +236,7 @@ impl AssetsFinder {
|
||||
*kind,
|
||||
path,
|
||||
schema.as_deref(),
|
||||
&sql,
|
||||
sql,
|
||||
);
|
||||
match sql_assets {
|
||||
Ok(Some(sql_assets)) => self.assets.extend(sql_assets),
|
||||
@@ -252,9 +264,10 @@ impl AssetsFinder {
|
||||
};
|
||||
|
||||
let arg_val = match arg {
|
||||
Arg(i, name) => node.args.get(i).or_else(|| {
|
||||
Arg(i, name) => node.arguments.args.get(i).or_else(|| {
|
||||
// Get arg by name
|
||||
node.keywords
|
||||
node.arguments
|
||||
.keywords
|
||||
.iter()
|
||||
.find(|kw| kw.arg.as_deref() == Some(name))
|
||||
.map(|kw| &kw.value)
|
||||
@@ -270,8 +283,9 @@ impl AssetsFinder {
|
||||
self.assets
|
||||
.push(ParseAssetsResult { kind, path, access_type, columns: None });
|
||||
}
|
||||
Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => {
|
||||
let path = parse_asset_syntax(&value, false)
|
||||
Some(Expr::StringLiteral(value)) => {
|
||||
let value = value.value.to_str();
|
||||
let path = parse_asset_syntax(value, false)
|
||||
.map(|(_, p)| p)
|
||||
.unwrap_or(std::borrow::Cow::Borrowed(value));
|
||||
self.assets.push(ParseAssetsResult {
|
||||
@@ -291,26 +305,24 @@ impl AssetsFinder {
|
||||
struct Arg(usize, &'static str);
|
||||
|
||||
/// Extract a string-literal keyword argument, e.g. `s3="value"` in a call.
|
||||
fn keyword_str_value(keywords: &[rustpython_ast::Keyword], name: &str) -> Option<String> {
|
||||
fn keyword_str_value(keywords: &[Keyword], name: &str) -> Option<String> {
|
||||
keywords
|
||||
.iter()
|
||||
.find(|kw| kw.arg.as_deref() == Some(name))
|
||||
.and_then(|kw| match &kw.value {
|
||||
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => Some(s.clone()),
|
||||
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Extract a string-literal value from a dict literal, e.g. `{"s3": "value"}`.
|
||||
fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option<String> {
|
||||
dict.keys
|
||||
fn dict_str_value(dict: &ExprDict, name: &str) -> Option<String> {
|
||||
dict.items
|
||||
.iter()
|
||||
.zip(dict.values.iter())
|
||||
.find_map(|(key, value)| match (key.as_ref()?, value) {
|
||||
(
|
||||
Expr::Constant(ExprConstant { value: Constant::Str(k), .. }),
|
||||
Expr::Constant(ExprConstant { value: Constant::Str(v), .. }),
|
||||
) if k.as_str() == name => Some(v.clone()),
|
||||
.find_map(|item| match (item.key.as_ref()?, &item.value) {
|
||||
(Expr::StringLiteral(k), Expr::StringLiteral(v)) if k.value.to_str() == name => {
|
||||
Some(v.value.to_str().to_string())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
@@ -328,12 +340,13 @@ fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option<String>
|
||||
/// matches the TS object form and the `# on s3:///…` trigger form exactly.
|
||||
fn s3_object_arg_path(expr: &Expr) -> Option<String> {
|
||||
let uri = match expr {
|
||||
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => {
|
||||
Expr::StringLiteral(s) => {
|
||||
let s = s.value.to_str();
|
||||
match s
|
||||
.strip_prefix("s3://")
|
||||
.and_then(|rest| rest.split_once('/'))
|
||||
{
|
||||
Some((_, key)) if !key.is_empty() => s.clone(),
|
||||
Some((_, key)) if !key.is_empty() => s.to_string(),
|
||||
_ => return None,
|
||||
}
|
||||
}
|
||||
@@ -347,8 +360,9 @@ fn s3_object_arg_path(expr: &Expr) -> Option<String> {
|
||||
if func_name != "S3Object" {
|
||||
return None;
|
||||
}
|
||||
let key = keyword_str_value(&call.keywords, "s3")?;
|
||||
let storage = keyword_str_value(&call.keywords, "storage").unwrap_or_default();
|
||||
let key = keyword_str_value(&call.arguments.keywords, "s3")?;
|
||||
let storage =
|
||||
keyword_str_value(&call.arguments.keywords, "storage").unwrap_or_default();
|
||||
format!("s3://{storage}/{key}")
|
||||
}
|
||||
Expr::Dict(dict) => {
|
||||
@@ -822,4 +836,62 @@ def main():
|
||||
},])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_py_asset_parser_recent_syntax() {
|
||||
let input = r#"
|
||||
import wmill
|
||||
lazy import polars
|
||||
|
||||
type Rows = list[dict]
|
||||
|
||||
def main[T](name: str):
|
||||
try:
|
||||
with wmill.load_s3_file_reader('s3:///in.csv') as f:
|
||||
f.read()
|
||||
except ValueError, TypeError:
|
||||
pass
|
||||
greeting = t"hello {name}"
|
||||
return f"{wmill.get_resource("u/user/pg")["host"]}"
|
||||
"#;
|
||||
let s = parse_assets(input).map(|o| o.assets);
|
||||
assert_eq!(
|
||||
s.map_err(|e| e.to_string()),
|
||||
Ok(vec![
|
||||
ParseAssetsResult {
|
||||
kind: AssetKind::S3Object,
|
||||
path: "/in.csv".to_string(),
|
||||
access_type: Some(R),
|
||||
columns: None,
|
||||
},
|
||||
ParseAssetsResult {
|
||||
kind: AssetKind::Resource,
|
||||
path: "u/user/pg".to_string(),
|
||||
access_type: None,
|
||||
columns: None,
|
||||
},
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_py_asset_parser_schema_qualified_db_without_table() {
|
||||
let input = r#"
|
||||
import wmill
|
||||
def main():
|
||||
db = wmill.datatable('dt:sch')
|
||||
db.query("ATTACH 'ducklake://other' AS o")
|
||||
db.query("SELECT * FROM t")
|
||||
"#;
|
||||
let s = parse_assets(input).map(|o| o.assets);
|
||||
assert_eq!(
|
||||
s.map_err(|e| e.to_string())
|
||||
.map(|a| a.into_iter().map(|a| (a.kind, a.path)).collect::<Vec<_>>()),
|
||||
Ok(vec![
|
||||
(AssetKind::DataTable, "dt".to_string()),
|
||||
(AssetKind::DataTable, "dt/sch.t".to_string()),
|
||||
(AssetKind::Ducklake, "other".to_string()),
|
||||
])
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,8 @@ tracing.workspace = true
|
||||
|
||||
[dependencies]
|
||||
windmill-parser.workspace = true
|
||||
rustpython-parser.workspace = true
|
||||
ruff_python_parser.workspace = true
|
||||
ruff_python_ast.workspace = true
|
||||
phf.workspace = true
|
||||
itertools.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -13,9 +13,9 @@ use async_recursion::async_recursion;
|
||||
use itertools::Itertools;
|
||||
use lazy_static::lazy_static;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::str::FromStr;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::collections::HashMap;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::str::FromStr;
|
||||
|
||||
use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
@@ -23,11 +23,8 @@ use regex::Regex;
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
use regex_lite::Regex;
|
||||
|
||||
use rustpython_parser::{
|
||||
ast::{Stmt, StmtImport, StmtImportFrom, Suite},
|
||||
text_size::TextRange,
|
||||
Parse,
|
||||
};
|
||||
use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom};
|
||||
use ruff_python_parser::parse_module;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use sqlx::{Pool, Postgres};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
@@ -160,6 +157,7 @@ pub struct ImportPin {
|
||||
}
|
||||
|
||||
pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImport>> {
|
||||
let full_code = code;
|
||||
// Use regex to safely find the main function definition
|
||||
let mut code = DEF_MAIN_RE
|
||||
.split(code)
|
||||
@@ -186,16 +184,22 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
|
||||
// This is needed because we've split off the real main function above
|
||||
let code_with_fake_main = format!("{}\n\ndef main(): pass", code);
|
||||
|
||||
let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| {
|
||||
anyhow::anyhow!("Error parsing code for imports: {}", e.to_string())
|
||||
})?;
|
||||
let (code, ast) = match parse_module(&code_with_fake_main) {
|
||||
Ok(parsed) => (code, parsed.into_suite()),
|
||||
// The split lands on any line reading like the definition of `main`, one inside a
|
||||
// string included: what precedes it is then not a module, while the whole file is.
|
||||
Err(e) => match parse_module(full_code) {
|
||||
Ok(parsed) => (full_code.to_string(), parsed.into_suite()),
|
||||
Err(_) => anyhow::bail!("Error parsing code for imports: {}", e.to_string()),
|
||||
},
|
||||
};
|
||||
// Note: We're still using the original code for finding pins,
|
||||
// as the TextRange values from the parsed AST would be based on code_with_fake_main
|
||||
// but we want to match against the original code
|
||||
let find_pin = |range: TextRange, key: String| {
|
||||
let find_pin = |stmt_end: usize, key: String| {
|
||||
let hs = code
|
||||
.chars()
|
||||
.skip(range.end().to_usize())
|
||||
.skip(stmt_end)
|
||||
.take_while(|e| *e != '\n')
|
||||
.collect::<String>();
|
||||
|
||||
@@ -226,9 +230,9 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
|
||||
let mut nimports: Vec<NImport> = ast
|
||||
.into_iter()
|
||||
.filter_map(|x| match x {
|
||||
Stmt::Import(StmtImport { names, range }) => names
|
||||
Stmt::Import(StmtImport { names, range, .. }) => names
|
||||
.get(0)
|
||||
.and_then(|al| find_pin(range, al.name.to_string()))
|
||||
.and_then(|al| find_pin(range.end().to_usize(), al.name.to_string()))
|
||||
.or(Some(
|
||||
names
|
||||
.into_iter()
|
||||
@@ -239,15 +243,11 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
|
||||
.flatten()
|
||||
.collect::<Vec<NImport>>(),
|
||||
)),
|
||||
Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => {
|
||||
Some(process_import(
|
||||
module.map(|x| x.to_string()),
|
||||
path,
|
||||
i.to_usize(),
|
||||
))
|
||||
}
|
||||
Stmt::ImportFrom(StmtImportFrom { level, module, .. }) if level > 0 => Some(
|
||||
process_import(module.map(|x| x.to_string()), path, level as usize),
|
||||
),
|
||||
Stmt::ImportFrom(StmtImportFrom { level: _, module, range, .. }) => find_pin(
|
||||
range,
|
||||
range.end().to_usize(),
|
||||
module.clone().map(|x| x.to_string()).unwrap_or_default(),
|
||||
)
|
||||
.or(Some(process_import(module.map(|x| x.to_string()), path, 0))),
|
||||
@@ -351,13 +351,16 @@ async fn parse_python_imports_inner(
|
||||
temp_script_refs: &Option<HashMap<String, String>>,
|
||||
) -> error::Result<HashMap<String, NImportResolved>> {
|
||||
tracing::debug!("Parsing python imports for path: {}", path);
|
||||
let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code);
|
||||
let PythonAnnotations { py310, py311, py312, py313, py314, py315, .. } =
|
||||
PythonAnnotations::parse(&code);
|
||||
tracing::debug!(
|
||||
"Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}",
|
||||
"Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}, py314: {}, py315: {}",
|
||||
py310,
|
||||
py311,
|
||||
py312,
|
||||
py313
|
||||
py313,
|
||||
py314,
|
||||
py315
|
||||
);
|
||||
|
||||
let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> {
|
||||
@@ -373,6 +376,8 @@ async fn parse_python_imports_inner(
|
||||
push_version_specifiers(py311, "==3.11.*".to_owned())?;
|
||||
push_version_specifiers(py312, "==3.12.*".to_owned())?;
|
||||
push_version_specifiers(py313, "==3.13.*".to_owned())?;
|
||||
push_version_specifiers(py314, "==3.14.*".to_owned())?;
|
||||
push_version_specifiers(py315, "==3.15.*".to_owned())?;
|
||||
|
||||
for x in code.lines() {
|
||||
if x.starts_with("# py:") || x.starts_with("#py:") {
|
||||
@@ -513,8 +518,14 @@ async fn parse_python_imports_inner(
|
||||
let mut nested = match n {
|
||||
NImport::Relative(rpath) => {
|
||||
// First try to get content from temp_script_refs cache if available
|
||||
let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) {
|
||||
tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash);
|
||||
let code_from_cache = if let Some(hash) =
|
||||
temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath))
|
||||
{
|
||||
tracing::debug!(
|
||||
"Found relative import '{}' in temp_script_refs with hash '{}'",
|
||||
rpath,
|
||||
hash
|
||||
);
|
||||
match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await {
|
||||
Ok(content) => Some(content),
|
||||
Err(e) => {
|
||||
@@ -529,19 +540,17 @@ async fn parse_python_imports_inner(
|
||||
// Use cached content if available, otherwise fall back to deployed script
|
||||
let code = match code_from_cache {
|
||||
Some(content) => content,
|
||||
None => {
|
||||
sqlx::query_scalar!(
|
||||
r#"
|
||||
None => sqlx::query_scalar!(
|
||||
r#"
|
||||
SELECT content FROM script WHERE path = $1 AND workspace_id = $2
|
||||
AND archived = false ORDER BY created_at DESC LIMIT 1
|
||||
"#,
|
||||
&rpath,
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
.unwrap_or_else(|| "".to_string())
|
||||
}
|
||||
&rpath,
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
.unwrap_or_else(|| "".to_string()),
|
||||
};
|
||||
|
||||
if already_visited.contains(&rpath) {
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
mod tests {
|
||||
|
||||
use sqlx::{Pool, Postgres};
|
||||
use windmill_parser_py_imports::parse_python_imports;
|
||||
use windmill_parser_py_imports::{
|
||||
parse_code_for_imports, parse_python_imports, ImportPin, NImport,
|
||||
};
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_parse_python_imports(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
@@ -117,4 +119,87 @@ def main():
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn auto(pkg: &str) -> NImport {
|
||||
NImport::Auto { pkg: pkg.to_string(), key: None }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_lazy_imports() -> anyhow::Result<()> {
|
||||
let code = "
|
||||
lazy import requests
|
||||
lazy import numpy # pin: numpy==2.1.0
|
||||
lazy from pandas import DataFrame
|
||||
lazy from f.shared import helper
|
||||
lazy = 1
|
||||
|
||||
def main():
|
||||
pass
|
||||
";
|
||||
assert_eq!(
|
||||
parse_code_for_imports(code, "f/foo/bar")?,
|
||||
vec![
|
||||
NImport::Pin {
|
||||
pins: vec![ImportPin {
|
||||
pkg: "numpy==2.1.0".to_string(),
|
||||
path: "f/foo/bar".to_string()
|
||||
}],
|
||||
key: "numpy".to_string()
|
||||
},
|
||||
auto("pandas"),
|
||||
auto("requests"),
|
||||
NImport::Relative("f/shared".to_string()),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_imports_recent_syntax_before_main() -> anyhow::Result<()> {
|
||||
let code = r#"
|
||||
import requests
|
||||
|
||||
type Alias = list[int]
|
||||
NAME = "x"
|
||||
D = {"a": 1}
|
||||
L = [[1], [2]]
|
||||
TEMPLATE = t"hello {NAME}"
|
||||
PEP701 = f"{D["a"]} {'\n'.join(["x"])}"
|
||||
PEP798 = [*a for a in L]
|
||||
|
||||
try:
|
||||
import ujson
|
||||
except ImportError, ValueError:
|
||||
pass
|
||||
|
||||
def main():
|
||||
pass
|
||||
"#;
|
||||
assert_eq!(
|
||||
parse_code_for_imports(code, "f/foo/bar")?,
|
||||
vec![auto("requests")]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_imports_main_lookalike_in_docstring() -> anyhow::Result<()> {
|
||||
for example in ["def main[T](x: T):", "def main(x):"] {
|
||||
let code = format!(
|
||||
"\"\"\"Usage example:\n{example}\n return x\n\"\"\"\nimport requests # pin: requests==2.32.0\n\ndef main(value: str):\n return value\n"
|
||||
);
|
||||
assert_eq!(
|
||||
parse_code_for_imports(&code, "f/foo/bar")?,
|
||||
vec![NImport::Pin {
|
||||
pins: vec![ImportPin {
|
||||
pkg: "requests==2.32.0".to_string(),
|
||||
path: "f/foo/bar".to_string()
|
||||
}],
|
||||
key: "requests".to_string()
|
||||
}],
|
||||
"{code}"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,8 +10,8 @@ path = "./src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
windmill-parser.workspace = true
|
||||
rustpython-parser.workspace = true
|
||||
ruff_python_parser.workspace = true
|
||||
ruff_python_ast.workspace = true
|
||||
itertools.workspace = true
|
||||
serde_json.workspace = true
|
||||
anyhow.workspace = true
|
||||
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
|
||||
|
||||
@@ -13,13 +13,11 @@ use itertools::Itertools;
|
||||
use serde_json::json;
|
||||
use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectType, Typ};
|
||||
|
||||
use rustpython_parser::{
|
||||
ast::{
|
||||
Constant, Expr, ExprAttribute, ExprConstant, ExprDict, ExprList, ExprName, Stmt,
|
||||
StmtAssign, StmtAsyncFunctionDef, StmtClassDef, StmtFunctionDef, Suite,
|
||||
},
|
||||
Parse,
|
||||
use ruff_python_ast::{
|
||||
Expr, ExprAttribute, ExprDict, ExprList, ExprName, Number, Parameters, Stmt, StmtAssign,
|
||||
StmtClassDef, StmtFunctionDef,
|
||||
};
|
||||
use ruff_python_parser::parse_module;
|
||||
|
||||
pub mod pydantic_parser;
|
||||
|
||||
@@ -81,22 +79,24 @@ fn should_parse_for_models(code: &str) -> bool {
|
||||
|| code.contains("import dataclasses")
|
||||
}
|
||||
|
||||
/// What follows `def <name>` or `async def <name>` on a line opening that function's
|
||||
/// parameters or type parameters.
|
||||
fn strip_def<'a>(line: &'a str, name: &str) -> Option<&'a str> {
|
||||
let rest = line
|
||||
.strip_prefix("async def ")
|
||||
.or_else(|| line.strip_prefix("def "))?
|
||||
.strip_prefix(name)?;
|
||||
(rest.starts_with('(') || rest.starts_with('[')).then_some(rest)
|
||||
}
|
||||
|
||||
fn filter_non_main(code: &str, main_name: &str) -> String {
|
||||
let def_main = format!("def {}(", main_name);
|
||||
let async_def_main = format!("async def {}(", main_name);
|
||||
let mut filtered_code = String::new();
|
||||
let mut code_iter = code.split("\n");
|
||||
let mut remaining: String = String::new();
|
||||
while let Some(line) = code_iter.next() {
|
||||
if line.starts_with(&async_def_main) {
|
||||
filtered_code += &async_def_main;
|
||||
remaining += line.strip_prefix(&async_def_main).unwrap();
|
||||
remaining += "\n";
|
||||
remaining += &code_iter.join("\n");
|
||||
break;
|
||||
} else if line.starts_with(&def_main) {
|
||||
filtered_code += &def_main;
|
||||
remaining += line.strip_prefix(&def_main).unwrap();
|
||||
if let Some(rest) = strip_def(line, main_name) {
|
||||
filtered_code += &line[..line.len() - rest.len()];
|
||||
remaining += rest;
|
||||
remaining += "\n";
|
||||
remaining += &code_iter.join("\n");
|
||||
break;
|
||||
@@ -106,6 +106,26 @@ fn filter_non_main(code: &str, main_name: &str) -> String {
|
||||
return String::new();
|
||||
}
|
||||
let mut chars = remaining.chars();
|
||||
|
||||
// PEP 695 type parameters, `def main[T](x: T)`, sit between the name and the parameters.
|
||||
if remaining.starts_with('[') {
|
||||
let mut open_brackets = 0;
|
||||
for c in chars.by_ref() {
|
||||
if c == '[' {
|
||||
open_brackets += 1;
|
||||
} else if c == ']' {
|
||||
open_brackets -= 1;
|
||||
}
|
||||
filtered_code.push(c);
|
||||
if open_brackets == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if chars.next() != Some('(') {
|
||||
return String::new();
|
||||
}
|
||||
filtered_code.push('(');
|
||||
let mut open_parens = 1;
|
||||
|
||||
while let Some(c) = chars.next() {
|
||||
@@ -183,15 +203,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
|
||||
|
||||
let filtered_code = filter_relevant_statements(code);
|
||||
|
||||
let ast = match Suite::parse(&filtered_code, "main.py") {
|
||||
Ok(ast) => ast,
|
||||
let ast = match parse_module(&filtered_code) {
|
||||
Ok(parsed) => parsed.into_suite(),
|
||||
Err(_) => return CodeMetadata { enums, descriptions },
|
||||
};
|
||||
|
||||
for stmt in ast {
|
||||
match stmt {
|
||||
Stmt::ClassDef(StmtClassDef { name, body, bases, .. }) if has_enum => {
|
||||
let is_enum = bases.iter().any(|base| {
|
||||
Stmt::ClassDef(class_def) if has_enum => {
|
||||
let is_enum = class_def.bases().iter().any(|base| {
|
||||
matches!(base, Expr::Name(ExprName { id, .. })
|
||||
if id == "Enum" || id == "IntEnum" || id == "StrEnum"
|
||||
|| id == "Flag" || id == "IntFlag")
|
||||
@@ -201,17 +221,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
|
||||
let mut values = Vec::new();
|
||||
let mut members = HashMap::new();
|
||||
|
||||
let StmtClassDef { name, body, .. } = class_def;
|
||||
for item in body {
|
||||
if let Stmt::Assign(StmtAssign { targets, value, .. }) = item {
|
||||
if let Some(Expr::Name(ExprName { id: target_name, .. })) =
|
||||
targets.first()
|
||||
{
|
||||
if !target_name.starts_with('_') {
|
||||
if let Expr::Constant(ExprConstant {
|
||||
value: Constant::Str(val),
|
||||
..
|
||||
}) = value.as_ref()
|
||||
{
|
||||
if let Expr::StringLiteral(val) = value.as_ref() {
|
||||
let val = val.value.to_str();
|
||||
values.push(val.to_string());
|
||||
members.insert(target_name.to_string(), val.to_string());
|
||||
}
|
||||
@@ -225,15 +243,13 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
|
||||
}
|
||||
}
|
||||
}
|
||||
Stmt::FunctionDef(StmtFunctionDef { name: func_name, body, .. }) if has_docstring => {
|
||||
if &func_name == main_name {
|
||||
Stmt::FunctionDef(StmtFunctionDef {
|
||||
name: func_name, body, is_async: false, ..
|
||||
}) if has_docstring => {
|
||||
if func_name.as_str() == main_name {
|
||||
if let Some(Stmt::Expr(expr_stmt)) = body.first() {
|
||||
if let Expr::Constant(ExprConstant {
|
||||
value: Constant::Str(docstring),
|
||||
..
|
||||
}) = expr_stmt.value.as_ref()
|
||||
{
|
||||
descriptions = parse_docstring_args(docstring);
|
||||
if let Expr::StringLiteral(docstring) = expr_stmt.value.as_ref() {
|
||||
descriptions = parse_docstring_args(docstring.value.to_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -294,6 +310,27 @@ fn parse_docstring_args(docstring: &str) -> HashMap<String, String> {
|
||||
descriptions
|
||||
}
|
||||
|
||||
/// A line that reads as the definition of `main_name` can also sit inside a string, a docstring
|
||||
/// showing an example for instance. When several lines do, only the parsed module tells which
|
||||
/// one is the function.
|
||||
fn main_params_among_lookalikes(code: &str, main_name: &str) -> Option<Parameters> {
|
||||
code.split('\n')
|
||||
.filter(|line| strip_def(line, main_name).is_some())
|
||||
.nth(1)?;
|
||||
parse_module(code)
|
||||
.ok()?
|
||||
.into_suite()
|
||||
.into_iter()
|
||||
.find_map(|x| match x {
|
||||
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
|
||||
if name.as_str() == main_name =>
|
||||
{
|
||||
Some(*parameters)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
/// skip_params is a micro optimization for when we just want to find the main
|
||||
/// function without parsing all the params.
|
||||
pub fn parse_python_signature(
|
||||
@@ -310,41 +347,39 @@ pub fn parse_python_signature(
|
||||
// - If no models: parse only the filtered main function
|
||||
let (params, module) = if should_parse_for_models(code) {
|
||||
// Parse full code once for both Pydantic detection and signature extraction
|
||||
let ast = Suite::parse(code, "main.py")
|
||||
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
|
||||
let ast = parse_module(code)
|
||||
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
|
||||
.into_suite();
|
||||
|
||||
// Extract main function from full AST
|
||||
let params = ast.iter().find_map(|x| match x {
|
||||
Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if name == &main_name => {
|
||||
Some(args.as_ref().clone())
|
||||
}
|
||||
Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. })
|
||||
if name == &main_name =>
|
||||
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
|
||||
if name.as_str() == main_name =>
|
||||
{
|
||||
Some(args.as_ref().clone())
|
||||
Some(parameters.as_ref().clone())
|
||||
}
|
||||
_ => None,
|
||||
});
|
||||
|
||||
// Keep AST for Pydantic/dataclass detection
|
||||
(params, Some(ast))
|
||||
} else if let Some(params) = main_params_among_lookalikes(code, &main_name) {
|
||||
(Some(params), None)
|
||||
} else {
|
||||
// No models detected - parse only the filtered main function
|
||||
let filtered_code = filter_non_main(code, &main_name);
|
||||
if filtered_code.is_empty() {
|
||||
(None, None)
|
||||
} else {
|
||||
let ast = Suite::parse(&filtered_code, "main.py")
|
||||
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
|
||||
let ast = parse_module(&filtered_code)
|
||||
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
|
||||
.into_suite();
|
||||
|
||||
let params = ast.into_iter().find_map(|x| match x {
|
||||
Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => {
|
||||
Some(*args)
|
||||
}
|
||||
Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. })
|
||||
if &name == &main_name =>
|
||||
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
|
||||
if name.as_str() == main_name =>
|
||||
{
|
||||
Some(*args)
|
||||
Some(*parameters)
|
||||
}
|
||||
_ => None,
|
||||
});
|
||||
@@ -378,8 +413,6 @@ pub fn parse_python_signature(
|
||||
|
||||
if !skip_params && params.is_some() {
|
||||
let params = params.unwrap();
|
||||
let def_arg_start = params.args.len() - params.defaults().count();
|
||||
|
||||
// Two-pass approach for lazy metadata extraction:
|
||||
// Pass 1: Parse types without enum info to determine if metadata is needed
|
||||
// Pass 2: Re-parse unknown types with metadata only if necessary
|
||||
@@ -392,11 +425,11 @@ pub fn parse_python_signature(
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, x)| {
|
||||
let arg_name = x.as_arg().arg.to_string();
|
||||
let arg_name = x.parameter.name.to_string();
|
||||
let (typ, has_default) = x
|
||||
.as_arg()
|
||||
.parameter
|
||||
.annotation
|
||||
.as_ref()
|
||||
.as_deref()
|
||||
.map_or((Typ::Unknown, false), |e| {
|
||||
parse_expr(e, &empty_enums, module_ref)
|
||||
});
|
||||
@@ -423,21 +456,16 @@ pub fn parse_python_signature(
|
||||
.into_iter()
|
||||
.map(|(i, arg_name, mut typ, mut has_default)| {
|
||||
if matches!(typ, Typ::Resource(_)) && !metadata.enums.is_empty() {
|
||||
if let Some(annotation) = params.args[i].as_arg().annotation.as_ref() {
|
||||
if let Some(annotation) = params.args[i].parameter.annotation.as_deref() {
|
||||
(typ, has_default) =
|
||||
parse_expr(annotation, &metadata.enums, module_ref);
|
||||
}
|
||||
}
|
||||
|
||||
let default = if i >= def_arg_start {
|
||||
params
|
||||
.defaults()
|
||||
.nth(i - def_arg_start)
|
||||
.map(|expr| to_value(expr, &metadata.enums))
|
||||
.flatten()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let default = params.args[i]
|
||||
.default
|
||||
.as_deref()
|
||||
.and_then(|expr| to_value(expr, &metadata.enums));
|
||||
|
||||
let should_get_type_from_default = match &typ {
|
||||
Typ::Unknown => true,
|
||||
@@ -465,10 +493,10 @@ pub fn parse_python_signature(
|
||||
// debounce accumulation support. Falls back to docstring
|
||||
// description if no union array pattern is found.
|
||||
let union_otyp = params.args[i]
|
||||
.as_arg()
|
||||
.parameter
|
||||
.annotation
|
||||
.as_ref()
|
||||
.and_then(|ann| detect_py_union_array_otyp(ann.as_ref()));
|
||||
.as_deref()
|
||||
.and_then(detect_py_union_array_otyp);
|
||||
|
||||
Arg {
|
||||
otyp: union_otyp.or_else(|| {
|
||||
@@ -509,12 +537,8 @@ pub fn parse_python_signature(
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_expr(
|
||||
e: &Box<Expr>,
|
||||
enums: &HashMap<String, EnumInfo>,
|
||||
module: Option<&[Stmt]>,
|
||||
) -> (Typ, bool) {
|
||||
match e.as_ref() {
|
||||
fn parse_expr(e: &Expr, enums: &HashMap<String, EnumInfo>, module: Option<&[Stmt]>) -> (Typ, bool) {
|
||||
match e {
|
||||
Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref(), enums, module), false),
|
||||
Expr::Attribute(x) => {
|
||||
if let Some(name) = x.value.as_name_expr() {
|
||||
@@ -531,12 +555,9 @@ fn parse_expr(
|
||||
}
|
||||
}
|
||||
Expr::BinOp(x) => {
|
||||
if matches!(
|
||||
x.right.as_ref(),
|
||||
Expr::Constant(ExprConstant { value: Constant::None, .. })
|
||||
) {
|
||||
if x.right.is_none_literal_expr() {
|
||||
(parse_expr(&x.left, enums, module).0, true)
|
||||
} else if detect_py_union_array_otyp(e.as_ref()).is_some() {
|
||||
} else if detect_py_union_array_otyp(e).is_some() {
|
||||
// T | list[T] — parsed type is Unknown; otyp is set separately
|
||||
(Typ::Unknown, false)
|
||||
} else {
|
||||
@@ -552,7 +573,7 @@ fn parse_expr(
|
||||
.elts
|
||||
.iter()
|
||||
.map(|x| match x {
|
||||
Expr::Constant(c) => c.value.as_str().map(|x| x.to_string()),
|
||||
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
|
||||
_ => None,
|
||||
})
|
||||
.filter_map(|x| x)
|
||||
@@ -625,15 +646,26 @@ fn map_resource_name(x: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_json::Value> {
|
||||
fn to_value(et: &Expr, enums: &HashMap<String, EnumInfo>) -> Option<serde_json::Value> {
|
||||
match et {
|
||||
Expr::Constant(ExprConstant { value, .. }) => Some(constant_to_value(value)),
|
||||
Expr::Dict(ExprDict { keys, values, .. }) => {
|
||||
let v = keys
|
||||
.into_iter()
|
||||
.zip(values)
|
||||
.map(|(k, v)| {
|
||||
let key = k
|
||||
Expr::NoneLiteral(_) => Some(json!(null)),
|
||||
Expr::BooleanLiteral(b) => Some(json!(b.value)),
|
||||
Expr::StringLiteral(s) => Some(json!(s.value.to_str())),
|
||||
Expr::BytesLiteral(b) => Some(json!(b.value.bytes().collect::<Vec<u8>>())),
|
||||
Expr::NumberLiteral(n) => Some(match &n.value {
|
||||
Number::Int(i) => {
|
||||
serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number"))
|
||||
}
|
||||
Number::Float(f) => json!(f),
|
||||
Number::Complex { real, imag } => json!([real, imag]),
|
||||
}),
|
||||
Expr::EllipsisLiteral(_) => Some(json!("...")),
|
||||
Expr::Dict(ExprDict { items, .. }) => {
|
||||
let v = items
|
||||
.iter()
|
||||
.map(|item| {
|
||||
let key = item
|
||||
.key
|
||||
.as_ref()
|
||||
.map(|e| to_value(e, enums))
|
||||
.flatten()
|
||||
@@ -642,7 +674,7 @@ fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or_else(|| "no_key".to_string());
|
||||
(key, to_value(&v, enums))
|
||||
(key, to_value(&item.value, enums))
|
||||
})
|
||||
.collect::<HashMap<String, _>>();
|
||||
Some(json!(v))
|
||||
@@ -672,20 +704,6 @@ fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_
|
||||
}
|
||||
}
|
||||
|
||||
fn constant_to_value(c: &Constant) -> serde_json::Value {
|
||||
match c {
|
||||
Constant::None => json!(null),
|
||||
Constant::Bool(b) => json!(b),
|
||||
Constant::Str(s) => json!(s),
|
||||
Constant::Bytes(b) => json!(b),
|
||||
Constant::Int(i) => serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")),
|
||||
Constant::Tuple(t) => json!(t.iter().map(constant_to_value).collect::<Vec<_>>()),
|
||||
Constant::Float(f) => json!(f),
|
||||
Constant::Complex { real, imag } => json!([real, imag]),
|
||||
Constant::Ellipsis => json!("..."),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
@@ -1323,4 +1341,89 @@ def main(color: Color = Color.RED):
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_python_sig_type_params() -> anyhow::Result<()> {
|
||||
for code in [
|
||||
"def main[T](x: str, y: int = 3):\n return x\n",
|
||||
"async def main[T: (int, str), *Ts](x: str, y: int = 3):\n return x\n",
|
||||
// parsed whole rather than pre-filtered down to `main`
|
||||
"from dataclasses import dataclass\n\ndef main[T](x: str, y: int = 3):\n return x\n",
|
||||
] {
|
||||
let result = parse_python_signature(code, None, false)?;
|
||||
assert_eq!(result.auto_kind, None, "{code}");
|
||||
assert_eq!(
|
||||
result
|
||||
.args
|
||||
.iter()
|
||||
.map(|a| (a.name.as_str(), &a.typ, &a.default))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
("x", &Typ::Str(None), &None),
|
||||
("y", &Typ::Int, &Some(json!(3)))
|
||||
],
|
||||
"{code}"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// A script with a model class is parsed whole, so syntax anywhere in it reaches the parser.
|
||||
#[test]
|
||||
fn test_parse_python_sig_recent_syntax() -> anyhow::Result<()> {
|
||||
let code = r#"
|
||||
from dataclasses import dataclass
|
||||
lazy import requests
|
||||
lazy from pandas import DataFrame
|
||||
|
||||
type Alias = list[int]
|
||||
|
||||
@dataclass
|
||||
class Point:
|
||||
x: int
|
||||
|
||||
try:
|
||||
import a
|
||||
except ImportError, ValueError:
|
||||
pass
|
||||
|
||||
D = {"a": 1}
|
||||
L = [[1], [2]]
|
||||
PEP701 = f"{D["a"]} {'\n'.join(["x"])}"
|
||||
PEP798 = [*a for a in L]
|
||||
|
||||
def main(p: Point, name: str = "world"):
|
||||
return t"hello {name}"
|
||||
"#;
|
||||
let result = parse_python_signature(code, None, false)?;
|
||||
assert_eq!(result.args.len(), 2);
|
||||
assert_eq!(
|
||||
result.args[0].typ,
|
||||
Typ::Object(ObjectType::new(
|
||||
Some("Point".to_string()),
|
||||
Some(vec![windmill_parser::ObjectProperty {
|
||||
key: "x".to_string(),
|
||||
typ: Box::new(Typ::Int)
|
||||
}])
|
||||
))
|
||||
);
|
||||
assert_eq!(result.args[1].default, Some(json!("world")));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_python_sig_main_lookalike_in_docstring() -> anyhow::Result<()> {
|
||||
for example in ["def main[T](x: T):", "def main(x: int, y: int):"] {
|
||||
let code = format!(
|
||||
"\"\"\"Usage example:\n{example}\n return x\n\"\"\"\n\ndef main(value: str):\n return value\n"
|
||||
);
|
||||
let result = parse_python_signature(&code, None, false)?;
|
||||
assert_eq!(
|
||||
result.args.iter().map(|a| &a.name).collect::<Vec<_>>(),
|
||||
vec!["value"],
|
||||
"{code}"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,8 +11,8 @@
|
||||
//! This module provides functionality to detect and parse Pydantic models and Python
|
||||
//! dataclasses from Python AST, enabling automatic UI generation for complex data structures.
|
||||
|
||||
use rustpython_parser::ast::{
|
||||
Constant, Expr, ExprAttribute, ExprCall, ExprConstant, ExprName, ExprTuple, Stmt, StmtAnnAssign,
|
||||
use ruff_python_ast::{
|
||||
Decorator, Expr, ExprAttribute, ExprCall, ExprName, ExprTuple, Stmt, StmtAnnAssign,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
use windmill_parser::{ObjectProperty, ObjectType, Typ};
|
||||
@@ -64,7 +64,7 @@ fn detect_model_type_impl(
|
||||
// Mark as being visited
|
||||
visited.insert(class_name.to_string());
|
||||
|
||||
let result = if is_pydantic_base(&class_def.bases) {
|
||||
let result = if is_pydantic_base(class_def.bases()) {
|
||||
// Pydantic BaseModel
|
||||
parse_model_fields(&class_def.body, class_def.name.as_str(), module, visited)
|
||||
} else if has_dataclass_decorator(&class_def.decorator_list)
|
||||
@@ -115,9 +115,9 @@ fn is_pydantic_base(bases: &[Expr]) -> bool {
|
||||
}
|
||||
|
||||
/// Checks if a class has @dataclass decorator (standard library)
|
||||
fn has_dataclass_decorator(decorators: &[Expr]) -> bool {
|
||||
fn has_dataclass_decorator(decorators: &[Decorator]) -> bool {
|
||||
for decorator in decorators {
|
||||
match decorator {
|
||||
match &decorator.expression {
|
||||
// Match: @dataclass
|
||||
Expr::Name(ExprName { id, .. }) if id.as_str() == "dataclass" => {
|
||||
return true;
|
||||
@@ -160,9 +160,9 @@ fn has_dataclass_decorator(decorators: &[Expr]) -> bool {
|
||||
}
|
||||
|
||||
/// Checks if a class has @pydantic.dataclasses.dataclass decorator (Pydantic v2)
|
||||
fn has_pydantic_dataclass_decorator(decorators: &[Expr]) -> bool {
|
||||
fn has_pydantic_dataclass_decorator(decorators: &[Decorator]) -> bool {
|
||||
for decorator in decorators {
|
||||
match decorator {
|
||||
match &decorator.expression {
|
||||
// Match: @pydantic.dataclasses.dataclass
|
||||
Expr::Attribute(ExprAttribute { attr, value, .. }) if attr.as_str() == "dataclass" => {
|
||||
if let Expr::Attribute(ExprAttribute {
|
||||
@@ -350,10 +350,10 @@ fn extract_field_type(
|
||||
|
||||
// String annotations: "ForwardRef" (forward references)
|
||||
// Not fully supported - return Unknown with warning
|
||||
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => {
|
||||
Expr::StringLiteral(s) => {
|
||||
eprintln!(
|
||||
"Forward references like \"{forward_ref}\" are not yet supported, treating as Unknown",
|
||||
forward_ref = s,
|
||||
forward_ref = s.value.to_str(),
|
||||
);
|
||||
Typ::Unknown
|
||||
}
|
||||
|
||||
@@ -20,13 +20,16 @@ pub fn parse_wmill_sdk_sql_assets(
|
||||
Ok(mut sql_assets) => {
|
||||
if let Some(schema) = schema {
|
||||
for asset in &mut sql_assets.assets {
|
||||
if asset.kind == kind && asset.path.starts_with(asset_name) {
|
||||
asset.path = format!(
|
||||
"{}/{}.{}",
|
||||
asset_name,
|
||||
schema,
|
||||
&asset.path[asset_name.len() + 1..]
|
||||
);
|
||||
if asset.kind != kind {
|
||||
continue;
|
||||
}
|
||||
// A statement can name the database alone, with no table to qualify.
|
||||
let table = asset
|
||||
.path
|
||||
.strip_prefix(asset_name)
|
||||
.and_then(|rest| rest.strip_prefix('/'));
|
||||
if let Some(table) = table {
|
||||
asset.path = format!("{asset_name}/{schema}.{table}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,8 +9,9 @@ name = "windmill_parser_wac"
|
||||
path = "./src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
rustpython-parser.workspace = true
|
||||
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
|
||||
ruff_python_parser.workspace = true
|
||||
ruff_python_ast.workspace = true
|
||||
ruff_text_size.workspace = true
|
||||
swc_common.workspace = true
|
||||
swc_ecma_parser.workspace = true
|
||||
swc_ecma_ast.workspace = true
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use rustpython_parser::{
|
||||
ast::{
|
||||
Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry,
|
||||
StmtTryStar, StmtWhile,
|
||||
},
|
||||
Parse,
|
||||
use ruff_python_ast::{
|
||||
CmpOp, ElifElseClause, ExceptHandler, Expr, ExprAttribute, ExprAwait, ExprCall, ExprName,
|
||||
Number, Parameters, Stmt, StmtExpr, StmtFor, StmtFunctionDef, StmtIf, StmtReturn, StmtTry,
|
||||
StmtWhile,
|
||||
};
|
||||
use ruff_python_parser::parse_module;
|
||||
use ruff_text_size::{Ranged, TextSize};
|
||||
|
||||
use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag};
|
||||
use crate::validation::{self, CompileError};
|
||||
@@ -42,9 +42,9 @@ type TaskFunctions = HashMap<String, Option<String>>;
|
||||
fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
|
||||
let mut tasks = HashMap::new();
|
||||
for stmt in stmts {
|
||||
if let Stmt::AsyncFunctionDef(func) = stmt {
|
||||
if let Stmt::FunctionDef(func @ StmtFunctionDef { is_async: true, .. }) = stmt {
|
||||
for dec in &func.decorator_list {
|
||||
match dec {
|
||||
match &dec.expression {
|
||||
// @task (bare decorator)
|
||||
Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => {
|
||||
tasks.insert(func.name.to_string(), None);
|
||||
@@ -68,13 +68,9 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
|
||||
if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
|
||||
if id.as_str() == "task_script" || id.as_str() == "task_flow" {
|
||||
// Extract the path from the first positional argument
|
||||
let path = call.args.first().and_then(|arg| {
|
||||
if let Expr::Constant(c) = arg {
|
||||
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
|
||||
return Some(s.to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
let path = call.arguments.args.first().and_then(|arg| match arg {
|
||||
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
|
||||
_ => None,
|
||||
});
|
||||
// Extract variable name from target
|
||||
if let Some(Expr::Name(ExprName { id: var_name, .. })) =
|
||||
@@ -92,13 +88,11 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
|
||||
|
||||
/// Extract the `path=` keyword argument from a `@task(path="...")` call.
|
||||
fn extract_task_path_kwarg(call: &ExprCall) -> Option<String> {
|
||||
for kw in &call.keywords {
|
||||
for kw in &call.arguments.keywords {
|
||||
if let Some(ref arg) = kw.arg {
|
||||
if arg.as_str() == "path" {
|
||||
if let Expr::Constant(c) = &kw.value {
|
||||
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
|
||||
return Some(s.to_string());
|
||||
}
|
||||
if let Expr::StringLiteral(s) = &kw.value {
|
||||
return Some(s.value.to_str().to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -150,10 +144,9 @@ impl WacWalker {
|
||||
|
||||
fn line_of_expr(&self, expr: &Expr) -> usize {
|
||||
let offset = match expr {
|
||||
Expr::Call(c) => c.range.start().to_usize(),
|
||||
Expr::Await(a) => a.range.start().to_usize(),
|
||||
Expr::Attribute(a) => a.range.start().to_usize(),
|
||||
Expr::Name(n) => n.range.start().to_usize(),
|
||||
Expr::Call(_) | Expr::Await(_) | Expr::Attribute(_) | Expr::Name(_) => {
|
||||
expr.start().to_usize()
|
||||
}
|
||||
_ => 0,
|
||||
};
|
||||
self.line_index.line_of(offset)
|
||||
@@ -161,17 +154,15 @@ impl WacWalker {
|
||||
|
||||
fn line_of_stmt(&self, stmt: &Stmt) -> usize {
|
||||
let offset = match stmt {
|
||||
Stmt::If(s) => s.range.start().to_usize(),
|
||||
Stmt::For(s) => s.range.start().to_usize(),
|
||||
Stmt::While(s) => s.range.start().to_usize(),
|
||||
Stmt::Return(s) => s.range.start().to_usize(),
|
||||
Stmt::Expr(s) => s.range.start().to_usize(),
|
||||
Stmt::Try(s) => s.range.start().to_usize(),
|
||||
Stmt::TryStar(s) => s.range.start().to_usize(),
|
||||
Stmt::Assign(s) => s.range.start().to_usize(),
|
||||
Stmt::AnnAssign(s) => s.range.start().to_usize(),
|
||||
Stmt::FunctionDef(s) => s.range.start().to_usize(),
|
||||
Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(),
|
||||
Stmt::If(_)
|
||||
| Stmt::For(_)
|
||||
| Stmt::While(_)
|
||||
| Stmt::Return(_)
|
||||
| Stmt::Expr(_)
|
||||
| Stmt::Try(_)
|
||||
| Stmt::Assign(_)
|
||||
| Stmt::AnnAssign(_)
|
||||
| Stmt::FunctionDef(_) => stmt.start().to_usize(),
|
||||
_ => 0,
|
||||
};
|
||||
self.line_index.line_of(offset)
|
||||
@@ -190,9 +181,7 @@ impl WacWalker {
|
||||
/// Check if an expression is `asyncio.gather(...)` call
|
||||
fn is_asyncio_gather_call(expr: &Expr) -> bool {
|
||||
if let Expr::Call(call) = expr {
|
||||
if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) =
|
||||
call.func.as_ref()
|
||||
{
|
||||
if let Expr::Attribute(ExprAttribute { value, attr, .. }) = call.func.as_ref() {
|
||||
if attr.as_str() == "gather" {
|
||||
if let Expr::Name(ExprName { id, .. }) = value.as_ref() {
|
||||
return id.as_str() == "asyncio";
|
||||
@@ -222,20 +211,20 @@ impl WacWalker {
|
||||
fn expr_to_source(expr: &Expr) -> String {
|
||||
match expr {
|
||||
Expr::Compare(c) => {
|
||||
let left = Self::expr_to_source(&c.left);
|
||||
if let Some(comparator) = c.comparators.first() {
|
||||
let left = Self::expr_to_source(&c.operands[0]);
|
||||
if let Some(comparator) = c.comparators().first() {
|
||||
let right = Self::expr_to_source(comparator);
|
||||
let op = match c.ops.first() {
|
||||
Some(rustpython_parser::ast::CmpOp::Gt) => ">",
|
||||
Some(rustpython_parser::ast::CmpOp::Lt) => "<",
|
||||
Some(rustpython_parser::ast::CmpOp::GtE) => ">=",
|
||||
Some(rustpython_parser::ast::CmpOp::LtE) => "<=",
|
||||
Some(rustpython_parser::ast::CmpOp::Eq) => "==",
|
||||
Some(rustpython_parser::ast::CmpOp::NotEq) => "!=",
|
||||
Some(rustpython_parser::ast::CmpOp::In) => "in",
|
||||
Some(rustpython_parser::ast::CmpOp::NotIn) => "not in",
|
||||
Some(rustpython_parser::ast::CmpOp::Is) => "is",
|
||||
Some(rustpython_parser::ast::CmpOp::IsNot) => "is not",
|
||||
Some(CmpOp::Gt) => ">",
|
||||
Some(CmpOp::Lt) => "<",
|
||||
Some(CmpOp::GtE) => ">=",
|
||||
Some(CmpOp::LtE) => "<=",
|
||||
Some(CmpOp::Eq) => "==",
|
||||
Some(CmpOp::NotEq) => "!=",
|
||||
Some(CmpOp::In) => "in",
|
||||
Some(CmpOp::NotIn) => "not in",
|
||||
Some(CmpOp::Is) => "is",
|
||||
Some(CmpOp::IsNot) => "is not",
|
||||
None => "?",
|
||||
};
|
||||
format!("{left} {op} {right}")
|
||||
@@ -253,14 +242,14 @@ impl WacWalker {
|
||||
format!("{value}.{}", a.attr)
|
||||
}
|
||||
Expr::Name(n) => n.id.to_string(),
|
||||
Expr::Constant(c) => match &c.value {
|
||||
rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""),
|
||||
rustpython_parser::ast::Constant::Int(i) => i.to_string(),
|
||||
rustpython_parser::ast::Constant::Float(f) => f.to_string(),
|
||||
rustpython_parser::ast::Constant::Bool(b) => b.to_string(),
|
||||
rustpython_parser::ast::Constant::None => "None".to_string(),
|
||||
_ => "...".to_string(),
|
||||
Expr::StringLiteral(s) => format!("\"{}\"", s.value.to_str()),
|
||||
Expr::NumberLiteral(n) => match &n.value {
|
||||
Number::Int(i) => i.to_string(),
|
||||
Number::Float(f) => f.to_string(),
|
||||
Number::Complex { .. } => "...".to_string(),
|
||||
},
|
||||
Expr::BooleanLiteral(b) => b.value.to_string(),
|
||||
Expr::NoneLiteral(_) => "None".to_string(),
|
||||
_ => "...".to_string(),
|
||||
}
|
||||
}
|
||||
@@ -279,8 +268,16 @@ impl WacWalker {
|
||||
match stmt {
|
||||
Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value),
|
||||
Stmt::Assign(a) => self.expr_contains_step(&a.value),
|
||||
Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
|
||||
Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
|
||||
Stmt::If(s) => {
|
||||
self.body_contains_step(&s.body)
|
||||
|| s.elif_else_clauses
|
||||
.iter()
|
||||
.any(|clause| self.body_contains_step(&clause.body))
|
||||
}
|
||||
// `async for` is not modelled: its body is skipped like any unsupported statement
|
||||
Stmt::For(s) if !s.is_async => {
|
||||
self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
|
||||
}
|
||||
Stmt::While(s) => {
|
||||
self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
|
||||
}
|
||||
@@ -289,19 +286,7 @@ impl WacWalker {
|
||||
|| self.body_contains_step(&s.orelse)
|
||||
|| self.body_contains_step(&s.finalbody)
|
||||
|| s.handlers.iter().any(|h| match h {
|
||||
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
|
||||
self.body_contains_step(&eh.body)
|
||||
}
|
||||
})
|
||||
}
|
||||
Stmt::TryStar(s) => {
|
||||
self.body_contains_step(&s.body)
|
||||
|| self.body_contains_step(&s.orelse)
|
||||
|| self.body_contains_step(&s.finalbody)
|
||||
|| s.handlers.iter().any(|h| match h {
|
||||
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
|
||||
self.body_contains_step(&eh.body)
|
||||
}
|
||||
ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body),
|
||||
})
|
||||
}
|
||||
Stmt::Return(_) => false,
|
||||
@@ -319,11 +304,12 @@ impl WacWalker {
|
||||
match expr {
|
||||
Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value),
|
||||
Expr::Call(call) => {
|
||||
if self.is_task_fn_call(&Expr::Call(call.clone())) {
|
||||
return true;
|
||||
}
|
||||
if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) {
|
||||
return call.args.iter().any(|a| self.expr_contains_step(a));
|
||||
if Self::is_asyncio_gather_call(expr) {
|
||||
return call
|
||||
.arguments
|
||||
.args
|
||||
.iter()
|
||||
.any(|a| self.expr_contains_step(a));
|
||||
}
|
||||
false
|
||||
}
|
||||
@@ -370,12 +356,11 @@ impl WacWalker {
|
||||
Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value),
|
||||
Stmt::Assign(a) => self.walk_expr_stmt(&a.value),
|
||||
Stmt::If(if_stmt) => self.walk_if(if_stmt),
|
||||
Stmt::For(for_stmt) => self.walk_for(for_stmt),
|
||||
Stmt::For(for_stmt) if !for_stmt.is_async => self.walk_for(for_stmt),
|
||||
Stmt::While(while_stmt) => self.walk_while(while_stmt),
|
||||
Stmt::Try(try_stmt) => self.walk_try(try_stmt),
|
||||
Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt),
|
||||
Stmt::Return(ret) => self.walk_return(ret),
|
||||
Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => {
|
||||
Stmt::FunctionDef(_) => {
|
||||
if self.stmt_contains_step(stmt) {
|
||||
self.errors.push(validation::error_step_in_nested_function(
|
||||
self.line_of_stmt(stmt),
|
||||
@@ -392,7 +377,7 @@ impl WacWalker {
|
||||
if let Expr::Await(ExprAwait { value, .. }) = expr {
|
||||
// await task_fn(...)
|
||||
if let Expr::Call(call) = value.as_ref() {
|
||||
if self.is_task_fn_call(&Expr::Call(call.clone())) {
|
||||
if self.is_task_fn_call(value) {
|
||||
return self.emit_step(call, expr);
|
||||
}
|
||||
// Check for SDK-level calls: step(), sleep(), wait_for_approval()
|
||||
@@ -430,15 +415,12 @@ impl WacWalker {
|
||||
"step" => {
|
||||
// step("name", fn) — extract the name from the first string argument
|
||||
let name = call
|
||||
.arguments
|
||||
.args
|
||||
.first()
|
||||
.and_then(|arg| {
|
||||
if let Expr::Constant(c) = arg {
|
||||
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
|
||||
return Some(s.to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
.and_then(|arg| match arg {
|
||||
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or_else(|| "step".to_string());
|
||||
let id = self.next_id();
|
||||
@@ -452,6 +434,7 @@ impl WacWalker {
|
||||
}
|
||||
"sleep" => {
|
||||
let seconds = call
|
||||
.arguments
|
||||
.args
|
||||
.first()
|
||||
.map(|arg| Self::expr_to_source(arg))
|
||||
@@ -517,10 +500,10 @@ impl WacWalker {
|
||||
});
|
||||
|
||||
let mut step_ids = Vec::new();
|
||||
for arg in &gather_call.args {
|
||||
for arg in &gather_call.arguments.args {
|
||||
// Each arg should be task_fn(...)
|
||||
if let Expr::Call(call) = arg {
|
||||
if self.is_task_fn_call(&Expr::Call(call.clone())) {
|
||||
if self.is_task_fn_call(arg) {
|
||||
let (name, script) = self
|
||||
.extract_step_info_from_task_call(call)
|
||||
.unwrap_or(("unknown".into(), "unknown".into()));
|
||||
@@ -553,15 +536,33 @@ impl WacWalker {
|
||||
}
|
||||
|
||||
fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> {
|
||||
let has_steps_in_body = self.body_contains_step(&if_stmt.body);
|
||||
let has_steps_in_else = self.body_contains_step(&if_stmt.orelse);
|
||||
self.walk_if_chain(
|
||||
if_stmt.start(),
|
||||
&if_stmt.test,
|
||||
&if_stmt.body,
|
||||
&if_stmt.elif_else_clauses,
|
||||
)
|
||||
}
|
||||
|
||||
/// An `elif` is laid out as an `if` nested alone in the `else` of the clause before it.
|
||||
fn walk_if_chain(
|
||||
&mut self,
|
||||
start: TextSize,
|
||||
test: &Expr,
|
||||
body: &[Stmt],
|
||||
clauses: &[ElifElseClause],
|
||||
) -> Option<(String, String)> {
|
||||
let has_steps_in_body = self.body_contains_step(body);
|
||||
let has_steps_in_else = clauses
|
||||
.iter()
|
||||
.any(|clause| self.body_contains_step(&clause.body));
|
||||
|
||||
if !has_steps_in_body && !has_steps_in_else {
|
||||
return None;
|
||||
}
|
||||
|
||||
let line = self.line_index.line_of(if_stmt.range.start().to_usize());
|
||||
let condition_source = Self::expr_to_source(&if_stmt.test);
|
||||
let line = self.line_index.line_of(start.to_usize());
|
||||
let condition_source = Self::expr_to_source(test);
|
||||
|
||||
let branch_id = self.next_id();
|
||||
let branch_node_id = self.add_node(DagNode {
|
||||
@@ -573,15 +574,19 @@ impl WacWalker {
|
||||
|
||||
let mut last_ids = Vec::new();
|
||||
|
||||
if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) {
|
||||
if let Some((true_first, true_last)) = self.walk_body(body) {
|
||||
self.add_edge(&branch_node_id, &true_first, Some("true".to_string()));
|
||||
last_ids.push(true_last);
|
||||
} else {
|
||||
last_ids.push(branch_node_id.clone());
|
||||
}
|
||||
|
||||
if !if_stmt.orelse.is_empty() {
|
||||
if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) {
|
||||
if let Some((clause, clauses)) = clauses.split_first() {
|
||||
let orelse = match &clause.test {
|
||||
Some(test) => self.walk_if_chain(clause.start(), test, &clause.body, clauses),
|
||||
None => self.walk_body(&clause.body),
|
||||
};
|
||||
if let Some((else_first, else_last)) = orelse {
|
||||
self.add_edge(&branch_node_id, &else_first, Some("false".to_string()));
|
||||
last_ids.push(else_last);
|
||||
} else {
|
||||
@@ -611,7 +616,7 @@ impl WacWalker {
|
||||
return None;
|
||||
}
|
||||
|
||||
let line = self.line_index.line_of(for_stmt.range.start().to_usize());
|
||||
let line = self.line_index.line_of(for_stmt.start().to_usize());
|
||||
let iter_source = Self::expr_to_source(&for_stmt.iter);
|
||||
|
||||
let start_id = self.next_id();
|
||||
@@ -644,7 +649,7 @@ impl WacWalker {
|
||||
return None;
|
||||
}
|
||||
|
||||
let line = self.line_index.line_of(while_stmt.range.start().to_usize());
|
||||
let line = self.line_index.line_of(while_stmt.start().to_usize());
|
||||
let condition = Self::expr_to_source(&while_stmt.test);
|
||||
|
||||
let start_id = self.next_id();
|
||||
@@ -677,39 +682,14 @@ impl WacWalker {
|
||||
|| self.body_contains_step(&try_stmt.orelse)
|
||||
|| self.body_contains_step(&try_stmt.finalbody)
|
||||
|| try_stmt.handlers.iter().any(|h| match h {
|
||||
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
|
||||
self.body_contains_step(&eh.body)
|
||||
}
|
||||
ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body),
|
||||
});
|
||||
|
||||
if !has_steps {
|
||||
return None;
|
||||
}
|
||||
|
||||
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
|
||||
self.emit_try_catch_branch(
|
||||
&try_stmt.body,
|
||||
&try_stmt.handlers,
|
||||
&try_stmt.finalbody,
|
||||
line,
|
||||
)
|
||||
}
|
||||
|
||||
fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> {
|
||||
let has_steps = self.body_contains_step(&try_stmt.body)
|
||||
|| self.body_contains_step(&try_stmt.orelse)
|
||||
|| self.body_contains_step(&try_stmt.finalbody)
|
||||
|| try_stmt.handlers.iter().any(|h| match h {
|
||||
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
|
||||
self.body_contains_step(&eh.body)
|
||||
}
|
||||
});
|
||||
|
||||
if !has_steps {
|
||||
return None;
|
||||
}
|
||||
|
||||
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
|
||||
let line = self.line_index.line_of(try_stmt.start().to_usize());
|
||||
self.emit_try_catch_branch(
|
||||
&try_stmt.body,
|
||||
&try_stmt.handlers,
|
||||
@@ -721,7 +701,7 @@ impl WacWalker {
|
||||
fn emit_try_catch_branch(
|
||||
&mut self,
|
||||
try_body: &[Stmt],
|
||||
handlers: &[rustpython_parser::ast::ExceptHandler],
|
||||
handlers: &[ExceptHandler],
|
||||
finally_body: &[Stmt],
|
||||
line: usize,
|
||||
) -> Option<(String, String)> {
|
||||
@@ -746,7 +726,7 @@ impl WacWalker {
|
||||
// Except handlers
|
||||
for handler in handlers {
|
||||
match handler {
|
||||
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
|
||||
ExceptHandler::ExceptHandler(eh) => {
|
||||
if let Some((catch_first, catch_last)) = self.walk_body(&eh.body) {
|
||||
self.add_edge(&branch_node_id, &catch_first, Some("except".to_string()));
|
||||
last_ids.push(catch_last);
|
||||
@@ -783,7 +763,7 @@ impl WacWalker {
|
||||
}
|
||||
|
||||
fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> {
|
||||
let line = self.line_index.line_of(ret.range.start().to_usize());
|
||||
let line = self.line_index.line_of(ret.start().to_usize());
|
||||
let id = self.next_id();
|
||||
let node_id = self.add_node(DagNode {
|
||||
id: id.clone(),
|
||||
@@ -796,12 +776,12 @@ impl WacWalker {
|
||||
}
|
||||
|
||||
/// Extract workflow function parameters (no longer skips ctx)
|
||||
fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
|
||||
fn extract_params(args: &Parameters) -> Vec<Param> {
|
||||
let mut params = Vec::new();
|
||||
for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) {
|
||||
let name = arg_with_default.def.arg.to_string();
|
||||
let name = arg_with_default.parameter.name.to_string();
|
||||
let typ = arg_with_default
|
||||
.def
|
||||
.parameter
|
||||
.annotation
|
||||
.as_ref()
|
||||
.map(|ann| WacWalker::expr_to_source(ann));
|
||||
@@ -811,59 +791,34 @@ fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
|
||||
}
|
||||
|
||||
pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError>> {
|
||||
let ast = rustpython_parser::ast::Suite::parse(code, "<workflow>")
|
||||
.map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?;
|
||||
let ast = parse_module(code)
|
||||
.map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?
|
||||
.into_suite();
|
||||
|
||||
// First pass: collect @task functions
|
||||
let task_functions = collect_task_functions(&ast);
|
||||
|
||||
// Find the @workflow async def
|
||||
let workflow_fn = ast.iter().find_map(|stmt| {
|
||||
if let Stmt::AsyncFunctionDef(func) = stmt {
|
||||
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
|
||||
if let Expr::Name(ExprName { id, .. }) = dec {
|
||||
id.as_str() == "workflow"
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
if has_workflow_decorator {
|
||||
return Some(func);
|
||||
let workflow_fns = || {
|
||||
ast.iter().filter_map(|stmt| match stmt {
|
||||
Stmt::FunctionDef(func)
|
||||
if func.decorator_list.iter().any(|dec| {
|
||||
matches!(&dec.expression, Expr::Name(ExprName { id, .. }) if id.as_str() == "workflow")
|
||||
}) =>
|
||||
{
|
||||
Some(func)
|
||||
}
|
||||
}
|
||||
// Also check non-async for error reporting
|
||||
if let Stmt::FunctionDef(func) = stmt {
|
||||
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
|
||||
if let Expr::Name(ExprName { id, .. }) = dec {
|
||||
id.as_str() == "workflow"
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
if has_workflow_decorator {
|
||||
return None; // Will be reported as not-async below
|
||||
}
|
||||
}
|
||||
None
|
||||
});
|
||||
_ => None,
|
||||
})
|
||||
};
|
||||
|
||||
// Check for non-async workflow function
|
||||
let non_async_workflow = ast.iter().find_map(|stmt| {
|
||||
if let Stmt::FunctionDef(func) = stmt {
|
||||
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
|
||||
if let Expr::Name(ExprName { id, .. }) = dec {
|
||||
id.as_str() == "workflow"
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
if has_workflow_decorator {
|
||||
let line_index = LineIndex::new(code);
|
||||
return Some(line_index.line_of(func.range.start().to_usize()));
|
||||
}
|
||||
}
|
||||
None
|
||||
});
|
||||
// Find the @workflow async def
|
||||
let workflow_fn = workflow_fns().find(|func| func.is_async);
|
||||
|
||||
// Check for non-async workflow function. The statement starts at its first decorator,
|
||||
// the name sits on the `def` line the error points at.
|
||||
let non_async_workflow = workflow_fns()
|
||||
.find(|func| !func.is_async)
|
||||
.map(|func| LineIndex::new(code).line_of(func.name.start().to_usize()));
|
||||
|
||||
if let Some(line) = non_async_workflow {
|
||||
if workflow_fn.is_none() {
|
||||
@@ -875,7 +830,7 @@ pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError
|
||||
vec![CompileError { message: "No @workflow async function found.".to_string(), line: 0 }]
|
||||
})?;
|
||||
|
||||
let params = extract_params(&workflow_fn.args);
|
||||
let params = extract_params(&workflow_fn.parameters);
|
||||
let source_hash = compute_source_hash(code);
|
||||
|
||||
let mut walker = WacWalker::new(code, task_functions);
|
||||
@@ -894,13 +849,3 @@ fn compute_source_hash(code: &str) -> String {
|
||||
hasher.update(code.as_bytes());
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
trait ToUsize {
|
||||
fn to_usize(self) -> usize;
|
||||
}
|
||||
|
||||
impl ToUsize for rustpython_parser::text_size::TextSize {
|
||||
fn to_usize(self) -> usize {
|
||||
u32::from(self) as usize
|
||||
}
|
||||
}
|
||||
|
||||
@@ -363,3 +363,77 @@ async def main(x: str):
|
||||
));
|
||||
assert!(matches!(dag.nodes[5].node_type, DagNodeType::Return));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_elif_chain_nests_branches() {
|
||||
let code = r#"
|
||||
from wmill import workflow, task
|
||||
|
||||
@task
|
||||
async def a(): ...
|
||||
@task
|
||||
async def b(): ...
|
||||
@task
|
||||
async def c(): ...
|
||||
|
||||
@workflow
|
||||
async def main(x: int):
|
||||
if x > 2:
|
||||
await a()
|
||||
elif x == 2:
|
||||
await b()
|
||||
else:
|
||||
await c()
|
||||
"#;
|
||||
|
||||
let dag = parse_python_workflow(code).expect("should parse");
|
||||
let branches: Vec<(&str, usize)> = dag
|
||||
.nodes
|
||||
.iter()
|
||||
.filter_map(|n| match &n.node_type {
|
||||
DagNodeType::Branch { condition_source } => Some((condition_source.as_str(), n.line)),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(branches, vec![("x > 2", 13), ("x == 2", 15)]);
|
||||
|
||||
// The `elif` branch hangs off the `false` edge of the `if` before it
|
||||
let (outer, inner) = (&dag.nodes[0].id, &dag.nodes[2].id);
|
||||
assert!(dag
|
||||
.edges
|
||||
.iter()
|
||||
.any(|e| &e.from == outer && &e.to == inner && e.label.as_deref() == Some("false")));
|
||||
assert_eq!(
|
||||
dag.nodes
|
||||
.iter()
|
||||
.filter(|n| matches!(n.node_type, DagNodeType::Merge))
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_workflow_with_recent_syntax() {
|
||||
let code = r#"
|
||||
from wmill import workflow, task
|
||||
lazy import polars
|
||||
|
||||
type Rows = list[dict]
|
||||
|
||||
@task
|
||||
async def load[T](rows: T): ...
|
||||
|
||||
@workflow
|
||||
async def main(name: str):
|
||||
try:
|
||||
await load(t"hello {name}")
|
||||
except ValueError, TypeError:
|
||||
pass
|
||||
"#;
|
||||
|
||||
let dag = parse_python_workflow(code).expect("should parse");
|
||||
assert!(dag
|
||||
.nodes
|
||||
.iter()
|
||||
.any(|n| matches!(&n.node_type, DagNodeType::Step { name, .. } if name == "load")));
|
||||
}
|
||||
|
||||
+356
-274
File diff suppressed because it is too large
Load Diff
@@ -12,7 +12,7 @@ resolver = "2"
|
||||
members = ["."]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.829.0"
|
||||
version = "1.830.0"
|
||||
edition = "2021"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
|
||||
|
||||
+4
-1
@@ -123,7 +123,8 @@ use windmill_worker::{
|
||||
BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS,
|
||||
DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR,
|
||||
POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR,
|
||||
RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR,
|
||||
PY314_CACHE_DIR, PY315_CACHE_DIR, RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR,
|
||||
TAR_JAVA_CACHE_DIR, UV_CACHE_DIR,
|
||||
};
|
||||
|
||||
use crate::monitor::{
|
||||
@@ -2179,6 +2180,8 @@ pub async fn run_workers(
|
||||
&*PY311_CACHE_DIR,
|
||||
&*PY312_CACHE_DIR,
|
||||
&*PY313_CACHE_DIR,
|
||||
&*PY314_CACHE_DIR,
|
||||
&*PY315_CACHE_DIR,
|
||||
&*BUN_BUNDLE_CACHE_DIR,
|
||||
&*GO_CACHE_DIR,
|
||||
&*GO_BIN_CACHE_DIR,
|
||||
|
||||
@@ -117,7 +117,7 @@ async fn test_ai_decision_answers_route_a_following_branch(
|
||||
"expr": "results.d.output.intent.choice === 'refund'",
|
||||
"modules": [{"id": "r", "value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(choice){ return 'handled ' + choice }",
|
||||
"input_transforms": {"choice": {
|
||||
"type": "javascript",
|
||||
|
||||
@@ -33,7 +33,7 @@ mod debounce_e2e {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export async function main(items: any[]) { return items }",
|
||||
"input_transforms": {
|
||||
"items": { "type": "javascript", "expr": "flow_input.items" }
|
||||
|
||||
@@ -38,8 +38,8 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<
|
||||
"subgrids": {},
|
||||
"hiddenInlineScripts": [{
|
||||
"name": "get_email",
|
||||
"language": "deno",
|
||||
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }",
|
||||
"path": "f/test/email_app/get_email"
|
||||
}]
|
||||
},
|
||||
@@ -53,7 +53,7 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<
|
||||
"one_of_inputs": {}
|
||||
},
|
||||
// SHA256 hash of raw_code content for anonymous execution
|
||||
"rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": {
|
||||
"rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": {
|
||||
"static_inputs": {},
|
||||
"one_of_inputs": {}
|
||||
}
|
||||
@@ -84,8 +84,8 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res
|
||||
"css": "",
|
||||
"inlineScripts": [{
|
||||
"name": "get_email",
|
||||
"language": "deno",
|
||||
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }"
|
||||
"language": "bun",
|
||||
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }"
|
||||
}]
|
||||
},
|
||||
"policy": {
|
||||
@@ -98,7 +98,7 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res
|
||||
"one_of_inputs": {}
|
||||
},
|
||||
// SHA256 hash of raw_code content for anonymous execution
|
||||
"rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": {
|
||||
"rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": {
|
||||
"static_inputs": {},
|
||||
"one_of_inputs": {}
|
||||
}
|
||||
@@ -197,8 +197,8 @@ async fn run_app_inline_script(
|
||||
"args": {},
|
||||
"component": "get_email",
|
||||
"raw_code": {
|
||||
"language": "deno",
|
||||
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }",
|
||||
"path": format!("{}/get_email", app_path)
|
||||
}
|
||||
});
|
||||
@@ -234,8 +234,8 @@ async fn run_raw_app_inline_script(
|
||||
"args": {},
|
||||
"component": "get_email",
|
||||
"raw_code": {
|
||||
"language": "deno",
|
||||
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }"
|
||||
"language": "bun",
|
||||
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }"
|
||||
}
|
||||
});
|
||||
if force_viewer {
|
||||
|
||||
@@ -104,7 +104,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
|
||||
VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '')
|
||||
VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\n//bun.lock\n<empty>')
|
||||
"#,
|
||||
error_handler_code
|
||||
)
|
||||
@@ -118,7 +118,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
|
||||
VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '')
|
||||
VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\n//bun.lock\n<empty>')
|
||||
"#,
|
||||
failing_script_hash,
|
||||
failing_script_code
|
||||
@@ -155,7 +155,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
priority: None,
|
||||
apply_preprocessor: false,
|
||||
concurrency_settings: ConcurrencySettings::default(),
|
||||
@@ -234,7 +234,7 @@ async fn test_error_handler_muted_on_script(db: Pool<Postgres>) -> anyhow::Resul
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
|
||||
VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return "handled"; }', 'deno', 'script', 'test-user', '{}', '', '', '')
|
||||
VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return "handled"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n<empty>')
|
||||
"#,
|
||||
)
|
||||
.execute(&db)
|
||||
@@ -245,7 +245,7 @@ async fn test_error_handler_muted_on_script(db: Pool<Postgres>) -> anyhow::Resul
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)
|
||||
VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error("fail"); }', 'deno', 'script', 'test-user', '{}', '', '', '', true)
|
||||
VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error("fail"); }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n<empty>', true)
|
||||
"#,
|
||||
failing_script_hash,
|
||||
)
|
||||
@@ -280,7 +280,7 @@ async fn test_error_handler_muted_on_script(db: Pool<Postgres>) -> anyhow::Resul
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
priority: None,
|
||||
apply_preprocessor: false,
|
||||
concurrency_settings: ConcurrencySettings::default(),
|
||||
@@ -331,7 +331,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool<Postgres>) -> anyh
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
|
||||
VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return "handled"; }', 'deno', 'script', 'test-user', '{}', '', '', '')
|
||||
VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return "handled"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n<empty>')
|
||||
"#,
|
||||
)
|
||||
.execute(&db)
|
||||
@@ -342,7 +342,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool<Postgres>) -> anyh
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
|
||||
VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return "ok"; }', 'deno', 'script', 'test-user', '{}', '', '', '')
|
||||
VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return "ok"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n<empty>')
|
||||
"#,
|
||||
success_script_hash,
|
||||
)
|
||||
@@ -377,7 +377,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool<Postgres>) -> anyh
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
priority: None,
|
||||
apply_preprocessor: false,
|
||||
concurrency_settings: ConcurrencySettings::default(),
|
||||
|
||||
+4
-4
@@ -40,9 +40,9 @@ VALUES (encode(sha256('NO_WS_TOKEN'::bytea), 'hex'), 'NO_WS_TOKE', 'NO_WS_TOKEN'
|
||||
INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, extra_perms)
|
||||
VALUES (
|
||||
'test-workspace', 'test-user',
|
||||
'export function main() { return Deno.env.get("WM_END_USER_EMAIL") || ""; }',
|
||||
'export function main() { return process.env.WM_END_USER_EMAIL || ""; }',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'deno', '', 'script',
|
||||
'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'bun', E'{}\n//bun.lock\n<empty>', 'script',
|
||||
'{"g/all": true}'
|
||||
);
|
||||
|
||||
@@ -51,7 +51,7 @@ INSERT INTO flow (workspace_id, summary, description, path, versions, schema, va
|
||||
VALUES (
|
||||
'test-workspace', 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email_flow', '{900002}',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}',
|
||||
'{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}]}',
|
||||
'test-user',
|
||||
'{"g/all": true}'
|
||||
);
|
||||
@@ -60,6 +60,6 @@ INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by)
|
||||
VALUES (
|
||||
900002, 'test-workspace', 'f/test/get_end_user_email_flow',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}',
|
||||
'{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}]}',
|
||||
'test-user'
|
||||
);
|
||||
|
||||
Vendored
+6
-6
@@ -11,7 +11,7 @@ export function main(world: string) {
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/hello', 123412, 'deno', '');
|
||||
'f/system/hello', 123412, 'bun', E'{}\n//bun.lock\n<empty>');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
@@ -28,7 +28,7 @@ export async function main(foo: string, bar: string) {
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/hello_with_preprocessor', 123413, 'deno', '');
|
||||
'f/system/hello_with_preprocessor', 123413, 'bun', E'{}\n//bun.lock\n<empty>');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
@@ -140,7 +140,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche
|
||||
'f/system/hello_with_nodes_flow',
|
||||
'{1443253234253454}',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
|
||||
E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}',
|
||||
E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"bun","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"bun","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}',
|
||||
'system'
|
||||
);
|
||||
|
||||
@@ -149,7 +149,7 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b
|
||||
'test-workspace',
|
||||
'f/system/hello_with_nodes_flow',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
|
||||
E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}',
|
||||
E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"bun","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"bun","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}',
|
||||
'system'
|
||||
);
|
||||
|
||||
@@ -160,7 +160,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche
|
||||
'f/system/hello_with_preprocessor',
|
||||
'{1443253234253456}',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}',
|
||||
E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record<string, string>\\n\\t\\t\\tquery: Record<string, string>\\n\\t\\t\\theaders: Record<string, string>\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record<string, string[]>\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}',
|
||||
E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"bun","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record<string, string>\\n\\t\\t\\tquery: Record<string, string>\\n\\t\\t\\theaders: Record<string, string>\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record<string, string[]>\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"bun","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}',
|
||||
'system'
|
||||
);
|
||||
|
||||
@@ -169,6 +169,6 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b
|
||||
'test-workspace',
|
||||
'f/system/hello_with_preprocessor',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}',
|
||||
E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record<string, string>\\n\\t\\t\\tquery: Record<string, string>\\n\\t\\t\\theaders: Record<string, string>\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record<string, string[]>\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}',
|
||||
E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"bun","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record<string, string>\\n\\t\\t\\tquery: Record<string, string>\\n\\t\\t\\theaders: Record<string, string>\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record<string, string[]>\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"bun","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}',
|
||||
'system'
|
||||
);
|
||||
|
||||
+26
@@ -98,6 +98,32 @@ INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, resu
|
||||
('18181818-1818-1818-1818-181818181818', 'test-workspace', 1000, 'success'::job_status,
|
||||
'{"agent": "AGENT_CHAT_RESULT"}');
|
||||
|
||||
-- A lock job for flow `f/shared/flow1`, as `wmill generate-metadata` queues it, and the
|
||||
-- token CI would lock with.
|
||||
INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES (
|
||||
encode(sha256('RUN_SCOPED_DEPS_TOKEN'::bytea), 'hex'), 'RUN_DEPS_', 'RUN_SCOPED_DEPS_TOKEN',
|
||||
'test2@windmill.dev', 'lock token', false,
|
||||
ARRAY['jobs:run:dependencies:f/shared/*']
|
||||
);
|
||||
INSERT INTO public.v2_job (
|
||||
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
|
||||
kind, runnable_path, tag, visible_to_owner
|
||||
) VALUES (
|
||||
'19191919-1919-1919-1919-191919191919', 'test-workspace', 'test-user-2',
|
||||
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
|
||||
'flowdependencies', 'f/shared/flow1', 'dependency', true
|
||||
), (
|
||||
-- Queued by a broader token with `*` as its path, which the request body allows.
|
||||
'20202020-2020-2020-2020-202020202020', 'test-workspace', 'test-user-2',
|
||||
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
|
||||
'dependencies', '*', 'dependency', true
|
||||
);
|
||||
INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES
|
||||
('19191919-1919-1919-1919-191919191919', 'test-workspace', 1000, 'success'::job_status,
|
||||
'{"lock": "DEPS_RESULT"}'),
|
||||
('20202020-2020-2020-2020-202020202020', 'test-workspace', 1000, 'success'::job_status,
|
||||
'{"lock": "WILDCARD_DEPS_RESULT"}');
|
||||
|
||||
-- A token pairing an app scope with a run scope, as someone driving an app's components
|
||||
-- programmatically would build. `APP_INLINE_JOB` is an inline-script component run: no
|
||||
-- `jobs:run` scope can name its kind, so only the `apps:run` half puts it in reach.
|
||||
|
||||
Vendored
+3
-3
@@ -13,7 +13,7 @@ export async function main(fail: boolean = true) {
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/failing_script', 12349, 'deno', '');
|
||||
'f/system/failing_script', 12349, 'bun', E'{}\n//bun.lock\n<empty>');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
@@ -26,7 +26,7 @@ export async function main() {
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/schedule_error_handler', 123410, 'deno', '');
|
||||
'f/system/schedule_error_handler', 123410, 'bun', E'{}\n//bun.lock\n<empty>');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
@@ -39,7 +39,7 @@ export async function main() {
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/schedule_recovery_handler', 123411, 'deno', '');
|
||||
'f/system/schedule_recovery_handler', 123411, 'bun', E'{}\n//bun.lock\n<empty>');
|
||||
|
||||
INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES (
|
||||
'test-workspace',
|
||||
|
||||
@@ -85,7 +85,7 @@ async fn test_flow_linear_input_transforms(db: Pool<Postgres>) -> anyhow::Result
|
||||
modules: vec![
|
||||
flow_module("a", FlowModuleValue::RawScript {
|
||||
input_transforms: [static_input("x", 10), static_input("y", 5)].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(x: number, y: number) {
|
||||
return {sum: x + y, product: x * y, items: [1, 2, 3, 4, 5]};
|
||||
@@ -105,7 +105,7 @@ export function main(x: number, y: number) {
|
||||
js_input("filtered", "results.a.items.filter(x => x > 2)"),
|
||||
js_input("from_flow_input", "flow_input.multiplier * results.a.sum"),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(total: number, doubled_items: number[], filtered: number[], from_flow_input: number) {
|
||||
return {total, doubled_items, filtered, from_flow_input};
|
||||
@@ -157,7 +157,7 @@ async fn test_flow_forloop_complex_expressions(db: Pool<Postgres>) -> anyhow::Re
|
||||
"a",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -199,7 +199,7 @@ export function main() {
|
||||
js_input("index", "flow_input.iter.index"),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(user_name: string, user_score: number, bonus: number, index: number) {
|
||||
return {name: user_name, final_score: user_score + bonus, position: index};
|
||||
@@ -261,7 +261,7 @@ async fn test_flow_branchone_conditions(db: Pool<Postgres>) -> anyhow::Result<()
|
||||
"a",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {status: "premium", score: 95, items: [1, 2, 3]};
|
||||
@@ -293,7 +293,7 @@ export function main() {
|
||||
js_input("score_from_a", "results.a.score"),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(discount: number, score_from_a: number) {
|
||||
return {branch: "premium_high", discount, score_from_a};
|
||||
@@ -320,7 +320,7 @@ export function main(discount: number, score_from_a: number) {
|
||||
"premium_low",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {branch: "premium_low", discount: 10};
|
||||
@@ -344,7 +344,7 @@ export function main() {
|
||||
"default_branch",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {branch: "default", discount: 0};
|
||||
@@ -398,7 +398,7 @@ async fn test_flow_previous_result_aggregation(db: Pool<Postgres>) -> anyhow::Re
|
||||
"a",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {value: 10, items: [1, 2, 3]};
|
||||
@@ -424,7 +424,7 @@ export function main() {
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(prev_value: number, prev_items_sum: number) {
|
||||
return {value: prev_value * 2, sum: prev_items_sum};
|
||||
@@ -452,7 +452,7 @@ export function main(prev_value: number, prev_items_sum: number) {
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(a_value: number, b_value: number, b_sum: number, combined: number) {
|
||||
return {a_value, b_value, b_sum, combined};
|
||||
@@ -504,7 +504,7 @@ async fn test_flow_nested_complexity(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
modules: vec![
|
||||
flow_module("data", FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -541,7 +541,7 @@ export function main() {
|
||||
js_input("values", "results.data.base_values.map(v => v * flow_input.iter.value.multiplier)"),
|
||||
js_input("sum", "results.data.base_values.reduce((a, b) => a + b, 0) * flow_input.iter.value.multiplier"),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(cat_name: string, multiplier: number, values: number[], sum: number) {
|
||||
return {category: cat_name, multiplier, computed_values: values, total: sum};
|
||||
@@ -602,7 +602,7 @@ async fn test_flow_object_transformations(db: Pool<Postgres>) -> anyhow::Result<
|
||||
modules: vec![
|
||||
flow_module("source", FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -633,7 +633,7 @@ export function main() {
|
||||
js_input("active_bonus", "results.source.config.activeBonus"),
|
||||
js_input("all_users", "results.source.users"),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(admins: string[], active_count: number, admin_bonus: number, active_bonus: number, all_users: any[]) {
|
||||
// Compute user_summary and total_bonus in the script since complex expressions in input_transforms
|
||||
@@ -702,7 +702,7 @@ async fn test_flow_skip_if_expressions(db: Pool<Postgres>) -> anyhow::Result<()>
|
||||
"check",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {should_skip: true, value: 100};
|
||||
@@ -722,7 +722,7 @@ export function main() {
|
||||
"maybe_skipped",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: [js_input("input_val", "results.check.value * 2")].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(input_val: number) {
|
||||
return {processed: input_val, was_run: true};
|
||||
@@ -750,7 +750,7 @@ export function main(input_val: number) {
|
||||
js_input("prev", "previous_result"),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(check_val: number, prev: any) {
|
||||
return {check_val, previous: prev};
|
||||
@@ -799,7 +799,7 @@ async fn test_flow_template_literals(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
modules: vec![
|
||||
flow_module("data", FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -825,7 +825,7 @@ export function main() {
|
||||
js_input("upper_name", "results.data.firstName.toUpperCase()"),
|
||||
js_input("items_formatted", "`Items: ${results.data.items.map(i => i.charAt(0).toUpperCase() + i.slice(1)).join(', ')}`"),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(full_name: string, greeting: string, items_str: string, upper_name: string, items_formatted: string) {
|
||||
return {full_name, greeting, items_str, upper_name, items_formatted};
|
||||
@@ -873,7 +873,7 @@ async fn test_flow_optional_chaining(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
modules: vec![
|
||||
flow_module("data", FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -905,7 +905,7 @@ export function main() {
|
||||
js_input("default_city", "results.data.partial_user?.address?.city ?? 'Unknown'"),
|
||||
js_input("default_country", "results.data.user?.address?.country ?? 'USA'"),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(city: string, missing_city: any, null_user_name: any, default_city: string, default_country: string) {
|
||||
return {city, missing_city, null_user_name, default_city, default_country};
|
||||
@@ -955,7 +955,7 @@ async fn test_flow_parallel_forloop(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"data",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {items: [1, 2, 3, 4, 5]};
|
||||
@@ -990,7 +990,7 @@ export function main() {
|
||||
js_input("cubed", "flow_input.iter.value.value ** 3"),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(original: number, squared: number, cubed: number) {
|
||||
return {original, squared, cubed};
|
||||
@@ -1089,7 +1089,7 @@ async fn test_flow_env_access(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
js_input("retry_count", "flow_env.CONFIG.retries"),
|
||||
js_input("has_auth", "flow_env.CONFIG.features.includes('auth')"),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(env_name: string, is_debug: boolean, timeout_val: number, api_url: string, retry_count: number, has_auth: boolean) {
|
||||
return {env_name, is_debug, timeout_val, api_url, retry_count, has_auth};
|
||||
@@ -1158,7 +1158,7 @@ async fn test_flow_input_and_env_combined(db: Pool<Postgres>) -> anyhow::Result<
|
||||
// Conditional based on both
|
||||
js_input("multiplier", "flow_env.ENV === 'production' ? flow_input.prod_mult : 1"),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(effective_limit: number, env_prefix: string, is_prod: boolean, doubled_input: number, multiplier: number) {
|
||||
return {effective_limit, env_prefix, is_prod, doubled_input, final_value: doubled_input * multiplier};
|
||||
@@ -1215,7 +1215,7 @@ async fn test_flow_results_optional_chaining(db: Pool<Postgres>) -> anyhow::Resu
|
||||
"a",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -1287,7 +1287,7 @@ export function main() {
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(
|
||||
user_name: string,
|
||||
@@ -1361,7 +1361,7 @@ async fn test_flow_large_integers(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"a",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -1401,7 +1401,7 @@ export function main() {
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(
|
||||
small: number, i32_max: number, over_i32: number,
|
||||
@@ -1459,7 +1459,7 @@ async fn test_flow_unicode_emoji(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
modules: vec![
|
||||
flow_module("a", FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -1489,7 +1489,7 @@ export function main() {
|
||||
js_input("mixed_upper", "results.a.mixed.toUpperCase()"),
|
||||
js_input("template", "`Welcome: ${results.a.greeting}`"),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(
|
||||
greeting: string, greeting_len: number, simple_str: string, has_world: boolean,
|
||||
@@ -1544,7 +1544,7 @@ async fn test_flow_complex_array_operations(db: Pool<Postgres>) -> anyhow::Resul
|
||||
modules: vec![
|
||||
flow_module("a", FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -1585,7 +1585,7 @@ export function main() {
|
||||
js_input("first_three", "results.a.numbers.slice(0, 3)"),
|
||||
js_input("last_two", "results.a.numbers.slice(-2)"),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(
|
||||
sorted_asc: number[], sorted_desc: number[], active_names: string[],
|
||||
@@ -1651,7 +1651,7 @@ async fn test_flow_multiline_expressions(db: Pool<Postgres>) -> anyhow::Result<(
|
||||
modules: vec![
|
||||
flow_module("a", FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -1710,7 +1710,7 @@ export function main() {
|
||||
}
|
||||
"#),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(
|
||||
subtotal: number, discounted_total: number,
|
||||
@@ -1768,7 +1768,7 @@ async fn test_flow_spread_with_results(db: Pool<Postgres>) -> anyhow::Result<()>
|
||||
modules: vec![
|
||||
flow_module("a", FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -1803,7 +1803,7 @@ export function main() {
|
||||
// Combining multiple spreads
|
||||
js_input("combined", "{config: {...results.a.config}, tags: [...results.a.tags], source: 'flow'}"),
|
||||
].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(
|
||||
merged_config: any, all_tags: string[], full_user: any,
|
||||
@@ -1869,7 +1869,7 @@ async fn test_flow_nested_forloop_results_access(db: Pool<Postgres>) -> anyhow::
|
||||
"a",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {
|
||||
@@ -1908,7 +1908,7 @@ export function main() {
|
||||
js_input("multiplier", "results.a.multiplier"), // Access outer step from inside for-loop
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(category: string, multiplier: number) {
|
||||
return {
|
||||
@@ -1939,7 +1939,7 @@ export function main(category: string, multiplier: number) {
|
||||
js_input("original_mult", "results.a.multiplier"), // Access outer step
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(items: any[], category: string, original_mult: number) {
|
||||
return {
|
||||
@@ -2013,7 +2013,7 @@ async fn test_flow_results_non_existent_step(db: Pool<Postgres>) -> anyhow::Resu
|
||||
"a",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return { value: 42 };
|
||||
@@ -2050,7 +2050,7 @@ export function main() {
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(
|
||||
existing: number,
|
||||
@@ -2126,7 +2126,7 @@ async fn test_flow_env_skip_if_inside_branchone(db: Pool<Postgres>) -> anyhow::R
|
||||
"inner",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {ran: true};
|
||||
@@ -2156,7 +2156,7 @@ export function main() {
|
||||
"branch_marker",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {marker: "branch-marker"};
|
||||
@@ -2193,7 +2193,7 @@ export function main() {
|
||||
"after",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: [js_input("prev", "previous_result")].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(prev: any) {
|
||||
return {prev};
|
||||
@@ -2258,7 +2258,7 @@ async fn test_flow_env_skip_if_nested_branchone(db: Pool<Postgres>) -> anyhow::R
|
||||
"leaf",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {ran: true};
|
||||
@@ -2282,7 +2282,7 @@ export function main() {
|
||||
"inner_marker",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {marker: "inner"};
|
||||
@@ -2337,7 +2337,7 @@ export function main() {
|
||||
"after",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: [js_input("prev", "previous_result")].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(prev: any) {
|
||||
return {prev};
|
||||
@@ -2414,7 +2414,7 @@ async fn test_flow_env_complex_input_transform_in_branch(db: Pool<Postgres>) ->
|
||||
"Math.min(flow_env.LIMIT, 10) + flow_env.OFFSET",
|
||||
)]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(value: number) {
|
||||
return {value};
|
||||
@@ -2487,7 +2487,7 @@ async fn test_flow_env_skip_if_in_parallel_forloop(db: Pool<Postgres>) -> anyhow
|
||||
"marker",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: [js_input("i", "flow_input.iter.value")].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(i: number) {
|
||||
return {marker: i};
|
||||
@@ -2508,7 +2508,7 @@ export function main(i: number) {
|
||||
"leaf",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {ran: true};
|
||||
@@ -2594,7 +2594,7 @@ async fn test_flow_env_imported_flow_uses_own_env(db: Pool<Postgres>) -> anyhow:
|
||||
"id": "leaf",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(key: string) { return {key}; }",
|
||||
"input_transforms": {
|
||||
"key": { "type": "javascript", "expr": "flow_env.KEY" }
|
||||
@@ -2679,7 +2679,7 @@ async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool<Postgres>) ->
|
||||
"id": "marker",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return {marker: \"from-imported\"}; }",
|
||||
"input_transforms": {}
|
||||
}
|
||||
@@ -2688,7 +2688,7 @@ async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool<Postgres>) ->
|
||||
"id": "leaf",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return {ran: true}; }",
|
||||
"input_transforms": {}
|
||||
},
|
||||
@@ -2791,7 +2791,7 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool<Postgres>) -> a
|
||||
"id": "marker",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return {marker: \"from-imported-branch\"}; }",
|
||||
"input_transforms": {}
|
||||
}
|
||||
@@ -2800,7 +2800,7 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool<Postgres>) -> a
|
||||
"id": "leaf",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return {ran: true}; }",
|
||||
"input_transforms": {}
|
||||
},
|
||||
@@ -2932,13 +2932,13 @@ async fn test_flow_env_marks_sub_flows_only_without_ancestor_env(
|
||||
"parallel": true,
|
||||
"modules": [{ "id": "leaf", "value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(key: any) { return key ?? null; }",
|
||||
"input_transforms": { "key": { "type": "javascript", "expr": "flow_env.KEY" } }
|
||||
}}, { "id": "tail", "value": {
|
||||
// A single simple step is inlined rather than stored as a flow node.
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(prev: any) { return prev; }",
|
||||
"input_transforms": { "prev": { "type": "javascript", "expr": "results.leaf" } }
|
||||
}}]
|
||||
@@ -3024,7 +3024,7 @@ async fn test_flow_env_in_stop_after_if(db: Pool<Postgres>) -> anyhow::Result<()
|
||||
"first",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {stage: "first"};
|
||||
@@ -3052,7 +3052,7 @@ export function main() {
|
||||
"second",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {stage: "second"};
|
||||
@@ -3110,7 +3110,7 @@ async fn test_stop_after_if_error_include_result(db: Pool<Postgres>) -> anyhow::
|
||||
"step",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return { userErrors: ["email taken"], ok: false };
|
||||
@@ -3200,7 +3200,7 @@ async fn test_flow_env_in_retry_if(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"fails",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
throw new Error("nope");
|
||||
@@ -3274,7 +3274,7 @@ async fn test_flow_env_in_stop_after_all_iters_if(db: Pool<Postgres>) -> anyhow:
|
||||
"iter_step",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: [js_input("i", "flow_input.iter.value")].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(i: number) {
|
||||
return {iter: i};
|
||||
@@ -3316,7 +3316,7 @@ export function main(i: number) {
|
||||
"after",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main() {
|
||||
return {stage: "after-loop"};
|
||||
@@ -3375,7 +3375,7 @@ async fn test_stop_after_all_iters_if_error_includes_result(
|
||||
"iter_step",
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: [js_input("i", "flow_input.iter.value")].into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"
|
||||
export function main(i: number) {
|
||||
return { iter: i };
|
||||
@@ -3453,8 +3453,8 @@ async fn test_flow_expr_step_tag(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
id,
|
||||
FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
content: "export function main() { return { lang: 'bun' } }".to_string(),
|
||||
language: ScriptLang::Bun,
|
||||
content: "export function main() { return { lang: 'python3' } }".to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: tag.map(str::to_string),
|
||||
@@ -3484,7 +3484,7 @@ async fn test_flow_expr_step_tag(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
.bind(job.id)
|
||||
.fetch_one(&db)
|
||||
.await?;
|
||||
assert_eq!(b_tag, "bun");
|
||||
assert_eq!(b_tag, "python3");
|
||||
|
||||
assert!(!job.success);
|
||||
let result = job.json_result().unwrap();
|
||||
|
||||
@@ -54,7 +54,7 @@ mod job_payload {
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
priority: None,
|
||||
apply_preprocessor: false,
|
||||
job_token_scopes: None,
|
||||
@@ -85,7 +85,7 @@ mod job_payload {
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
priority: None,
|
||||
apply_preprocessor: true,
|
||||
concurrency_settings:
|
||||
@@ -165,7 +165,7 @@ mod job_payload {
|
||||
let test = || async {
|
||||
let result = RunJob::from(JobPayload::FlowScript {
|
||||
id: flow_scripts[0],
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default(),
|
||||
cache_ttl: None,
|
||||
@@ -185,7 +185,7 @@ mod job_payload {
|
||||
let test = || async {
|
||||
let result = RunJob::from(JobPayload::FlowScript {
|
||||
id: flow_scripts[1],
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
concurrency_settings:
|
||||
windmill_common::runnable_settings::ConcurrencySettings::default(),
|
||||
cache_ttl: None,
|
||||
@@ -263,7 +263,7 @@ mod job_payload {
|
||||
let result = RunJob::from(JobPayload::Dependencies {
|
||||
path: "f/system/hello".to_string(),
|
||||
hash: ScriptHash(123412),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
debouncing_settings: Default::default(),
|
||||
dedicated_worker: None,
|
||||
})
|
||||
@@ -351,7 +351,7 @@ mod job_payload {
|
||||
console.log(greet)
|
||||
return greet
|
||||
}"#,
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"world": { "type": "javascript", "expr": "flow_input.world" }
|
||||
}
|
||||
@@ -407,7 +407,7 @@ mod job_payload {
|
||||
return greet
|
||||
}"#
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
})
|
||||
.run_until_complete(&db, false, port)
|
||||
.await
|
||||
@@ -416,7 +416,10 @@ mod job_payload {
|
||||
|
||||
assert_eq!(
|
||||
result,
|
||||
json!({ "lock": "", "status": "Successful lock file generation" })
|
||||
json!({
|
||||
"lock": "{\n \"dependencies\": {}\n}\n//bun.lock\n<empty>",
|
||||
"status": "Successful lock file generation"
|
||||
})
|
||||
);
|
||||
};
|
||||
test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
|
||||
@@ -626,7 +629,7 @@ mod job_payload {
|
||||
console.log(greet)
|
||||
return greet
|
||||
}"#,
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"world": { "type": "javascript", "expr": "flow_input.world" }
|
||||
}
|
||||
@@ -673,7 +676,7 @@ mod job_payload {
|
||||
"content": r#"export function main(world: string) {
|
||||
return `Hello ${world}!`;
|
||||
}"#,
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"world": { "type": "javascript", "expr": "flow_input.world" }
|
||||
}
|
||||
@@ -685,7 +688,7 @@ mod job_payload {
|
||||
"content": r#"export function main(world: string, a: string) {
|
||||
return `${a} ${world}!`;
|
||||
}"#,
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"world": { "type": "javascript", "expr": "flow_input.world" },
|
||||
"a": { "type": "javascript", "expr": "results.a" }
|
||||
@@ -704,7 +707,7 @@ mod job_payload {
|
||||
"x": { "type": "javascript", "expr": "flow_input.iter.value" }
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": r#"export function main(world: string, b: string, x: string) {
|
||||
return `${x}: ${b} ${world}!`;
|
||||
}"#,
|
||||
@@ -857,7 +860,7 @@ mod job_payload {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"world": { "type": "javascript", "expr": "flow_input.world" }
|
||||
},
|
||||
@@ -874,7 +877,7 @@ mod job_payload {
|
||||
"id": "inner_first",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"world": { "type": "javascript", "expr": "flow_input.world" }
|
||||
},
|
||||
@@ -884,7 +887,7 @@ mod job_payload {
|
||||
"id": "inner_second",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"world": { "type": "javascript", "expr": "flow_input.world" },
|
||||
"first": { "type": "javascript", "expr": "results.inner_first" }
|
||||
@@ -1045,7 +1048,7 @@ mod job_payload {
|
||||
"id": "inner",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"iter_val": { "type": "javascript", "expr": "flow_input.iter.value" },
|
||||
"tag": { "type": "javascript", "expr": "flow_input.tag" }
|
||||
@@ -1244,7 +1247,7 @@ mod job_payload {
|
||||
"id": "leaf",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"tag": { "type": "javascript", "expr": "flow_input.tag" },
|
||||
"iv": { "type": "javascript", "expr": "flow_input.iter.value" }
|
||||
@@ -1388,7 +1391,7 @@ mod job_payload {
|
||||
"id": "leaf",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"tag": { "type": "javascript", "expr": "flow_input.tag" }
|
||||
},
|
||||
|
||||
@@ -476,6 +476,44 @@ async fn test_single_job_read_authorization(db: Pool<Postgres>) -> anyhow::Resul
|
||||
);
|
||||
}
|
||||
|
||||
// A lock job belongs to `jobs:run:dependencies`: its path is whatever the caller sent,
|
||||
// so a token scoped to run the flow at that path must not read it.
|
||||
let path = "completed/get_result/19191919-1919-1919-1919-191919191919";
|
||||
let (status, body) = get(&base, path, Some("RUN_SCOPED_DEPS_TOKEN")).await;
|
||||
assert!(
|
||||
status.is_success() && body.contains("DEPS_RESULT"),
|
||||
"dependencies-scoped token must read its lock job (got {status}): {body}"
|
||||
);
|
||||
let (status, body) = get(&base, path, Some("RUN_SCOPED_TOKEN")).await;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"a flow-scoped token must not read a lock job (got {status}): {body}"
|
||||
);
|
||||
let (status, body) = get(
|
||||
&base,
|
||||
&format!("completed/get_result/{FLOW_JOB}"),
|
||||
Some("RUN_SCOPED_DEPS_TOKEN"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"dependencies-scoped token must not read a flow run at its path (got {status}): {body}"
|
||||
);
|
||||
// A lock job filed at `*` matches no path-scoped grant: the stored path is not a pattern.
|
||||
let (status, body) = get(
|
||||
&base,
|
||||
"completed/get_result/20202020-2020-2020-2020-202020202020",
|
||||
Some("RUN_SCOPED_DEPS_TOKEN"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"a path-scoped token must not read a lock job filed at `*` (got {status}): {body}"
|
||||
);
|
||||
|
||||
// An `apps:run:<app>` scope is a start grant too: the inline-script component run it
|
||||
// launched — a kind no `jobs:run` scope can name — stays readable to a token scoped
|
||||
// to that app, and stays out of reach for one that is only scoped to run jobs.
|
||||
|
||||
@@ -217,7 +217,7 @@ async fn test_api_restart_at_step_rejects_parallel_loop(db: Pool<Postgres>) -> a
|
||||
"id": "inner",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {},
|
||||
"content": "export function main() { return 'ok' }"
|
||||
}
|
||||
|
||||
+16
-8
@@ -70,12 +70,20 @@ mod retry {
|
||||
|
||||
fn inner_step() -> &'static str {
|
||||
r#"
|
||||
import { connect } from "node:net";
|
||||
|
||||
export async function main(index, port) {
|
||||
const buf = new Uint8Array([0]);
|
||||
const sock = await Deno.connect({ port });
|
||||
await sock.write(new Uint8Array([index]));
|
||||
if (await sock.read(buf) != 1) throw Error("read");
|
||||
return buf[0];
|
||||
return await new Promise((resolve, reject) => {
|
||||
const sock = connect({ port, host: "127.0.0.1" }, () => {
|
||||
sock.write(new Uint8Array([index]));
|
||||
});
|
||||
sock.on("data", (data) => {
|
||||
resolve(data[0]);
|
||||
sock.destroy();
|
||||
});
|
||||
sock.on("error", reject);
|
||||
sock.on("close", () => reject(Error("read")));
|
||||
});
|
||||
}
|
||||
"#
|
||||
}
|
||||
@@ -105,7 +113,7 @@ def main(last, port):
|
||||
"port": { "type": "javascript", "expr": "flow_input.port" },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": inner_step(),
|
||||
},
|
||||
}],
|
||||
@@ -281,7 +289,7 @@ def main(last, port):
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return \"ok\" }",
|
||||
},
|
||||
}, {
|
||||
@@ -293,7 +301,7 @@ def main(last, port):
|
||||
"port": { "type": "javascript", "expr": "flow_input.port" },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": inner_step(),
|
||||
},
|
||||
"retry": { "constant": { "attempts": 1, "seconds": 0 } },
|
||||
|
||||
@@ -123,7 +123,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
|
||||
VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')
|
||||
VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\n//bun.lock\n<empty>')
|
||||
"#,
|
||||
success_handler_code
|
||||
)
|
||||
@@ -138,7 +138,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
|
||||
VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')
|
||||
VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\n//bun.lock\n<empty>')
|
||||
"#,
|
||||
test_script_hash,
|
||||
test_script_code
|
||||
@@ -175,7 +175,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
priority: None,
|
||||
apply_preprocessor: false,
|
||||
concurrency_settings: ConcurrencySettings::default(),
|
||||
|
||||
@@ -63,11 +63,11 @@ mod suspend_resume {
|
||||
"op": { "type": "javascript", "expr": "flow_input.op ?? 'resume'", },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "\
|
||||
export async function main(n, port, op) {\
|
||||
const job = Deno.env.get('WM_JOB_ID');
|
||||
const token = Deno.env.get('WM_TOKEN');
|
||||
const job = process.env.WM_JOB_ID;
|
||||
const token = process.env.WM_TOKEN;
|
||||
const r = await fetch(
|
||||
`http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\
|
||||
{\
|
||||
@@ -102,7 +102,7 @@ mod suspend_resume {
|
||||
"resumes": { "type": "javascript", "expr": "resumes", },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(n, resume, resumes) { return { n: n + 1, resume, resumes } }"
|
||||
},
|
||||
"suspend": {
|
||||
@@ -116,7 +116,7 @@ mod suspend_resume {
|
||||
"resumes": { "type": "javascript", "expr": "resumes", },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(last, resume, resumes) { return { last, resume, resumes } }"
|
||||
},
|
||||
}],
|
||||
@@ -256,11 +256,11 @@ mod suspend_resume {
|
||||
"port": { "type": "javascript", "expr": "flow_input.port" },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "\
|
||||
export async function main(port) {\
|
||||
const job = Deno.env.get('WM_JOB_ID');\
|
||||
const token = Deno.env.get('WM_TOKEN');\
|
||||
const job = process.env.WM_JOB_ID;\
|
||||
const token = process.env.WM_TOKEN;\
|
||||
const secret = await (await fetch(\
|
||||
`http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\
|
||||
{ headers: { 'Authorization': `Bearer ${token}` } }\
|
||||
@@ -278,7 +278,7 @@ mod suspend_resume {
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'b ran' }",
|
||||
},
|
||||
// The gate holds `b` back, so `b` never runs and its error policy describes
|
||||
@@ -290,7 +290,7 @@ mod suspend_resume {
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'handled' }",
|
||||
},
|
||||
},
|
||||
@@ -343,7 +343,7 @@ mod suspend_resume {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'step1'; }"
|
||||
},
|
||||
"suspend": {
|
||||
@@ -355,7 +355,7 @@ mod suspend_resume {
|
||||
"id": "b",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'step2 - after approval'; }"
|
||||
}
|
||||
}]
|
||||
@@ -454,7 +454,7 @@ mod suspend_resume {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'step1'; }"
|
||||
},
|
||||
"suspend": {
|
||||
@@ -466,7 +466,7 @@ mod suspend_resume {
|
||||
"id": "b",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'step2 - after approval'; }"
|
||||
}
|
||||
}]
|
||||
@@ -556,7 +556,7 @@ mod suspend_resume {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'step1'; }"
|
||||
},
|
||||
"suspend": {
|
||||
@@ -567,7 +567,7 @@ mod suspend_resume {
|
||||
"id": "b",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'step2 - after approval'; }"
|
||||
}
|
||||
}]
|
||||
@@ -658,7 +658,7 @@ mod suspend_resume {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'step1'; }"
|
||||
},
|
||||
"suspend": {
|
||||
@@ -670,7 +670,7 @@ mod suspend_resume {
|
||||
"id": "b",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'step2 - after approval'; }"
|
||||
}
|
||||
}]
|
||||
@@ -764,7 +764,7 @@ mod suspend_resume {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'step1'; }"
|
||||
},
|
||||
"suspend": {
|
||||
@@ -776,7 +776,7 @@ mod suspend_resume {
|
||||
"id": "b",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 'step2 - after approval'; }"
|
||||
}
|
||||
}]
|
||||
@@ -933,7 +933,7 @@ mod suspend_resume {
|
||||
"suspend": { "required_events": 1, "timeout": 86400 },
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export async function main() { return 1 }",
|
||||
"input_transforms": {},
|
||||
},
|
||||
@@ -942,7 +942,7 @@ mod suspend_resume {
|
||||
"id": "b",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export async function main() { return 42 }",
|
||||
"input_transforms": {},
|
||||
},
|
||||
@@ -980,7 +980,7 @@ mod suspend_resume {
|
||||
"suspend": { "required_events": APPROVALS, "timeout": 86400 },
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export async function main() { return 1 }",
|
||||
"input_transforms": {},
|
||||
},
|
||||
@@ -989,7 +989,7 @@ mod suspend_resume {
|
||||
"id": "b",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(resumes) { return resumes.length }",
|
||||
"input_transforms": {
|
||||
"resumes": { "type": "javascript", "expr": "resumes" },
|
||||
|
||||
+153
-45
@@ -166,7 +166,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_deno_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
async fn test_bun_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
@@ -183,7 +183,7 @@ async fn test_deno_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
id: "a".to_string(),
|
||||
value: FlowModuleValue::RawScript {
|
||||
input_transforms: Default::default(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: numbers.to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
@@ -232,7 +232,7 @@ async fn test_deno_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
},
|
||||
)]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: doubles.to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
@@ -296,7 +296,7 @@ async fn test_deno_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
let port = server.addr.port();
|
||||
|
||||
for i in 0..50 {
|
||||
println!("deno flow iteration: {}", i);
|
||||
println!("bun flow iteration: {}", i);
|
||||
let job = run_job_in_new_worker_until_complete(&db, false, job.clone(), port).await;
|
||||
// println!("job: {:#?}", job.flow_status);
|
||||
let result = job.json_result().unwrap();
|
||||
@@ -346,13 +346,13 @@ async fn test_identity(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
async fn test_bun_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
|
||||
let write_file = r#"export async function main(loop: boolean, i: number, path: string) {
|
||||
await Deno.writeTextFile(`./shared/${path}`, `${loop} ${i}`);
|
||||
await Bun.write(`./shared/${path}`, `${loop} ${i}`);
|
||||
}"#
|
||||
.to_string();
|
||||
|
||||
@@ -373,7 +373,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: write_file.clone(),
|
||||
path: None,
|
||||
lock: None,
|
||||
@@ -432,7 +432,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: write_file,
|
||||
path: None,
|
||||
lock: None,
|
||||
@@ -472,9 +472,9 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
InputTransform::Static { value: windmill_common::worker::to_raw_value(&"outer.txt") },
|
||||
)]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"export async function main(path: string, path2: string) {
|
||||
return await Deno.readTextFile(`./shared/${path}`) + "," + await Deno.readTextFile(`./shared/${path2}`);
|
||||
return await Bun.file(`./shared/${path}`).text() + "," + await Bun.file(`./shared/${path2}`).text();
|
||||
}"#
|
||||
.to_string(),
|
||||
path: None,
|
||||
@@ -546,9 +546,9 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
language: ScriptLang::Deno,
|
||||
language: ScriptLang::Bun,
|
||||
content: r#"export async function main(path: string, loops: string[], path2: string) {
|
||||
return await Deno.readTextFile(`./shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`./shared/${path2}`);
|
||||
return await Bun.file(`./shared/${path}`).text() + "," + loops + "," + await Bun.file(`./shared/${path2}`).text();
|
||||
}"#
|
||||
.to_string(),
|
||||
path: None,
|
||||
@@ -618,7 +618,7 @@ async fn test_same_worker_survives_empty_branch(db: Pool<Postgres>) -> anyhow::R
|
||||
"id": "c",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return 1 }",
|
||||
}
|
||||
}]
|
||||
@@ -630,7 +630,7 @@ async fn test_same_worker_survives_empty_branch(db: Pool<Postgres>) -> anyhow::R
|
||||
"id": "b",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return 42 }",
|
||||
}
|
||||
}
|
||||
@@ -671,7 +671,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return 42 }",
|
||||
}
|
||||
},
|
||||
@@ -686,7 +686,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"value": {
|
||||
"input_transforms": {"v": {"type": "javascript", "expr": "results.a"}},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(v){ return v }",
|
||||
}
|
||||
|
||||
@@ -3857,7 +3857,7 @@ fn module_add_item_to_list(i: i32, id: &str) -> serde_json::Value {
|
||||
}
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(array, i){ array.push(i); return array }",
|
||||
}
|
||||
})
|
||||
@@ -3868,7 +3868,7 @@ fn module_failure() -> serde_json::Value {
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ throw Error('failure') }",
|
||||
}
|
||||
})
|
||||
@@ -3886,7 +3886,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return [1] }",
|
||||
}
|
||||
},
|
||||
@@ -3924,7 +3924,7 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return [1] }",
|
||||
}
|
||||
},
|
||||
@@ -3962,7 +3962,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return [1] }",
|
||||
}
|
||||
},
|
||||
@@ -4002,7 +4002,7 @@ async fn test_branchall_simple(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return [1] }",
|
||||
}
|
||||
},
|
||||
@@ -4050,7 +4050,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return [1] }",
|
||||
}
|
||||
},
|
||||
@@ -4086,7 +4086,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return [1] }",
|
||||
}
|
||||
},
|
||||
@@ -4132,7 +4132,7 @@ async fn test_branchone_nested(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return [] }",
|
||||
}
|
||||
},
|
||||
@@ -4191,7 +4191,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(){ return [1] }",
|
||||
}
|
||||
},
|
||||
@@ -4257,7 +4257,7 @@ async fn test_failure_module(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"n": { "type": "javascript", "expr": "flow_input.n", },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(n, l) { if (n == 0) throw Error(JSON.stringify(l)); return { l: [...l, 0] } }",
|
||||
},
|
||||
}, {
|
||||
@@ -4268,7 +4268,7 @@ async fn test_failure_module(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"n": { "type": "javascript", "expr": "flow_input.n", },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(n, l) { if (n == 1) throw Error(JSON.stringify(l)); return { l: [...l, 1] } }",
|
||||
},
|
||||
}, {
|
||||
@@ -4278,7 +4278,7 @@ async fn test_failure_module(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"n": { "type": "javascript", "expr": "flow_input.n", },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(n, l) { if (n == 2) throw Error(JSON.stringify(l)); return { l: [...l, 2] } }",
|
||||
},
|
||||
}],
|
||||
@@ -4286,7 +4286,7 @@ async fn test_failure_module(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
"value": {
|
||||
"input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } },
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(error) { return { 'from failure module': error } }",
|
||||
}
|
||||
},
|
||||
@@ -4475,7 +4475,7 @@ fn hanging_step_value() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export async function main() { await new Promise((r) => setTimeout(r, 600000)); }",
|
||||
})
|
||||
}
|
||||
@@ -4487,7 +4487,7 @@ fn marker_failure_module() -> serde_json::Value {
|
||||
"value": {
|
||||
"input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } },
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main(error) { return { handled_unrecoverable: true, error } }",
|
||||
}
|
||||
})
|
||||
@@ -4585,7 +4585,7 @@ async fn test_unrecoverable_failure_on_continue_on_error_runs_failure_module(
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return { ran_b: true }; }",
|
||||
},
|
||||
},
|
||||
@@ -4624,7 +4624,7 @@ async fn test_run_wait_result_early_return_with_failure_module(
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {},
|
||||
"content": "export function main() { throw new Error('boom'); }",
|
||||
},
|
||||
@@ -4632,7 +4632,7 @@ async fn test_run_wait_result_early_return_with_failure_module(
|
||||
"failure_module": {
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {},
|
||||
"content": "export function main() { return { recovered: true } }",
|
||||
},
|
||||
@@ -4720,7 +4720,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { throw new Error('boom'); }",
|
||||
},
|
||||
}],
|
||||
@@ -4734,7 +4734,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return { ran_b: true } }",
|
||||
},
|
||||
}],
|
||||
@@ -4742,7 +4742,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return { handled: true, recover: true } }",
|
||||
},
|
||||
},
|
||||
@@ -4797,14 +4797,23 @@ async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::
|
||||
server.close().await.unwrap();
|
||||
|
||||
#[cfg(feature = "quickjs")]
|
||||
assert!(!chaining_error.success, "an error no handler recovered must keep the flow failed");
|
||||
assert!(
|
||||
!chaining_error.success,
|
||||
"an error no handler recovered must keep the flow failed"
|
||||
);
|
||||
|
||||
assert!(stopped.success, "a recovered failure inside a loop should end the flow as a success");
|
||||
assert!(
|
||||
stopped.success,
|
||||
"a recovered failure inside a loop should end the flow as a success"
|
||||
);
|
||||
assert!(
|
||||
stopped.json_result().unwrap().get("ran_b").is_none(),
|
||||
"the step after the loop must not run, as without recovery"
|
||||
);
|
||||
assert_eq!(stopped_handler_runs, 1, "the loop must stop at the first failed iteration");
|
||||
assert_eq!(
|
||||
stopped_handler_runs, 1,
|
||||
"the loop must stop at the first failed iteration"
|
||||
);
|
||||
|
||||
assert!(skipping.success);
|
||||
assert_eq!(
|
||||
@@ -4815,7 +4824,11 @@ async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::
|
||||
assert_eq!(skipping_handler_runs, 3);
|
||||
|
||||
assert!(
|
||||
!parallel.json_result().unwrap().to_string().contains("ran_b"),
|
||||
!parallel
|
||||
.json_result()
|
||||
.unwrap()
|
||||
.to_string()
|
||||
.contains("ran_b"),
|
||||
"the parallel loop fails as without recovery, so the step after it must not run"
|
||||
);
|
||||
Ok(())
|
||||
@@ -4987,6 +5000,101 @@ async fn test_flow_lock_all(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A deployed flow runs a multi-step loop body as the flow node its deploy stored, which
|
||||
/// carries its own copy of the flow's `preserve_step_tags`. A preview never loads that node.
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_saved_flow_preserves_nested_step_tags(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
use futures::StreamExt;
|
||||
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
let client = reqwest::Client::new();
|
||||
let flow = json!({
|
||||
"path": "g/all/saved_step_tags", "summary": "", "description": "", "tag": "python3",
|
||||
"schema": { "type": "object", "properties": {}, "required": [] },
|
||||
"value": {
|
||||
"preserve_step_tags": true,
|
||||
"modules": [{
|
||||
"id": "loop",
|
||||
"value": {
|
||||
"type": "forloopflow", "skip_failures": false,
|
||||
"iterator": { "type": "static", "value": [2, 3] },
|
||||
"modules": [{
|
||||
"id": "own_tag",
|
||||
"value": {
|
||||
"type": "rawscript", "language": "bun", "lock": "", "tag": "bun",
|
||||
"content": "export function main(n: number) { return n * n; }",
|
||||
"input_transforms": {
|
||||
"n": { "type": "javascript", "expr": "flow_input.iter.value" }
|
||||
}
|
||||
}
|
||||
}, {
|
||||
"id": "untagged",
|
||||
"value": {
|
||||
"type": "rawscript", "language": "bun", "lock": "",
|
||||
"content": "export function main(n: number) { return n + 1; }",
|
||||
"input_transforms": {
|
||||
"n": { "type": "javascript", "expr": "results.own_tag" }
|
||||
}
|
||||
}
|
||||
}]
|
||||
}
|
||||
}]
|
||||
}
|
||||
});
|
||||
let mut completed = listen_for_completed_jobs(&db).await;
|
||||
client
|
||||
.post(format!(
|
||||
"http://localhost:{port}/api/w/test-workspace/flows/create"
|
||||
))
|
||||
.bearer_auth("SECRET_TOKEN")
|
||||
.json(&flow)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
let deployment = in_test_worker(&db, completed.next(), port).await.unwrap();
|
||||
let deployed = completed_job(deployment, &db).await;
|
||||
assert!(deployed.success, "{:?}", deployed.result);
|
||||
|
||||
let job_id = client
|
||||
.post(format!(
|
||||
"http://localhost:{port}/api/w/test-workspace/jobs/run/f/g/all/saved_step_tags"
|
||||
))
|
||||
.bearer_auth("SECRET_TOKEN")
|
||||
.json(&json!({}))
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.text()
|
||||
.await?
|
||||
.parse::<Uuid>()?;
|
||||
in_test_worker(&db, completed.find(&job_id), port).await;
|
||||
let job = completed_job(job_id, &db).await;
|
||||
assert!(job.success, "{:?}", job.result);
|
||||
let tags = sqlx::query_as::<_, (String, String)>(
|
||||
"WITH RECURSIVE descendants AS (
|
||||
SELECT id FROM v2_job WHERE id = $1
|
||||
UNION ALL
|
||||
SELECT child.id FROM v2_job child JOIN descendants ON child.parent_job = descendants.id
|
||||
)
|
||||
SELECT DISTINCT flow_step_id, tag FROM v2_job
|
||||
WHERE id IN (SELECT id FROM descendants) AND flow_step_id IN ('own_tag', 'untagged')
|
||||
ORDER BY flow_step_id, tag",
|
||||
)
|
||||
.bind(job_id)
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
tags,
|
||||
vec![
|
||||
("own_tag".to_owned(), "bun".to_owned()),
|
||||
("untagged".to_owned(), "python3".to_owned())
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
|
||||
@@ -5687,7 +5795,7 @@ async fn test_job_labels(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
console.log(greet)
|
||||
return { greet, wm_labels: ["yolo", "greet", "greet", world] };
|
||||
}"#,
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"world": { "type": "javascript", "expr": "flow_input.world" }
|
||||
}
|
||||
@@ -5896,7 +6004,7 @@ async fn test_flow_substep_tag_availability_check(db: Pool<Postgres>) -> anyhow:
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return 42; }",
|
||||
"tag": "restricted-tag",
|
||||
},
|
||||
@@ -5956,7 +6064,7 @@ async fn test_flow_substep_tag_checked_on_resolved_value(db: Pool<Postgres>) ->
|
||||
"id": id,
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"content": "export function main() { return { lang: 'bun' } }",
|
||||
"tag": tag,
|
||||
},
|
||||
|
||||
@@ -78,7 +78,7 @@ async fn test_between_steps_zombie_restart_reuses_all_children(
|
||||
"id": "inner",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
|
||||
},
|
||||
@@ -91,7 +91,7 @@ async fn test_between_steps_zombie_restart_reuses_all_children(
|
||||
"id": "after",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"loop_res": { "type": "javascript", "expr": "results.fanout" }
|
||||
},
|
||||
@@ -210,7 +210,7 @@ async fn test_mid_iteration_zombie_not_reused(db: Pool<Postgres>) -> anyhow::Res
|
||||
"id": "inner",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
|
||||
},
|
||||
@@ -223,7 +223,7 @@ async fn test_mid_iteration_zombie_not_reused(db: Pool<Postgres>) -> anyhow::Res
|
||||
"id": "after",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"language": "bun",
|
||||
"input_transforms": {
|
||||
"loop_res": { "type": "javascript", "expr": "results.fanout" }
|
||||
},
|
||||
@@ -331,7 +331,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse(
|
||||
{
|
||||
"id": "inner_first",
|
||||
"value": {
|
||||
"type": "rawscript", "language": "deno",
|
||||
"type": "rawscript", "language": "bun",
|
||||
"input_transforms": {},
|
||||
"content": "export function main() { return 'first' }"
|
||||
}
|
||||
@@ -339,7 +339,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse(
|
||||
{
|
||||
"id": "inner_second",
|
||||
"value": {
|
||||
"type": "rawscript", "language": "deno",
|
||||
"type": "rawscript", "language": "bun",
|
||||
"input_transforms": {
|
||||
"first": { "type": "javascript", "expr": "results.inner_first" }
|
||||
},
|
||||
@@ -353,7 +353,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse(
|
||||
{
|
||||
"id": "after",
|
||||
"value": {
|
||||
"type": "rawscript", "language": "deno",
|
||||
"type": "rawscript", "language": "bun",
|
||||
"input_transforms": {
|
||||
"b": { "type": "javascript", "expr": "results.branch" }
|
||||
},
|
||||
@@ -466,7 +466,7 @@ async fn test_raw_flow_restart_does_not_reuse_edited_step(
|
||||
"modules": [{
|
||||
"id": "inner",
|
||||
"value": {
|
||||
"type": "rawscript", "language": "deno",
|
||||
"type": "rawscript", "language": "bun",
|
||||
"input_transforms": {
|
||||
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
|
||||
},
|
||||
@@ -478,7 +478,7 @@ async fn test_raw_flow_restart_does_not_reuse_edited_step(
|
||||
{
|
||||
"id": "after",
|
||||
"value": {
|
||||
"type": "rawscript", "language": "deno",
|
||||
"type": "rawscript", "language": "bun",
|
||||
"input_transforms": {
|
||||
"loop_res": { "type": "javascript", "expr": "results.fanout" }
|
||||
},
|
||||
@@ -578,7 +578,7 @@ async fn test_non_monitor_cancel_is_not_reused(db: Pool<Postgres>) -> anyhow::Re
|
||||
"modules": [{
|
||||
"id": "inner",
|
||||
"value": {
|
||||
"type": "rawscript", "language": "deno",
|
||||
"type": "rawscript", "language": "bun",
|
||||
"input_transforms": {
|
||||
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
|
||||
},
|
||||
@@ -590,7 +590,7 @@ async fn test_non_monitor_cancel_is_not_reused(db: Pool<Postgres>) -> anyhow::Re
|
||||
{
|
||||
"id": "after",
|
||||
"value": {
|
||||
"type": "rawscript", "language": "deno",
|
||||
"type": "rawscript", "language": "bun",
|
||||
"input_transforms": {
|
||||
"loop_res": { "type": "javascript", "expr": "results.fanout" }
|
||||
},
|
||||
|
||||
@@ -28,6 +28,7 @@ eventsource-stream.workspace = true
|
||||
futures.workspace = true
|
||||
http.workspace = true
|
||||
mime_guess.workspace = true
|
||||
rand.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -477,6 +477,29 @@ pub async fn assume_role_with_oidc_token(
|
||||
// Error Formatting
|
||||
// ============================================================================
|
||||
|
||||
/// The error for a Bedrock stream that failed after it opened, retrying the exceptions the
|
||||
/// Vercel AI SDK retries (`amazon-bedrock-stream-error.ts`) and a broken connection. The
|
||||
/// SDK's own retries cover the opening call alone.
|
||||
pub fn bedrock_stream_error<R: std::fmt::Debug>(
|
||||
error: &aws_sdk_bedrockruntime::error::SdkError<
|
||||
aws_sdk_bedrockruntime::types::error::ConverseStreamOutputError,
|
||||
R,
|
||||
>,
|
||||
) -> Error {
|
||||
let message = format!("Bedrock stream error: {}", format_bedrock_error(error));
|
||||
match error.as_service_error() {
|
||||
Some(e)
|
||||
if !(e.is_internal_server_exception()
|
||||
|| e.is_model_stream_error_exception()
|
||||
|| e.is_service_unavailable_exception()
|
||||
|| e.is_throttling_exception()) =>
|
||||
{
|
||||
Error::AIError(message)
|
||||
}
|
||||
_ => crate::retry::transient_error(message, None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format AWS SDK errors with detailed information
|
||||
pub fn format_bedrock_error<E, R>(error: &aws_sdk_bedrockruntime::error::SdkError<E, R>) -> String
|
||||
where
|
||||
|
||||
@@ -326,6 +326,17 @@ pub struct GeminiSSEEvent {
|
||||
pub candidates: Option<Vec<GeminiSSECandidate>>,
|
||||
#[serde(rename = "usageMetadata")]
|
||||
pub usage_metadata: Option<GeminiUsageMetadata>,
|
||||
#[serde(default)]
|
||||
pub error: Option<crate::retry::StreamErrorBody>,
|
||||
#[serde(rename = "promptFeedback", default)]
|
||||
pub prompt_feedback: Option<GeminiPromptFeedback>,
|
||||
}
|
||||
|
||||
/// Set when Gemini refuses the prompt itself, in which case no candidate follows.
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiPromptFeedback {
|
||||
#[serde(rename = "blockReason", default)]
|
||||
pub block_reason: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -550,10 +561,40 @@ pub fn openai_tools_to_gemini(
|
||||
}
|
||||
}
|
||||
|
||||
/// Follows a Gemini stream to tell a complete turn from one cut short.
|
||||
#[derive(Default)]
|
||||
pub struct GeminiTurn {
|
||||
ended: bool,
|
||||
has_output: bool,
|
||||
}
|
||||
|
||||
impl GeminiTurn {
|
||||
pub fn observe(&mut self, event: &GeminiParsedEvent) {
|
||||
// A function call arrives whole and may come without a `finishReason`, so it
|
||||
// counts too, as in gemini-cli's stream validation.
|
||||
self.ended |= event.finish_reason.is_some() || !event.tool_calls.is_empty();
|
||||
self.has_output |= event.text.is_some() || event.reasoning.is_some();
|
||||
}
|
||||
|
||||
/// Whether the turn's final event arrived.
|
||||
pub fn ended(&self) -> bool {
|
||||
self.ended
|
||||
}
|
||||
|
||||
/// Whether a stream that closed cleanly here holds the whole turn. Gemini 2.5 closes
|
||||
/// its stream without a single event when thinking spends the whole
|
||||
/// `maxOutputTokens`: an empty answer, which a retry would only get again, rather
|
||||
/// than a cut one.
|
||||
pub fn complete_at_close(&self) -> bool {
|
||||
self.ended || !self.has_output
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse one Gemini SSE data line into a [`GeminiParsedEvent`].
|
||||
///
|
||||
/// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`).
|
||||
/// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating.
|
||||
/// An error the stream reports is returned as one.
|
||||
pub fn parse_gemini_sse_event(data: &str) -> Result<Option<GeminiParsedEvent>, Error> {
|
||||
if data.is_empty() || data == "[DONE]" {
|
||||
return Ok(None);
|
||||
@@ -567,6 +608,21 @@ pub fn parse_gemini_sse_event(data: &str) -> Result<Option<GeminiParsedEvent>, E
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(error) = event.error {
|
||||
return Err(error.into_error("Gemini"));
|
||||
}
|
||||
|
||||
// `BLOCK_REASON_UNSPECIFIED` is the protobuf enum's placeholder, not a block.
|
||||
if let Some(reason) = event
|
||||
.prompt_feedback
|
||||
.and_then(|f| f.block_reason)
|
||||
.filter(|r| !matches!(r.as_str(), "" | "BLOCK_REASON_UNSPECIFIED"))
|
||||
{
|
||||
return Err(Error::AIError(format!(
|
||||
"Gemini blocked the prompt: {reason}"
|
||||
)));
|
||||
}
|
||||
|
||||
let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() };
|
||||
|
||||
let Some(candidates) = event.candidates else {
|
||||
|
||||
@@ -11,6 +11,7 @@ pub mod model_context;
|
||||
pub mod providers;
|
||||
pub mod proxy;
|
||||
pub mod query_builder;
|
||||
pub mod retry;
|
||||
pub mod sse;
|
||||
pub mod types;
|
||||
pub mod utils;
|
||||
|
||||
@@ -9,11 +9,12 @@
|
||||
use super::{anthropic_model_rejects_sampling_params, REASONING_OFF_SENTINEL};
|
||||
use crate::{
|
||||
ai_bedrock::{
|
||||
bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop,
|
||||
bedrock_stream_event_to_reasoning_delta, bedrock_stream_event_to_text,
|
||||
bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_delta_with_block_index,
|
||||
bedrock_stream_event_to_tool_start, bedrock_stream_event_to_tool_start_with_block_index,
|
||||
build_tool_config, create_inference_config, format_bedrock_error, json_to_document,
|
||||
bedrock_model_supports_prompt_caching, bedrock_stream_error,
|
||||
bedrock_stream_event_is_block_stop, bedrock_stream_event_to_reasoning_delta,
|
||||
bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta,
|
||||
bedrock_stream_event_to_tool_delta_with_block_index, bedrock_stream_event_to_tool_start,
|
||||
bedrock_stream_event_to_tool_start_with_block_index, build_tool_config,
|
||||
create_inference_config, format_bedrock_error, json_to_document,
|
||||
openai_messages_to_bedrock, streaming_tool_calls_to_openai, BearerTokenProvider,
|
||||
BedrockClient, StreamingToolCall,
|
||||
},
|
||||
@@ -24,6 +25,7 @@ use crate::{
|
||||
image_handler::prepare_messages_for_api,
|
||||
proxy::ProxyBuildArgs,
|
||||
query_builder::{ParsedResponse, StreamEventSink},
|
||||
retry::{openai_stream_error_chunk, truncated_stream_error},
|
||||
types::{OpenAIMessage, StreamingEvent, TokenUsage, ToolDef},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
@@ -461,26 +463,31 @@ pub fn sdk_stream_to_sse(
|
||||
async_stream::stream! {
|
||||
let mut stream = stream;
|
||||
let mut state = BedrockSseStreamState::new(id, model, created);
|
||||
let mut message_stopped = false;
|
||||
|
||||
loop {
|
||||
let failure = loop {
|
||||
match stream.recv().await {
|
||||
Ok(Some(event)) => {
|
||||
if matches!(
|
||||
event,
|
||||
aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(_)
|
||||
) {
|
||||
message_stopped = true;
|
||||
}
|
||||
for chunk in bedrock_sse_chunks_for_event(&event, &mut state) {
|
||||
yield Ok(chunk);
|
||||
}
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
yield Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
e.to_string(),
|
||||
));
|
||||
break;
|
||||
}
|
||||
Ok(None) => break (!message_stopped).then(truncated_stream_error),
|
||||
Err(_) if message_stopped => break None,
|
||||
Err(e) => break Some(bedrock_stream_error(&e)),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
yield Ok(Bytes::from("data: [DONE]\n\n"));
|
||||
match failure {
|
||||
Some(error) => yield Ok(Bytes::from(openai_stream_error_chunk(&error))),
|
||||
None => yield Ok(Bytes::from("data: [DONE]\n\n")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1081,11 +1088,19 @@ impl BedrockQueryBuilder {
|
||||
// Claude reasoning block for the turn (only populated when thinking is on),
|
||||
// attached to the first tool call for replay before toolUse.
|
||||
let mut reasoning: Option<crate::ai_types::BedrockExtraContent> = None;
|
||||
let mut message_stopped = false;
|
||||
|
||||
// Process stream events using shared parsing functions
|
||||
loop {
|
||||
match stream.recv().await {
|
||||
Ok(Some(event)) => {
|
||||
if matches!(
|
||||
event,
|
||||
aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(_)
|
||||
) {
|
||||
message_stopped = true;
|
||||
}
|
||||
|
||||
// Fold reasoning deltas into the turn's reasoning block (for
|
||||
// replay before toolUse, required by Claude when thinking is
|
||||
// on) and stream the readable summary as a thinking affordance.
|
||||
@@ -1161,11 +1176,13 @@ impl BedrockQueryBuilder {
|
||||
}
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
return Err(Error::internal_err(format!("Bedrock stream error: {}", e)));
|
||||
}
|
||||
Err(_) if message_stopped => break,
|
||||
Err(e) => return Err(bedrock_stream_error(&e)),
|
||||
}
|
||||
}
|
||||
if !message_stopped {
|
||||
return Err(truncated_stream_error());
|
||||
}
|
||||
|
||||
// Send tool call events to stream processor
|
||||
if let Some(processor) = stream_event_sink.as_ref() {
|
||||
|
||||
@@ -5,17 +5,20 @@ use crate::{
|
||||
parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google,
|
||||
GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiImageContent, GeminiImageRequest,
|
||||
GeminiImageResponse, GeminiInlineData, GeminiPart, GeminiPredictContent, GeminiTextRequest,
|
||||
GeminiThinkingConfig, GeminiTool,
|
||||
GeminiThinkingConfig, GeminiTool, GeminiTurn,
|
||||
},
|
||||
image_handler::{download_and_encode_s3_image, prepare_messages_for_api},
|
||||
proxy::{ProxyBuildArgs, ProxyRequest},
|
||||
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink},
|
||||
retry::{
|
||||
openai_stream_error_chunk, send_with_retries, transient_error, truncated_stream_error,
|
||||
},
|
||||
sse::{GeminiSSEParser, SSEParser},
|
||||
types::*,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use eventsource_stream::Eventsource;
|
||||
use eventsource_stream::{EventStreamError, Eventsource};
|
||||
use futures::{stream::BoxStream, StreamExt};
|
||||
use http::{header, HeaderMap, HeaderValue, Method, StatusCode};
|
||||
use serde::Deserialize;
|
||||
@@ -336,9 +339,7 @@ pub async fn handle_google_ai_chat_proxy(
|
||||
) -> Result<GoogleAIProxyResponse, Error> {
|
||||
let GoogleAIProxyRequest { request, model, stream } = build_google_ai_chat_proxy_request(args)?;
|
||||
|
||||
let response =
|
||||
send_google_ai_proxy_request(client, request, "Failed to send request to Gemini API")
|
||||
.await?;
|
||||
let response = send_google_ai_proxy_request(client, request).await?;
|
||||
|
||||
if stream {
|
||||
Ok(convert_streaming_response(response, &model))
|
||||
@@ -356,8 +357,7 @@ pub async fn handle_google_ai_models_proxy(
|
||||
args: &ProxyBuildArgs<'_>,
|
||||
) -> Result<GoogleAIProxyResponse, Error> {
|
||||
let request = build_google_ai_models_proxy_request(args);
|
||||
let response =
|
||||
send_google_ai_proxy_request(client, request, "Failed to fetch Gemini models").await?;
|
||||
let response = send_google_ai_proxy_request(client, request).await?;
|
||||
|
||||
let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| {
|
||||
Error::internal_err(format!("Failed to parse Gemini models response: {}", e))
|
||||
@@ -505,26 +505,13 @@ fn add_google_ai_auth_header(headers: &mut Vec<(String, String)>, api_key: &str,
|
||||
async fn send_google_ai_proxy_request(
|
||||
client: &reqwest::Client,
|
||||
proxy_request: ProxyRequest,
|
||||
send_error_message: &str,
|
||||
) -> Result<reqwest::Response, Error> {
|
||||
let mut request = client.request(proxy_request.method.clone(), &proxy_request.url);
|
||||
for (header_name, header_value) in &proxy_request.headers {
|
||||
request = request.header(header_name.as_str(), header_value.as_str());
|
||||
}
|
||||
|
||||
let response = request
|
||||
.body(proxy_request.body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::internal_err(format!("{}: {}", send_error_message, e)))?;
|
||||
|
||||
if let Err(e) = response.error_for_status_ref() {
|
||||
let status = e.status().map(|s| s.to_string()).unwrap_or_default();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(Error::AIError(format!("{}: {}", status, body)));
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
send_with_retries(request.body(proxy_request.body)).await
|
||||
}
|
||||
|
||||
fn convert_streaming_response(response: reqwest::Response, model: &str) -> GoogleAIProxyResponse {
|
||||
@@ -535,10 +522,15 @@ fn convert_streaming_response(response: reqwest::Response, model: &str) -> Googl
|
||||
let openai_sse_stream = async_stream::stream! {
|
||||
tokio::pin!(gemini_sse_stream);
|
||||
let mut tool_call_index: usize = 0;
|
||||
while let Some(event) = gemini_sse_stream.next().await {
|
||||
let mut turn = GeminiTurn::default();
|
||||
let failure = loop {
|
||||
let Some(event) = gemini_sse_stream.next().await else {
|
||||
break (!turn.complete_at_close()).then(truncated_stream_error);
|
||||
};
|
||||
match event {
|
||||
Ok(event) => match parse_gemini_sse_event(&event.data) {
|
||||
Ok(Some(parsed)) => {
|
||||
turn.observe(&parsed);
|
||||
for chunk in gemini_event_to_openai_sse_chunks(
|
||||
&parsed, &id, &model, &mut tool_call_index,
|
||||
) {
|
||||
@@ -546,12 +538,23 @@ fn convert_streaming_response(response: reqwest::Response, model: &str) -> Googl
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e),
|
||||
Err(e) => break Some(e),
|
||||
},
|
||||
Err(EventStreamError::Transport(_)) if turn.ended() => break None,
|
||||
Err(EventStreamError::Transport(e)) => {
|
||||
break Some(transient_error(
|
||||
format!("The connection to Gemini broke off mid-response: {e}"),
|
||||
None,
|
||||
));
|
||||
}
|
||||
Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e),
|
||||
}
|
||||
}
|
||||
yield Ok::<Bytes, reqwest::Error>(Bytes::from("data: [DONE]\n\n"));
|
||||
};
|
||||
let last = match failure {
|
||||
Some(error) => openai_stream_error_chunk(&error),
|
||||
None => "data: [DONE]\n\n".to_string(),
|
||||
};
|
||||
yield Ok::<Bytes, reqwest::Error>(Bytes::from(last));
|
||||
}
|
||||
.boxed();
|
||||
|
||||
|
||||
@@ -266,6 +266,7 @@ impl QueryBuilder for OtherQueryBuilder {
|
||||
mut events_str,
|
||||
stream_event_processor,
|
||||
usage: openai_usage,
|
||||
..
|
||||
} = openai_sse_parser;
|
||||
|
||||
// Process streaming events with error handling
|
||||
|
||||
@@ -0,0 +1,474 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::{header::HeaderMap, RequestBuilder, Response, StatusCode};
|
||||
use windmill_common::error::{to_anyhow, Error};
|
||||
|
||||
/// How many times a provider call is retried after its first attempt.
|
||||
pub const MAX_RETRIES: u32 = 3;
|
||||
const BASE_DELAY: Duration = Duration::from_secs(1);
|
||||
const MAX_BACKOFF: Duration = Duration::from_secs(10);
|
||||
/// A provider that asks for a longer wait is out of capacity for longer than anyone should
|
||||
/// block on (a spent daily quota, say), so its error is returned instead of waited out.
|
||||
const MAX_RETRY_AFTER: Duration = Duration::from_secs(60);
|
||||
|
||||
/// A provider call that failed for a reason a later attempt may not meet: the provider was
|
||||
/// overloaded, throttled or unreachable, or its response broke off part way.
|
||||
#[derive(Debug)]
|
||||
pub struct TransientAIError {
|
||||
pub message: String,
|
||||
/// The wait the provider asked for, when it named one.
|
||||
pub retry_after: Option<Duration>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for TransientAIError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for TransientAIError {}
|
||||
|
||||
pub fn transient_error(message: impl Into<String>, retry_after: Option<Duration>) -> Error {
|
||||
to_anyhow(TransientAIError { message: message.into(), retry_after }).into()
|
||||
}
|
||||
|
||||
pub fn as_transient(error: &Error) -> Option<&TransientAIError> {
|
||||
match error {
|
||||
Error::Anyhow { error, .. } => error.downcast_ref::<TransientAIError>(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The error to report once a transient failure is no longer retried, without the source
|
||||
/// location `Error::Anyhow` renders.
|
||||
pub fn into_final_error(error: Error) -> Error {
|
||||
match as_transient(&error) {
|
||||
Some(transient) => Error::AIError(transient.message.clone()),
|
||||
None => error,
|
||||
}
|
||||
}
|
||||
|
||||
/// The statuses the OpenAI and Anthropic SDKs and the Vercel AI SDK retry.
|
||||
fn is_retryable_status(status: u16) -> bool {
|
||||
matches!(status, 408 | 409 | 429 | 500..=599)
|
||||
}
|
||||
|
||||
/// The error for a provider's error response. The rules follow the OpenAI and Anthropic
|
||||
/// SDKs (`x-should-retry`, then the status) and gemini-cli (a daily quota is spent until
|
||||
/// tomorrow, and the wait Gemini asks for is in its body).
|
||||
pub fn response_error(
|
||||
status: StatusCode,
|
||||
headers: &HeaderMap,
|
||||
body: &str,
|
||||
message: String,
|
||||
) -> Error {
|
||||
let retryable = match headers.get("x-should-retry").and_then(|v| v.to_str().ok()) {
|
||||
Some("true") => true,
|
||||
Some("false") => false,
|
||||
_ => {
|
||||
is_retryable_status(status.as_u16())
|
||||
// OpenAI answers a spent balance with the 429 it also uses for rate limits.
|
||||
&& !body.contains("insufficient_quota")
|
||||
&& !google_quota_spent_for_the_day(body)
|
||||
}
|
||||
};
|
||||
if retryable {
|
||||
transient_error(
|
||||
message,
|
||||
retry_after(headers).or_else(|| google_retry_delay(body)),
|
||||
)
|
||||
} else {
|
||||
Error::AIError(message)
|
||||
}
|
||||
}
|
||||
|
||||
/// The `details` of a Google API error body, where Gemini puts its quota and retry hints.
|
||||
fn google_error_details(body: &str) -> Vec<serde_json::Value> {
|
||||
serde_json::from_str::<serde_json::Value>(body)
|
||||
.ok()
|
||||
.and_then(|v| v.get("error")?.get("details")?.as_array().cloned())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn google_quota_spent_for_the_day(body: &str) -> bool {
|
||||
google_error_details(body).iter().any(|detail| {
|
||||
detail
|
||||
.get("violations")
|
||||
.and_then(|v| v.as_array())
|
||||
.is_some_and(|violations| {
|
||||
violations.iter().any(|violation| {
|
||||
violation
|
||||
.get("quotaId")
|
||||
.and_then(|id| id.as_str())
|
||||
.is_some_and(|id| id.contains("PerDay") || id.contains("Daily"))
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// The `RetryInfo.retryDelay` of a Google API error, a protobuf duration such as `"53s"`.
|
||||
fn google_retry_delay(body: &str) -> Option<Duration> {
|
||||
google_error_details(body).iter().find_map(|detail| {
|
||||
let delay = detail.get("retryDelay")?.as_str()?.strip_suffix('s')?;
|
||||
let seconds = delay
|
||||
.parse::<f64>()
|
||||
.ok()
|
||||
.filter(|v| v.is_finite() && *v >= 0.0)?;
|
||||
Some(Duration::try_from_secs_f64(seconds).unwrap_or(Duration::MAX))
|
||||
})
|
||||
}
|
||||
|
||||
/// The wait a provider asked for in its `retry-after-ms` or `retry-after` header. An
|
||||
/// HTTP-date `retry-after` is ignored, which falls back to the computed backoff.
|
||||
fn retry_after(headers: &HeaderMap) -> Option<Duration> {
|
||||
let seconds = |name: &str, scale: f64| {
|
||||
headers
|
||||
.get(name)?
|
||||
.to_str()
|
||||
.ok()?
|
||||
.trim()
|
||||
.parse::<f64>()
|
||||
.ok()
|
||||
.filter(|v| v.is_finite() && *v >= 0.0)
|
||||
.map(|v| Duration::try_from_secs_f64(v / scale).unwrap_or(Duration::MAX))
|
||||
};
|
||||
seconds("retry-after-ms", 1000.0).or_else(|| seconds("retry-after", 1.0))
|
||||
}
|
||||
|
||||
/// The error object a provider sends inside a stream: OpenAI-compatible chunks carry
|
||||
/// `type` and a string or status `code`, Anthropic `type`, Gemini a status `code` and
|
||||
/// `status`.
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct StreamErrorBody {
|
||||
#[serde(default)]
|
||||
pub message: Option<String>,
|
||||
#[serde(default, rename = "type")]
|
||||
pub kind: Option<String>,
|
||||
#[serde(default)]
|
||||
pub code: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
pub status: Option<String>,
|
||||
}
|
||||
|
||||
impl StreamErrorBody {
|
||||
/// The error for a failure an OpenAI-compatible or Gemini stream reported after opening
|
||||
/// with a 200, classified as the Vercel AI SDK classifies OpenAI stream errors
|
||||
/// (`openai-stream-error.ts`): the status `code` holds, else the status its code and type
|
||||
/// name, else a server error.
|
||||
pub fn into_error(self, provider: &str) -> Error {
|
||||
let code = self.code_str();
|
||||
let discriminator = [code.as_deref(), self.kind.as_deref()]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.to_lowercase();
|
||||
let has = |terms: &[&str]| terms.iter().any(|term| discriminator.contains(term));
|
||||
let status = match code.as_deref().and_then(|code| code.parse::<u16>().ok()) {
|
||||
Some(status @ 400..=599) => status,
|
||||
_ if has(&["insufficient_quota", "rate_limit"]) => 429,
|
||||
_ if has(&["authentication"]) => 401,
|
||||
_ if has(&["permission"]) => 403,
|
||||
_ if has(&["not_found"]) => 404,
|
||||
_ if has(&["invalid", "bad_request", "context_length"]) => 400,
|
||||
_ if has(&["overload"]) => 503,
|
||||
_ if has(&["timeout"]) => 504,
|
||||
_ => 500,
|
||||
};
|
||||
let retryable = !has(&["insufficient_quota"]) && is_retryable_status(status);
|
||||
self.into_classified_error(provider, retryable)
|
||||
}
|
||||
|
||||
/// The error for a failure an Anthropic stream reported after opening with a 200. The
|
||||
/// kinds retried are those the Vercel AI SDK retries, plus `timeout_error`, which
|
||||
/// Anthropic documents as its 504; any other kind is a fault in the request or account.
|
||||
pub fn into_anthropic_error(self) -> Error {
|
||||
let retryable = matches!(
|
||||
self.kind.as_deref(),
|
||||
Some("api_error" | "overloaded_error" | "rate_limit_error" | "timeout_error")
|
||||
);
|
||||
self.into_classified_error("Anthropic", retryable)
|
||||
}
|
||||
|
||||
fn code_str(&self) -> Option<String> {
|
||||
match self.code.as_ref()? {
|
||||
serde_json::Value::String(code) => Some(code.clone()),
|
||||
serde_json::Value::Number(code) => Some(code.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn into_classified_error(self, provider: &str, retryable: bool) -> Error {
|
||||
let kind = [self.kind.clone(), self.code_str(), self.status]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
.join("/");
|
||||
let message = format!(
|
||||
"{provider} stream error{}: {}",
|
||||
(!kind.is_empty())
|
||||
.then(|| format!(" ({kind})"))
|
||||
.unwrap_or_default(),
|
||||
self.message.as_deref().unwrap_or("no details")
|
||||
);
|
||||
if retryable {
|
||||
transient_error(message, None)
|
||||
} else {
|
||||
Error::AIError(message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The error for a stream that closed before the provider sent the event ending a
|
||||
/// complete response: cut off by the provider or by a proxy in between.
|
||||
pub fn truncated_stream_error() -> Error {
|
||||
transient_error(
|
||||
"The AI provider's response stream ended before the response was complete",
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
/// An OpenAI-format stream chunk reporting `error`, which ends a response converted to that
|
||||
/// format after it started streaming: the chat's OpenAI SDK throws on a chunk carrying
|
||||
/// `error`, where a bare end of stream would read as a finished answer.
|
||||
pub fn openai_stream_error_chunk(error: &Error) -> String {
|
||||
let message = as_transient(error).map_or_else(|| error.to_string(), |t| t.message.clone());
|
||||
format!(
|
||||
"data: {}\n\n",
|
||||
serde_json::json!({ "error": { "message": message } })
|
||||
)
|
||||
}
|
||||
|
||||
/// Sends a provider request, retrying a transient failure while nothing of the response
|
||||
/// has been read: a failed connection or an error status. Once retries are spent the error
|
||||
/// carries the provider's answer, as a single attempt's would.
|
||||
pub async fn send_with_retries(request: RequestBuilder) -> Result<Response, Error> {
|
||||
let mut backoff = Backoff::default();
|
||||
loop {
|
||||
// A streamed body cannot be replayed, so such a request is sent once.
|
||||
let Some(attempt) = request.try_clone() else {
|
||||
return send_once(request).await.map_err(into_final_error);
|
||||
};
|
||||
let error = match send_once(attempt).await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) => error,
|
||||
};
|
||||
let Some(delay) = as_transient(&error).and_then(|t| backoff.next_delay(t)) else {
|
||||
return Err(into_final_error(error));
|
||||
};
|
||||
tracing::warn!(
|
||||
"AI provider call failed ({error}); retrying in {:.1}s ({}/{MAX_RETRIES})",
|
||||
delay.as_secs_f64(),
|
||||
backoff.retries()
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// The error for a provider request that failed before any response arrived. The request
|
||||
/// timeout is the caller's whole budget, so a timed-out call leaves no time for another
|
||||
/// attempt; a connect timeout has its own, shorter one.
|
||||
pub fn send_error(e: reqwest::Error, context: &str) -> Error {
|
||||
let message = format!("{context}: {e}");
|
||||
if e.is_builder() || (e.is_timeout() && !e.is_connect()) {
|
||||
Error::AIError(message)
|
||||
} else {
|
||||
transient_error(message, None)
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_once(request: RequestBuilder) -> Result<Response, Error> {
|
||||
let response = request
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| send_error(e, "Failed to reach the AI provider"))?;
|
||||
let status = response.status();
|
||||
if status.is_client_error() || status.is_server_error() {
|
||||
let headers = response.headers().clone();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(response_error(status, &headers, &body, body.clone()));
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Counts the retries of one provider call and spaces them out.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Backoff {
|
||||
retries: u32,
|
||||
}
|
||||
|
||||
impl Backoff {
|
||||
/// The wait before retrying a call that failed with `error`, or `None` once it should
|
||||
/// fail instead.
|
||||
pub fn next_delay(&mut self, error: &TransientAIError) -> Option<Duration> {
|
||||
if self.retries >= MAX_RETRIES {
|
||||
return None;
|
||||
}
|
||||
let delay = match error.retry_after {
|
||||
Some(wait) if wait > MAX_RETRY_AFTER => return None,
|
||||
// A zero wait would retry back to back, so it gets the computed backoff.
|
||||
Some(wait) if !wait.is_zero() => wait,
|
||||
// Jittered, so callers throttled together do not come back together.
|
||||
_ => (BASE_DELAY * 2u32.pow(self.retries))
|
||||
.min(MAX_BACKOFF)
|
||||
.mul_f64(0.5 + rand::random::<f64>() * 0.5),
|
||||
};
|
||||
self.retries += 1;
|
||||
Some(delay)
|
||||
}
|
||||
|
||||
/// Retries granted so far.
|
||||
pub fn retries(&self) -> u32 {
|
||||
self.retries
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Error responses as providers send them, each with the retry decision of the client
|
||||
/// the rule is taken from.
|
||||
#[test]
|
||||
fn classifies_error_responses() {
|
||||
const GEMINI_QUOTA: &str = r#"{"error":{"code":429,"status":"RESOURCE_EXHAUSTED","details":[{"@type":"type.googleapis.com/google.rpc.QuotaFailure","violations":[{"quotaId":"QUOTA_ID"}]},{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"53s"}]}}"#;
|
||||
let cases: &[(u16, &[(&str, &str)], &str, Option<Option<u64>>)] = &[
|
||||
(
|
||||
429,
|
||||
&[],
|
||||
r#"{"error":{"code":"rate_limit_exceeded"}}"#,
|
||||
Some(None),
|
||||
),
|
||||
(
|
||||
429,
|
||||
&[],
|
||||
r#"{"error":{"type":"insufficient_quota","code":"insufficient_quota"}}"#,
|
||||
None,
|
||||
),
|
||||
(
|
||||
529,
|
||||
&[],
|
||||
r#"{"type":"error","error":{"type":"overloaded_error"}}"#,
|
||||
Some(None),
|
||||
),
|
||||
(501, &[], "", Some(None)),
|
||||
(400, &[], "", None),
|
||||
(401, &[], "", None),
|
||||
(400, &[("x-should-retry", "true")], "", Some(None)),
|
||||
(503, &[("x-should-retry", "false")], "", None),
|
||||
(429, &[("retry-after", "2")], "", Some(Some(2))),
|
||||
(
|
||||
429,
|
||||
&[],
|
||||
&GEMINI_QUOTA.replace("QUOTA_ID", "GenerateRequestsPerMinutePerProjectPerModel"),
|
||||
Some(Some(53)),
|
||||
),
|
||||
(
|
||||
429,
|
||||
&[],
|
||||
&GEMINI_QUOTA.replace(
|
||||
"QUOTA_ID",
|
||||
"GenerateRequestsPerDayPerProjectPerModel-FreeTier",
|
||||
),
|
||||
None,
|
||||
),
|
||||
];
|
||||
for (status, headers, body, expected) in cases {
|
||||
let headers = headers
|
||||
.iter()
|
||||
.map(|(k, v)| (k.parse().unwrap(), v.parse().unwrap()))
|
||||
.collect::<HeaderMap>();
|
||||
let error = response_error(
|
||||
StatusCode::from_u16(*status).unwrap(),
|
||||
&headers,
|
||||
body,
|
||||
String::new(),
|
||||
);
|
||||
let got = as_transient(&error).map(|t| t.retry_after.map(|d| d.as_secs()));
|
||||
assert_eq!(&got, expected, "{status} {headers:?} {body}");
|
||||
}
|
||||
}
|
||||
|
||||
/// In-stream error objects as providers send them, after the 200 opened the stream.
|
||||
#[test]
|
||||
fn classifies_stream_errors() {
|
||||
let retried = |json: &str, anthropic: bool| {
|
||||
let body: StreamErrorBody = serde_json::from_str(json).unwrap();
|
||||
let error = if anthropic {
|
||||
body.into_anthropic_error()
|
||||
} else {
|
||||
body.into_error("AI provider")
|
||||
};
|
||||
as_transient(&error).is_some()
|
||||
};
|
||||
for (json, expected) in [
|
||||
(r#"{"message":"m","type":"server_error"}"#, true),
|
||||
(
|
||||
r#"{"message":"m","type":"requests","code":"rate_limit_exceeded"}"#,
|
||||
true,
|
||||
),
|
||||
(
|
||||
r#"{"message":"m","type":"insufficient_quota","code":"insufficient_quota"}"#,
|
||||
false,
|
||||
),
|
||||
(
|
||||
r#"{"message":"m","type":"invalid_request_error","code":"context_length_exceeded"}"#,
|
||||
false,
|
||||
),
|
||||
// LiteLLM's proxy puts the upstream status in `code` as a string.
|
||||
(r#"{"message":"m","type":"None","code":"400"}"#, false),
|
||||
(r#"{"message":"m","code":"503"}"#, true),
|
||||
(r#"{"code":503,"message":"m","status":"UNAVAILABLE"}"#, true),
|
||||
(
|
||||
r#"{"code":400,"message":"m","status":"INVALID_ARGUMENT"}"#,
|
||||
false,
|
||||
),
|
||||
(r#"{"message":"m"}"#, true),
|
||||
] {
|
||||
assert_eq!(retried(json, false), expected, "{json}");
|
||||
}
|
||||
for (kind, expected) in [
|
||||
("overloaded_error", true),
|
||||
("api_error", true),
|
||||
("rate_limit_error", true),
|
||||
("timeout_error", true),
|
||||
("billing_error", false),
|
||||
("invalid_request_error", false),
|
||||
("some_new_error", false),
|
||||
] {
|
||||
let json = format!(r#"{{"type":"{kind}","message":"m"}}"#);
|
||||
assert_eq!(retried(&json, true), expected, "{kind}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn honours_retry_after_up_to_a_bound() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("retry-after", "2".parse().unwrap());
|
||||
assert_eq!(retry_after(&headers), Some(Duration::from_secs(2)));
|
||||
headers.insert("retry-after-ms", "1500".parse().unwrap());
|
||||
assert_eq!(retry_after(&headers), Some(Duration::from_millis(1500)));
|
||||
headers.remove("retry-after-ms");
|
||||
headers.insert("retry-after", "1e30".parse().unwrap());
|
||||
assert_eq!(retry_after(&headers), Some(Duration::MAX));
|
||||
|
||||
let mut backoff = Backoff::default();
|
||||
let asked = |secs| TransientAIError {
|
||||
message: String::new(),
|
||||
retry_after: Some(Duration::from_secs(secs)),
|
||||
};
|
||||
assert_eq!(backoff.next_delay(&asked(5)), Some(Duration::from_secs(5)));
|
||||
assert_eq!(backoff.next_delay(&asked(3600)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stops_after_max_retries() {
|
||||
let mut backoff = Backoff::default();
|
||||
let error = TransientAIError { message: String::new(), retry_after: None };
|
||||
for _ in 0..MAX_RETRIES {
|
||||
let delay = backoff.next_delay(&error).expect("retry");
|
||||
assert!(delay <= MAX_BACKOFF);
|
||||
}
|
||||
assert_eq!(backoff.next_delay(&error), None);
|
||||
}
|
||||
}
|
||||
+231
-13
@@ -1,6 +1,6 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use eventsource_stream::Eventsource;
|
||||
use eventsource_stream::{EventStreamError, Eventsource};
|
||||
use indexmap::IndexMap;
|
||||
use reqwest::Response;
|
||||
use serde::Deserialize;
|
||||
@@ -8,12 +8,13 @@ use tokio_stream::StreamExt;
|
||||
use windmill_common::{error::Error, utils::rd_string};
|
||||
|
||||
use crate::{
|
||||
ai_google::{parse_gemini_sse_event, GeminiUsageMetadata},
|
||||
ai_google::{parse_gemini_sse_event, GeminiTurn, GeminiUsageMetadata},
|
||||
ai_types::UrlCitation,
|
||||
ai_types::{
|
||||
AnthropicExtraContent, ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall,
|
||||
},
|
||||
query_builder::StreamEventSink,
|
||||
retry::{transient_error, truncated_stream_error, StreamErrorBody},
|
||||
types::{StreamingEvent, TokenUsage},
|
||||
};
|
||||
|
||||
@@ -43,6 +44,8 @@ pub struct OpenAIChoiceDelta {
|
||||
#[derive(Deserialize)]
|
||||
pub struct OpenAIChoice {
|
||||
pub delta: Option<OpenAIChoiceDelta>,
|
||||
#[serde(default)]
|
||||
pub finish_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Nested prompt token details returned by the Chat Completions API.
|
||||
@@ -87,6 +90,8 @@ pub struct OpenAISSEEvent {
|
||||
pub choices: Option<Vec<OpenAIChoice>>,
|
||||
#[serde(default)]
|
||||
pub usage: Option<OpenAIChatUsage>,
|
||||
#[serde(default)]
|
||||
pub error: Option<StreamErrorBody>,
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
@@ -99,6 +104,14 @@ lazy_static::lazy_static! {
|
||||
pub trait SSEParser {
|
||||
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>;
|
||||
|
||||
/// Whether the stream delivered the event that ends a complete response.
|
||||
fn is_complete(&self) -> bool;
|
||||
|
||||
/// Whether a stream that closed cleanly holds a complete response.
|
||||
fn is_complete_at_close(&self) -> bool {
|
||||
self.is_complete()
|
||||
}
|
||||
|
||||
async fn parse_events(&mut self, response: Response) -> Result<(), Error> {
|
||||
let mut stream = response.bytes_stream().eventsource();
|
||||
let mut consecutive_errors = 0;
|
||||
@@ -113,6 +126,18 @@ pub trait SSEParser {
|
||||
|
||||
self.parse_event_data(&event.data).await?;
|
||||
}
|
||||
// The body ends right after a transport error, so it cannot be skipped
|
||||
// like a malformed event. A timeout spent the caller's whole budget.
|
||||
Err(EventStreamError::Transport(_)) if self.is_complete() => break,
|
||||
Err(EventStreamError::Transport(e)) => {
|
||||
let message =
|
||||
format!("The connection to the AI provider broke off mid-response: {e}");
|
||||
return Err(if e.is_timeout() {
|
||||
Error::AIError(message)
|
||||
} else {
|
||||
transient_error(message, None)
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
consecutive_errors += 1;
|
||||
tracing::error!("Failed to parse SSE event: {}", e);
|
||||
@@ -130,6 +155,9 @@ pub trait SSEParser {
|
||||
}
|
||||
}
|
||||
|
||||
if !self.is_complete_at_close() {
|
||||
return Err(truncated_stream_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -145,6 +173,9 @@ pub struct OpenAISSEParser {
|
||||
pub stream_event_processor: Box<dyn StreamEventSink>,
|
||||
/// Token usage from final chunk (when stream_options.include_usage is true)
|
||||
pub usage: Option<OpenAIChatUsage>,
|
||||
/// Set by `[DONE]` or by a choice's `finish_reason`: OpenAI-compatible gateways do not
|
||||
/// all send `[DONE]`.
|
||||
complete: bool,
|
||||
}
|
||||
|
||||
impl OpenAISSEParser {
|
||||
@@ -156,13 +187,19 @@ impl OpenAISSEParser {
|
||||
events_str: String::new(),
|
||||
stream_event_processor,
|
||||
usage: None,
|
||||
complete: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SSEParser for OpenAISSEParser {
|
||||
fn is_complete(&self) -> bool {
|
||||
self.complete
|
||||
}
|
||||
|
||||
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
|
||||
if data == "[DONE]" {
|
||||
self.complete = true;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -173,13 +210,21 @@ impl SSEParser for OpenAISSEParser {
|
||||
.ok();
|
||||
|
||||
if let Some(event) = event {
|
||||
if let Some(error) = event.error {
|
||||
return Err(error.into_error("AI provider"));
|
||||
}
|
||||
|
||||
// Extract usage from final chunk (when stream_options.include_usage is true)
|
||||
if let Some(usage) = event.usage {
|
||||
self.usage = Some(usage);
|
||||
}
|
||||
|
||||
if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) {
|
||||
if let Some(delta) = choices.remove(0).delta {
|
||||
let choice = choices.remove(0);
|
||||
if choice.finish_reason.is_some() {
|
||||
self.complete = true;
|
||||
}
|
||||
if let Some(delta) = choice.delta {
|
||||
if let Some(reasoning) = delta.reasoning_content.filter(|s| !s.is_empty()) {
|
||||
self.accumulated_reasoning.push_str(&reasoning);
|
||||
let event = StreamingEvent::ReasoningTokenDelta { content: reasoning };
|
||||
@@ -345,6 +390,8 @@ pub enum AnthropicSSEEvent {
|
||||
MessageDelta {
|
||||
#[serde(default)]
|
||||
usage: Option<AnthropicUsage>,
|
||||
#[serde(default)]
|
||||
delta: Option<AnthropicMessageDelta>,
|
||||
},
|
||||
#[serde(rename = "message_stop")]
|
||||
MessageStop {},
|
||||
@@ -353,12 +400,18 @@ pub enum AnthropicSSEEvent {
|
||||
#[serde(rename = "error")]
|
||||
Error {
|
||||
#[serde(default)]
|
||||
message: Option<String>,
|
||||
error: StreamErrorBody,
|
||||
},
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct AnthropicMessageDelta {
|
||||
#[serde(default)]
|
||||
pub stop_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Tracks state of a content block during streaming
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)]
|
||||
@@ -391,6 +444,9 @@ pub struct AnthropicSSEParser {
|
||||
/// can be replayed before `tool_use` (required by Claude when thinking is on).
|
||||
pending_reasoning: Option<AnthropicExtraContent>,
|
||||
reasoning_attached: bool,
|
||||
/// Set by `message_stop` or by the `stop_reason` of the `message_delta` sent just
|
||||
/// before it, so a gateway that drops `message_stop` does not fail a whole answer.
|
||||
complete: bool,
|
||||
}
|
||||
|
||||
impl AnthropicSSEParser {
|
||||
@@ -407,11 +463,16 @@ impl AnthropicSSEParser {
|
||||
usage: None,
|
||||
pending_reasoning: None,
|
||||
reasoning_attached: false,
|
||||
complete: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SSEParser for AnthropicSSEParser {
|
||||
fn is_complete(&self) -> bool {
|
||||
self.complete
|
||||
}
|
||||
|
||||
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
|
||||
let event: Option<AnthropicSSEEvent> = serde_json::from_str(data)
|
||||
.inspect_err(|e| {
|
||||
@@ -583,9 +644,8 @@ impl SSEParser for AnthropicSSEParser {
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
AnthropicSSEEvent::Error { message } => {
|
||||
let error_msg = message.unwrap_or_else(|| "Unknown error".to_string());
|
||||
tracing::error!("Anthropic streaming error: {}", error_msg);
|
||||
AnthropicSSEEvent::Error { error } => {
|
||||
return Err(error.into_anthropic_error());
|
||||
}
|
||||
AnthropicSSEEvent::MessageStart { message } => {
|
||||
// The only event carrying the prompt-side counts. `message_delta`
|
||||
@@ -595,7 +655,10 @@ impl SSEParser for AnthropicSSEParser {
|
||||
self.usage = Some(usage);
|
||||
}
|
||||
}
|
||||
AnthropicSSEEvent::MessageDelta { usage } => {
|
||||
AnthropicSSEEvent::MessageDelta { usage, delta } => {
|
||||
if delta.is_some_and(|d| d.stop_reason.is_some()) {
|
||||
self.complete = true;
|
||||
}
|
||||
if let Some(u) = usage {
|
||||
match &mut self.usage {
|
||||
// Field by field, so the prompt counts from `message_start`
|
||||
@@ -614,10 +677,9 @@ impl SSEParser for AnthropicSSEParser {
|
||||
}
|
||||
}
|
||||
}
|
||||
AnthropicSSEEvent::MessageStop {} => self.complete = true,
|
||||
// Ignore other events
|
||||
AnthropicSSEEvent::MessageStop {}
|
||||
| AnthropicSSEEvent::Ping {}
|
||||
| AnthropicSSEEvent::Unknown => {}
|
||||
AnthropicSSEEvent::Ping {} | AnthropicSSEEvent::Unknown => {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -645,6 +707,7 @@ pub struct GeminiSSEParser {
|
||||
pub annotations: Vec<UrlCitation>,
|
||||
pub used_websearch: bool,
|
||||
pub usage: Option<GeminiUsageMetadata>,
|
||||
turn: GeminiTurn,
|
||||
}
|
||||
|
||||
impl GeminiSSEParser {
|
||||
@@ -659,15 +722,25 @@ impl GeminiSSEParser {
|
||||
annotations: Vec::new(),
|
||||
used_websearch: false,
|
||||
usage: None,
|
||||
turn: GeminiTurn::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SSEParser for GeminiSSEParser {
|
||||
fn is_complete(&self) -> bool {
|
||||
self.turn.ended()
|
||||
}
|
||||
|
||||
fn is_complete_at_close(&self) -> bool {
|
||||
self.turn.complete_at_close()
|
||||
}
|
||||
|
||||
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
|
||||
let Some(parsed) = parse_gemini_sse_event(data)? else {
|
||||
return Ok(());
|
||||
};
|
||||
self.turn.observe(&parsed);
|
||||
|
||||
if let Some(reasoning) = parsed.reasoning.filter(|s| !s.is_empty()) {
|
||||
self.accumulated_reasoning.push_str(&reasoning);
|
||||
@@ -797,6 +870,8 @@ impl OpenAIResponsesUsage {
|
||||
pub struct OpenAIResponsesResponse {
|
||||
#[serde(default)]
|
||||
pub usage: Option<OpenAIResponsesUsage>,
|
||||
#[serde(default)]
|
||||
pub error: Option<StreamErrorBody>,
|
||||
}
|
||||
|
||||
/// SSE event types for OpenAI Responses API streaming
|
||||
@@ -833,6 +908,20 @@ pub enum OpenAIResponsesSSEEvent {
|
||||
#[serde(rename = "response.incomplete")]
|
||||
Incomplete { response: OpenAIResponsesResponse },
|
||||
|
||||
#[serde(rename = "response.failed")]
|
||||
Failed { response: OpenAIResponsesResponse },
|
||||
|
||||
/// Flat (`code`, `message`) from OpenAI, nested under `error` from some compatible servers.
|
||||
#[serde(rename = "error")]
|
||||
Error {
|
||||
#[serde(default)]
|
||||
code: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
message: Option<String>,
|
||||
#[serde(default)]
|
||||
error: Option<StreamErrorBody>,
|
||||
},
|
||||
|
||||
/// Response created
|
||||
#[serde(rename = "response.created")]
|
||||
Created {},
|
||||
@@ -890,6 +979,7 @@ pub struct OpenAIResponsesSSEParser {
|
||||
pub usage: Option<OpenAIResponsesUsage>,
|
||||
/// Reasoning summary parts seen so far, to separate them as paragraphs
|
||||
reasoning_summary_parts: usize,
|
||||
response_ended: bool,
|
||||
}
|
||||
|
||||
impl OpenAIResponsesSSEParser {
|
||||
@@ -906,11 +996,16 @@ impl OpenAIResponsesSSEParser {
|
||||
used_websearch: false,
|
||||
usage: None,
|
||||
reasoning_summary_parts: 0,
|
||||
response_ended: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SSEParser for OpenAIResponsesSSEParser {
|
||||
fn is_complete(&self) -> bool {
|
||||
self.response_ended
|
||||
}
|
||||
|
||||
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
|
||||
let event: Option<OpenAIResponsesSSEEvent> = serde_json::from_str(data)
|
||||
.inspect_err(|e| {
|
||||
@@ -1010,11 +1105,22 @@ impl SSEParser for OpenAIResponsesSSEParser {
|
||||
|
||||
OpenAIResponsesSSEEvent::Completed { response }
|
||||
| OpenAIResponsesSSEEvent::Incomplete { response } => {
|
||||
self.response_ended = true;
|
||||
if let Some(usage) = response.usage {
|
||||
self.usage = Some(usage);
|
||||
}
|
||||
}
|
||||
|
||||
OpenAIResponsesSSEEvent::Failed { response } => {
|
||||
return Err(response.error.unwrap_or_default().into_error("OpenAI"));
|
||||
}
|
||||
|
||||
OpenAIResponsesSSEEvent::Error { code, message, error } => {
|
||||
return Err(error
|
||||
.unwrap_or(StreamErrorBody { code, message, ..Default::default() })
|
||||
.into_error("OpenAI"));
|
||||
}
|
||||
|
||||
OpenAIResponsesSSEEvent::ReasoningSummaryPartAdded {} => {
|
||||
self.reasoning_summary_parts += 1;
|
||||
if self.reasoning_summary_parts > 1 {
|
||||
@@ -1037,9 +1143,10 @@ impl SSEParser for OpenAIResponsesSSEParser {
|
||||
}
|
||||
}
|
||||
|
||||
OpenAIResponsesSSEEvent::Done {} => self.response_ended = true,
|
||||
|
||||
// Ignore other event types
|
||||
OpenAIResponsesSSEEvent::Done {}
|
||||
| OpenAIResponsesSSEEvent::Created {}
|
||||
OpenAIResponsesSSEEvent::Created {}
|
||||
| OpenAIResponsesSSEEvent::InProgress {}
|
||||
| OpenAIResponsesSSEEvent::OutputItemDone {}
|
||||
| OpenAIResponsesSSEEvent::ContentPartAdded {}
|
||||
@@ -1093,6 +1200,117 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn sse_response(chunks: Vec<Result<&'static str, std::io::Error>>) -> Response {
|
||||
let body = reqwest::Body::wrap_stream(futures::stream::iter(chunks));
|
||||
Response::from(http::Response::new(body))
|
||||
}
|
||||
|
||||
/// A stream that breaks off, or reports a failure inside its 200, must fail the call
|
||||
/// rather than hand back what arrived as if it were the whole answer.
|
||||
#[tokio::test]
|
||||
async fn stream_failures_are_errors_not_partial_answers() {
|
||||
use crate::retry::as_transient;
|
||||
|
||||
const STARTED: &str = concat!(
|
||||
"data: {\"type\":\"message_start\"}\n\n",
|
||||
"data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n",
|
||||
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hel\"}}\n\n",
|
||||
);
|
||||
let anthropic = |chunks| async move {
|
||||
AnthropicSSEParser::new(Box::new(NoopSink))
|
||||
.parse_events(sse_response(chunks))
|
||||
.await
|
||||
};
|
||||
|
||||
let overloaded = anthropic(vec![
|
||||
Ok(STARTED),
|
||||
Ok("data: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded\"}}\n\n"),
|
||||
])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(as_transient(&overloaded).is_some(), "{overloaded}");
|
||||
|
||||
let refused = anthropic(vec![
|
||||
Ok(STARTED),
|
||||
Ok("data: {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"bad\"}}\n\n"),
|
||||
])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(as_transient(&refused).is_none(), "{refused}");
|
||||
|
||||
let truncated = anthropic(vec![Ok(STARTED)]).await.unwrap_err();
|
||||
assert!(as_transient(&truncated).is_some(), "{truncated}");
|
||||
|
||||
let dropped = anthropic(vec![
|
||||
Ok(STARTED),
|
||||
Err(std::io::Error::new(
|
||||
std::io::ErrorKind::ConnectionReset,
|
||||
"reset",
|
||||
)),
|
||||
])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(as_transient(&dropped).is_some(), "{dropped}");
|
||||
|
||||
anthropic(vec![
|
||||
Ok(STARTED),
|
||||
Ok("data: {\"type\":\"message_stop\"}\n\n"),
|
||||
])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
anthropic(vec![
|
||||
Ok(STARTED),
|
||||
Ok("data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n"),
|
||||
])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let truncated = OpenAISSEParser::new(Box::new(NoopSink))
|
||||
.parse_events(sse_response(vec![Ok(
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"Hi\"}}]}\n\n",
|
||||
)]))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(as_transient(&truncated).is_some(), "{truncated}");
|
||||
|
||||
let quota = OpenAIResponsesSSEParser::new(Box::new(NoopSink))
|
||||
.parse_events(sse_response(vec![Ok(
|
||||
"data: {\"type\":\"error\",\"error\":{\"type\":\"insufficient_quota\",\"code\":\"insufficient_quota\",\"message\":\"m\"}}\n\n",
|
||||
)]))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(as_transient("a).is_none(), "{quota}");
|
||||
|
||||
GeminiSSEParser::new(Box::new(NoopSink))
|
||||
.parse_events(sse_response(vec![Ok(
|
||||
"data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"functionCall\":{\"name\":\"get_weather\",\"args\":{}}}]}}]}\n\n",
|
||||
)]))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let gemini = |chunks| async move {
|
||||
GeminiSSEParser::new(Box::new(NoopSink))
|
||||
.parse_events(sse_response(chunks))
|
||||
.await
|
||||
};
|
||||
// Thinking spent the whole `maxOutputTokens`: a clean close with no event.
|
||||
gemini(vec![]).await.unwrap();
|
||||
let dropped = gemini(vec![Err(std::io::Error::new(
|
||||
std::io::ErrorKind::ConnectionReset,
|
||||
"reset",
|
||||
))])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(as_transient(&dropped).is_some(), "{dropped}");
|
||||
let truncated = gemini(vec![Ok(
|
||||
"data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"Hi\"}]}}]}\n\n",
|
||||
)])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(as_transient(&truncated).is_some(), "{truncated}");
|
||||
}
|
||||
|
||||
/// The prompt-side counts arrive only on `message_start` and the completion total
|
||||
/// only on `message_delta`; a parser that keeps just the last one reports a request
|
||||
/// with no input tokens at all.
|
||||
|
||||
@@ -170,10 +170,16 @@ const FLOW_JOBS: [&'static str; 6] = [
|
||||
/// scoped by the agent's path like a deployed runnable's, as `jobs:run:agents:<path>`.
|
||||
const AGENT_JOBS: [&'static str; 1] = ["jobs/run/agent"];
|
||||
|
||||
/// Lock resolution for scripts, apps and flows, scoped as `jobs:run:dependencies:<path>`.
|
||||
/// Resolving a package can run its build or install scripts on the worker, so this is
|
||||
/// still code execution: the scope only takes away the direct ways to run code.
|
||||
const DEPENDENCY_JOBS: [&'static str; 2] = ["jobs/run/dependencies", "jobs/run/flow_dependencies"];
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref RUN_PATH_ACTIONS: Vec<&'static str> = {
|
||||
let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies", "apps_u/execute_component", "apps_u/upload_s3_file"];
|
||||
let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "apps_u/execute_component", "apps_u/upload_s3_file"];
|
||||
|
||||
v.extend(DEPENDENCY_JOBS);
|
||||
v.extend(SCRIPT_JOBS);
|
||||
v.extend(FLOW_JOBS);
|
||||
v.extend(AGENT_JOBS);
|
||||
@@ -227,11 +233,13 @@ fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction {
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks the route path to determine the runnable kind ("agents", "flows" or "scripts").
|
||||
/// Checks the route path to determine the runnable kind ("dependencies", "agents", "flows"
|
||||
/// or "scripts").
|
||||
///
|
||||
/// The order of checks is important:
|
||||
/// - Flow-related paths are checked first to avoid false positives, as some flow paths
|
||||
/// (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`).
|
||||
/// - Dependency paths are checked before flow paths: `flow_dependencies` starts with `f`.
|
||||
/// - Flow-related paths are checked before script paths to avoid false positives, as some
|
||||
/// flow paths (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`).
|
||||
///
|
||||
/// Returns the kind based on the match, or `None` if no match is found.
|
||||
fn determine_kind_from_route(route_path: &str) -> Option<String> {
|
||||
@@ -247,6 +255,12 @@ fn determine_kind_from_route(route_path: &str) -> Option<String> {
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if DEPENDENCY_JOBS
|
||||
.iter()
|
||||
.any(|path| route_path.starts_with(path))
|
||||
{
|
||||
return Some("dependencies".to_string());
|
||||
}
|
||||
if AGENT_JOBS.iter().any(|path| route_path.starts_with(path)) {
|
||||
return Some("agents".to_string());
|
||||
}
|
||||
@@ -586,17 +600,23 @@ pub struct JobReadConfinement {
|
||||
}
|
||||
|
||||
impl JobReadConfinement {
|
||||
/// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows` or `agents`)
|
||||
/// is inside the confinement. A read path admits every kind; a run scope only its own.
|
||||
/// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows`, `agents` or
|
||||
/// `dependencies`) is inside the confinement. A read path admits every kind; a run
|
||||
/// scope only its own.
|
||||
pub fn admits(&self, kind: &str, runnable_path: &str) -> bool {
|
||||
if self.read_paths.allows(runnable_path) {
|
||||
return true;
|
||||
}
|
||||
// Lock jobs and previews store the path their caller sent. The matcher reads `*`
|
||||
// in it as a pattern, and `,`/`:` are scope syntax: such a path needs a grant
|
||||
// covering every path, as enqueuing it did.
|
||||
let resource = windmill_common::auth::is_scope_literal_path(runnable_path)
|
||||
.then(|| vec![runnable_path.to_string()]);
|
||||
let required = ScopeDefinition::new(
|
||||
ScopeDomain::Jobs.as_str(),
|
||||
ScopeAction::Run.as_str(),
|
||||
Some(kind),
|
||||
Some(vec![runnable_path.to_string()]),
|
||||
resource,
|
||||
);
|
||||
self.run.iter().any(|scope| scope.includes(&required))
|
||||
}
|
||||
@@ -1715,6 +1735,24 @@ mod tests {
|
||||
scope_for_route("POST", "/api/w/ws/jobs/run/agent/u/x/y").as_deref(),
|
||||
Some("jobs:run:agents")
|
||||
);
|
||||
for path in [
|
||||
"/api/w/ws/jobs/run/dependencies",
|
||||
"/api/w/ws/jobs/run/dependencies_async",
|
||||
"/api/w/ws/jobs/run/flow_dependencies",
|
||||
"/api/w/ws/jobs/run/flow_dependencies_async",
|
||||
] {
|
||||
assert_eq!(
|
||||
scope_for_route("POST", path).as_deref(),
|
||||
Some("jobs:run:dependencies"),
|
||||
"{path}"
|
||||
);
|
||||
assert!(check_route_access(&["jobs:run:dependencies".into()], path, "POST").is_ok());
|
||||
assert!(check_route_access(&["jobs:run:flows".into()], path, "POST").is_err());
|
||||
}
|
||||
// A dependencies grant reaches no other run route.
|
||||
for path in ["/api/w/ws/jobs/run/preview", "/api/w/ws/jobs/run/p/u/x/y"] {
|
||||
assert!(check_route_access(&["jobs:run:dependencies".into()], path, "POST").is_err());
|
||||
}
|
||||
|
||||
// Preview/bundle runs have no deployed path and their handlers require the
|
||||
// broad `jobs:run` scope, so the derived scope must not carry a kind.
|
||||
|
||||
@@ -20,7 +20,8 @@ use windmill_api_auth::{
|
||||
require_owner_of_path, ApiAuthed,
|
||||
};
|
||||
use windmill_common::workspaces::{
|
||||
check_deploy_rules, check_operator_can_build_flows, operator_can_build_flows, RuleCheckResult,
|
||||
check_deploy_rules, check_operator_can_build, operator_builder_rights, BuilderKind,
|
||||
RuleCheckResult,
|
||||
};
|
||||
use windmill_common::{
|
||||
user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay},
|
||||
@@ -243,7 +244,7 @@ async fn list_flows(
|
||||
|
||||
// Append the authed user's drafts at paths with no deployed flow; see scripts.rs.
|
||||
if lq.include_draft_only.unwrap_or(false)
|
||||
&& (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?)
|
||||
&& (!authed.is_operator || operator_builder_rights(&db, &w_id).await?.flows)
|
||||
&& offset == 0
|
||||
&& lq.path_start.is_none()
|
||||
&& lq.path_exact.is_none()
|
||||
@@ -731,7 +732,14 @@ async fn create_flow(
|
||||
Path(w_id): Path<String>,
|
||||
Json(mut nf): Json<NewFlow>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "create flows").await?;
|
||||
check_operator_can_build(
|
||||
&db,
|
||||
&w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Flows,
|
||||
"create flows",
|
||||
)
|
||||
.await?;
|
||||
check_scopes(&authed, || format!("flows:write:{}", nf.path))?;
|
||||
|
||||
// A `<= 0` flow timeout is "unset", not a 0-second limit that kills every run instantly.
|
||||
@@ -1344,7 +1352,14 @@ async fn update_flow(
|
||||
Path((w_id, flow_path)): Path<(String, StripPath)>,
|
||||
Json(ef): Json<EditFlow>,
|
||||
) -> Result<String> {
|
||||
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "update flows").await?;
|
||||
check_operator_can_build(
|
||||
&db,
|
||||
&w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Flows,
|
||||
"update flows",
|
||||
)
|
||||
.await?;
|
||||
let flow_path = flow_path.to_path();
|
||||
// The URL identifies the flow being updated; the body path is only needed to rename.
|
||||
let mut nf = ef.into_new_flow(flow_path);
|
||||
@@ -2038,7 +2053,14 @@ async fn archive_flow_by_path(
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(archived): Json<Archived>,
|
||||
) -> Result<String> {
|
||||
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "archive flows").await?;
|
||||
check_operator_can_build(
|
||||
&db,
|
||||
&w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Flows,
|
||||
"archive flows",
|
||||
)
|
||||
.await?;
|
||||
let path = path.to_path();
|
||||
check_scopes(&authed, || format!("flows:write:{}", path))?;
|
||||
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
|
||||
@@ -2179,7 +2201,14 @@ async fn delete_flow_by_path(
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Query(query): Query<DeleteFlowQuery>,
|
||||
) -> Result<String> {
|
||||
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "delete flows").await?;
|
||||
check_operator_can_build(
|
||||
&db,
|
||||
&w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Flows,
|
||||
"delete flows",
|
||||
)
|
||||
.await?;
|
||||
let path = path.to_path();
|
||||
check_scopes(&authed, || format!("flows:write:{}", path))?;
|
||||
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
|
||||
|
||||
@@ -0,0 +1,570 @@
|
||||
use serde_json::json;
|
||||
use serial_test::serial;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use windmill_common::workspaces::invalidate_operator_rights_cache;
|
||||
use windmill_test_utils::*;
|
||||
|
||||
/// Every test here must be `#[serial]`: the rights cache is process-global and keyed by workspace id
|
||||
/// alone, so parallel tests under this same id read each other's rights.
|
||||
const WS: &str = "test-workspace";
|
||||
|
||||
fn client(token: &str) -> reqwest::Client {
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
headers.insert(
|
||||
reqwest::header::AUTHORIZATION,
|
||||
reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")).unwrap(),
|
||||
);
|
||||
reqwest::ClientBuilder::new()
|
||||
.default_headers(headers)
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn set_builder(db: &Pool<Postgres>, flows: bool, apps: bool) -> anyhow::Result<()> {
|
||||
sqlx::query(
|
||||
"UPDATE workspace_settings SET operator_settings = $1::text::jsonb WHERE workspace_id = $2",
|
||||
)
|
||||
.bind(format!(
|
||||
r#"{{"builder_flows": {flows}, "builder_apps": {apps}}}"#
|
||||
))
|
||||
.bind(WS)
|
||||
.execute(db)
|
||||
.await?;
|
||||
// The rights are read through a process-global 60s cache keyed by workspace id.
|
||||
invalidate_operator_rights_cache(WS);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn add_script(db: &Pool<Postgres>, hash: i64, path: &str, owner: &str) -> anyhow::Result<()> {
|
||||
sqlx::query(
|
||||
"INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema,
|
||||
summary, description, lock, extra_perms)
|
||||
VALUES ($1, $2, $3, 'x', 'bun', 'script', $4, '{}', '', '', '', '{}')",
|
||||
)
|
||||
.bind(WS)
|
||||
.bind(hash)
|
||||
.bind(path)
|
||||
.bind(owner)
|
||||
.execute(db)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A full-code app is deployed multipart, the way the editor and the CLI use it.
|
||||
fn raw_app_form(app: serde_json::Value) -> reqwest::multipart::Form {
|
||||
reqwest::multipart::Form::new()
|
||||
.part(
|
||||
"app",
|
||||
reqwest::multipart::Part::text(app.to_string())
|
||||
.mime_str("application/json")
|
||||
.unwrap(),
|
||||
)
|
||||
.part(
|
||||
"js",
|
||||
reqwest::multipart::Part::text("console.log(1)").file_name("app.js"),
|
||||
)
|
||||
}
|
||||
|
||||
fn raw_app(path: &str, mode: &str, runnable_path: &str) -> reqwest::multipart::Form {
|
||||
raw_app_form(json!({
|
||||
"path": path,
|
||||
"summary": "",
|
||||
"value": {"files": {}, "runnables": {"r": {
|
||||
"name": "r", "type": "runnableByPath", "runType": "script",
|
||||
"path": runnable_path
|
||||
}}},
|
||||
"policy": {"execution_mode": mode, "triggerables_v2": {}}
|
||||
}))
|
||||
}
|
||||
|
||||
fn composition_flow(path: &str) -> serde_json::Value {
|
||||
json!({
|
||||
"path": path, "summary": "", "description": "", "schema": {},
|
||||
"value": {"modules": [{
|
||||
"id": "a",
|
||||
"value": {"type": "script", "path": "u/operator/some_script", "input_transforms": {}}
|
||||
}]}
|
||||
})
|
||||
}
|
||||
|
||||
/// The app half of the builder boundary, plus the proof that the two rights are independent: a
|
||||
/// gate reading "either" would authorize the kind its workspace never granted, which is the whole
|
||||
/// point of splitting them.
|
||||
#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))]
|
||||
#[serial]
|
||||
async fn test_operator_builder_apps_boundary(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
let api = format!("http://localhost:{port}/api/w/{WS}");
|
||||
let c = client("OPERATOR_TOKEN_1");
|
||||
|
||||
add_script(&db, 4241, "u/operator/some_script", "operator").await?;
|
||||
// `permissions_test` gives the operator fixture no rights on `u/alice/**`.
|
||||
add_script(&db, 4243, "u/alice/private", "alice").await?;
|
||||
// A version shared with the operator, then archived, and a newer one at the same path that is
|
||||
// not: the newer one is what runs.
|
||||
add_script(&db, 4244, "u/alice/reused", "alice").await?;
|
||||
sqlx::query(
|
||||
"UPDATE script SET archived = true, extra_perms = '{\"u/operator\": false}',
|
||||
created_at = now() - interval '1 day' WHERE hash = 4244",
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
add_script(&db, 4245, "u/alice/reused", "alice").await?;
|
||||
|
||||
set_builder(&db, false, false).await?;
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/create_raw"))
|
||||
.multipart(raw_app(
|
||||
"u/operator/a0",
|
||||
"publisher",
|
||||
"u/operator/some_script",
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
!resp.status().is_success(),
|
||||
"an operator without the builder right must not create a full-code app"
|
||||
);
|
||||
|
||||
set_builder(&db, true, true).await?;
|
||||
|
||||
// `*_raw_source` compiles caller-supplied sources with a bundler job on a worker. It sits
|
||||
// beside `create_app_raw`, which builders MAY use, so it is the gate most likely to be opened
|
||||
// by mistake later.
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/create_raw_source"))
|
||||
.json(&json!({
|
||||
"path": "u/operator/a1", "summary": "",
|
||||
"value": {"files": {"index.ts": "console.log(1)"}, "runnables": {}},
|
||||
"policy": {"execution_mode": "publisher"}
|
||||
}))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
!resp.status().is_success(),
|
||||
"a builder must not compile app sources on a worker"
|
||||
);
|
||||
|
||||
// Low-code apps carry inline scripts, so they stay shut too.
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/create"))
|
||||
.json(&json!({
|
||||
"path": "u/operator/a2", "summary": "",
|
||||
"value": {"grid": []}, "policy": {"execution_mode": "publisher"}
|
||||
}))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
!resp.status().is_success(),
|
||||
"a builder must not create a low-code app"
|
||||
);
|
||||
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/create_raw"))
|
||||
.multipart(raw_app("u/operator/a3", "publisher", "u/alice/private"))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
!resp.status().is_success(),
|
||||
"a builder must not deploy an app referencing a runnable it cannot read"
|
||||
);
|
||||
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/create_raw"))
|
||||
.multipart(raw_app("u/operator/a4", "viewer", "u/operator/some_script"))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
!resp.status().is_success(),
|
||||
"a builder must not deploy a viewer-mode app: the policy stops bounding what it can invoke"
|
||||
);
|
||||
|
||||
// Storage strips a NUL, turning this key into `inlineScript`.
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/create_raw"))
|
||||
.multipart(raw_app_form(json!({
|
||||
"path": "u/operator/a_nul", "summary": "",
|
||||
"value": {"files": {}, "runnables": {"r": {
|
||||
"type": "inline",
|
||||
"inline\u{0}Script": {"content": "x", "language": "bun"}
|
||||
}}},
|
||||
"policy": {"execution_mode": "publisher", "triggerables_v2": {}}
|
||||
})))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
403,
|
||||
"a NUL in a key must not hide an inline script: {}",
|
||||
resp.text().await?
|
||||
);
|
||||
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/create_raw"))
|
||||
.multipart(raw_app(
|
||||
"u/operator/a5",
|
||||
"publisher",
|
||||
"u/operator/some_script",
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
resp.status().is_success(),
|
||||
"a builder must be able to deploy a full-code app over readable runnables: {}",
|
||||
resp.text().await?
|
||||
);
|
||||
let sandbox: Option<bool> = sqlx::query_scalar(
|
||||
"SELECT (policy->>'sandbox')::boolean FROM app WHERE workspace_id = $1 AND path = $2",
|
||||
)
|
||||
.bind(WS)
|
||||
.bind("u/operator/a5")
|
||||
.fetch_one(&db)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
sandbox,
|
||||
Some(true),
|
||||
"a builder-authored app must be stored sandboxed"
|
||||
);
|
||||
|
||||
// `execute_component` looks up `<component>:<path>` with an unrestricted component string, so
|
||||
// a key with a second colon still resolves at run time and must not slip past validation.
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/create_raw"))
|
||||
.multipart(raw_app_form(json!({
|
||||
"path": "u/operator/a6", "summary": "",
|
||||
"value": {"files": {}, "runnables": {}},
|
||||
"policy": {"execution_mode": "publisher", "triggerables_v2": {
|
||||
"x:y:script/u/alice/private": {"static_inputs": {}, "one_of_inputs": {}}
|
||||
}}
|
||||
})))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
!resp.status().is_success(),
|
||||
"a multi-colon triggerable key must not hide an unreadable runnable from validation"
|
||||
);
|
||||
|
||||
// The two rights are independent.
|
||||
set_builder(&db, true, false).await?;
|
||||
let resp = c
|
||||
.post(format!("{api}/flows/create"))
|
||||
.json(&composition_flow("u/operator/flows_only"))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
resp.status().is_success(),
|
||||
"flows-only must still create a flow: {}",
|
||||
resp.text().await?
|
||||
);
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/create_raw"))
|
||||
.multipart(raw_app(
|
||||
"u/operator/flows_only_app",
|
||||
"publisher",
|
||||
"u/operator/some_script",
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
!resp.status().is_success(),
|
||||
"flows-only must not deploy a full-code app"
|
||||
);
|
||||
|
||||
set_builder(&db, false, true).await?;
|
||||
let resp = c
|
||||
.post(format!("{api}/flows/create"))
|
||||
.json(&composition_flow("u/operator/apps_only"))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
!resp.status().is_success(),
|
||||
"apps-only must not create a flow"
|
||||
);
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/create_raw"))
|
||||
.multipart(raw_app(
|
||||
"u/operator/apps_only_app",
|
||||
"publisher",
|
||||
"u/operator/some_script",
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
resp.status().is_success(),
|
||||
"apps-only must still deploy a full-code app: {}",
|
||||
resp.text().await?
|
||||
);
|
||||
|
||||
// The deploy panel saves access and frontend scopes through the settings-only endpoint.
|
||||
for (body, expected) in [
|
||||
(
|
||||
json!({"policy": {"execution_mode": "anonymous", "triggerables_v2": {}}}),
|
||||
200,
|
||||
),
|
||||
(
|
||||
json!({"policy": {"execution_mode": "publisher", "sandbox": false, "triggerables_v2": {}}}),
|
||||
403,
|
||||
),
|
||||
(
|
||||
json!({"value": {"files": {}, "runnables": {}}, "policy": {"execution_mode": "publisher"}}),
|
||||
403,
|
||||
),
|
||||
] {
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/update/u/operator/apps_only_app"))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await?;
|
||||
let status = resp.status();
|
||||
assert_eq!(
|
||||
status,
|
||||
expected,
|
||||
"settings update {body}: {}",
|
||||
resp.text().await?
|
||||
);
|
||||
}
|
||||
let sandbox: Option<bool> = sqlx::query_scalar(
|
||||
"SELECT (policy->>'sandbox')::boolean FROM app WHERE workspace_id = $1 AND path = $2",
|
||||
)
|
||||
.bind(WS)
|
||||
.bind("u/operator/apps_only_app")
|
||||
.fetch_one(&db)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
sandbox,
|
||||
Some(true),
|
||||
"a settings update must not turn a builder app's sandbox off"
|
||||
);
|
||||
|
||||
// The editor runs the runnables a draft names as whoever opens it, and its frontend code
|
||||
// with their session unless the preview is sandboxed.
|
||||
for (runnables, policy, expected) in [
|
||||
(json!({}), json!({"sandbox": true}), 200),
|
||||
(
|
||||
json!({"a": {"type": "inline", "inlineScript": {"content": "x", "language": "bun"}}}),
|
||||
json!({"sandbox": true}),
|
||||
403,
|
||||
),
|
||||
(json!({}), json!({"sandbox": false}), 403),
|
||||
(
|
||||
json!({}),
|
||||
json!({"sandbox": true, "execution_mode": "viewer"}),
|
||||
403,
|
||||
),
|
||||
(json!({}), json!(null), 403),
|
||||
(
|
||||
json!({}),
|
||||
json!({"sandbox": true, "on_behalf_of": "u/alice", "on_behalf_of_email": "alice@windmill.dev"}),
|
||||
403,
|
||||
),
|
||||
(
|
||||
json!({}),
|
||||
json!({"sandbox": true, "on_behalf_of": "u/operator"}),
|
||||
200,
|
||||
),
|
||||
(
|
||||
json!({}),
|
||||
json!({"sandbox": true, "triggerables": {
|
||||
format!("{}script/u/operator/some_script", "a:".repeat(300)): {}
|
||||
}}),
|
||||
400,
|
||||
),
|
||||
(
|
||||
json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/operator/some_script"}}),
|
||||
json!({"sandbox": true}),
|
||||
200,
|
||||
),
|
||||
(
|
||||
json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/alice/private"}}),
|
||||
json!({"sandbox": true}),
|
||||
403,
|
||||
),
|
||||
(
|
||||
json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/alice/reused"}}),
|
||||
json!({"sandbox": true}),
|
||||
403,
|
||||
),
|
||||
(
|
||||
json!({"r": {"type": "path", "runType": "hubscript", "path": "hub/1/x/y"}}),
|
||||
json!({"sandbox": true}),
|
||||
403,
|
||||
),
|
||||
(
|
||||
json!({"r": {"type": "path", "runType": "script/u/alice", "path": "u/operator/some_script"}}),
|
||||
json!({"sandbox": true}),
|
||||
400,
|
||||
),
|
||||
(
|
||||
json!({"r": {"type": "path", "runType": "script", "path": ["u/alice/private"]}}),
|
||||
json!({"sandbox": true}),
|
||||
400,
|
||||
),
|
||||
(
|
||||
json!({"r": {"type": ["path"], "runType": "script", "path": "u/alice/private"}}),
|
||||
json!({"sandbox": true}),
|
||||
400,
|
||||
),
|
||||
(
|
||||
json!([{"type": "path", "runType": "script", "path": "u/alice/private"}]),
|
||||
json!({"sandbox": true}),
|
||||
400,
|
||||
),
|
||||
(
|
||||
json!({"r": {"type": "path", "runType": "script", "path": "u/operator/some_script",
|
||||
"fields": {"x": {"type": "static", "value": {"type": "path", "path": "/srv/out"}}}}}),
|
||||
json!({"sandbox": true}),
|
||||
200,
|
||||
),
|
||||
(
|
||||
json!({}),
|
||||
json!({"sandbox": true, "triggerables": {"x:rawscript/abc": {}}}),
|
||||
403,
|
||||
),
|
||||
(
|
||||
json!({}),
|
||||
json!({"sandbox": true, "triggerables": {"x:script/u/alice/private": {}}}),
|
||||
403,
|
||||
),
|
||||
(
|
||||
json!({}),
|
||||
json!({"sandbox": true, "triggerables_v2": {
|
||||
"x:script/u/alice/private": {"static_inputs": {}, "one_of_inputs": {}},
|
||||
"y:rawscript/abc": {"static_inputs": {}, "one_of_inputs": {}}
|
||||
}}),
|
||||
200,
|
||||
),
|
||||
] {
|
||||
let resp = c
|
||||
.post(format!("{api}/drafts/update/raw_app/u/operator/d1"))
|
||||
.json(&json!({
|
||||
"value": {"files": {}, "runnables": runnables, "summary": "", "policy": policy},
|
||||
"force": true
|
||||
}))
|
||||
.send()
|
||||
.await?;
|
||||
let status = resp.status();
|
||||
assert_eq!(
|
||||
status,
|
||||
expected,
|
||||
"raw app draft {runnables} {policy}: {}",
|
||||
resp.text().await?
|
||||
);
|
||||
}
|
||||
|
||||
// The component preview runs a deployed runnable for a builder; inline code and hub
|
||||
// scripts are unreviewed code.
|
||||
for (preview, expected) in [
|
||||
(json!({"path": "script/u/operator/some_script"}), 200),
|
||||
(json!({"path": "script/hub/1/x/y"}), 403),
|
||||
(
|
||||
json!({"raw_code": {"content": "x", "language": "bun"}}),
|
||||
403,
|
||||
),
|
||||
] {
|
||||
let mut body = json!({"component": "c", "args": {}, "force_viewer_static_fields": {}});
|
||||
body.as_object_mut()
|
||||
.unwrap()
|
||||
.extend(preview.as_object().unwrap().clone());
|
||||
let resp = c
|
||||
.post(format!(
|
||||
"{api}/apps_u/execute_component/u/operator/apps_only_app"
|
||||
))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await?;
|
||||
let status = resp.status();
|
||||
assert_eq!(
|
||||
status,
|
||||
expected,
|
||||
"component preview {preview}: {}",
|
||||
resp.text().await?
|
||||
);
|
||||
}
|
||||
|
||||
// Delete and redeploy each serve both kinds; builder rights cover only full-code apps.
|
||||
let resp = client("SECRET_TOKEN")
|
||||
.post(format!("{api}/apps/create"))
|
||||
.json(&json!({
|
||||
"path": "u/operator/low_code", "summary": "",
|
||||
"value": {"grid": []}, "policy": {"execution_mode": "publisher"}
|
||||
}))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(resp.status().is_success(), "{}", resp.text().await?);
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/update_raw/u/operator/low_code"))
|
||||
.multipart(raw_app_form(json!({
|
||||
"policy": {"execution_mode": "publisher", "triggerables_v2": {}}
|
||||
})))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
403,
|
||||
"policy-only update of a low-code app: {}",
|
||||
resp.text().await?
|
||||
);
|
||||
// Deployment history restores a full-code version with allow_kind_change set.
|
||||
let resp = c
|
||||
.post(format!("{api}/apps/update_raw/u/operator/apps_only_app"))
|
||||
.multipart(raw_app_form(json!({
|
||||
"value": {"files": {}, "runnables": {}},
|
||||
"policy": {"execution_mode": "publisher", "triggerables_v2": {}},
|
||||
"allow_kind_change": true
|
||||
})))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
200,
|
||||
"restore of a full-code version: {}",
|
||||
resp.text().await?
|
||||
);
|
||||
for (path, expected) in [("low_code", 403), ("apps_only_app", 200)] {
|
||||
let resp = c
|
||||
.delete(format!("{api}/apps/delete/u/operator/{path}"))
|
||||
.send()
|
||||
.await?;
|
||||
let status = resp.status();
|
||||
assert_eq!(status, expected, "delete {path}: {}", resp.text().await?);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The readability check must not hold its RLS transaction while it takes a second connection,
|
||||
/// or a one-connection pool (`DATABASE_CONNECTIONS=1`) stalls every builder draft naming a script.
|
||||
#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))]
|
||||
#[serial]
|
||||
async fn test_operator_builder_app_check_fits_one_connection(
|
||||
pool_opts: sqlx::postgres::PgPoolOptions,
|
||||
connect_opts: sqlx::postgres::PgConnectOptions,
|
||||
) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let db = pool_opts
|
||||
.max_connections(1)
|
||||
.acquire_timeout(std::time::Duration::from_secs(5))
|
||||
.connect_with(connect_opts)
|
||||
.await?;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let api = format!("http://localhost:{}/api/w/{WS}", server.addr.port());
|
||||
add_script(&db, 4241, "u/operator/some_script", "operator").await?;
|
||||
set_builder(&db, false, true).await?;
|
||||
|
||||
let resp = client("OPERATOR_TOKEN_1")
|
||||
.post(format!("{api}/drafts/update/raw_app/u/operator/d1"))
|
||||
.json(&json!({
|
||||
"value": {"files": {}, "summary": "", "runnables": {"r": {
|
||||
"name": "r", "type": "runnableByPath", "runType": "script",
|
||||
"path": "u/operator/some_script"
|
||||
}}, "policy": {"sandbox": true, "triggerables_v2": {}}},
|
||||
"force": true
|
||||
}))
|
||||
.send()
|
||||
.await?;
|
||||
let status = resp.status();
|
||||
assert_eq!(status, 200, "{}", resp.text().await?);
|
||||
Ok(())
|
||||
}
|
||||
@@ -11608,10 +11608,13 @@ struct ChangeOperatorSettings {
|
||||
/// withdraw or grant the right on every push.
|
||||
///
|
||||
/// `builder_flows` lets every operator of this workspace compose flows out of already-deployed
|
||||
/// runnables, and makes each of them consume a full author seat instead of half of one.
|
||||
/// runnables, `builder_apps` full-code apps. Either makes each operator consume a full author
|
||||
/// seat instead of half of one.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
builder_flows: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
builder_apps: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
manage_schedules: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
manage_triggers: Option<bool>,
|
||||
@@ -11626,9 +11629,10 @@ async fn update_operator_settings(
|
||||
require_admin(authed.is_admin, &authed.username)?;
|
||||
|
||||
// Every operator of the workspace turns into a full seat, which an offline license may not
|
||||
// cover. It is a no-op delta when the right is already on.
|
||||
// cover. Either right costs the same seat, so this prices "any", and it is a no-op delta when
|
||||
// one of them is already on.
|
||||
#[cfg(feature = "enterprise")]
|
||||
if settings.builder_flows == Some(true) {
|
||||
if settings.builder_flows == Some(true) || settings.builder_apps == Some(true) {
|
||||
if let Some(msg) =
|
||||
windmill_common::ee_oss::check_seat_cap_for_operator_builder(&db, &w_id).await?
|
||||
{
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: "3.0.3"
|
||||
|
||||
info:
|
||||
version: 1.829.0
|
||||
version: 1.830.0
|
||||
title: Windmill API
|
||||
|
||||
contact:
|
||||
@@ -25310,9 +25310,12 @@ paths:
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
- $ref: "#/components/parameters/TablePreviewS3ResourcePath"
|
||||
- $ref: "#/components/parameters/TablePreviewFilters"
|
||||
responses:
|
||||
"200":
|
||||
description: Parquet Preview
|
||||
description: >
|
||||
Parquet Preview: `columns`, `column_types` (Arrow type names) and `rows`.
|
||||
content:
|
||||
application/json: {}
|
||||
|
||||
@@ -25337,6 +25340,13 @@ paths:
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
- $ref: "#/components/parameters/TablePreviewS3ResourcePath"
|
||||
- name: csv_separator
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
- $ref: "#/components/parameters/TablePreviewCsvHasHeader"
|
||||
- $ref: "#/components/parameters/TablePreviewFilters"
|
||||
responses:
|
||||
"200":
|
||||
description: Table count
|
||||
@@ -25385,13 +25395,20 @@ paths:
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
- $ref: "#/components/parameters/TablePreviewS3ResourcePath"
|
||||
- name: csv_separator
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
- $ref: "#/components/parameters/TablePreviewCsvHasHeader"
|
||||
- $ref: "#/components/parameters/TablePreviewFilters"
|
||||
responses:
|
||||
"200":
|
||||
description: Csv Preview
|
||||
description: >
|
||||
Csv Preview: `columns`, `column_types` (Arrow type names), `rows`, and the
|
||||
`csv_separator` the file was read with, guessed from its first bytes when
|
||||
none is given.
|
||||
A `.jsonl` or `.ndjson` file is read as newline-delimited JSON.
|
||||
content:
|
||||
application/json: {}
|
||||
|
||||
@@ -27751,6 +27768,28 @@ components:
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
TablePreviewS3ResourcePath:
|
||||
name: s3_resource_path
|
||||
description: Read the file from this object storage resource instead of the workspace storage.
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
TablePreviewCsvHasHeader:
|
||||
name: csv_has_header
|
||||
description: Whether the first CSV line names the columns. Defaults to true.
|
||||
in: query
|
||||
schema:
|
||||
type: boolean
|
||||
TablePreviewFilters:
|
||||
name: filters
|
||||
description: >
|
||||
JSON object, column name to filter. A number column takes an optional comparison
|
||||
operator and a number (`>=10`), a boolean column a boolean, and any other column a
|
||||
case-insensitive substring, or `=value` for an exact match. A `search_term` without
|
||||
a `search_col` matches any column, case-insensitively.
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
HubPublishFolder:
|
||||
name: folder
|
||||
in: query
|
||||
@@ -38007,6 +38046,9 @@ components:
|
||||
builder_flows:
|
||||
type: boolean
|
||||
description: Whether operators can compose flows out of existing runnables (consumes a full seat). Omitting the field leaves the stored value unchanged.
|
||||
builder_apps:
|
||||
type: boolean
|
||||
description: Whether operators can compose full-code apps out of existing runnables (consumes a full seat). Omitting the field leaves the stored value unchanged.
|
||||
manage_schedules:
|
||||
type: boolean
|
||||
description: Whether operators can create, edit and delete schedules. Granted unless withdrawn; omitting the field leaves the stored value unchanged.
|
||||
|
||||
@@ -40,6 +40,7 @@ use windmill_ai::proxy::{
|
||||
fim::maybe_transform_fim_request, proxy_execution_mode, ProxyBuildArgs, ProxyExecutionMode,
|
||||
ProxyRequest,
|
||||
};
|
||||
use windmill_ai::retry::send_with_retries;
|
||||
use windmill_audit::{audit_oss::audit_log, ActionKind};
|
||||
use windmill_common::db::UserDB;
|
||||
use windmill_common::error::{to_anyhow, Error, Result};
|
||||
@@ -1188,15 +1189,11 @@ async fn global_proxy(
|
||||
}
|
||||
};
|
||||
|
||||
let response = request.send().await.map_err(to_anyhow)?;
|
||||
let response = send_with_retries(request).await;
|
||||
|
||||
audit_global_ai_request(&db, &authed).await?;
|
||||
|
||||
if response.error_for_status_ref().is_err() {
|
||||
let err_msg = response.text().await.unwrap_or("".to_string());
|
||||
return Err(Error::AIError(err_msg));
|
||||
}
|
||||
|
||||
let response = response?;
|
||||
let status_code = response.status();
|
||||
let headers = response.headers().clone();
|
||||
let stream = response.bytes_stream();
|
||||
@@ -1576,7 +1573,7 @@ async fn proxy(
|
||||
}
|
||||
};
|
||||
|
||||
let response = request.send().await.map_err(to_anyhow)?;
|
||||
let response = send_with_retries(request).await;
|
||||
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
@@ -1592,11 +1589,7 @@ async fn proxy(
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
if response.error_for_status_ref().is_err() {
|
||||
let err_msg = response.text().await.unwrap_or("".to_string());
|
||||
return Err(Error::AIError(err_msg));
|
||||
}
|
||||
|
||||
let response = response?;
|
||||
let status_code = response.status();
|
||||
let headers = response.headers().clone();
|
||||
let is_sse = is_sse_response(&headers);
|
||||
|
||||
@@ -56,7 +56,10 @@ use std::str;
|
||||
use windmill_audit::audit_oss::{audit_log, AuditAuthorable};
|
||||
use windmill_audit::ActionKind;
|
||||
use windmill_common::{
|
||||
apps::{AppScriptId, ListAppQuery, APP_WORKSPACED_ROUTE},
|
||||
apps::{
|
||||
app_value_has_inline_script, app_value_runnable_paths, AppScriptId, ListAppQuery,
|
||||
APP_WORKSPACED_ROUTE,
|
||||
},
|
||||
auth::{APP_EMBED_TOKEN_LABEL_PREFIX, RAW_APP_SDK_TOKEN_LABEL_PREFIX, TOKEN_PREFIX_LEN},
|
||||
cache::{self, future::FutureCachedExt},
|
||||
db::{DbWithOptAuthed, UserDB},
|
||||
@@ -75,7 +78,8 @@ use windmill_common::{
|
||||
variables::{build_crypt, build_crypt_with_key_suffix, encrypt},
|
||||
worker::{to_raw_value, CLOUD_HOSTED},
|
||||
workspaces::{
|
||||
check_deploy_rules, check_user_against_rule, ProtectionRuleKind, RuleCheckResult,
|
||||
check_deploy_rules, check_operator_can_build, check_user_against_rule,
|
||||
operator_builder_rights, BuilderKind, ProtectionRuleKind, RuleCheckResult,
|
||||
},
|
||||
HUB_BASE_URL,
|
||||
};
|
||||
@@ -749,7 +753,7 @@ async fn list_apps(
|
||||
// Append the authed user's `app`/`raw_app` drafts at paths with no deployed app;
|
||||
// see scripts.rs.
|
||||
if lq.include_draft_only.unwrap_or(false)
|
||||
&& !authed.is_operator
|
||||
&& (!authed.is_operator || operator_builder_rights(&db, &w_id).await?.apps)
|
||||
&& offset == 0
|
||||
&& lq.path_start.is_none()
|
||||
&& lq.path_exact.is_none()
|
||||
@@ -1740,11 +1744,14 @@ async fn mint_preview_sdk_token(
|
||||
Path(w_id): Path<String>,
|
||||
Json(req): Json<PreviewSdkTokenRequest>,
|
||||
) -> Result<String> {
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot preview raw apps".to_string(),
|
||||
));
|
||||
}
|
||||
check_operator_can_build(
|
||||
&db,
|
||||
&w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Apps,
|
||||
"preview raw apps",
|
||||
)
|
||||
.await?;
|
||||
check_scopes(&authed, || format!("apps:write:{}", req.path))?;
|
||||
let (token, _expiration) =
|
||||
mint_raw_app_sdk_token(&db, &w_id, &req.path, &authed, &req.scopes, job_id).await?;
|
||||
@@ -2102,18 +2109,23 @@ async fn get_id_from_secret(
|
||||
prefix: Option<&str>,
|
||||
) -> Result<i64> {
|
||||
let mc = build_crypt(db, w_id).await?;
|
||||
let decrypted = mc
|
||||
.decrypt_bytes_to_bytes(&(hex::decode(secret)?))
|
||||
.map_err(|e| Error::internal_err(e.to_string()))?;
|
||||
let mut bytes = str::from_utf8(&decrypted).map_err(to_anyhow)?;
|
||||
if let Some(prefix) = prefix {
|
||||
if !bytes.starts_with(prefix) {
|
||||
return Err(Error::BadRequest("Invalid secret".to_string()));
|
||||
}
|
||||
bytes = bytes.strip_prefix(prefix).unwrap_or("");
|
||||
id_from_secret(&mc, &secret, prefix)
|
||||
.ok_or_else(|| Error::BadRequest("Invalid secret".to_string()))
|
||||
}
|
||||
|
||||
/// The secret comes from the URL of unauthenticated routes, so every way it can be invalid
|
||||
/// must produce the same response: a caller must never learn why a secret was rejected.
|
||||
fn id_from_secret(
|
||||
mc: &magic_crypt::MagicCrypt256,
|
||||
secret: &str,
|
||||
prefix: Option<&str>,
|
||||
) -> Option<i64> {
|
||||
let decrypted = mc.decrypt_bytes_to_bytes(&hex::decode(secret).ok()?).ok()?;
|
||||
let plaintext = String::from_utf8(decrypted).ok()?;
|
||||
match prefix {
|
||||
Some(prefix) => plaintext.strip_prefix(prefix)?.parse().ok(),
|
||||
None => plaintext.parse().ok(),
|
||||
}
|
||||
let id: i64 = bytes.parse().map_err(to_anyhow)?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
async fn get_public_resource(
|
||||
@@ -2262,6 +2274,260 @@ async fn store_raw_app_file<'a>(
|
||||
|
||||
Ok(())
|
||||
}
|
||||
/// A builder app write is held to one set of rules whether it deploys or saves a draft: the
|
||||
/// editor previews a draft as whoever opens it, and an admin may deploy it as it stands. A deploy
|
||||
/// fills in what a draft must state, since it stores the policy it checked.
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
pub(crate) enum BuilderAppWrite {
|
||||
Deploy,
|
||||
Draft,
|
||||
}
|
||||
|
||||
/// The half of the builder-app check that needs no DB (see `docs/operator-builder-rights.md`).
|
||||
/// Returns every runnable the app can invoke, as `(is_flow, path)`, from both the policy's
|
||||
/// triggerables and the value's path runnables: they are not the same list.
|
||||
fn check_operator_composed_app(
|
||||
write: BuilderAppWrite,
|
||||
raw_app: bool,
|
||||
value: Option<&RawValue>,
|
||||
policy: Option<&mut Policy>,
|
||||
) -> Result<Vec<(bool, String)>> {
|
||||
let verb = if write == BuilderAppWrite::Deploy {
|
||||
"deploy"
|
||||
} else {
|
||||
"save"
|
||||
};
|
||||
if !raw_app {
|
||||
return Err(Error::PermissionDenied(
|
||||
"Operators with builder rights can only author full-code apps".to_string(),
|
||||
));
|
||||
}
|
||||
let mut referenced: Vec<(bool, String)> = Vec::new();
|
||||
if let Some(value) = value {
|
||||
// The text stored, NULs stripped: a stripped NUL can rename a key into one this reads.
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_str(&strip_json_nul(value.get())).map_err(to_anyhow)?;
|
||||
if app_value_has_inline_script(&value) {
|
||||
return Err(Error::PermissionDenied(format!(
|
||||
"Operators with builder rights cannot {verb} an app carrying inline scripts"
|
||||
)));
|
||||
}
|
||||
referenced.extend(app_value_runnable_paths(&value)?);
|
||||
}
|
||||
let Some(policy) = policy else {
|
||||
return Err(match write {
|
||||
BuilderAppWrite::Deploy => Error::BadRequest(
|
||||
"Operators with builder rights must deploy an app with its policy".to_string(),
|
||||
),
|
||||
BuilderAppWrite::Draft => Error::PermissionDenied(
|
||||
"Operators with builder rights can only save sandboxed apps".to_string(),
|
||||
),
|
||||
});
|
||||
};
|
||||
// Every deploy of a draft rebuilds `triggerables_v2` from the runnables, so a draft's copy,
|
||||
// stale since the last deploy, must not block autosave. A deploy path that skips the rebuild
|
||||
// must stop exempting it here.
|
||||
if write == BuilderAppWrite::Draft {
|
||||
policy.triggerables_v2 = None;
|
||||
}
|
||||
referenced.extend(checked_policy_runnable_paths(policy)?);
|
||||
match (policy.sandbox, write) {
|
||||
(Some(true), _) => {}
|
||||
(None, BuilderAppWrite::Deploy) => policy.sandbox = Some(true),
|
||||
_ => {
|
||||
return Err(Error::PermissionDenied(format!(
|
||||
"Operators with builder rights can only {verb} sandboxed apps"
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
// Viewer mode would make the triggerables collected above non-exhaustive (see the docs), and
|
||||
// the deploy panel shows it as members-only, so an admin deploying a draft would not see it.
|
||||
// Pin an omitted mode on deploy: `update_app_internal` resolves it to the deployed app's, so a
|
||||
// redeploy over an admin's viewer-mode app would otherwise inherit `Viewer`.
|
||||
match policy.stated_execution_mode() {
|
||||
Some(ExecutionMode::Viewer) => {
|
||||
return Err(Error::PermissionDenied(format!(
|
||||
"Operators with builder rights cannot {verb} an app that runs as its viewer"
|
||||
)))
|
||||
}
|
||||
None if write == BuilderAppWrite::Deploy => {
|
||||
policy.set_execution_mode(ExecutionMode::Publisher)
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
referenced.sort();
|
||||
referenced.dedup();
|
||||
refuse_hub_runnables(&referenced)?;
|
||||
Ok(referenced)
|
||||
}
|
||||
|
||||
/// The runnables a builder app's policy authorizes, as `(is_flow, path)`; refuses a policy that
|
||||
/// pins inline code.
|
||||
fn checked_policy_runnable_paths(policy: &Policy) -> Result<Vec<(bool, String)>> {
|
||||
// A `rawscript/<sha>` triggerable is the deployed app's authorization to run caller-supplied
|
||||
// `raw_code` hashing to it: pinning one hands a builder arbitrary code execution through an
|
||||
// app whose value passed the inline-script check. A composition-only app has none.
|
||||
fn pins_raw_script<T>(map: &Option<HashMap<String, T>>) -> bool {
|
||||
map.iter()
|
||||
.flat_map(|m| m.keys())
|
||||
.any(|k| k.starts_with("rawscript/") || k.contains(":rawscript/"))
|
||||
}
|
||||
if pins_raw_script(&policy.triggerables) || pins_raw_script(&policy.triggerables_v2) {
|
||||
return Err(Error::PermissionDenied(
|
||||
"Operators with builder rights cannot save or deploy an app whose policy pins inline code"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
// `execute_component` looks up `{component}:{path}` with an unrestricted component, so every
|
||||
// colon is a possible split (`a:b:script/x` resolves for `component = "a:b"`): check every
|
||||
// suffix that parses as a runnable. That copies a key once per colon, so bound the key first:
|
||||
// a 255-char path plus a component name is far below the cap.
|
||||
const MAX_KEY_LEN: usize = 512;
|
||||
let keys = || {
|
||||
policy
|
||||
.triggerables
|
||||
.iter()
|
||||
.flat_map(|t| t.keys())
|
||||
.chain(policy.triggerables_v2.iter().flat_map(|t| t.keys()))
|
||||
};
|
||||
if keys().any(|k| k.len() > MAX_KEY_LEN) {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"App policy triggerable keys must be at most {MAX_KEY_LEN} bytes"
|
||||
)));
|
||||
}
|
||||
Ok(keys()
|
||||
.flat_map(|key| {
|
||||
std::iter::once(key.as_str())
|
||||
.chain(key.match_indices(':').map(|(i, _)| &key[i + 1..]))
|
||||
.filter_map(|candidate| {
|
||||
candidate
|
||||
.strip_prefix("script/")
|
||||
.map(|p| (false, p.to_string()))
|
||||
.or_else(|| {
|
||||
candidate
|
||||
.strip_prefix("flow/")
|
||||
.map(|p| (true, p.to_string()))
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn refuse_hub_runnables(referenced: &[(bool, String)]) -> Result<()> {
|
||||
for (_, path) in referenced {
|
||||
if path.starts_with("hub/") {
|
||||
return Err(Error::PermissionDenied(format!(
|
||||
"Operators with builder rights cannot reference the hub runnable {path}. Deploy it to the workspace first."
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs on every app write by an operator with builder rights, from inside the create/update
|
||||
/// internals so both raw-app endpoints are covered by one check.
|
||||
///
|
||||
/// `execute_component` resolves the runnable it picks with the root DB handle, so a path the
|
||||
/// builder cannot read would still run, and in a `Viewer` app it would run as whoever opened it.
|
||||
/// RLS on this transaction is the check; refusing `Viewer` above is what makes it exhaustive.
|
||||
pub(crate) async fn validate_operator_composed_app(
|
||||
authed: &ApiAuthed,
|
||||
db: &DB,
|
||||
user_db: &UserDB,
|
||||
w_id: &str,
|
||||
write: BuilderAppWrite,
|
||||
raw_app: bool,
|
||||
value: Option<&RawValue>,
|
||||
policy: Option<&mut Policy>,
|
||||
) -> Result<()> {
|
||||
// A deploy resets the run identity to its builder (unless they are in `wm_deployers`, see
|
||||
// docs/operator-builder-rights.md); a draft keeps whatever it names, which an admin's
|
||||
// one-click draft deploy preserves without showing it.
|
||||
if let (BuilderAppWrite::Draft, Some(policy)) = (write, policy.as_deref()) {
|
||||
let names_another = policy
|
||||
.on_behalf_of
|
||||
.as_deref()
|
||||
.is_some_and(|p| p != username_to_permissioned_as(&authed.username))
|
||||
|| policy
|
||||
.on_behalf_of_email
|
||||
.as_deref()
|
||||
.is_some_and(|e| e != authed.email);
|
||||
if names_another {
|
||||
return Err(Error::PermissionDenied(
|
||||
"Operators with builder rights cannot save an app that runs as someone else"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let referenced = check_operator_composed_app(write, raw_app, value, policy)?;
|
||||
require_runnables_readable(authed, db, user_db, w_id, referenced).await
|
||||
}
|
||||
|
||||
async fn require_runnables_readable(
|
||||
authed: &ApiAuthed,
|
||||
db: &DB,
|
||||
user_db: &UserDB,
|
||||
w_id: &str,
|
||||
referenced: Vec<(bool, String)>,
|
||||
) -> Result<()> {
|
||||
if referenced.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
// Each script version carries its own grants, and a path reused after its versions were
|
||||
// archived starts with none. Check the version a run would pick, found without RLS, not
|
||||
// whichever version the caller happens to see. Resolved before the transaction opens so no
|
||||
// request holds two pool connections at once.
|
||||
let mut hashes = Vec::with_capacity(referenced.len());
|
||||
for (is_flow, path) in &referenced {
|
||||
hashes.push(if *is_flow {
|
||||
None
|
||||
} else {
|
||||
windmill_common::get_latest_script_hash(db, path, w_id).await?
|
||||
});
|
||||
}
|
||||
let mut tx = user_db.clone().begin(authed).await?;
|
||||
for ((is_flow, path), hash) in referenced.into_iter().zip(hashes) {
|
||||
let readable = if is_flow {
|
||||
sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2)",
|
||||
w_id,
|
||||
path,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?
|
||||
.unwrap_or(false)
|
||||
} else {
|
||||
match hash {
|
||||
Some(hash) => sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)",
|
||||
w_id,
|
||||
hash,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?
|
||||
.unwrap_or(false),
|
||||
None => false,
|
||||
}
|
||||
};
|
||||
// One message for every cause: telling "no runnable version" apart from "not readable"
|
||||
// would reveal that a path the caller cannot read exists.
|
||||
if !readable {
|
||||
return Err(Error::PermissionDenied(if is_flow {
|
||||
format!("Flow {path} does not exist or is not readable by you")
|
||||
} else {
|
||||
format!(
|
||||
"Script {path} does not exist, is not readable by you, or has not finished deploying"
|
||||
)
|
||||
}));
|
||||
}
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
macro_rules! process_app_multipart {
|
||||
($authed:expr, $user_db:expr, $db:expr, $w_id:expr, $path:expr, $multipart:expr, $internal_fn:expr) => {
|
||||
async {
|
||||
@@ -2338,11 +2604,14 @@ async fn create_app_raw<'a>(
|
||||
Path(w_id): Path<String>,
|
||||
multipart: Multipart,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot create apps for security reasons".to_string(),
|
||||
));
|
||||
}
|
||||
check_operator_can_build(
|
||||
&db,
|
||||
&w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Apps,
|
||||
"create apps",
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
|
||||
&w_id,
|
||||
@@ -2519,6 +2788,19 @@ async fn create_app_internal<'a>(
|
||||
if raw_app {
|
||||
validate_raw_app_path_keys(&app.value.0)?;
|
||||
}
|
||||
if authed.is_operator {
|
||||
validate_operator_composed_app(
|
||||
&authed,
|
||||
&db,
|
||||
&user_db,
|
||||
w_id,
|
||||
BuilderAppWrite::Deploy,
|
||||
raw_app,
|
||||
Some(&app.value.0),
|
||||
Some(&mut app.policy),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
if *CLOUD_HOSTED {
|
||||
let nb_apps =
|
||||
sqlx::query_scalar!("SELECT COUNT(*) FROM app WHERE workspace_id = $1", &w_id)
|
||||
@@ -2825,13 +3107,23 @@ async fn delete_app(
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> Result<String> {
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot delete apps for security reasons".to_string(),
|
||||
));
|
||||
}
|
||||
check_operator_can_build(
|
||||
&db,
|
||||
&w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Apps,
|
||||
"delete apps",
|
||||
)
|
||||
.await?;
|
||||
let path = path.to_path();
|
||||
check_scopes(&authed, || format!("apps:write:{}", path))?;
|
||||
// One endpoint deletes both kinds, and builder rights only cover full-code apps.
|
||||
if authed.is_operator && deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true)
|
||||
{
|
||||
return Err(Error::PermissionDenied(
|
||||
"Operators with builder rights can only delete full-code apps".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if path == "g/all/setup_app" && w_id == "admins" {
|
||||
return Err(Error::BadRequest(
|
||||
@@ -3019,13 +3311,20 @@ async fn update_app(
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(ns): Json<EditApp>,
|
||||
) -> JsonResult<AppDeployed> {
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot update apps for security reasons".to_string(),
|
||||
));
|
||||
}
|
||||
// create_app_internal(authed, user_db, db, &w_id, &mut app).await?;
|
||||
let path = path.to_path();
|
||||
// The deploy panel saves a deployed full-code app's access and frontend scopes here. A
|
||||
// builder may change those settings; the app itself only through `update_app_raw`.
|
||||
if authed.is_operator {
|
||||
check_operator_can_build(&db, &w_id, true, BuilderKind::Apps, "update apps").await?;
|
||||
if ns.value.is_some()
|
||||
|| deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true)
|
||||
{
|
||||
return Err(Error::PermissionDenied(
|
||||
"Operators with builder rights can only change the settings of full-code apps"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
check_scopes(&authed, || format!("apps:write:{}", path))?;
|
||||
|
||||
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
|
||||
@@ -3042,8 +3341,10 @@ async fn update_app(
|
||||
|
||||
let opath = path.to_string();
|
||||
let db2 = db.clone();
|
||||
// Settings-only for an operator, of an app checked above to be full-code.
|
||||
let raw_app = authed.is_operator;
|
||||
let (new_tx, npath, v_id) =
|
||||
update_app_internal(authed, db, user_db, &w_id, path, false, ns, None).await?;
|
||||
update_app_internal(authed, db, user_db, &w_id, path, raw_app, ns, None).await?;
|
||||
new_tx.commit().await?;
|
||||
|
||||
tally_app_rename(&db2, &w_id, &opath, &npath, v_id).await;
|
||||
@@ -3072,6 +3373,10 @@ async fn update_app_raw_source(
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(ns): Json<EditApp>,
|
||||
) -> JsonResult<AppDeployed> {
|
||||
// Stays closed to operators even with builder rights, unlike `create_app_raw`/`update_app_raw`
|
||||
// next to it: this path compiles caller-supplied sources with a bundler job on a worker, which
|
||||
// is arbitrary code execution. Builders lose nothing: the browser and the CLI both bundle
|
||||
// locally and deploy through the multipart endpoints.
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot update apps for security reasons".to_string(),
|
||||
@@ -3232,6 +3537,7 @@ async fn create_app_raw_source(
|
||||
Path(w_id): Path<String>,
|
||||
Json(mut app): Json<CreateApp>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
// Closed to builders too, for the reason on `update_app_raw_source`.
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot create apps for security reasons".to_string(),
|
||||
@@ -3397,11 +3703,14 @@ async fn update_app_raw<'a>(
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
multipart: Multipart,
|
||||
) -> JsonResult<AppDeployed> {
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot update apps for security reasons".to_string(),
|
||||
));
|
||||
}
|
||||
check_operator_can_build(
|
||||
&db,
|
||||
&w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Apps,
|
||||
"update apps",
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
|
||||
&w_id,
|
||||
@@ -3417,6 +3726,14 @@ async fn update_app_raw<'a>(
|
||||
|
||||
let path = path.to_path();
|
||||
check_scopes(&authed, || format!("apps:write:{}", path))?;
|
||||
// `update_app_internal` only compares kinds when a new value is deployed, so a
|
||||
// policy-only update would otherwise let a builder rewrite a low-code app's policy.
|
||||
if authed.is_operator && deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true)
|
||||
{
|
||||
return Err(Error::PermissionDenied(
|
||||
"Operators with builder rights can only update full-code apps".to_string(),
|
||||
));
|
||||
}
|
||||
let opath = path.to_string();
|
||||
let db2 = db.clone();
|
||||
let (npath, v_id) = process_app_multipart!(
|
||||
@@ -3484,6 +3801,23 @@ async fn update_app_internal<'a>(
|
||||
}
|
||||
}
|
||||
|
||||
if authed.is_operator {
|
||||
// Restoring a full-code version always sends allow_kind_change. A builder converts
|
||||
// nothing, so drop it and let the locked kind check below refuse an actual conversion.
|
||||
ns.allow_kind_change = None;
|
||||
validate_operator_composed_app(
|
||||
&authed,
|
||||
&db,
|
||||
&user_db,
|
||||
w_id,
|
||||
BuilderAppWrite::Deploy,
|
||||
raw_app,
|
||||
ns.value.as_ref().map(|v| v.0.as_ref()),
|
||||
ns.policy.as_mut(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Resolved on the (non-RLS) pool before the RLS transaction opens, for the reason
|
||||
// `create_app` states, with the same known, accepted rename race (see
|
||||
// `resolve_on_behalf_of`). Submitting a policy is how a deployer claims the app's execution
|
||||
@@ -4140,10 +4474,26 @@ async fn execute_component(
|
||||
let authed = opt_authed.as_ref().ok_or_else(|| {
|
||||
Error::NotAuthorized("App component preview requires authentication".to_string())
|
||||
})?;
|
||||
// A builder testing the app it is composing only ever previews a deployed runnable
|
||||
// (`path`), confined below to what it may read. Inline `raw_code`, which an `id` also
|
||||
// requires, is authoring code, so it stays closed to every operator. So is a hub
|
||||
// path, which `require_path_read_access_for_preview` admits for everyone and
|
||||
// `get_payload_tag_from_prefixed_path` then downloads and enqueues: it is exactly the
|
||||
// unreviewed code the composition check refuses in a flow.
|
||||
let previews_hub = payload.path.as_deref().is_some_and(|p| {
|
||||
p.strip_prefix("script/")
|
||||
.or_else(|| p.strip_prefix("flow/"))
|
||||
.unwrap_or(p)
|
||||
.starts_with("hub/")
|
||||
});
|
||||
if authed.is_operator {
|
||||
return Err(Error::NotAuthorized(
|
||||
"Operators cannot run preview jobs for security reasons".to_string(),
|
||||
));
|
||||
let msg = "Operators cannot run preview jobs for security reasons".to_string();
|
||||
if !operator_builder_rights(&db, &w_id).await?.apps {
|
||||
return Err(Error::NotAuthorized(msg));
|
||||
}
|
||||
if payload.raw_code.is_some() || previews_hub {
|
||||
return Err(Error::PermissionDenied(msg));
|
||||
}
|
||||
}
|
||||
check_scopes(authed, || format!("jobs:run"))?;
|
||||
if let Some(p) = payload.path.as_deref() {
|
||||
@@ -5498,6 +5848,10 @@ impl AppLoadCountQuery {
|
||||
search_col: self.search_col,
|
||||
search_term: self.search_term,
|
||||
storage: self.storage,
|
||||
s3_resource_path: None,
|
||||
csv_separator: None,
|
||||
csv_has_header: None,
|
||||
filters: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -5531,6 +5885,9 @@ impl AppLoadPreviewQuery {
|
||||
search_term: self.search_term,
|
||||
storage: self.storage,
|
||||
csv_separator: self.csv_separator,
|
||||
s3_resource_path: None,
|
||||
csv_has_header: None,
|
||||
filters: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -5618,7 +5975,9 @@ async fn app_load_table_count(
|
||||
let job_authed =
|
||||
app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?;
|
||||
let resp =
|
||||
crate::job_helpers_oss::load_table_count_internal(job_authed, &db, &w_id, file_key, inner)
|
||||
crate::job_helpers_oss::load_table_count_internal(
|
||||
job_authed, &db, None, &w_id, file_key, inner,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(resp).into_response())
|
||||
}
|
||||
@@ -5637,7 +5996,7 @@ async fn app_load_parquet_preview(
|
||||
let job_authed =
|
||||
app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?;
|
||||
let resp = crate::job_helpers_oss::load_preview_internal(
|
||||
job_authed, &db, &w_id, file_key, inner, true,
|
||||
job_authed, &db, None, &w_id, file_key, inner, true,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(resp).into_response())
|
||||
@@ -5657,7 +6016,7 @@ async fn app_load_csv_preview(
|
||||
let job_authed =
|
||||
app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?;
|
||||
let resp = crate::job_helpers_oss::load_preview_internal(
|
||||
job_authed, &db, &w_id, file_key, inner, false,
|
||||
job_authed, &db, None, &w_id, file_key, inner, false,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(resp).into_response())
|
||||
@@ -6346,6 +6705,127 @@ mod embed_token_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod operator_app_tests {
|
||||
use super::{check_operator_composed_app, BuilderAppWrite, Policy};
|
||||
use windmill_common::error::Result;
|
||||
use windmill_common::worker::to_raw_value;
|
||||
|
||||
fn builder_policy(triggerables_v2: serde_json::Value) -> Policy {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"execution_mode": "publisher",
|
||||
"triggerables_v2": triggerables_v2,
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn path_runnable(path: &str, run_type: &str) -> serde_json::Value {
|
||||
serde_json::json!({"files": {}, "runnables": {
|
||||
"a": {"name": "a", "type": "runnableByPath", "path": path, "runType": run_type}
|
||||
}})
|
||||
}
|
||||
|
||||
fn composed_app(value: serde_json::Value, policy: &mut Policy) -> Result<Vec<(bool, String)>> {
|
||||
let value = to_raw_value(&value);
|
||||
check_operator_composed_app(BuilderAppWrite::Deploy, true, Some(&value), Some(policy))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operator_app_check_forces_the_sandbox_and_refuses_code() {
|
||||
let clean = serde_json::json!({"files": {}, "runnables": {
|
||||
"a": {"name": "a", "type": "runnableByPath", "path": "f/x/s", "runType": "script"}
|
||||
}});
|
||||
|
||||
// A composition-only app goes through, sandboxed whether or not it asked to be.
|
||||
let mut policy = builder_policy(
|
||||
serde_json::json!({"a:script/f/x/s": {"static_inputs": {}, "one_of_inputs": {}}}),
|
||||
);
|
||||
let referenced = composed_app(clean.clone(), &mut policy).unwrap();
|
||||
assert_eq!(policy.sandbox, Some(true));
|
||||
// The runnables the policy authorizes are handed back for the caller to authorize.
|
||||
assert_eq!(referenced, vec![(false, "f/x/s".to_string())]);
|
||||
|
||||
// A hub reference is unreviewed code, refused like it is in a flow.
|
||||
let mut policy = builder_policy(
|
||||
serde_json::json!({"a:script/hub/1/x": {"static_inputs": {}, "one_of_inputs": {}}}),
|
||||
);
|
||||
assert!(composed_app(clean.clone(), &mut policy).is_err());
|
||||
|
||||
let mut policy = builder_policy(serde_json::json!({}));
|
||||
policy.sandbox = Some(false);
|
||||
assert!(composed_app(clean.clone(), &mut policy).is_err());
|
||||
|
||||
// An inline script is refused even with no `language`, which the locking traversal skips.
|
||||
let mut policy = builder_policy(serde_json::json!({}));
|
||||
assert!(composed_app(
|
||||
serde_json::json!({"runnables": {"a": {"inlineScript": {"content": "x"}}}}),
|
||||
&mut policy
|
||||
)
|
||||
.is_err());
|
||||
|
||||
// A `rawscript/<sha>` triggerable is what authorizes caller-supplied `raw_code` on the
|
||||
// deployed app, so pinning one would be arbitrary code execution behind a clean value.
|
||||
for key in ["rawscript/abc", "a:rawscript/abc"] {
|
||||
let mut policy = builder_policy(
|
||||
serde_json::json!({ key: {"static_inputs": {}, "one_of_inputs": {}} }),
|
||||
);
|
||||
assert!(
|
||||
composed_app(clean.clone(), &mut policy).is_err(),
|
||||
"{key} must be refused"
|
||||
);
|
||||
}
|
||||
|
||||
// What the deployed bundle actually asks to run comes from the value's `runnableByPath`
|
||||
// entries, which are a separate surface from the policy: both are reported.
|
||||
let mut policy = builder_policy(serde_json::json!({}));
|
||||
assert_eq!(
|
||||
composed_app(path_runnable("f/x/s", "script"), &mut policy).unwrap(),
|
||||
vec![(false, "f/x/s".to_string())]
|
||||
);
|
||||
let mut policy = builder_policy(serde_json::json!({}));
|
||||
assert_eq!(
|
||||
composed_app(path_runnable("f/x/f", "flow"), &mut policy).unwrap(),
|
||||
vec![(true, "f/x/f".to_string())]
|
||||
);
|
||||
let mut policy = builder_policy(serde_json::json!({}));
|
||||
assert!(composed_app(path_runnable("hub/1/x", "hubscript"), &mut policy).is_err());
|
||||
|
||||
// `execute_component` looks up `<component>:<path>` with an unrestricted component, so a
|
||||
// second colon must not hide the path from validation: `x:y:script/hub/…` resolves at run
|
||||
// time for `component = "x:y"`.
|
||||
let mut policy = builder_policy(
|
||||
serde_json::json!({"x:y:script/hub/123/foo": {"static_inputs": {}, "one_of_inputs": {}}}),
|
||||
);
|
||||
assert!(composed_app(clean.clone(), &mut policy).is_err());
|
||||
let mut policy = builder_policy(
|
||||
serde_json::json!({"x:y:script/f/x/s": {"static_inputs": {}, "one_of_inputs": {}}}),
|
||||
);
|
||||
assert_eq!(
|
||||
composed_app(clean.clone(), &mut policy).unwrap(),
|
||||
vec![(false, "f/x/s".to_string())]
|
||||
);
|
||||
|
||||
// `Viewer` mode makes `execute_component` accept any script/flow path, triggerables or
|
||||
// not, and run it as the viewer, so the checks above would stop binding.
|
||||
let mut policy: Policy = serde_json::from_value(serde_json::json!({
|
||||
"execution_mode": "viewer", "triggerables_v2": {}
|
||||
}))
|
||||
.unwrap();
|
||||
assert!(composed_app(clean.clone(), &mut policy).is_err());
|
||||
|
||||
// Low-code apps stay closed.
|
||||
let value = to_raw_value(&clean);
|
||||
let mut policy = builder_policy(serde_json::json!({}));
|
||||
assert!(check_operator_composed_app(
|
||||
BuilderAppWrite::Deploy,
|
||||
false,
|
||||
Some(&value),
|
||||
Some(&mut policy)
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod policy_tests {
|
||||
/// `execution_mode` is optional in the OpenAPI schema the API clients and the
|
||||
@@ -6368,6 +6848,39 @@ mod policy_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod app_secret_tests {
|
||||
use super::{id_from_secret, BUNDLE_SECRET_PREFIX};
|
||||
use magic_crypt::MagicCryptTrait;
|
||||
|
||||
#[test]
|
||||
fn every_invalid_secret_is_rejected_alike() {
|
||||
let mc = magic_crypt::new_magic_crypt!("workspace-key", 256);
|
||||
let other = magic_crypt::new_magic_crypt!("another-key", 256);
|
||||
let valid = hex::encode(mc.encrypt_str_to_bytes("42"));
|
||||
assert_eq!(id_from_secret(&mc, &valid, None), Some(42));
|
||||
let bundle = hex::encode(mc.encrypt_str_to_bytes(format!("{BUNDLE_SECRET_PREFIX}7")));
|
||||
assert_eq!(
|
||||
id_from_secret(&mc, &bundle, Some(BUNDLE_SECRET_PREFIX)),
|
||||
Some(7)
|
||||
);
|
||||
|
||||
let mut tampered = mc.encrypt_str_to_bytes("42");
|
||||
*tampered.last_mut().unwrap() ^= 1;
|
||||
for bad in [
|
||||
"not hex".to_string(),
|
||||
hex::encode(&tampered),
|
||||
hex::encode(other.encrypt_str_to_bytes("42")),
|
||||
hex::encode(mc.encrypt_bytes_to_bytes(&[0xff, 0xfe])),
|
||||
hex::encode(mc.encrypt_str_to_bytes("not a number")),
|
||||
valid.clone(),
|
||||
] {
|
||||
let prefix = (bad == valid).then_some(BUNDLE_SECRET_PREFIX);
|
||||
assert_eq!(id_from_secret(&mc, &bad, prefix), None, "{bad}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod raw_app_file_key_tests {
|
||||
use super::validate_raw_app_path_keys;
|
||||
|
||||
@@ -24,7 +24,7 @@ use windmill_common::{
|
||||
users::resolve_username_to_email,
|
||||
utils::{check_proper_path, require_admin, strip_json_nul},
|
||||
variables::{build_crypt, encrypt},
|
||||
workspaces::operator_can_build_flows,
|
||||
workspaces::{operator_builder_rights, BuilderKind},
|
||||
};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
@@ -112,7 +112,7 @@ async fn list_drafts(
|
||||
// Without builder rights an operator has no drafts of their own (they can't write any, see
|
||||
// `require_can_write_path`), so this list is always empty for them. They can still READ some
|
||||
// collaborators' drafts via `/drafts/get`.
|
||||
if authed.is_operator && !operator_can_build_flows(&db, &w_id).await? {
|
||||
if authed.is_operator && !operator_builder_rights(&db, &w_id).await?.any() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
let all_users = query.all_users.unwrap_or(false);
|
||||
@@ -501,6 +501,31 @@ async fn update_draft(
|
||||
}
|
||||
}
|
||||
|
||||
// A raw-app draft carries the app value and its policy side by side.
|
||||
if authed.is_operator && kind == UserDraftItemKind::RawApp {
|
||||
if let Some(value) = &req.value {
|
||||
let draft: serde_json::Value = serde_json::from_str(&strip_json_nul(value.0.get()))
|
||||
.map_err(|e| Error::BadRequest(format!("Invalid app draft: {e}")))?;
|
||||
let mut policy = draft
|
||||
.get("policy")
|
||||
.filter(|p| !p.is_null())
|
||||
.map(|p| serde_json::from_value::<crate::apps::Policy>(p.clone()))
|
||||
.transpose()
|
||||
.map_err(|e| Error::BadRequest(format!("Invalid app draft policy: {e}")))?;
|
||||
crate::apps::validate_operator_composed_app(
|
||||
&authed,
|
||||
&db,
|
||||
&user_db,
|
||||
&w_id,
|
||||
crate::apps::BuilderAppWrite::Draft,
|
||||
true,
|
||||
Some(&value.0),
|
||||
policy.as_mut(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
let applied = if let Some(value) = &req.value {
|
||||
// Secret variable values must never sit in `draft.value` in plaintext
|
||||
// (see `encrypt_secret_variable_value`).
|
||||
@@ -1245,14 +1270,21 @@ pub(crate) async fn require_can_write_path(
|
||||
if authed.is_admin {
|
||||
return Ok(());
|
||||
}
|
||||
// Operators are read-only and never WRITE drafts, except a flow draft where the workspace
|
||||
// granted the builder right: the kind has to be checked, or the right would open drafts of
|
||||
// kinds it says nothing about. Read access is deliberately asymmetric:
|
||||
// Operators are read-only and never WRITE drafts, except of a kind the workspace granted them
|
||||
// the matching builder right for: a flows-only workspace must not get raw-app drafts through
|
||||
// here. Read access is deliberately asymmetric:
|
||||
// `require_can_read_path` has no operator block, so an operator can still READ a draft they
|
||||
// can read via `/drafts/get`, mirroring their read access to deployed content. Intended.
|
||||
if authed.is_operator {
|
||||
let granted =
|
||||
matches!(kind, UserDraftItemKind::Flow) && operator_can_build_flows(db, w_id).await?;
|
||||
let allowed = match kind {
|
||||
UserDraftItemKind::Flow => Some(BuilderKind::Flows),
|
||||
UserDraftItemKind::RawApp => Some(BuilderKind::Apps),
|
||||
_ => None,
|
||||
};
|
||||
let granted = match allowed {
|
||||
Some(kind) => operator_builder_rights(db, w_id).await?.has(kind),
|
||||
None => false,
|
||||
};
|
||||
if !granted {
|
||||
return Err(Error::PermissionDenied(
|
||||
"operators cannot save drafts".to_string(),
|
||||
|
||||
@@ -263,6 +263,10 @@ mod app_s3_display_stubs {
|
||||
pub search_col: Option<String>,
|
||||
pub search_term: Option<String>,
|
||||
pub storage: Option<String>,
|
||||
pub s3_resource_path: Option<String>,
|
||||
pub csv_separator: Option<String>,
|
||||
pub csv_has_header: Option<bool>,
|
||||
pub filters: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -278,7 +282,10 @@ mod app_s3_display_stubs {
|
||||
pub search_col: Option<String>,
|
||||
pub search_term: Option<String>,
|
||||
pub storage: Option<String>,
|
||||
pub s3_resource_path: Option<String>,
|
||||
pub csv_separator: Option<String>,
|
||||
pub csv_has_header: Option<bool>,
|
||||
pub filters: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn load_file_metadata_internal(
|
||||
@@ -308,6 +315,7 @@ mod app_s3_display_stubs {
|
||||
pub async fn load_table_count_internal(
|
||||
_authed: OptJobAuthed,
|
||||
_db: &DB,
|
||||
_user_db: Option<UserDB>,
|
||||
_w_id: &str,
|
||||
_file_key: String,
|
||||
_query: LoadCountQuery,
|
||||
@@ -320,6 +328,7 @@ mod app_s3_display_stubs {
|
||||
pub async fn load_preview_internal(
|
||||
_authed: OptJobAuthed,
|
||||
_db: &DB,
|
||||
_user_db: Option<UserDB>,
|
||||
_w_id: &str,
|
||||
_file_key: String,
|
||||
_query: LoadPreviewQuery,
|
||||
|
||||
@@ -55,7 +55,8 @@ use windmill_common::workspace_dependencies::{
|
||||
RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES,
|
||||
};
|
||||
use windmill_common::workspaces::{
|
||||
check_operator_can_build_flows, check_user_against_rule, ProtectionRuleKind, RuleCheckResult,
|
||||
check_operator_can_build, check_user_against_rule, BuilderKind, ProtectionRuleKind,
|
||||
RuleCheckResult,
|
||||
};
|
||||
use windmill_common::DYNAMIC_INPUT_CACHE;
|
||||
#[cfg(all(feature = "enterprise", feature = "instance_smtp"))]
|
||||
@@ -2010,9 +2011,11 @@ async fn require_job_within_read_scope(
|
||||
}
|
||||
}
|
||||
// `scope_kind` is the runnable kind a `jobs:run:<kind>:<path>` scope can name, or
|
||||
// NULL for a job no such scope reaches directly (previews, dependency jobs,
|
||||
// flow-inlined scripts) — those are still readable as a step of a matching flow,
|
||||
// through their ancestors. A `singlestepflow` wraps either a script or a flow, so it
|
||||
// NULL for a job no such scope reaches directly (previews, flow-inlined scripts) —
|
||||
// those are still readable as a step of a matching flow, through their ancestors.
|
||||
// A dependency job's path is the one its `jobs:run:dependencies:<path>` check
|
||||
// admitted, but it is filed under `dependencies`, so it never passes for a run of
|
||||
// the deployed runnable at that path. A `singlestepflow` wraps either a script or a flow, so it
|
||||
// projects onto the wrapped runnable the same way the batch-rerun query does. An agent
|
||||
// run is a preview of the one-step flow `agent_runs::agent_step_flow` builds, filed under the
|
||||
// agent's path (`<path>.chat` for a chat turn); the editor's own runs of it look the same,
|
||||
@@ -2031,6 +2034,7 @@ async fn require_job_within_read_scope(
|
||||
WHEN a.agent THEN 'agents'
|
||||
WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'
|
||||
WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'
|
||||
WHEN j.kind IN ('dependencies', 'flowdependencies') THEN 'dependencies'
|
||||
WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN
|
||||
CASE WHEN COALESCE(
|
||||
(SELECT m->'value'->>'type'
|
||||
@@ -4628,14 +4632,8 @@ async fn cancel_selection(
|
||||
} else {
|
||||
ids
|
||||
};
|
||||
let Json(mut w_cancelled) = cancel_jobs(
|
||||
ids,
|
||||
&db,
|
||||
&authed,
|
||||
workspace_id.as_str(),
|
||||
force_cancel,
|
||||
)
|
||||
.await?;
|
||||
let Json(mut w_cancelled) =
|
||||
cancel_jobs(ids, &db, &authed, workspace_id.as_str(), force_cancel).await?;
|
||||
cancelled.append(&mut w_cancelled);
|
||||
}
|
||||
|
||||
@@ -9529,13 +9527,23 @@ pub struct RunDependenciesResponse {
|
||||
pub dependencies: String,
|
||||
}
|
||||
|
||||
/// The path comes from the request body: one carrying scope syntax (`,`, `:`, `*`) could
|
||||
/// put a granted path beside an ungranted one, so it needs a grant covering every path.
|
||||
fn check_dependencies_scope(authed: &ApiAuthed, path: &str) -> error::Result<()> {
|
||||
if windmill_common::auth::is_scope_literal_path(path) {
|
||||
check_scopes(authed, || format!("jobs:run:dependencies:{path}"))
|
||||
} else {
|
||||
check_scopes(authed, || "jobs:run:dependencies".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
async fn push_dependencies_job(
|
||||
authed: &ApiAuthed,
|
||||
db: &DB,
|
||||
w_id: &str,
|
||||
req: RunDependenciesRequest,
|
||||
) -> error::Result<Uuid> {
|
||||
check_scopes(authed, || format!("jobs:run"))?;
|
||||
check_dependencies_scope(authed, &req.entrypoint)?;
|
||||
if authed.is_operator {
|
||||
return Err(error::Error::NotAuthorized(
|
||||
"Operators cannot run dependencies jobs for security reasons".to_string(),
|
||||
@@ -9658,8 +9666,15 @@ async fn push_flow_dependencies_job(
|
||||
w_id: &str,
|
||||
req: RunFlowDependenciesRequest,
|
||||
) -> error::Result<Uuid> {
|
||||
check_scopes(authed, || format!("jobs:run"))?;
|
||||
check_operator_can_build_flows(db, w_id, authed.is_operator, "run dependencies jobs").await?;
|
||||
check_dependencies_scope(authed, &req.path)?;
|
||||
check_operator_can_build(
|
||||
db,
|
||||
w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Flows,
|
||||
"run dependencies jobs",
|
||||
)
|
||||
.await?;
|
||||
// The dependency job locks whatever inline code this request carries, on a worker. A
|
||||
// composition-only flow has none, so validating here costs a builder nothing and keeps the
|
||||
// lock step from becoming the way to run code the write path refuses.
|
||||
@@ -10074,7 +10089,14 @@ async fn run_preview_flow_job(
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
Json(raw_flow): Json<PreviewFlow>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "run preview jobs").await?;
|
||||
check_operator_can_build(
|
||||
&db,
|
||||
&w_id,
|
||||
authed.is_operator,
|
||||
BuilderKind::Flows,
|
||||
"run preview jobs",
|
||||
)
|
||||
.await?;
|
||||
// Flow preview runs an arbitrary, request-supplied flow definition; require the broad
|
||||
// jobs:run scope so a narrowly-scoped token cannot escape its scope. See run_preview_script.
|
||||
check_scopes(&authed, || format!("jobs:run"))?;
|
||||
|
||||
@@ -34,7 +34,7 @@ use std::collections::HashMap;
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::{Error, JsonResult},
|
||||
workspaces::operator_can_build_flows,
|
||||
workspaces::{operator_builder_rights, OperatorBuilderRights},
|
||||
};
|
||||
use windmill_types::scripts::ScriptHash;
|
||||
use windmill_types::user_drafts::DraftUserRef;
|
||||
@@ -574,9 +574,14 @@ async fn list_runnables(
|
||||
// Draft-only rows are the caller's own work in progress: never archived, so they
|
||||
// have no place in the archived view, and carrying no labels of their own they are
|
||||
// out of scope of a label filter (as in the per-kind endpoints). Operators don't
|
||||
// see other people's drafts and have none of their own, except a builder's flows.
|
||||
// see other people's drafts and have none of their own, except of what they may build.
|
||||
let builder = if authed.is_operator {
|
||||
operator_builder_rights(&db, &w_id).await?
|
||||
} else {
|
||||
OperatorBuilderRights::default()
|
||||
};
|
||||
let include_drafts = q.include_draft_only.unwrap_or(false)
|
||||
&& (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?)
|
||||
&& (!authed.is_operator || builder.any())
|
||||
&& !show_archived
|
||||
&& q.label.as_ref().filter(|s| !s.is_empty()).is_none();
|
||||
let draft_extras_for = |kind: &str| -> Vec<String> {
|
||||
@@ -662,7 +667,10 @@ async fn list_runnables(
|
||||
keyset: Option<&str>,
|
||||
limit: Option<usize>|
|
||||
-> Option<String> {
|
||||
if !include_drafts || !kinds.contains(&kind) || (authed.is_operator && kind != "flow") {
|
||||
if !include_drafts
|
||||
|| !kinds.contains(&kind)
|
||||
|| (authed.is_operator && !operator_drafts_kind(builder, kind))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
// `fav` is ignored: with no favorite join there is nothing to filter on, and the
|
||||
@@ -990,6 +998,15 @@ async fn count_runnables_by_owner(
|
||||
Ok(Json(RunnableCountsResponse { counts }))
|
||||
}
|
||||
|
||||
/// Whether an operator with `builder` rights can have drafts of a `/list` kind.
|
||||
fn operator_drafts_kind(builder: OperatorBuilderRights, kind: &str) -> bool {
|
||||
match kind {
|
||||
"flow" => builder.flows,
|
||||
"app" => builder.apps,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Adds the caller's draft-only rows to `counts`, in the same shape `/list`
|
||||
/// returns them so a badge never disagrees with the rows behind it.
|
||||
///
|
||||
@@ -1010,13 +1027,21 @@ async fn add_draft_counts(
|
||||
if !q.include_draft_only.unwrap_or(false) {
|
||||
return Ok(());
|
||||
}
|
||||
// An operator has no drafts of their own, except a builder's flows.
|
||||
// An operator has no drafts of their own, except of what they may build.
|
||||
let builder_kinds: Vec<&str>;
|
||||
let kinds: &[&str] = if !authed.is_operator {
|
||||
kinds
|
||||
} else if kinds.contains(&"flow") && operator_can_build_flows(db, w_id).await? {
|
||||
&["flow"]
|
||||
} else {
|
||||
return Ok(());
|
||||
let builder = operator_builder_rights(db, w_id).await?;
|
||||
builder_kinds = kinds
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|k| operator_drafts_kind(builder, k))
|
||||
.collect();
|
||||
if builder_kinds.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
&builder_kinds
|
||||
};
|
||||
// $1 = workspace, $2 = the caller's email.
|
||||
let mut binds: Vec<String> = vec![];
|
||||
|
||||
@@ -215,6 +215,11 @@ lazy_static! {
|
||||
label: "Run flows".to_string(),
|
||||
requires_resource_path: true,
|
||||
},
|
||||
ScopeOption {
|
||||
value: "jobs:run:dependencies".to_string(),
|
||||
label: "Generate dependency locks".to_string(),
|
||||
requires_resource_path: true,
|
||||
},
|
||||
],
|
||||
}];
|
||||
|
||||
|
||||
@@ -18,6 +18,59 @@ lazy_static::lazy_static! {
|
||||
pub static ref APP_WORKSPACED_ROUTE: AtomicBool = AtomicBool::new(false);
|
||||
}
|
||||
|
||||
/// Whether the app value carries an `inlineScript` anywhere.
|
||||
///
|
||||
/// Deliberately not built on [`traverse_app_inline_scripts`], which only reports a script whose
|
||||
/// `language` parses and stops descending as soon as it sees an `inlineScript` key. That is right
|
||||
/// for locking (nothing to lock without a language) and wrong for an authorization check, where
|
||||
/// the author picks the fields: this refuses the key itself, whatever it contains.
|
||||
pub fn app_value_has_inline_script(value: &Value) -> bool {
|
||||
match value {
|
||||
Value::Object(object) => {
|
||||
object.contains_key("inlineScript") || object.values().any(app_value_has_inline_script)
|
||||
}
|
||||
Value::Array(array) => array.iter().any(app_value_has_inline_script),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every workspace runnable the app value points a component at, as `(is_flow, path)`.
|
||||
///
|
||||
/// This is what the deployed bundle actually asks `execute_component` to run: it resolves a
|
||||
/// `runnable_id` against the stored `runnables` and sends `{runType}/{path}`. The policy's
|
||||
/// triggerables are a separate surface, so both have to be authorized. Anything not plain text
|
||||
/// is refused, never skipped: the editor compares `type` loosely (`["path"] == "path"`) and
|
||||
/// interpolates `path` (`["u/x"]` → `u/x`), so a skipped entry still runs.
|
||||
pub fn app_value_runnable_paths(value: &Value) -> error::Result<Vec<(bool, String)>> {
|
||||
let bad = |msg: String| error::Error::BadRequest(msg);
|
||||
let runnables = match value.get("runnables") {
|
||||
None | Some(Value::Null) => return Ok(vec![]),
|
||||
Some(Value::Object(runnables)) => runnables,
|
||||
Some(_) => return Err(bad("App runnables must be an object".to_string())),
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
for (id, runnable) in runnables {
|
||||
let Value::Object(runnable) = runnable else {
|
||||
return Err(bad(format!("Runnable {id} must be an object")));
|
||||
};
|
||||
match runnable.get("type") {
|
||||
Some(Value::String(t)) if t == "runnableByPath" || t == "path" => {}
|
||||
None | Some(Value::String(_)) => continue,
|
||||
Some(_) => return Err(bad(format!("Runnable {id} has a non-text type"))),
|
||||
}
|
||||
let Some(Value::String(path)) = runnable.get("path") else {
|
||||
return Err(bad(format!("Runnable {id} must name its path as text")));
|
||||
};
|
||||
let is_flow = match runnable.get("runType") {
|
||||
Some(Value::String(t)) if t == "flow" => true,
|
||||
Some(Value::String(t)) if t == "script" || t == "hubscript" => false,
|
||||
_ => return Err(bad(format!("Unsupported runType for runnable {path}"))),
|
||||
};
|
||||
out.push((is_flow, path.clone()));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Traverse FlowValue while invoking provided by caller callback on leafs
|
||||
// #[async_recursion::async_recursion(?Send)]
|
||||
pub fn traverse_app_inline_scripts<
|
||||
|
||||
@@ -1201,6 +1201,8 @@ pub struct PythonAnnotations {
|
||||
pub py311: bool,
|
||||
pub py312: bool,
|
||||
pub py313: bool,
|
||||
pub py314: bool,
|
||||
pub py315: bool,
|
||||
pub sandbox: bool,
|
||||
}
|
||||
|
||||
@@ -1805,6 +1807,7 @@ pub struct CpuStat {
|
||||
pub throttled_usec: u64,
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn cpu_stat_field(stat: &str, key: &str) -> Option<u64> {
|
||||
stat.lines().find_map(|line| {
|
||||
let (k, v) = line.split_once(' ')?;
|
||||
@@ -1812,6 +1815,7 @@ fn cpu_stat_field(stat: &str, key: &str) -> Option<u64> {
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn parse_cgroup_v2_cpu_stat(stat: &str) -> Option<CpuStat> {
|
||||
Some(CpuStat {
|
||||
usage_usec: cpu_stat_field(stat, "usage_usec")?,
|
||||
@@ -1822,6 +1826,7 @@ fn parse_cgroup_v2_cpu_stat(stat: &str) -> Option<CpuStat> {
|
||||
}
|
||||
|
||||
/// cgroup v1 splits the same figures over two controllers and counts time in nanoseconds.
|
||||
#[cfg(not(windows))]
|
||||
fn parse_cgroup_v1_cpu_stat(cpuacct_usage: &str, stat: &str) -> Option<CpuStat> {
|
||||
Some(CpuStat {
|
||||
usage_usec: cpuacct_usage.trim().parse::<u64>().ok()? / 1000,
|
||||
@@ -2889,6 +2894,8 @@ pub enum PyVAlias {
|
||||
#[default]
|
||||
Py312,
|
||||
Py313,
|
||||
Py314,
|
||||
Py315,
|
||||
}
|
||||
|
||||
impl Into<pep440_rs::Version> for PyVAlias {
|
||||
@@ -2906,13 +2913,20 @@ impl Into<u32> for PyVAlias {
|
||||
impl PyVAlias {
|
||||
pub fn all<T: From<PyVAlias>>() -> Vec<T> {
|
||||
use PyVAlias::*;
|
||||
vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()]
|
||||
vec![
|
||||
Py310.into(),
|
||||
Py311.into(),
|
||||
Py312.into(),
|
||||
Py313.into(),
|
||||
Py314.into(),
|
||||
Py315.into(),
|
||||
]
|
||||
}
|
||||
// Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH)
|
||||
fn major(&self) -> u32 {
|
||||
use PyVAlias::*;
|
||||
match self {
|
||||
Py310 | Py311 | Py312 | Py313 => 3,
|
||||
Py310 | Py311 | Py312 | Py313 | Py314 | Py315 => 3,
|
||||
// Py400 | Py401 => 4
|
||||
}
|
||||
}
|
||||
@@ -2927,6 +2941,8 @@ impl PyVAlias {
|
||||
"311" => Some(Py311),
|
||||
"312" => Some(Py312),
|
||||
"313" => Some(Py313),
|
||||
"314" => Some(Py314),
|
||||
"315" => Some(Py315),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -2986,6 +3002,7 @@ mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[test]
|
||||
fn cgroup_cpu_stat_is_normalized_to_microseconds() {
|
||||
let v2 = "usage_usec 2500000\nuser_usec 2000000\nsystem_usec 500000\n\
|
||||
|
||||
@@ -1067,7 +1067,7 @@ pub async fn billable_seats(db: &crate::DB, w_id: &str) -> Result<BillableSeats>
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.map_err(|e| Error::internal_err(format!("counting billable seats of {w_id}: {e:#}")))?;
|
||||
let (developers, operators) = if operator_can_build_flows(db, w_id).await? {
|
||||
let (developers, operators) = if operator_builder_rights(db, w_id).await?.any() {
|
||||
(row.developers + row.operators, 0)
|
||||
} else {
|
||||
(row.developers, row.operators)
|
||||
@@ -1205,14 +1205,47 @@ lazy_static::lazy_static! {
|
||||
|
||||
/// Every operator right of a workspace, read and cached together because they share one
|
||||
/// `operator_settings` column and one invalidation. The two groups have opposite polarity:
|
||||
/// `builder_flows` is granted on request and costs a seat, the `manage` rights are held by
|
||||
/// the builder rights are granted on request and cost a seat, the `manage` rights are held by
|
||||
/// default and cost nothing.
|
||||
#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct OperatorRights {
|
||||
pub builder_flows: bool,
|
||||
pub builder: OperatorBuilderRights,
|
||||
pub manage: OperatorManageRights,
|
||||
}
|
||||
|
||||
/// The two things a workspace can let its operators compose. Granted independently: composing a
|
||||
/// flow is fully checked server-side, while a full-code app is a browser-built bundle no check can
|
||||
/// read, so an admin may reasonably want the first without the second.
|
||||
#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct OperatorBuilderRights {
|
||||
pub flows: bool,
|
||||
pub apps: bool,
|
||||
}
|
||||
|
||||
/// Which capability a gate needs. Naming a kind at each call site is what keeps the two rights
|
||||
/// from collapsing back into one: a gate that takes "either" authorizes an app write on a
|
||||
/// flows-only workspace.
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum BuilderKind {
|
||||
Flows,
|
||||
Apps,
|
||||
}
|
||||
|
||||
impl OperatorBuilderRights {
|
||||
/// Whether either right is granted. Only for surfaces that are genuinely not per-kind: the
|
||||
/// seat a membership costs, and listing drafts.
|
||||
pub fn any(&self) -> bool {
|
||||
self.flows || self.apps
|
||||
}
|
||||
|
||||
pub fn has(&self, kind: BuilderKind) -> bool {
|
||||
match kind {
|
||||
BuilderKind::Flows => self.flows,
|
||||
BuilderKind::Apps => self.apps,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes an operator may perform unless the workspace withdraws them. Unlike the visibility
|
||||
/// flags beside them in `operator_settings`, these are enforced; unlike a right a workspace
|
||||
/// grants, they are held by default and cost no seat.
|
||||
@@ -1291,6 +1324,7 @@ async fn operator_rights(db: &DB, workspace_id: &str) -> Result<OperatorRights>
|
||||
// defaults have to differ here rather than at the call sites.
|
||||
let row = sqlx::query!(
|
||||
"SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\",
|
||||
COALESCE((operator_settings->>'builder_apps')::boolean, false) AS \"apps!\",
|
||||
COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\",
|
||||
COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\"
|
||||
FROM workspace_settings WHERE workspace_id = $1",
|
||||
@@ -1306,7 +1340,7 @@ async fn operator_rights(db: &DB, workspace_id: &str) -> Result<OperatorRights>
|
||||
|
||||
let rights = row
|
||||
.map(|r| OperatorRights {
|
||||
builder_flows: r.flows,
|
||||
builder: OperatorBuilderRights { flows: r.flows, apps: r.apps },
|
||||
manage: OperatorManageRights { schedules: r.schedules, triggers: r.triggers },
|
||||
})
|
||||
.unwrap_or_default();
|
||||
@@ -1316,21 +1350,22 @@ async fn operator_rights(db: &DB, workspace_id: &str) -> Result<OperatorRights>
|
||||
Ok(rights)
|
||||
}
|
||||
|
||||
/// Whether operators of this workspace may compose flows out of already-deployed runnables. Per
|
||||
/// workspace, not per user: every operator gets it, and consumes a full seat for it.
|
||||
pub async fn operator_can_build_flows(db: &DB, workspace_id: &str) -> Result<bool> {
|
||||
Ok(operator_rights(db, workspace_id).await?.builder_flows)
|
||||
/// What operators of this workspace may compose out of already-deployed runnables. Per workspace,
|
||||
/// not per user: every operator gets whatever is granted, and consumes a full seat for it.
|
||||
pub async fn operator_builder_rights(db: &DB, workspace_id: &str) -> Result<OperatorBuilderRights> {
|
||||
Ok(operator_rights(db, workspace_id).await?.builder)
|
||||
}
|
||||
|
||||
/// Gate for a write only a workspace that granted builder rights lets operators perform. `action`
|
||||
/// completes "Operators cannot {action} for security reasons".
|
||||
pub async fn check_operator_can_build_flows(
|
||||
/// Gate for a write only a workspace that granted the builder right covering `kind` lets operators
|
||||
/// perform. `action` completes "Operators cannot {action} for security reasons".
|
||||
pub async fn check_operator_can_build(
|
||||
db: &DB,
|
||||
workspace_id: &str,
|
||||
is_operator: bool,
|
||||
kind: BuilderKind,
|
||||
action: &str,
|
||||
) -> Result<()> {
|
||||
if is_operator && !operator_can_build_flows(db, workspace_id).await? {
|
||||
if is_operator && !operator_builder_rights(db, workspace_id).await?.has(kind) {
|
||||
return Err(Error::PermissionDenied(format!(
|
||||
"Operators cannot {action} for security reasons"
|
||||
)));
|
||||
@@ -1338,14 +1373,15 @@ pub async fn check_operator_can_build_flows(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether a membership consumes an operator (half) seat rather than an author seat. A builder
|
||||
/// right makes an operator an author of deployable artifacts, so it weighs a full seat.
|
||||
/// Whether a membership consumes an operator (half) seat rather than an author seat. Either
|
||||
/// builder right makes an operator an author of deployable artifacts, so both weigh the same full
|
||||
/// seat: there is no half-author.
|
||||
pub async fn consumes_operator_seat(
|
||||
db: &DB,
|
||||
workspace_id: &str,
|
||||
is_operator: bool,
|
||||
) -> Result<bool> {
|
||||
Ok(is_operator && !operator_can_build_flows(db, workspace_id).await?)
|
||||
Ok(is_operator && !operator_builder_rights(db, workspace_id).await?.any())
|
||||
}
|
||||
|
||||
/// Invalidate the operator rights cache for a workspace
|
||||
|
||||
@@ -143,6 +143,20 @@ async fn paid_seats_and_fork_count(db: Pool<Postgres>) {
|
||||
(4, 0, 1, 5),
|
||||
"builder rights bill every operator as a developer, service accounts stay half a seat"
|
||||
);
|
||||
sqlx::query(
|
||||
"UPDATE workspace_settings SET operator_settings = '{\"builder_apps\": true}'
|
||||
WHERE workspace_id = 'seat-root'",
|
||||
)
|
||||
.execute(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
windmill_common::workspaces::invalidate_operator_rights_cache("seat-root");
|
||||
let breakdown = billable_seats(&db, "seat-root").await.unwrap();
|
||||
assert_eq!(
|
||||
(breakdown.developers, breakdown.operators, breakdown.seats),
|
||||
(4, 0, 5),
|
||||
"the app right alone also bills every operator as a developer"
|
||||
);
|
||||
|
||||
insert_ws(&db, "seat-fork1", Some("seat-root"), false).await;
|
||||
insert_ws(&db, "seat-fork2", Some("seat-root"), false).await;
|
||||
|
||||
@@ -22,16 +22,15 @@ deno_webidl.workspace = true
|
||||
deno_web.workspace = true
|
||||
deno_crypto.workspace = true
|
||||
deno_net.workspace = true
|
||||
deno_console.workspace = true
|
||||
deno_url.workspace = true
|
||||
deno_core.workspace = true
|
||||
deno_ast.workspace = true
|
||||
deno_tls.workspace = true
|
||||
deno_permissions.workspace = true
|
||||
deno_io.workspace = true
|
||||
deno_fs.workspace = true
|
||||
deno_telemetry.workspace = true
|
||||
deno_error.workspace = true
|
||||
sys_traits.workspace = true
|
||||
# Not used directly: depending on it is what makes the workspace pin bind.
|
||||
v8.workspace = true
|
||||
winapi.workspace = true
|
||||
|
||||
itertools.workspace = true
|
||||
@@ -58,14 +57,8 @@ deno_webidl.workspace = true
|
||||
deno_web.workspace = true
|
||||
deno_crypto.workspace = true
|
||||
deno_net.workspace = true
|
||||
deno_console.workspace = true
|
||||
deno_url.workspace = true
|
||||
deno_core.workspace = true
|
||||
deno_ast.workspace = true
|
||||
deno_tls.workspace = true
|
||||
deno_permissions.workspace = true
|
||||
deno_io.workspace = true
|
||||
deno_fs.workspace = true
|
||||
deno_telemetry.workspace = true
|
||||
deno_error.workspace = true
|
||||
winapi.workspace = true
|
||||
|
||||
@@ -1,107 +1,11 @@
|
||||
use deno_ast::{MediaType, ParseParams};
|
||||
use deno_core::{ModuleCodeString, ModuleName, SourceMapData};
|
||||
use deno_core::{Extension, ModuleCodeString, ModuleName, SourceMapData};
|
||||
use deno_error::JsErrorBox;
|
||||
use deno_fetch::FetchPermissions;
|
||||
use deno_net::NetPermissions;
|
||||
use deno_web::{BlobStore, TimersPermission};
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashSet;
|
||||
use std::env;
|
||||
use std::fmt::Write as _;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct PermissionsContainer;
|
||||
|
||||
impl FetchPermissions for PermissionsContainer {
|
||||
#[inline(always)]
|
||||
fn check_net_url(
|
||||
&mut self,
|
||||
_url: &deno_core::url::Url,
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn check_read<'a>(
|
||||
&mut self,
|
||||
_path: Cow<'a, Path>,
|
||||
_api_name: &str,
|
||||
_get_path: &'a dyn deno_fs::GetPath,
|
||||
) -> Result<deno_fs::CheckedPath<'a>, deno_io::fs::FsError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn check_write<'a>(
|
||||
&mut self,
|
||||
_path: Cow<'a, Path>,
|
||||
_api_name: &str,
|
||||
_get_path: &'a dyn deno_fs::GetPath,
|
||||
) -> Result<deno_fs::CheckedPath<'a>, deno_io::fs::FsError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn check_net_vsock(
|
||||
&mut self,
|
||||
_cid: u32,
|
||||
_port: u32,
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
}
|
||||
|
||||
impl TimersPermission for PermissionsContainer {
|
||||
#[inline(always)]
|
||||
fn allow_hrtime(&mut self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl NetPermissions for PermissionsContainer {
|
||||
fn check_read(
|
||||
&mut self,
|
||||
_p: &str,
|
||||
_api_name: &str,
|
||||
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
fn check_write(
|
||||
&mut self,
|
||||
_p: &str,
|
||||
_api_name: &str,
|
||||
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
fn check_net<T: AsRef<str>>(
|
||||
&mut self,
|
||||
_host: &(T, Option<u16>),
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
fn check_write_path<'a>(
|
||||
&mut self,
|
||||
_p: Cow<'a, Path>,
|
||||
_api_name: &str,
|
||||
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
fn check_vsock(
|
||||
&mut self,
|
||||
_cid: u32,
|
||||
_port: u32,
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
}
|
||||
|
||||
deno_core::extension!(
|
||||
fetch,
|
||||
@@ -109,38 +13,29 @@ deno_core::extension!(
|
||||
esm = ["src/runtime.js"],
|
||||
);
|
||||
|
||||
// `extension_transpiler` callback for `deno_core::snapshot::create_snapshot`.
|
||||
// `extension_transpiler` callback for `deno_core::snapshot::create_snapshot`,
|
||||
// also applied to the residual lazy sources below.
|
||||
//
|
||||
// Specialized to our snapshot's inputs. Of the eight deno_* extensions
|
||||
// we register via `init()`, seven ship pre-built `.js` files
|
||||
// in their `esm` lists (webidl/url/console/web/crypto/fetch/net) — only
|
||||
// `deno_telemetry`'s `extension!` macro lists `.ts` files
|
||||
// (`telemetry.ts`, `util.ts`), so the TypeScript branch is needed
|
||||
// solely for that crate. Our local `fetch` extension contributes
|
||||
// `src/runtime.js` (pure JS). No `node:` imports happen at snapshot
|
||||
// build time, no `.mjs`, no user-supplied modules. So:
|
||||
// Specialized to our inputs: the deno_* extensions ship `.js`, except
|
||||
// `deno_telemetry` whose lazy scripts are `.ts`. Our local `fetch` extension
|
||||
// contributes `src/runtime.js`. No `node:` imports, no `.mjs`, no
|
||||
// user-supplied modules. So:
|
||||
// - `.js` → pass through.
|
||||
// - `.ts` → transpile via deno_ast (deno_telemetry only).
|
||||
// - anything else → build bug (deno shipping an unexpected file type
|
||||
// or us mislabelling one), panic loudly rather than emit a broken
|
||||
// snapshot.
|
||||
//
|
||||
// No source maps: the snapshot is a binary blob the runtime loads — source
|
||||
// maps would never be consumed.
|
||||
// No source maps: nothing consumes them.
|
||||
//
|
||||
// The signature still returns `Result<_, JsErrorBox>` because that's what
|
||||
// `extension_transpiler` expects, but we never construct one — parse and
|
||||
// transpile failures are build-time bugs in deno's own .ts internals (or
|
||||
// in our runtime.js, if we ever change its extension), so they panic.
|
||||
// The signature returns `Result<_, JsErrorBox>` because that's what
|
||||
// `extension_transpiler` expects, but we never construct one: parse and
|
||||
// transpile failures are build-time bugs, so they panic.
|
||||
//
|
||||
// This replaces a call to `deno_runtime::transpile::maybe_transpile_source`
|
||||
// from `deno_runtime 0.198.0`. The original is more general (handles
|
||||
// `node:` modules, `.mjs`, emits source maps in debug builds, plumbs
|
||||
// errors via `JsErrorBox`); none of that surface is reachable in our
|
||||
// build. Dropping the `deno_runtime` dep eliminates a
|
||||
// `deno_cache → rusqlite → libsqlite3-sys 0.35` transitive chain that
|
||||
// collides with sqlx-sqlite's `libsqlite3-sys 0.30` (cargo's
|
||||
// `links = "sqlite3"` rule).
|
||||
// `deno_runtime::transpile::maybe_transpile_source` is the general version;
|
||||
// depending on `deno_runtime` would pull a
|
||||
// `deno_cache → rusqlite → libsqlite3-sys` chain that collides with
|
||||
// sqlx-sqlite's `libsqlite3-sys` (cargo's `links = "sqlite3"` rule).
|
||||
fn maybe_transpile_source(
|
||||
name: ModuleName,
|
||||
source: ModuleCodeString,
|
||||
@@ -154,7 +49,7 @@ fn maybe_transpile_source(
|
||||
|
||||
let parsed = deno_ast::parse_module(ParseParams {
|
||||
specifier: deno_core::url::Url::parse(&name).unwrap(),
|
||||
text: source.into(),
|
||||
text: source.as_str().into(),
|
||||
media_type,
|
||||
capture_tokens: false,
|
||||
scope_analysis: false,
|
||||
@@ -169,7 +64,13 @@ fn maybe_transpile_source(
|
||||
..Default::default()
|
||||
},
|
||||
&deno_ast::TranspileModuleOptions::default(),
|
||||
&deno_ast::EmitOptions::default(),
|
||||
// The default appends an inline source map comment, which is a
|
||||
// syntax error where deno_core evaluates a lazy script as an
|
||||
// expression.
|
||||
&deno_ast::EmitOptions {
|
||||
source_map: deno_ast::SourceMapOption::None,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap_or_else(|e| panic!("snapshot transpile: emit failed for {name}: {e}"))
|
||||
.into_source();
|
||||
@@ -177,27 +78,66 @@ fn maybe_transpile_source(
|
||||
Ok((transpiled.text.into(), None))
|
||||
}
|
||||
|
||||
/// One `(specifier, source)` table of the extension scripts the snapshot did
|
||||
/// not evaluate. The snapshot only carries scripts that ran while it was built,
|
||||
/// so a script first loaded in a live isolate has to come from here.
|
||||
///
|
||||
/// deno_core binary-searches the table and reads each source as ASCII without
|
||||
/// checking, so it must be sorted by specifier, free of duplicates, and ASCII.
|
||||
fn residual_table(
|
||||
name: &str,
|
||||
files: impl Iterator<Item = deno_core::ExtensionFileSource>,
|
||||
consumed: &HashSet<String>,
|
||||
) -> String {
|
||||
let mut files: Vec<_> = files.filter(|f| !consumed.contains(f.specifier)).collect();
|
||||
files.sort_by_key(|f| f.specifier);
|
||||
files.dedup_by_key(|f| f.specifier);
|
||||
|
||||
let mut out = format!("pub static {name}: &[(&str, &str)] = &[\n");
|
||||
for file in files {
|
||||
let source = file
|
||||
.load()
|
||||
.unwrap_or_else(|e| panic!("could not load {}: {e}", file.specifier));
|
||||
let (code, _) =
|
||||
maybe_transpile_source(ModuleName::from_static(file.specifier), source).unwrap();
|
||||
assert!(
|
||||
code.as_str().is_ascii(),
|
||||
"{} is not ASCII after transpiling",
|
||||
file.specifier
|
||||
);
|
||||
writeln!(out, " ({:?}, {:?}),", file.specifier, code.as_str()).unwrap();
|
||||
}
|
||||
out.push_str("];\n");
|
||||
out
|
||||
}
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
|
||||
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
|
||||
|
||||
let exts = vec![
|
||||
deno_telemetry::deno_telemetry::init(),
|
||||
deno_webidl::deno_webidl::init(),
|
||||
deno_url::deno_url::init(),
|
||||
deno_console::deno_console::init(),
|
||||
deno_web::deno_web::init::<PermissionsContainer>(Arc::new(BlobStore::default()), None),
|
||||
deno_crypto::deno_crypto::init(None),
|
||||
deno_fetch::deno_fetch::init::<PermissionsContainer>(Default::default()),
|
||||
deno_net::deno_net::init::<PermissionsContainer>(None, None),
|
||||
// Same list and order as `create_nativets_runtime`, which supplies their
|
||||
// state through `lazy_init_extensions`.
|
||||
let exts: Vec<Extension> = vec![
|
||||
deno_telemetry::deno_telemetry::lazy_init(),
|
||||
deno_webidl::deno_webidl::lazy_init(),
|
||||
deno_web::deno_web::lazy_init(),
|
||||
deno_crypto::deno_crypto::lazy_init(),
|
||||
deno_fetch::deno_fetch::lazy_init(),
|
||||
deno_net::deno_net::lazy_init(),
|
||||
fetch::init(),
|
||||
];
|
||||
|
||||
// Build the file path to the snapshot.
|
||||
let lazy_js: Vec<_> = exts
|
||||
.iter()
|
||||
.flat_map(|e| e.lazy_loaded_js_files.iter().cloned())
|
||||
.collect();
|
||||
let lazy_esm: Vec<_> = exts
|
||||
.iter()
|
||||
.flat_map(|e| e.lazy_loaded_esm_files.iter().cloned())
|
||||
.collect();
|
||||
|
||||
let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
|
||||
let snapshot_path = o.join("FETCH_SNAPSHOT.bin");
|
||||
|
||||
// Create the snapshot.
|
||||
let output = deno_core::snapshot::create_snapshot(
|
||||
deno_core::snapshot::CreateSnapshotOptions {
|
||||
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
|
||||
@@ -213,9 +153,17 @@ fn main() {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut file = std::fs::File::create(snapshot_path).unwrap();
|
||||
let mut file = std::fs::File::create(o.join("FETCH_SNAPSHOT.bin")).unwrap();
|
||||
file.write_all(&output.output).unwrap();
|
||||
|
||||
let consumed: HashSet<String> = output.consumed_lazy_specifiers.into_iter().collect();
|
||||
let residual = format!(
|
||||
"{}{}",
|
||||
residual_table("RESIDUAL_LAZY_JS", lazy_js.into_iter(), &consumed),
|
||||
residual_table("RESIDUAL_LAZY_ESM", lazy_esm.into_iter(), &consumed),
|
||||
);
|
||||
std::fs::write(o.join("residual_lazy_sources.rs"), residual).unwrap();
|
||||
|
||||
for path in output.files_loaded_during_snapshot {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
|
||||
@@ -22,9 +22,7 @@ mod smoke_tests;
|
||||
mod cert_tests;
|
||||
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
cell::RefCell,
|
||||
path::PathBuf,
|
||||
rc::Rc,
|
||||
sync::{Arc, LazyLock, Mutex},
|
||||
};
|
||||
@@ -32,17 +30,55 @@ use std::{
|
||||
// Re-export deno_telemetry for use by windmill-worker's otel proxy
|
||||
pub use deno_telemetry;
|
||||
|
||||
/// The real environment and filesystem, except that the OTLP protocol always
|
||||
/// reads as HTTP.
|
||||
///
|
||||
/// The caller points `OTEL_EXPORTER_OTLP_ENDPOINT` at its own HTTP collector
|
||||
/// only while [`init_telemetry`] runs, and the instance's own exporter shares
|
||||
/// `OTEL_EXPORTER_OTLP_PROTOCOL`. Honouring `grpc` there would build an exporter
|
||||
/// that reads the endpoint again on every export, and `console` one that never
|
||||
/// reads it, so either would send job telemetry past that collector.
|
||||
struct HttpOtlpSys;
|
||||
|
||||
impl sys_traits::BaseEnvVar for HttpOtlpSys {
|
||||
fn base_env_var_os(&self, key: &std::ffi::OsStr) -> Option<std::ffi::OsString> {
|
||||
if key == "OTEL_EXPORTER_OTLP_PROTOCOL" {
|
||||
return Some("http/protobuf".into());
|
||||
}
|
||||
sys_traits::impls::RealSys.base_env_var_os(key)
|
||||
}
|
||||
}
|
||||
|
||||
impl sys_traits::BaseFsRead for HttpOtlpSys {
|
||||
fn base_fs_read(
|
||||
&self,
|
||||
path: &std::path::Path,
|
||||
) -> std::io::Result<std::borrow::Cow<'static, [u8]>> {
|
||||
sys_traits::impls::RealSys.base_fs_read(path)
|
||||
}
|
||||
}
|
||||
|
||||
/// `deno_telemetry::init` for the isolates of this process.
|
||||
pub fn init_telemetry(
|
||||
rt_config: deno_telemetry::OtelRuntimeConfig,
|
||||
config: deno_telemetry::OtelConfig,
|
||||
) -> anyhow::Result<()> {
|
||||
deno_telemetry::init(&HttpOtlpSys, rt_config, config)
|
||||
}
|
||||
|
||||
use deno_ast::ParseParams;
|
||||
use deno_core::{
|
||||
error::CoreErrorKind,
|
||||
op2, serde_v8, url,
|
||||
v8::{self, IsolateHandle},
|
||||
Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions,
|
||||
};
|
||||
use deno_error::JsErrorBox;
|
||||
use deno_fetch::FetchPermissions;
|
||||
use deno_net::NetPermissions;
|
||||
use deno_permissions::{
|
||||
Permissions, PermissionsContainer, PermissionsOptions, RuntimePermissionDescriptorParser,
|
||||
};
|
||||
use deno_tls::{rustls::pki_types::CertificateDer, rustls::RootCertStore, RootCertStoreProvider};
|
||||
use deno_web::{BlobStore, TimersPermission};
|
||||
use deno_web::{BlobStore, InMemoryBroadcastChannel};
|
||||
use itertools::Itertools;
|
||||
use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
@@ -57,156 +93,55 @@ use windmill_common::worker::{write_file, Connection, WINDMILL_DIR};
|
||||
|
||||
// ── Snapshot-matched extensions ──────────────────────────────────────
|
||||
//
|
||||
// `deno_core` 0.352 validates that the snapshot's extension list is a
|
||||
// *prefix* of the runtime's extension list (snapshot does not need an
|
||||
// exact match — runtime is allowed to add extensions at the tail, but
|
||||
// must not reorder or omit any that the snapshot baked in).
|
||||
// The snapshot (build.rs) is built from the deno_* extensions in their
|
||||
// `lazy_init()` form followed by this local `fetch` ext. deno_core requires the
|
||||
// runtime's extension list to start with that same list in that same order; it
|
||||
// may append more, which is how the windmill `ext` carrying our own ops gets in.
|
||||
//
|
||||
// Our snapshot (in build.rs) is the same eight deno_* extensions ending
|
||||
// with this local `fetch` ext. The runtime adds one extra entry at the
|
||||
// end — the windmill `ext` carrying our own ops — which is fine because
|
||||
// it's after the snapshot prefix.
|
||||
//
|
||||
// This local `fetch` extension declaration must be present in both
|
||||
// build.rs and lib.rs so the type passes through the `init()` macro.
|
||||
// The ESM is already in the snapshot, so this `init()` call at runtime
|
||||
// is a no-op for esm — the registration just records the ext.
|
||||
// This declaration must exist in both build.rs and lib.rs. The ESM is already
|
||||
// in the snapshot, so `init()` at runtime only records the extension.
|
||||
deno_core::extension!(
|
||||
fetch,
|
||||
esm_entry_point = "ext:fetch/src/runtime.js",
|
||||
esm = ["src/runtime.js"],
|
||||
);
|
||||
|
||||
// ── Permission container ─────────────────────────────────────────────
|
||||
include!(concat!(env!("OUT_DIR"), "/residual_lazy_sources.rs"));
|
||||
|
||||
/// `no_network` is what `//no_network` promises: no outbound traffic of any kind,
|
||||
/// the Windmill API included (the client module reaches it through `fetch`).
|
||||
/// Enforced here and by disabling the deno_net ops, never in JS: user code can
|
||||
/// call `Deno.core.ops.*` directly and skip any JS-level guard.
|
||||
pub struct PermissionsContainer {
|
||||
pub no_network: bool,
|
||||
}
|
||||
|
||||
const NO_NETWORK_ANNOTATION: &str = "no_network";
|
||||
// ── Permissions ──────────────────────────────────────────────────────
|
||||
|
||||
/// Every op deno_net registers, see `create_nativets_runtime`. Read from the
|
||||
/// extension itself so an upgrade that adds an op cannot leave it enabled.
|
||||
static NO_NETWORK_DISABLED_OPS: LazyLock<std::collections::HashSet<&'static str>> =
|
||||
LazyLock::new(|| {
|
||||
deno_net::deno_net::init::<PermissionsContainer>(None, None)
|
||||
deno_net::deno_net::lazy_init()
|
||||
.ops
|
||||
.iter()
|
||||
.map(|op| op.name)
|
||||
.collect()
|
||||
});
|
||||
|
||||
impl PermissionsContainer {
|
||||
fn deny_net(&self, target: &str) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
if self.no_network {
|
||||
Err(deno_permissions::PermissionDeniedError::Fatal {
|
||||
access: format!(
|
||||
"net access to {target} (the script is annotated //{NO_NETWORK_ANNOTATION})"
|
||||
),
|
||||
}
|
||||
.into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FetchPermissions for PermissionsContainer {
|
||||
#[inline(always)]
|
||||
fn check_net_url(
|
||||
&mut self,
|
||||
url: &deno_core::url::Url,
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
self.deny_net(&format!("\"{}\"", url.host_str().unwrap_or_default()))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn check_read<'a>(
|
||||
&mut self,
|
||||
path: Cow<'a, std::path::Path>,
|
||||
_api_name: &str,
|
||||
_get_path: &'a dyn deno_fs::GetPath,
|
||||
) -> Result<deno_fs::CheckedPath<'a>, deno_io::fs::FsError> {
|
||||
Ok(deno_fs::CheckedPath::Unresolved(path))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn check_write<'a>(
|
||||
&mut self,
|
||||
path: Cow<'a, std::path::Path>,
|
||||
_api_name: &str,
|
||||
_get_path: &'a dyn deno_fs::GetPath,
|
||||
) -> Result<deno_fs::CheckedPath<'a>, deno_io::fs::FsError> {
|
||||
Ok(deno_fs::CheckedPath::Unresolved(path))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn check_net_vsock(
|
||||
&mut self,
|
||||
cid: u32,
|
||||
port: u32,
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
self.deny_net(&format!("vsock {cid}:{port}"))
|
||||
}
|
||||
}
|
||||
|
||||
impl TimersPermission for PermissionsContainer {
|
||||
#[inline(always)]
|
||||
fn allow_hrtime(&mut self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl NetPermissions for PermissionsContainer {
|
||||
fn check_read(
|
||||
&mut self,
|
||||
p: &str,
|
||||
_api_name: &str,
|
||||
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
|
||||
self.deny_net(&format!("unix socket \"{p}\""))?;
|
||||
Ok(PathBuf::from(p))
|
||||
}
|
||||
|
||||
fn check_write(
|
||||
&mut self,
|
||||
p: &str,
|
||||
_api_name: &str,
|
||||
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
|
||||
self.deny_net(&format!("unix socket \"{p}\""))?;
|
||||
Ok(PathBuf::from(p))
|
||||
}
|
||||
|
||||
fn check_net<T: AsRef<str>>(
|
||||
&mut self,
|
||||
host: &(T, Option<u16>),
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
self.deny_net(&format!("\"{}\"", host.0.as_ref()))
|
||||
}
|
||||
|
||||
fn check_write_path<'a>(
|
||||
&mut self,
|
||||
p: Cow<'a, std::path::Path>,
|
||||
_api_name: &str,
|
||||
) -> Result<Cow<'a, std::path::Path>, deno_permissions::PermissionCheckError> {
|
||||
self.deny_net(&format!("unix socket \"{}\"", p.display()))?;
|
||||
Ok(p)
|
||||
}
|
||||
|
||||
fn check_vsock(
|
||||
&mut self,
|
||||
cid: u32,
|
||||
port: u32,
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
self.deny_net(&format!("vsock {cid}:{port}"))
|
||||
}
|
||||
/// What an isolate may do: reach the network, unless `no_network`, and nothing
|
||||
/// else (no filesystem, environment, subprocess or FFI). Nothing prompts.
|
||||
///
|
||||
/// `no_network` is what `//no_network` promises: no outbound traffic of any
|
||||
/// kind, the Windmill API included (the client module reaches it through
|
||||
/// `fetch`). Enforced here and by disabling the deno_net ops, never in JS: user
|
||||
/// code can call `Deno.core.ops.*` directly and skip any JS-level guard.
|
||||
fn isolate_permissions(no_network: bool) -> anyhow::Result<PermissionsContainer> {
|
||||
let parser = Arc::new(RuntimePermissionDescriptorParser::new(
|
||||
sys_traits::impls::RealSys,
|
||||
));
|
||||
// An empty list means "all hosts", for a grant and for a denial alike.
|
||||
let options = PermissionsOptions {
|
||||
allow_net: (!no_network).then(Vec::new),
|
||||
deny_net: no_network.then(Vec::new),
|
||||
prompt: false,
|
||||
..Default::default()
|
||||
};
|
||||
let permissions = Permissions::from_options(parser.as_ref(), &options)
|
||||
.map_err(|e| anyhow::anyhow!("invalid isolate permissions: {e}"))?;
|
||||
Ok(PermissionsContainer::new(parser, permissions))
|
||||
}
|
||||
|
||||
// ── Types ────────────────────────────────────────────────────────────
|
||||
@@ -228,7 +163,7 @@ pub struct NativeAnnotation {
|
||||
/// script; `None` leaves the default in force.
|
||||
pub fetch_response_timeout_secs: Option<u64>,
|
||||
/// `//no_network`: the isolate may open no connection at all, see
|
||||
/// [`PermissionsContainer`].
|
||||
/// [`isolate_permissions`].
|
||||
pub no_network: bool,
|
||||
}
|
||||
|
||||
@@ -413,13 +348,10 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> {
|
||||
// deno_fetch requires a TLS provider; install ring as default (idempotent).
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
|
||||
let unrecognized_v8_flags = deno_core::v8_set_flags(vec![
|
||||
"--stack-size=1024".to_string(),
|
||||
"--no-harmony-import-assertions".to_string(),
|
||||
])
|
||||
.into_iter()
|
||||
.skip(1)
|
||||
.collect::<Vec<_>>();
|
||||
let unrecognized_v8_flags = deno_core::v8_set_flags(vec!["--stack-size=1024".to_string()])
|
||||
.into_iter()
|
||||
.skip(1)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if !unrecognized_v8_flags.is_empty() {
|
||||
init_err = Some(format!(
|
||||
@@ -435,7 +367,7 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> {
|
||||
// SIGSEGV in WasmCodePointerTable::AllocateUninitializedEntry() on x86_64 Linux.
|
||||
// See: https://github.com/denoland/deno_core/issues/952
|
||||
let platform = deno_core::v8::new_unprotected_default_platform(0, false).make_shared();
|
||||
deno_core::JsRuntime::init_platform(Some(platform), false);
|
||||
deno_core::JsRuntime::init_platform(Some(platform));
|
||||
});
|
||||
|
||||
if let Some(msg) = init_err {
|
||||
@@ -666,17 +598,15 @@ pub(crate) fn create_nativets_runtime(
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Same list and order as the snapshot (build.rs); their state follows in
|
||||
// `lazy_init_extensions` below, which must match it too.
|
||||
let exts: Vec<Extension> = vec![
|
||||
deno_telemetry::deno_telemetry::init(),
|
||||
deno_webidl::deno_webidl::init(),
|
||||
deno_url::deno_url::init(),
|
||||
deno_console::deno_console::init(),
|
||||
deno_web::deno_web::init::<PermissionsContainer>(Arc::new(BlobStore::default()), None),
|
||||
// Registered after deno_web to keep the snapshot (build.rs) a prefix of
|
||||
// the runtime extension list; deno_crypto declares deps = [deno_webidl, deno_web].
|
||||
deno_crypto::deno_crypto::init(None),
|
||||
deno_fetch::deno_fetch::init::<PermissionsContainer>(fetch_options),
|
||||
deno_net::deno_net::init::<PermissionsContainer>(None, None),
|
||||
deno_telemetry::deno_telemetry::lazy_init(),
|
||||
deno_webidl::deno_webidl::lazy_init(),
|
||||
deno_web::deno_web::lazy_init(),
|
||||
deno_crypto::deno_crypto::lazy_init(),
|
||||
deno_fetch::deno_fetch::lazy_init(),
|
||||
deno_net::deno_net::lazy_init(),
|
||||
fetch::init(),
|
||||
ext,
|
||||
];
|
||||
@@ -688,6 +618,8 @@ pub(crate) fn create_nativets_runtime(
|
||||
deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128),
|
||||
),
|
||||
startup_snapshot: Some(RUNTIME_SNAPSHOT),
|
||||
residual_lazy_js_sources: RESIDUAL_LAZY_JS,
|
||||
residual_lazy_esm_sources: RESIDUAL_LAZY_ESM,
|
||||
module_loader: Some(Rc::new(deno_core::FsModuleLoader)),
|
||||
extension_transpiler: None,
|
||||
..Default::default()
|
||||
@@ -714,12 +646,28 @@ pub(crate) fn create_nativets_runtime(
|
||||
y * 2
|
||||
});
|
||||
|
||||
js_runtime
|
||||
.lazy_init_extensions(vec![
|
||||
deno_telemetry::deno_telemetry::args(),
|
||||
deno_webidl::deno_webidl::args(),
|
||||
deno_web::deno_web::args(
|
||||
Arc::new(BlobStore::default()),
|
||||
None,
|
||||
false,
|
||||
InMemoryBroadcastChannel::default(),
|
||||
),
|
||||
deno_crypto::deno_crypto::args(None),
|
||||
deno_fetch::deno_fetch::args(fetch_options),
|
||||
deno_net::deno_net::args(None, None),
|
||||
])
|
||||
.map_err(windmill_common::error::to_anyhow)?;
|
||||
|
||||
let (log_sender, log_receiver) = mpsc::unbounded_channel::<String>();
|
||||
|
||||
{
|
||||
let op_state = js_runtime.op_state();
|
||||
let mut op_state = op_state.borrow_mut();
|
||||
op_state.put(PermissionsContainer { no_network });
|
||||
op_state.put(isolate_permissions(no_network)?);
|
||||
op_state.put(MainArgs { args: initial_args });
|
||||
op_state.put(LogString { s: log_sender });
|
||||
}
|
||||
@@ -731,7 +679,7 @@ pub(crate) fn create_nativets_runtime(
|
||||
.execute_script(
|
||||
"<wm_init>",
|
||||
format!(
|
||||
"globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms} }})"
|
||||
"globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms}, noNetwork: {no_network} }})"
|
||||
),
|
||||
)
|
||||
.map_err(windmill_common::error::to_anyhow)?;
|
||||
@@ -777,7 +725,7 @@ pub(crate) fn extract_global_string(
|
||||
js_runtime: &mut JsRuntime,
|
||||
global: v8::Global<v8::Value>,
|
||||
) -> Result<Box<RawValue>, String> {
|
||||
let scope = &mut js_runtime.handle_scope();
|
||||
deno_core::scope!(scope, js_runtime);
|
||||
let local = v8::Local::new(scope, global);
|
||||
match serde_v8::from_v8::<Option<String>>(scope, local) {
|
||||
Ok(s) => Ok(unsafe_raw(s.unwrap_or_else(|| "null".to_string()))),
|
||||
@@ -1169,7 +1117,10 @@ import("file:///eval.ts").then((module) => module.{main_fn}(...args))
|
||||
Ok(global) => {
|
||||
extract_global_string(js_runtime, global).map_err(|e| ExecuteError::Script(e))
|
||||
}
|
||||
Err(deno_core::error::CoreError::Js(e)) => {
|
||||
Err(e) if matches!(e.as_kind(), CoreErrorKind::Js(_)) => {
|
||||
let CoreErrorKind::Js(e) = e.into_kind() else {
|
||||
unreachable!("matched as a JS error above")
|
||||
};
|
||||
let source = e.frames.first().and_then(|f| {
|
||||
f.file_name
|
||||
.as_ref()
|
||||
|
||||
@@ -1,39 +1,42 @@
|
||||
import * as abortSignal from "ext:deno_web/03_abort_signal.js";
|
||||
import * as domException from "ext:deno_web/01_dom_exception.js";
|
||||
import * as base64 from "ext:deno_web/05_base64.js";
|
||||
import * as console from "ext:deno_console/01_console.js";
|
||||
import * as encoding from "ext:deno_web/08_text_encoding.js";
|
||||
import * as event from "ext:deno_web/02_event.js";
|
||||
import * as fetch from "ext:deno_fetch/26_fetch.js";
|
||||
import * as file from "ext:deno_web/09_file.js";
|
||||
import * as fileReader from "ext:deno_web/10_filereader.js";
|
||||
import * as formData from "ext:deno_fetch/21_formdata.js";
|
||||
import * as headers from "ext:deno_fetch/20_headers.js";
|
||||
import * as streams from "ext:deno_web/06_streams.js";
|
||||
import * as timers from "ext:deno_web/02_timers.js";
|
||||
import * as url from "ext:deno_url/00_url.js";
|
||||
import * as net from "ext:deno_net/01_net.js";
|
||||
import * as tls from "ext:deno_net/02_tls.js";
|
||||
import * as urlPattern from "ext:deno_url/01_urlpattern.js";
|
||||
import * as webidl from "ext:deno_webidl/00_webidl.js";
|
||||
import * as crypto from "ext:deno_crypto/00_crypto.js";
|
||||
import * as response from "ext:deno_fetch/23_response.js";
|
||||
import * as request from "ext:deno_fetch/23_request.js";
|
||||
import "ext:deno_web/02_structured_clone.js";
|
||||
import * as globalInterfaces from "ext:deno_web/04_global_interfaces.js";
|
||||
// Namespace imports (not side-effect-only) so their constructors are reachable
|
||||
// for the globalThis wiring below. The module bodies still execute on
|
||||
// evaluation, so their side effects apply.
|
||||
import * as messagePort from "ext:deno_web/13_message_port.js";
|
||||
import * as compression from "ext:deno_web/14_compression.js";
|
||||
import * as performance from "ext:deno_web/15_performance.js";
|
||||
import "ext:deno_web/16_image_data.js";
|
||||
import "ext:deno_fetch/27_eventsource.js";
|
||||
import { core, internals } from "ext:core/mod.js";
|
||||
|
||||
// deno_crypto brands CryptoKeys with this symbol and reads it from `internals`
|
||||
// when it evaluates; in Deno a deno_node module sets it, which is not loaded here.
|
||||
internals.kKeyObject ??= Symbol("kKeyObject");
|
||||
|
||||
// Every script loaded here runs at snapshot-build time and is baked into the
|
||||
// snapshot. One that is first loaded at run time comes from the residual table
|
||||
// build.rs emits instead.
|
||||
const abortSignal = core.loadExtScript("ext:deno_web/03_abort_signal.js");
|
||||
const domException = core.loadExtScript("ext:deno_web/01_dom_exception.js");
|
||||
const base64 = core.loadExtScript("ext:deno_web/05_base64.js");
|
||||
const console = core.loadExtScript("ext:deno_web/01_console.js");
|
||||
const encoding = core.loadExtScript("ext:deno_web/08_text_encoding.js");
|
||||
const event = core.loadExtScript("ext:deno_web/02_event.js");
|
||||
const fetch = core.loadExtScript("ext:deno_fetch/26_fetch.js");
|
||||
const file = core.loadExtScript("ext:deno_web/09_file.js");
|
||||
const fileReader = core.loadExtScript("ext:deno_web/10_filereader.js");
|
||||
const formData = core.loadExtScript("ext:deno_fetch/21_formdata.js");
|
||||
const headers = core.loadExtScript("ext:deno_fetch/20_headers.js");
|
||||
const streams = core.loadExtScript("ext:deno_web/06_streams.js");
|
||||
const timers = core.loadExtScript("ext:deno_web/02_timers.js");
|
||||
const url = core.loadExtScript("ext:deno_web/00_url.js");
|
||||
const urlPattern = core.loadExtScript("ext:deno_web/01_urlpattern.js");
|
||||
const webidl = core.loadExtScript("ext:deno_webidl/00_webidl.js");
|
||||
const crypto = core.loadExtScript("ext:deno_crypto/00_crypto.js");
|
||||
const response = core.loadExtScript("ext:deno_fetch/23_response.js");
|
||||
const request = core.loadExtScript("ext:deno_fetch/23_request.js");
|
||||
const globalInterfaces = core.loadExtScript(
|
||||
"ext:deno_web/04_global_interfaces.js",
|
||||
);
|
||||
const messagePort = core.loadExtScript("ext:deno_web/13_message_port.js");
|
||||
const compression = core.loadExtScript("ext:deno_web/14_compression.js");
|
||||
const performance = core.loadExtScript("ext:deno_web/15_performance.js");
|
||||
|
||||
// deno_fetch applies no deadline, so a peer that accepts a request and then
|
||||
// never answers leaves `await fetch(...)` pending until the job timeout, which
|
||||
// self-hosted defaults to 7 days.
|
||||
const ORIGINAL_FETCH = fetch.fetch;
|
||||
const DENO_FETCH = fetch.fetch;
|
||||
|
||||
// deno_web's timers reject any `this` other than undefined/globalThis, so
|
||||
// `timers.setTimeout(...)` passes the module namespace and throws "Illegal
|
||||
@@ -50,6 +53,14 @@ const abortControllerAbort = Function.prototype.call.bind(
|
||||
);
|
||||
const ReflectApply = Reflect.apply;
|
||||
|
||||
// Forwards the original argument count, like every caller below relies on. The
|
||||
// extra promise hop under `no_network` costs the same-tick settling of an
|
||||
// aborted fetch, which only matters where a request could have been made.
|
||||
function ORIGINAL_FETCH() {
|
||||
const p = ReflectApply(DENO_FETCH, undefined, arguments);
|
||||
return noNetwork ? promiseThen(p, undefined, rewordNoNetworkDenial) : p;
|
||||
}
|
||||
|
||||
// Installed per isolate by __wmInitPerIsolate; 0 disables. Only a backstop
|
||||
// for the impossible case of fetch running before that init.
|
||||
let fetchResponseTimeoutMs = 900_000;
|
||||
@@ -74,6 +85,21 @@ function fetchResponseTimeoutError(requestUrl, timeoutMs) {
|
||||
);
|
||||
}
|
||||
|
||||
// Set per isolate for a `//no_network` script. The denial itself is the
|
||||
// isolate's permissions (Rust); this only replaces deno's "run again with the
|
||||
// --allow-net flag" hint, which names a flag a script author cannot pass.
|
||||
let noNetwork = false;
|
||||
|
||||
function rewordNoNetworkDenial(e) {
|
||||
if (e?.name === "NotCapable" && typeof e.message === "string") {
|
||||
e.message = e.message.replace(
|
||||
", run again with the --allow-net flag",
|
||||
": the script is annotated //no_network",
|
||||
);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
globalThis.atob = base64.atob;
|
||||
globalThis.btoa = base64.btoa;
|
||||
// Not `async`, for the same reason deno_fetch's own outer fetch isn't: WPT
|
||||
@@ -152,7 +178,22 @@ globalThis.console = new console.Console((msg, level) =>
|
||||
);
|
||||
globalThis.AbortController = abortSignal.AbortController;
|
||||
globalThis.AbortSignal = abortSignal.AbortSignal;
|
||||
globalThis.crypto = crypto.crypto;
|
||||
// A getter, not a value: `crypto.crypto` mints a cppgc object, and the cppgc
|
||||
// heap is not attached while this module runs at snapshot-build time.
|
||||
Object.defineProperty(globalThis, "crypto", {
|
||||
configurable: true,
|
||||
enumerable: true,
|
||||
get: () => crypto.crypto,
|
||||
// Assignment has to keep working, as it does on a plain property.
|
||||
set(value) {
|
||||
Object.defineProperty(globalThis, "crypto", {
|
||||
configurable: true,
|
||||
enumerable: true,
|
||||
writable: true,
|
||||
value,
|
||||
});
|
||||
},
|
||||
});
|
||||
globalThis.Crypto = crypto.Crypto;
|
||||
globalThis.CryptoKey = crypto.CryptoKey;
|
||||
globalThis.SubtleCrypto = crypto.SubtleCrypto;
|
||||
@@ -233,6 +274,7 @@ globalThis.__wmInitPerIsolate = (config) => {
|
||||
if (config != null && typeof config.fetchResponseTimeoutMs === "number") {
|
||||
fetchResponseTimeoutMs = config.fetchResponseTimeoutMs;
|
||||
}
|
||||
noNetwork = config?.noNetwork === true;
|
||||
|
||||
// setTimeOrigin() seeds performance.timeOrigin from the isolate's wall clock;
|
||||
// without it timeOrigin is undefined and `timeOrigin + performance.now()` is NaN.
|
||||
@@ -256,13 +298,14 @@ globalThis.__wmInitPerIsolate = (config) => {
|
||||
};
|
||||
|
||||
// Expose bootstrapOtel globally so it can be called from Rust after runtime creation.
|
||||
// We use dynamic import so deno_telemetry isn't loaded during snapshot creation.
|
||||
// Loaded on first call so deno_telemetry isn't evaluated during snapshot creation.
|
||||
// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic]
|
||||
// consoleConfig: 0=ignore, 1=capture, 2=replace
|
||||
globalThis.__bootstrapOtel = () => {
|
||||
import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => {
|
||||
bootstrap([1, 0, 1, 0]);
|
||||
// Expose enterSpan for setting parent trace context
|
||||
globalThis.__enterSpan = enterSpan;
|
||||
});
|
||||
const { bootstrap, enterSpan } = core.loadExtScript(
|
||||
"ext:deno_telemetry/telemetry.ts",
|
||||
);
|
||||
bootstrap([1, 0, 1, 0]);
|
||||
// Expose enterSpan for setting parent trace context
|
||||
globalThis.__enterSpan = enterSpan;
|
||||
};
|
||||
|
||||
@@ -451,6 +451,10 @@ export async function main(): Promise<{ uuid: string; nonzero: boolean; sha256:
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
|
||||
// `crypto` is installed as an accessor; polyfill-style assignment must
|
||||
// still work rather than throw for want of a setter.
|
||||
(globalThis as any).crypto = globalThis.crypto;
|
||||
|
||||
return { uuid, nonzero, sha256 };
|
||||
}
|
||||
"#;
|
||||
|
||||
@@ -55,6 +55,9 @@ async function attempt(f) {{
|
||||
export async function main() {{
|
||||
return {{
|
||||
fetch: await attempt(() => fetch("http://127.0.0.1:{port}/").then((r) => r.text())),
|
||||
data_url: await attempt(() => fetch("data:text/plain,ok").then((r) => r.text())),
|
||||
blob_url: await attempt(() => fetch(URL.createObjectURL(new Blob(["ok"]))).then((r) => r.text())),
|
||||
raw_fetch: await attempt(() => Deno.core.ops.op_fetch("GET", "http://127.0.0.1:{port}/", [], null, false, null, null)),
|
||||
tcp: await attempt(() => Deno.core.ops.op_net_connect_tcp({{ hostname: "127.0.0.1", port: {port} }}, null, null)),
|
||||
unix: await attempt(() => Deno.core.ops.op_net_connect_unix("/var/run/docker.sock")),
|
||||
quic: await attempt(() => Deno.core.ops.op_quic_endpoint_create({{ hostname: "localhost", port: 0 }}, true)),
|
||||
@@ -76,6 +79,15 @@ async fn no_network_denies_fetch_and_raw_socket_ops() {
|
||||
fetch.contains("no_network"),
|
||||
"fetch was not denied: {fetch}"
|
||||
);
|
||||
// Neither opens a connection, so neither is network access.
|
||||
assert_eq!(out["data_url"], "allowed", "{out}");
|
||||
assert_eq!(out["blob_url"], "allowed", "{out}");
|
||||
// The global `fetch` only words the refusal; the op under it is what refuses.
|
||||
let raw_fetch = out["raw_fetch"].as_str().unwrap();
|
||||
assert!(
|
||||
raw_fetch.contains("Requires net access"),
|
||||
"the fetch op was not denied: {raw_fetch}"
|
||||
);
|
||||
for key in ["tcp", "unix", "quic"] {
|
||||
let msg = out[key].as_str().unwrap();
|
||||
assert!(
|
||||
|
||||
@@ -71,7 +71,7 @@ async fn spawn_capturing_http(captured: Arc<Mutex<Vec<Vec<u8>>>>) -> u16 {
|
||||
/// drifts away from `tracing_enabled + Capture`, this test breaks
|
||||
/// before the customer-facing panic does.
|
||||
fn init_with_ee_otel_config() {
|
||||
deno_telemetry::init(
|
||||
windmill_runtime_nativets::init_telemetry(
|
||||
deno_telemetry::OtelRuntimeConfig {
|
||||
runtime_name: "windmill-nativets".into(),
|
||||
runtime_version: "0".into(),
|
||||
@@ -157,13 +157,12 @@ export async function main(): Promise<number> {{
|
||||
"fetch target should have been hit exactly once"
|
||||
);
|
||||
|
||||
// 4. Force the BatchSpanProcessor to flush so the exporter posts
|
||||
// to our mock collector synchronously (default flush interval
|
||||
// is ~5s; tests can't wait that long).
|
||||
deno_telemetry::flush();
|
||||
// Exporter is async over the OTel runtime; give it a beat to
|
||||
// actually send the HTTP request.
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
// 4. The BatchSpanProcessor exports on its own schedule (about 5s by
|
||||
// default) and deno_telemetry exposes no flush, so wait for it.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
|
||||
while otlp_hits.lock().await.is_empty() && tokio::time::Instant::now() < deadline {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
let otlp_captured = otlp_hits.lock().await;
|
||||
assert!(
|
||||
|
||||
@@ -46,7 +46,7 @@ async fn ee_otel_init_config_populates_otel_globals() {
|
||||
// Step 1: reproduce the footgun. `OtelConfig::default()` has
|
||||
// tracing/metrics off and `console = Ignore`, which trips the
|
||||
// 0.31 early-return — Ok(()) but OnceCell stays empty.
|
||||
deno_telemetry::init(
|
||||
windmill_runtime_nativets::init_telemetry(
|
||||
deno_telemetry::OtelRuntimeConfig {
|
||||
runtime_name: "windmill-nativets-test".into(),
|
||||
runtime_version: "0".into(),
|
||||
@@ -63,10 +63,17 @@ async fn ee_otel_init_config_populates_otel_globals() {
|
||||
tracing_enabled/console fields."
|
||||
);
|
||||
|
||||
// The instance's own exporter shares this variable and may be set to gRPC,
|
||||
// or to anything else. Native telemetry must not read it: a value
|
||||
// deno_telemetry rejects outright shows it is never consulted.
|
||||
// SAFETY: the only test in this binary, so nothing reads the environment
|
||||
// concurrently.
|
||||
unsafe { std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", "not-a-protocol") };
|
||||
|
||||
// Step 2: re-call init with the exact config shape that the EE
|
||||
// `load_internal_otel_exporter` ships — this is what production
|
||||
// hits, so the test exercises the actual prod call path.
|
||||
deno_telemetry::init(
|
||||
windmill_runtime_nativets::init_telemetry(
|
||||
deno_telemetry::OtelRuntimeConfig {
|
||||
runtime_name: "windmill-nativets".into(),
|
||||
runtime_version: "0".into(),
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
use std::sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
Arc,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use windmill_ai::{query_builder::StreamEventSink, types::*};
|
||||
use windmill_common::{error::Error, worker::Connection};
|
||||
@@ -10,11 +15,13 @@ use crate::job_logger::append_result_stream;
|
||||
pub struct StreamEventProcessor {
|
||||
tx: Option<tokio::sync::mpsc::Sender<String>>,
|
||||
pub handle: Option<tokio::task::JoinHandle<()>>,
|
||||
/// Shared by every clone, so a sink lent to a parser counts into its owner.
|
||||
streamed: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Clone for StreamEventProcessor {
|
||||
fn clone(&self) -> Self {
|
||||
Self { tx: self.tx.clone(), handle: None }
|
||||
Self { tx: self.tx.clone(), handle: None, streamed: self.streamed.clone() }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,13 +55,19 @@ impl StreamEventProcessor {
|
||||
}
|
||||
});
|
||||
|
||||
Self { tx: Some(tx), handle: Some(handle) }
|
||||
Self { tx: Some(tx), handle: Some(handle), streamed: Arc::default() }
|
||||
}
|
||||
|
||||
/// Create a silent StreamEventProcessor that only accumulates events locally
|
||||
/// without persisting them to the database. Used when streaming is disabled.
|
||||
pub fn new_silent() -> Self {
|
||||
Self { tx: None, handle: None }
|
||||
Self { tx: None, handle: None, streamed: Arc::default() }
|
||||
}
|
||||
|
||||
/// How many events have been streamed to the job's readers. A silent processor
|
||||
/// streams none.
|
||||
pub fn streamed_events(&self) -> usize {
|
||||
self.streamed.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn to_handle(self) -> Option<tokio::task::JoinHandle<()>> {
|
||||
@@ -79,6 +92,7 @@ impl StreamEventSink for StreamEventProcessor {
|
||||
Ok(event_json) => {
|
||||
let event_json = format!("{}\n", event_json);
|
||||
events_str.push_str(&event_json);
|
||||
self.streamed.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
if let Err(err) = tx
|
||||
.send(event_json.clone())
|
||||
|
||||
@@ -43,6 +43,7 @@ use windmill_ai::{
|
||||
common_outbound_headers, needs_unavailable_oauth_exchange, retain_effective_credentials,
|
||||
},
|
||||
query_builder::{BuildRequestArgs, ParsedResponse},
|
||||
retry::{as_transient, into_final_error, response_error, send_error, Backoff, MAX_RETRIES},
|
||||
types::*,
|
||||
utils::{pinned_ai_client_for, should_use_structured_output_tool},
|
||||
};
|
||||
@@ -1659,10 +1660,17 @@ pub async fn run_agent(
|
||||
.await;
|
||||
|
||||
let mut retried_context = false;
|
||||
let mut backoff = Backoff::default();
|
||||
let streamed_events = || {
|
||||
stream_event_processor
|
||||
.as_ref()
|
||||
.map_or(0, |processor| processor.streamed_events())
|
||||
};
|
||||
let (parsed, request_message_count) = loop {
|
||||
// How many messages this request carries, so the provider's prompt count can
|
||||
// later be told apart from what the response and its tool results add.
|
||||
let request_message_count = messages.context().len();
|
||||
let streamed_before = streamed_events();
|
||||
|
||||
// Handle AWS Bedrock provider specially using the official SDK
|
||||
let attempt: error::Result<_> = async {
|
||||
@@ -1818,7 +1826,7 @@ pub async fn run_agent(
|
||||
let resp = build_http_request(&endpoint, &auth_headers, request_body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?;
|
||||
.map_err(|e| send_error(e, "Failed to call API"))?;
|
||||
|
||||
match resp.error_for_status_ref() {
|
||||
Ok(_) => {
|
||||
@@ -1829,6 +1837,7 @@ pub async fn run_agent(
|
||||
}
|
||||
Err(e) => {
|
||||
let status = resp.status();
|
||||
let headers = resp.headers().clone();
|
||||
let text = resp
|
||||
.text()
|
||||
.await
|
||||
@@ -1898,10 +1907,9 @@ pub async fn run_agent(
|
||||
query_builder = create_chat_completions_query_builder(&credentials);
|
||||
include_usage = true;
|
||||
} else {
|
||||
return Err(Error::internal_err(format!(
|
||||
"API error calling {}: {} - {}",
|
||||
endpoint, e, text
|
||||
)));
|
||||
let message =
|
||||
format!("API error calling {}: {} - {}", endpoint, e, text);
|
||||
return Err(response_error(status, &headers, &text, message));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1961,7 +1969,45 @@ pub async fn run_agent(
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
Err(error) => {
|
||||
let Some(transient) = as_transient(&error) else {
|
||||
return Err(error);
|
||||
};
|
||||
// Readers have no way to take back what they were shown, so a call
|
||||
// that already streamed part of its answer fails as it is.
|
||||
if streamed_events() != streamed_before {
|
||||
append_logs(
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
"AI provider call failed after part of its answer had streamed; not retried.\n".to_string(),
|
||||
conn,
|
||||
)
|
||||
.await;
|
||||
return Err(into_final_error(error));
|
||||
}
|
||||
let Some(delay) = backoff.next_delay(transient) else {
|
||||
return Err(into_final_error(error));
|
||||
};
|
||||
append_logs(
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
format!(
|
||||
"AI provider call failed: {}. Retrying in {:.1}s (retry {}/{MAX_RETRIES}).\n",
|
||||
transient.message,
|
||||
delay.as_secs_f64(),
|
||||
backoff.retries()
|
||||
),
|
||||
conn,
|
||||
)
|
||||
.await;
|
||||
let mut cancelled = cancel_rx.clone();
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(delay) => {}
|
||||
Ok(_) = cancelled.wait_for(|cancelled| *cancelled) => {
|
||||
return Err(Error::ExecutionErr("Job cancelled".to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1962,6 +1962,18 @@ pub async fn save_in_cache(
|
||||
CACHED_RESULTS.insert(cached_path, Arc::new(store_cache_resource));
|
||||
}
|
||||
|
||||
pub(crate) fn deno_not_installed_error() -> Error {
|
||||
Error::ExecutionErr(format!(
|
||||
"Deno is not installed on this worker (looked for it at {}). The standard Windmill images \
|
||||
no longer ship Deno: official support for it on those images was discontinued after Deno \
|
||||
announced the end of development of the Deno runtime (https://deno.com/blog/cloudflare). \
|
||||
We recommend migrating this script to Bun. To keep running Deno scripts, use the \
|
||||
windmill-full or windmill-ee-full image on every worker that accepts the deno tag, or \
|
||||
install Deno on them and point DENO_PATH at it.",
|
||||
crate::DENO_PATH.as_str()
|
||||
))
|
||||
}
|
||||
|
||||
fn tentatively_improve_error(err: Error, executable: &str) -> Error {
|
||||
#[cfg(unix)]
|
||||
let err_msgs = vec!["os error 2", "os error 3", "No such file or directory"];
|
||||
@@ -1970,6 +1982,11 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error {
|
||||
let err_msgs = vec!["program not found", "os error 2", "os error 3"];
|
||||
|
||||
if err_msgs.iter().any(|msg| err.to_string().contains(msg)) {
|
||||
// An absolute DENO_PATH is checked before spawning, where a missing job dir
|
||||
// cannot be mistaken for a missing binary.
|
||||
if executable == crate::DENO_PATH.as_str() && !Path::new(executable).is_absolute() {
|
||||
return deno_not_installed_error();
|
||||
}
|
||||
return Error::internal_err(format!(
|
||||
"Executable {executable} not found on worker. PATH: {}",
|
||||
*PATH_ENV
|
||||
|
||||
@@ -7,7 +7,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
|
||||
|
||||
use crate::{
|
||||
common::{
|
||||
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
|
||||
build_command_with_isolation, create_args_and_out_file, deno_not_installed_error,
|
||||
get_reserved_variables,
|
||||
parse_npm_config, read_file, read_result, start_child_process, OccupancyMetrics,
|
||||
StreamNotifier,
|
||||
},
|
||||
@@ -155,6 +156,17 @@ async fn get_common_deno_proc_envs(
|
||||
return deno_envs;
|
||||
}
|
||||
|
||||
/// A spawn failure only names Deno when Deno itself is the spawned executable; behind an
|
||||
/// isolation wrapper or a dedicated worker it surfaces as an opaque exit. A bare `DENO_PATH`
|
||||
/// is resolved through PATH at spawn time, so it is left to the spawn error.
|
||||
fn ensure_deno_installed() -> Result<()> {
|
||||
let path = std::path::Path::new(DENO_PATH.as_str());
|
||||
if path.is_absolute() && !path.exists() {
|
||||
return Err(deno_not_installed_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn generate_deno_lock(
|
||||
job_id: &Uuid,
|
||||
code: &str,
|
||||
@@ -167,6 +179,7 @@ pub async fn generate_deno_lock(
|
||||
base_internal_url: &str,
|
||||
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
|
||||
) -> error::Result<String> {
|
||||
ensure_deno_installed()?;
|
||||
let _ = write_file(job_dir, "main.ts", code)?;
|
||||
|
||||
let import_map_path = format!("{job_dir}/import_map.json");
|
||||
@@ -249,6 +262,7 @@ pub async fn handle_deno_job(
|
||||
occupancy_metrics: &mut OccupancyMetrics,
|
||||
has_stream: &mut bool,
|
||||
) -> error::Result<Box<RawValue>> {
|
||||
ensure_deno_installed()?;
|
||||
let annotations = TypeScriptAnnotations::parse(inner_content);
|
||||
|
||||
// let mut start = Instant::now();
|
||||
@@ -745,6 +759,7 @@ pub async fn start_worker(
|
||||
|
||||
use crate::common::build_envs_map;
|
||||
|
||||
ensure_deno_installed()?;
|
||||
let _ = write_file(job_dir, "main.ts", inner_content)?;
|
||||
let common_deno_proc_envs = get_common_deno_proc_envs(
|
||||
&token,
|
||||
|
||||
@@ -215,6 +215,8 @@ lazy_static::lazy_static! {
|
||||
pub static ref PY311_CACHE_DIR: String = format!("{}python_3_11", *ROOT_CACHE_DIR);
|
||||
pub static ref PY312_CACHE_DIR: String = format!("{}python_3_12", *ROOT_CACHE_DIR);
|
||||
pub static ref PY313_CACHE_DIR: String = format!("{}python_3_13", *ROOT_CACHE_DIR);
|
||||
pub static ref PY314_CACHE_DIR: String = format!("{}python_3_14", *ROOT_CACHE_DIR);
|
||||
pub static ref PY315_CACHE_DIR: String = format!("{}python_3_15", *ROOT_CACHE_DIR);
|
||||
|
||||
pub static ref TAR_JAVA_CACHE_DIR: String = format!("{}tar/java", *ROOT_CACHE_DIR);
|
||||
|
||||
|
||||
@@ -1244,6 +1244,7 @@ pub async fn handle_flow_dependency_job(
|
||||
&job.workspace_id,
|
||||
flow.failure_module.as_ref(),
|
||||
flow.same_worker,
|
||||
flow.preserve_step_tags,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -2134,6 +2135,7 @@ async fn insert_flow_modules<'c>(
|
||||
workspace_id: &str,
|
||||
failure_module: Option<&Box<FlowModule>>,
|
||||
same_worker: bool,
|
||||
preserve_step_tags: bool,
|
||||
modules: &mut Vec<FlowModule>,
|
||||
modules_node: &mut Option<FlowNodeId>,
|
||||
) -> Result<sqlx::Transaction<'c, sqlx::Postgres>> {
|
||||
@@ -2144,6 +2146,7 @@ async fn insert_flow_modules<'c>(
|
||||
workspace_id,
|
||||
failure_module,
|
||||
same_worker,
|
||||
preserve_step_tags,
|
||||
))
|
||||
.await?;
|
||||
if modules.is_empty() || crate::worker_flow::is_simple_modules(modules, failure_module) {
|
||||
@@ -2160,6 +2163,7 @@ async fn insert_flow_modules<'c>(
|
||||
modules: std::mem::take(modules),
|
||||
failure_module: failure_module.cloned(),
|
||||
same_worker,
|
||||
preserve_step_tags,
|
||||
..Default::default()
|
||||
}))),
|
||||
None,
|
||||
@@ -2176,6 +2180,7 @@ async fn reduce_flow<'c>(
|
||||
workspace_id: &str,
|
||||
failure_module: Option<&Box<FlowModule>>,
|
||||
same_worker: bool,
|
||||
preserve_step_tags: bool,
|
||||
) -> Result<sqlx::Transaction<'c, sqlx::Postgres>> {
|
||||
use FlowModuleValue::*;
|
||||
for module in &mut *modules {
|
||||
@@ -2234,6 +2239,7 @@ async fn reduce_flow<'c>(
|
||||
workspace_id,
|
||||
failure_module,
|
||||
same_worker,
|
||||
preserve_step_tags,
|
||||
modules,
|
||||
modules_node,
|
||||
)
|
||||
@@ -2247,6 +2253,7 @@ async fn reduce_flow<'c>(
|
||||
workspace_id,
|
||||
failure_module,
|
||||
same_worker,
|
||||
preserve_step_tags,
|
||||
&mut branch.modules,
|
||||
&mut branch.modules_node,
|
||||
)
|
||||
@@ -2258,6 +2265,7 @@ async fn reduce_flow<'c>(
|
||||
workspace_id,
|
||||
failure_module,
|
||||
same_worker,
|
||||
preserve_step_tags,
|
||||
default,
|
||||
default_node,
|
||||
)
|
||||
@@ -2271,6 +2279,7 @@ async fn reduce_flow<'c>(
|
||||
workspace_id,
|
||||
failure_module,
|
||||
same_worker,
|
||||
preserve_step_tags,
|
||||
&mut branch.modules,
|
||||
&mut branch.modules_node,
|
||||
)
|
||||
|
||||
@@ -22,12 +22,12 @@ deno run -A benchmark_suite.ts -c suite_wac.json
|
||||
|
||||
### Script benchmarks
|
||||
- `noop` — Empty jobs (measures pure scheduling overhead)
|
||||
- `deno`, `bun`, `python`, `go`, `bash` — Language runtimes
|
||||
- `bun`, `python`, `go`, `bash` — Language runtimes
|
||||
- `nativets` — BunNative (no isolation)
|
||||
- `dedicated`, `dedicated_nativets` — Dedicated worker mode
|
||||
|
||||
### Flow benchmarks
|
||||
- `2steps` — 2-step flow (deno + identity)
|
||||
- `2steps` — 2-step flow (bun + identity)
|
||||
- `bigscriptinflow` — Flow with large raw bash script
|
||||
- `flow_seq_2_bun` — 2 sequential bun steps
|
||||
- `flow_par_2_bun` — 2 parallel bun steps (branchall)
|
||||
|
||||
@@ -37,7 +37,7 @@ async function verifyOutputs(uuids: string[], workspace: string) {
|
||||
console.log(`Incorrect results: ${incorrectResults}`);
|
||||
}
|
||||
|
||||
export const NON_TEST_TAGS = ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets", "flow"]
|
||||
export const NON_TEST_TAGS = ["python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets", "flow"]
|
||||
|
||||
const FLOW_COMPARISON_KINDS = ["flow_seq_2_bun", "flow_par_2_bun", "flow_seq_3_bun"];
|
||||
export async function main({
|
||||
@@ -148,7 +148,7 @@ export async function main({
|
||||
}
|
||||
|
||||
if (
|
||||
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
|
||||
["python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
|
||||
kind
|
||||
)
|
||||
) {
|
||||
@@ -169,7 +169,7 @@ export async function main({
|
||||
kind: "noop",
|
||||
});
|
||||
} else if (
|
||||
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
|
||||
["python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
|
||||
kind
|
||||
)
|
||||
) {
|
||||
@@ -414,7 +414,7 @@ if (import.meta.main) {
|
||||
)
|
||||
.option(
|
||||
"--kind <kind:string>",
|
||||
"Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps, nativets, dedicated_nativets, wac_seq_2, wac_par_2, wac_seq_3, wac_inline_2, flow_seq_2_bun, flow_par_2_bun, flow_seq_3_bun",
|
||||
"Specifiy the benchmark kind among: identity, python, go, bash, dedicated, bun, noop, 2steps, nativets, dedicated_nativets, wac_seq_2, wac_par_2, wac_seq_3, wac_inline_2, flow_seq_2_bun, flow_par_2_bun, flow_seq_3_bun",
|
||||
{
|
||||
required: true,
|
||||
}
|
||||
|
||||
@@ -49,26 +49,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"graph_title": "deno throughput benchmark",
|
||||
"benchmarks": [
|
||||
{
|
||||
"kind": "deno",
|
||||
"workers": 1,
|
||||
"label": "1 worker"
|
||||
},
|
||||
{
|
||||
"kind": "deno",
|
||||
"workers": 4,
|
||||
"label": "4 workers"
|
||||
},
|
||||
{
|
||||
"kind": "deno",
|
||||
"workers": 8,
|
||||
"label": "8 workers"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"graph_title": "bun throughput benchmark",
|
||||
"benchmarks": [
|
||||
@@ -170,7 +150,7 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"graph_title": "go vs python vs deno vs bun vs bash vs nativets",
|
||||
"graph_title": "go vs python vs bun vs bash vs nativets",
|
||||
"benchmarks": [
|
||||
{
|
||||
"kind": "go",
|
||||
@@ -182,11 +162,6 @@
|
||||
"workers": 1,
|
||||
"label": "python"
|
||||
},
|
||||
{
|
||||
"kind": "deno",
|
||||
"workers": 1,
|
||||
"label": "deno"
|
||||
},
|
||||
{
|
||||
"kind": "bun",
|
||||
"workers": 1,
|
||||
|
||||
+10
-14
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
|
||||
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
|
||||
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
|
||||
|
||||
export const VERSION = "v1.829.0";
|
||||
export const VERSION = "v1.830.0";
|
||||
|
||||
export async function login(email: string, password: string): Promise<string> {
|
||||
return await windmill.UserService.login({
|
||||
@@ -87,10 +87,6 @@ export async function createBenchScript(
|
||||
schemaProperties = {
|
||||
uuid: { default: null, description: "", type: "string" },
|
||||
};
|
||||
} else if (scriptPattern === "deno") {
|
||||
scriptContent =
|
||||
'export function main(){ return Deno.env.get("WM_JOB_ID"); }';
|
||||
language = "deno";
|
||||
} else if (scriptPattern === "nativets") {
|
||||
scriptContent =
|
||||
'//native\nexport async function main(){ return (await fetch(BASE_URL + "/api/version")).text() }';
|
||||
@@ -256,10 +252,10 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => {
|
||||
id: "a",
|
||||
value: {
|
||||
input_transforms: {},
|
||||
language: api.RawScript.language.DENO,
|
||||
language: "bun" as api.RawScript.language,
|
||||
type: "rawscript",
|
||||
content:
|
||||
'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
|
||||
'export function main(){ return Bun.env["WM_FLOW_JOB_ID"]; }',
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -277,7 +273,7 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => {
|
||||
expr: "results.a",
|
||||
},
|
||||
},
|
||||
language: api.RawScript.language.DENO,
|
||||
language: "bun" as api.RawScript.language,
|
||||
type: "rawscript",
|
||||
content: "export function main(x: string){ return x; }",
|
||||
},
|
||||
@@ -298,10 +294,10 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => {
|
||||
id: "a",
|
||||
value: {
|
||||
input_transforms: {},
|
||||
language: api.RawScript.language.DENO,
|
||||
language: "bun" as api.RawScript.language,
|
||||
type: "rawscript",
|
||||
content:
|
||||
'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
|
||||
'export function main(){ return Bun.env["WM_FLOW_JOB_ID"]; }',
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -321,7 +317,7 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => {
|
||||
expr: "results.a",
|
||||
},
|
||||
},
|
||||
language: api.RawScript.language.DENO,
|
||||
language: "bun" as api.RawScript.language,
|
||||
type: "rawscript",
|
||||
content: "export function main(x: string){ return x; }",
|
||||
},
|
||||
@@ -339,7 +335,7 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => {
|
||||
expr: "results.a",
|
||||
},
|
||||
},
|
||||
language: api.RawScript.language.DENO,
|
||||
language: "bun" as api.RawScript.language,
|
||||
type: "rawscript",
|
||||
content: "export function main(x: string){ return x; }",
|
||||
},
|
||||
@@ -490,10 +486,10 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => {
|
||||
id: "a",
|
||||
value: {
|
||||
input_transforms: {},
|
||||
language: api.RawScript.language.DENO,
|
||||
language: "bun" as api.RawScript.language,
|
||||
type: "rawscript",
|
||||
content:
|
||||
'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
|
||||
'export function main(){ return Bun.env["WM_FLOW_JOB_ID"]; }',
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user