mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-18 16:02:29 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e033c73b79 | ||
|
|
8c3ac22d8d | ||
|
|
21398e5447 | ||
|
|
980cbcccf0 | ||
|
|
dd422fcc5d | ||
|
|
876a9cfc8e |
@@ -1,65 +0,0 @@
|
||||
# Project display name in the dashboard
|
||||
name: Windmill
|
||||
|
||||
workspace:
|
||||
mainBranch: main
|
||||
worktreeRoot: ../windmill__worktrees
|
||||
defaultAgent: claude
|
||||
|
||||
startupEnvs:
|
||||
CARGO_FEATURES: "quickjs"
|
||||
WM_CLONE_DB: false
|
||||
USE_RUST_PLUGIN: false
|
||||
|
||||
lifecycleHooks:
|
||||
postCreate: bash ./scripts/post-create.sh
|
||||
preRemove: bash ./scripts/pre-remove.sh
|
||||
|
||||
auto_name:
|
||||
model: gemini-2.5-flash-lite
|
||||
|
||||
# Each service defines a port env var that webmux injects into pane and agent
|
||||
# process environments when creating a worktree. Ports are auto-assigned:
|
||||
# base + (slot x step).
|
||||
services:
|
||||
- name: backend
|
||||
portEnv: BACKEND_PORT
|
||||
portStart: 8000
|
||||
portStep: 10
|
||||
- name: frontend
|
||||
portEnv: FRONTEND_PORT
|
||||
portStart: 3000
|
||||
portStep: 10
|
||||
|
||||
profiles:
|
||||
default:
|
||||
runtime: host
|
||||
yolo: true
|
||||
envPassthrough: []
|
||||
systemPrompt: >
|
||||
You are running inside a tmux session with other panes running services.
|
||||
Pane layout (current window):
|
||||
- Pane 0: this pane (claude agent)
|
||||
- Pane 1: backend (cargo watch -x run)
|
||||
- Pane 2: frontend (npm run dev)
|
||||
To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (backend) or \`tmux capture-pane -t .2 -p -S -50\` (frontend).
|
||||
When restarting backend or frontend, make sure to use ${BACKEND_PORT} and ${FRONTEND_PORT}.
|
||||
Because we are running backend with cargo watch, to verify your changes, just check the logs in the backend pane. No need for cargo check.
|
||||
panes:
|
||||
- id: agent
|
||||
kind: agent
|
||||
focus: true
|
||||
- id: backend
|
||||
kind: command
|
||||
split: right
|
||||
command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/backend" && PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}"
|
||||
- id: frontend
|
||||
kind: command
|
||||
split: bottom
|
||||
command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/frontend" && npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0
|
||||
|
||||
integrations:
|
||||
github:
|
||||
linkedRepos: []
|
||||
linear:
|
||||
enabled: true
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
name: Windmill
|
||||
|
||||
startupEnvs:
|
||||
CARGO_FEATURES: "quickjs"
|
||||
WM_CLONE_DB: false
|
||||
USE_RUST_PLUGIN: false
|
||||
|
||||
services:
|
||||
- name: BE
|
||||
portEnv: BACKEND_PORT
|
||||
- name: FE
|
||||
portEnv: FRONTEND_PORT
|
||||
|
||||
profiles:
|
||||
default:
|
||||
name: default
|
||||
|
||||
sandbox:
|
||||
name: sandbox
|
||||
image: windmill-sandbox
|
||||
envPassthrough:
|
||||
- AWS_ACCESS_KEY_ID
|
||||
- AWS_SECRET_ACCESS_KEY
|
||||
- R2_ENDPOINT
|
||||
- R2_BUCKET
|
||||
- R2_PUBLIC_URL
|
||||
extraMounts:
|
||||
- hostPath: ~/.ssh
|
||||
guestPath: /root/.ssh
|
||||
writable: true
|
||||
- hostPath: ~/.codex
|
||||
guestPath: /root/.codex
|
||||
writable: true
|
||||
- hostPath: ~/windmill-ee-private
|
||||
writable: true
|
||||
- hostPath: ~/windmill-ee-private__worktrees
|
||||
writable: true
|
||||
systemPrompt: >
|
||||
You are running inside a sandboxed container with full permissions.
|
||||
This worktree is configured with the following ports:
|
||||
|
||||
- Backend: port ${BACKEND_PORT}.
|
||||
Start with: cd backend && PORT=${BACKEND_PORT}
|
||||
DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill
|
||||
cargo watch -x run
|
||||
|
||||
- Frontend: port ${FRONTEND_PORT}.
|
||||
Start with: cd frontend && REMOTE=http://localhost:${BACKEND_PORT}
|
||||
npm run dev -- --port ${FRONTEND_PORT} --host 0.0.0.0
|
||||
|
||||
--- Screenshots ---
|
||||
You can take screenshots of the frontend UI and upload them to R2
|
||||
for use in PR descriptions.
|
||||
1) Take a screenshot:
|
||||
bunx playwright screenshot --browser chromium
|
||||
http://localhost:${FRONTEND_PORT}/path/to/page /tmp/screenshot.png
|
||||
2) Upload to R2:
|
||||
aws s3 cp /tmp/screenshot.png
|
||||
"s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/screenshot.png"
|
||||
--endpoint-url "$(printenv R2_ENDPOINT)"
|
||||
3) The public URL will be:
|
||||
$(printenv R2_PUBLIC_URL)/<branch>/screenshot.png
|
||||
4) Include in PR descriptions using markdown image syntax.
|
||||
|
||||
--- Terminal Recordings (asciinema) ---
|
||||
You can record terminal sessions and upload them for sharing.
|
||||
asciinema is available on PATH.
|
||||
|
||||
1) Write a shell script with the commands to demo. Add sleep
|
||||
delays for readable pacing:
|
||||
- 0.5s after printing a "$ command" line (lets viewer read it)
|
||||
- 1.5-2s after command output (lets viewer absorb the result)
|
||||
- Set GIT_PAGER=cat and PAGER=cat to prevent pager hangs
|
||||
|
||||
2) Record headlessly:
|
||||
asciinema rec --headless --overwrite \
|
||||
-c "bash /tmp/demo.sh" \
|
||||
--window-size 120x50 \
|
||||
--title "Description of demo" \
|
||||
/tmp/demo.cast
|
||||
|
||||
3) Upload to asciinema.org:
|
||||
XDG_DATA_HOME=/tmp/.local/share \
|
||||
asciinema upload --server-url https://asciinema.org /tmp/demo.cast
|
||||
|
||||
--- Mermaid Diagrams ---
|
||||
You can render Mermaid diagrams to SVG using the pre-installed mmdc CLI.
|
||||
The puppeteer config (no-sandbox + Chromium path) is at /root/.puppeteerrc.json.
|
||||
|
||||
1) Write a .mmd file with your diagram:
|
||||
cat > /tmp/diagram.mmd << 'EOF'
|
||||
graph TD
|
||||
A[Start] --> B[End]
|
||||
EOF
|
||||
|
||||
2) Render to SVG (the -p flag is required):
|
||||
mmdc -i /tmp/diagram.mmd -o /tmp/diagram.svg -p /root/.puppeteerrc.json
|
||||
|
||||
3) Upload to R2:
|
||||
aws s3 cp /tmp/diagram.svg
|
||||
"s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/diagram.svg"
|
||||
--endpoint-url "$(printenv R2_ENDPOINT)"
|
||||
|
||||
4) The public URL will be:
|
||||
$(printenv R2_PUBLIC_URL)/<branch>/diagram.svg
|
||||
|
||||
5) Include in PR descriptions using markdown image syntax.
|
||||
|
||||
IMPORTANT: Read docs/autonomous-mode.md before starting any work.
|
||||
|
||||
linkedRepos:
|
||||
- repo: windmill-labs/windmill-ee-private
|
||||
alias: ee
|
||||
@@ -1,30 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## [1.652.0](https://github.com/windmill-labs/windmill/compare/v1.651.1...v1.652.0) (2026-03-09)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add secretKeyRef support for package registry and storage credentials ([#8275](https://github.com/windmill-labs/windmill/issues/8275)) ([73d27e9](https://github.com/windmill-labs/windmill/commit/73d27e92dd6ced1602f6328f245fec0fa96860e1))
|
||||
* expose OTEL trace context as env vars in job execution ([#8277](https://github.com/windmill-labs/windmill/issues/8277)) ([93f75ad](https://github.com/windmill-labs/windmill/commit/93f75ada5e49036f0d998e3d3d53de4dc2c2e83f))
|
||||
* workflow-as-code (WAC) v2 ([#8172](https://github.com/windmill-labs/windmill/issues/8172)) ([a6d4390](https://github.com/windmill-labs/windmill/commit/a6d4390790d21d535df1e9d525bffd577c50d8dc))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* cli: support deleting linked resources-variables without throwing ([#8248](https://github.com/windmill-labs/windmill/issues/8248)) ([7859bca](https://github.com/windmill-labs/windmill/commit/7859bca6ae80d32a73a46910960afc6812e64115))
|
||||
* Database studio fixes ([#8251](https://github.com/windmill-labs/windmill/issues/8251)) ([1d78589](https://github.com/windmill-labs/windmill/commit/1d785899404e8636a206cda9a2914df32a1a5269))
|
||||
* **frontend:** unsaved changes dialog when flow already saved ([#8259](https://github.com/windmill-labs/windmill/issues/8259)) ([0330993](https://github.com/windmill-labs/windmill/commit/0330993cb66cdabffcd6e552a0f85a9a3931c62d))
|
||||
* gracefully handle uninitialized OTEL tracing proxy port ([#8274](https://github.com/windmill-labs/windmill/issues/8274)) ([8b1fe8f](https://github.com/windmill-labs/windmill/commit/8b1fe8f9de7b0c03655558d0c46cfff71a4b2047))
|
||||
* guard iteration picker VirtualList against empty items array ([#8273](https://github.com/windmill-labs/windmill/issues/8273)) ([c97cf60](https://github.com/windmill-labs/windmill/commit/c97cf604ab4a902d89fe873b90dbeb9dabc940eb)), closes [#8272](https://github.com/windmill-labs/windmill/issues/8272)
|
||||
* mask secrets in OAuth config debug/log output ([#8269](https://github.com/windmill-labs/windmill/issues/8269)) ([e75763d](https://github.com/windmill-labs/windmill/commit/e75763dbe5ffe08e6cde082203596d510c2c3b29))
|
||||
* parallel branchall hang on bad stop_after_all_iters_if + results.x.length null ([#8276](https://github.com/windmill-labs/windmill/issues/8276)) ([41e523f](https://github.com/windmill-labs/windmill/commit/41e523f827c4e3d5db525a1f14e24936b0b8af46))
|
||||
* redact secrets in set_global_setting log line ([#8270](https://github.com/windmill-labs/windmill/issues/8270)) ([6a0473c](https://github.com/windmill-labs/windmill/commit/6a0473c5783dc0fef2ae82dc5345a5f0596f124d))
|
||||
* remove $bindable() fallback values causing props_invalid_value error in oauth settings ([#8265](https://github.com/windmill-labs/windmill/issues/8265)) ([037035e](https://github.com/windmill-labs/windmill/commit/037035e094937827305dad29bd76a495d78bc46f))
|
||||
* skip down migrations in potentially_stale checksum comparison ([#8271](https://github.com/windmill-labs/windmill/issues/8271)) ([5ba4029](https://github.com/windmill-labs/windmill/commit/5ba4029d8692b2e6054fca7f45ed4cfded4738ef))
|
||||
* sql input horizontal scroll missing after switching flow steps ([#8249](https://github.com/windmill-labs/windmill/issues/8249)) ([ce8ac9c](https://github.com/windmill-labs/windmill/commit/ce8ac9cf52dc17061673b9b72556279c48c26f8e))
|
||||
* wmill workspace whoami output ([#8246](https://github.com/windmill-labs/windmill/issues/8246)) ([1ac391a](https://github.com/windmill-labs/windmill/commit/1ac391a795585747fe5911ac41b157556569fedb))
|
||||
|
||||
## [1.651.1](https://github.com/windmill-labs/windmill/compare/v1.651.0...v1.651.1) (2026-03-05)
|
||||
|
||||
|
||||
|
||||
@@ -26,27 +26,6 @@ Open-source platform for internal tools, workflows, API integrations, background
|
||||
- **Login**: `admin@windmill.dev` / `changeme`
|
||||
- **Instance settings**: navigate to `/#superadmin-settings`
|
||||
|
||||
## Banned Patterns
|
||||
|
||||
### `$bindable(default_value)` on optional props
|
||||
|
||||
Using `$bindable(default_value)` on props that can be `undefined` is **banned**. This pattern causes subtle bugs because the default value masks the `undefined` state.
|
||||
|
||||
**Bad:**
|
||||
```svelte
|
||||
let { my_prop = $bindable(default_value) }: { my_prop?: string } = $props()
|
||||
```
|
||||
|
||||
**Correct alternatives:**
|
||||
|
||||
1. **Use `$derived` with nullish coalescing** — handle the potential `undefined` at the usage site:
|
||||
```svelte
|
||||
let { my_prop = $bindable() }: { my_prop?: string } = $props()
|
||||
let effective_value = $derived(my_prop ?? default_value)
|
||||
```
|
||||
|
||||
2. **Create a `useMyPropState()` helper** — encapsulate the undefined-handling logic in a reusable function and call it higher in the component tree, so the child component always receives a defined value.
|
||||
|
||||
## Core Principles
|
||||
|
||||
- Search for existing code to reuse before writing new code
|
||||
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_queue SET canceled_by = $2, canceled_reason = $3 WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699"
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Int4"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6"
|
||||
}
|
||||
+2
-8
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n WITH job_info AS (\n SELECT id, kind::text AS kind, parent_job\n FROM v2_job\n WHERE id = $1\n )\n SELECT\n q.id AS \"id!\",\n s.flow_status,\n q.suspend AS \"suspend!\",\n j.runnable_path AS script_path,\n j.permissioned_as_email AS email,\n (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\",\n (ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS \"is_wac!\"\n FROM job_info ji\n JOIN v2_job_queue q ON q.id = CASE\n WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n ELSE COALESCE(ji.parent_job, ji.id)\n END\n JOIN v2_job j ON j.id = q.id\n LEFT JOIN v2_job_status s ON s.id = q.id\n FOR UPDATE OF q\n ",
|
||||
"query": "\n WITH job_info AS (\n SELECT id, kind::text AS kind, parent_job\n FROM v2_job\n WHERE id = $1\n )\n SELECT\n q.id AS \"id!\",\n s.flow_status,\n q.suspend AS \"suspend!\",\n j.runnable_path AS script_path,\n j.permissioned_as_email AS email,\n (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\"\n FROM job_info ji\n JOIN v2_job_queue q ON q.id = CASE\n WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n ELSE ji.parent_job\n END\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_status s ON s.id = q.id\n FOR UPDATE OF q\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -32,11 +32,6 @@
|
||||
"ordinal": 5,
|
||||
"name": "is_flow_level!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "is_wac!",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
@@ -50,9 +45,8 @@
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353"
|
||||
"hash": "1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82"
|
||||
}
|
||||
+1
-1
@@ -15,7 +15,7 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91"
|
||||
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)\n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Int8",
|
||||
"Int8",
|
||||
"Text",
|
||||
"Bool",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "3e8afd021088a99a24f27fa6f0a1b7f3edba3e9b834c814b464305bc2eb6ba80"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int8",
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5"
|
||||
}
|
||||
+1
-1
@@ -15,7 +15,7 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
|
||||
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int4",
|
||||
"TextArray",
|
||||
"Float4",
|
||||
"Int8",
|
||||
"Int8",
|
||||
"Text",
|
||||
"Int8",
|
||||
"Int8",
|
||||
"Float4",
|
||||
"Float4",
|
||||
"Float4",
|
||||
"Bool",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "6cd099d458ac380d5da27b9e69da035755496ea50f2b78fb9b1cd3a2eb7e7625"
|
||||
}
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4"
|
||||
}
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5"
|
||||
}
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e"
|
||||
}
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1 RETURNING suspend",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "suspend",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a"
|
||||
}
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL\n RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS \"job_ids: serde_json::Value\"",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "job_ids: serde_json::Value",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int8",
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340"
|
||||
}
|
||||
+1
-1
@@ -15,7 +15,7 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06"
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Float8"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20"
|
||||
}
|
||||
Generated
+89
-106
@@ -7103,7 +7103,7 @@ dependencies = [
|
||||
"libc",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"socket2 0.6.3",
|
||||
"socket2 0.6.2",
|
||||
"system-configuration",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
@@ -7974,9 +7974,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.183"
|
||||
version = "0.2.182"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
|
||||
checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
|
||||
|
||||
[[package]]
|
||||
name = "libffi"
|
||||
@@ -10664,7 +10664,7 @@ dependencies = [
|
||||
"quinn-udp",
|
||||
"rustc-hash 2.1.1",
|
||||
"rustls 0.23.35",
|
||||
"socket2 0.6.3",
|
||||
"socket2 0.6.2",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -10673,9 +10673,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quinn-proto"
|
||||
version = "0.11.14"
|
||||
version = "0.11.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
|
||||
checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"bytes",
|
||||
@@ -10702,7 +10702,7 @@ dependencies = [
|
||||
"cfg_aliases 0.2.1",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2 0.6.3",
|
||||
"socket2 0.6.2",
|
||||
"tracing",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
@@ -12677,12 +12677,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.6.3"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14462,7 +14462,7 @@ dependencies = [
|
||||
"indexmap 2.11.1",
|
||||
"toml_datetime 0.7.0",
|
||||
"toml_parser",
|
||||
"winnow 0.7.15",
|
||||
"winnow 0.7.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14471,7 +14471,7 @@ version = "1.0.9+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4"
|
||||
dependencies = [
|
||||
"winnow 0.7.15",
|
||||
"winnow 0.7.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -15741,7 +15741,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-nats",
|
||||
@@ -15808,7 +15808,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-alerting"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -15821,7 +15821,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
@@ -15849,7 +15849,6 @@ dependencies = [
|
||||
"dashmap 6.1.0",
|
||||
"datafusion",
|
||||
"ed25519-dalek",
|
||||
"eventsource-stream",
|
||||
"flate2",
|
||||
"futures",
|
||||
"git-version",
|
||||
@@ -15961,7 +15960,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-agent-workers"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -15984,7 +15983,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-assets"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -15997,7 +15996,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-auth"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16023,7 +16022,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-client"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
@@ -16033,7 +16032,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-configs"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16050,7 +16049,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-debug"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"base64 0.22.1",
|
||||
@@ -16073,7 +16072,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-embeddings"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16096,7 +16095,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-flow-conversations"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16112,7 +16111,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-flows"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16132,7 +16131,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-groups"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16152,7 +16151,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-inputs"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16166,7 +16165,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-integration-tests"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-nats",
|
||||
@@ -16193,7 +16192,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-jobs"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16218,7 +16217,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-npm-proxy"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"flate2",
|
||||
@@ -16236,7 +16235,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-openapi"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16257,7 +16256,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-schedule"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16277,7 +16276,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-scripts"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16307,7 +16306,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-settings"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16334,7 +16333,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-sse"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"serde",
|
||||
@@ -16346,7 +16345,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-users"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"axum 0.7.9",
|
||||
@@ -16369,7 +16368,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-workers"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16383,7 +16382,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-workspaces"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"axum 0.7.9",
|
||||
"chrono",
|
||||
@@ -16414,7 +16413,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-audit"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"lazy_static",
|
||||
@@ -16428,7 +16427,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-autoscaling"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -16447,7 +16446,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-common"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -16546,7 +16545,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-dep-map"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"itertools 0.14.0",
|
||||
@@ -16565,7 +16564,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-git-sync"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"serde",
|
||||
@@ -16580,7 +16579,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-indexer"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -16604,7 +16603,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-jseval"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"futures",
|
||||
@@ -16621,7 +16620,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-macros"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"itertools 0.14.0",
|
||||
"lazy_static",
|
||||
@@ -16637,7 +16636,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-mcp"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -16658,7 +16657,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-native-triggers"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -16689,7 +16688,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-oauth"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-oauth2",
|
||||
@@ -16713,7 +16712,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-object-store"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-stream",
|
||||
@@ -16747,7 +16746,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-operator"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"futures",
|
||||
@@ -16765,7 +16764,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"convert_case 0.6.0",
|
||||
"serde",
|
||||
@@ -16774,7 +16773,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-bash"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -16786,7 +16785,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-csharp"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
@@ -16798,7 +16797,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-go"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"gosyn",
|
||||
@@ -16810,7 +16809,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-graphql"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -16822,7 +16821,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-java"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
@@ -16834,7 +16833,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-nu"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"nu-parser",
|
||||
@@ -16845,7 +16844,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-php"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
@@ -16856,7 +16855,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-py"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
@@ -16869,7 +16868,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-py-imports"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
@@ -16893,7 +16892,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-ruby"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -16907,7 +16906,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-rust"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"convert_case 0.6.0",
|
||||
@@ -16924,7 +16923,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-sql"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -16939,7 +16938,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-ts"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -16956,25 +16955,9 @@ dependencies = [
|
||||
"windmill-parser-sql",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-wac"
|
||||
version = "1.652.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"rustpython-ast",
|
||||
"rustpython-parser",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"swc_common",
|
||||
"swc_ecma_ast",
|
||||
"swc_ecma_parser",
|
||||
"swc_ecma_visit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-yaml"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde",
|
||||
@@ -16985,7 +16968,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-queue"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
@@ -17022,7 +17005,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-runtime-nativets"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"const_format",
|
||||
@@ -17060,7 +17043,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-sql-datatype-parser-wasm"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
"wasm-bindgen",
|
||||
@@ -17071,7 +17054,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-store"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
@@ -17100,7 +17083,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-test-utils"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum 0.7.9",
|
||||
@@ -17123,7 +17106,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17156,7 +17139,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-email"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17176,7 +17159,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-gcp"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17210,7 +17193,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-http"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17245,7 +17228,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-kafka"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17268,7 +17251,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-mqtt"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17292,7 +17275,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-nats"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-nats",
|
||||
@@ -17316,7 +17299,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-postgres"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17351,7 +17334,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-sqs"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17379,7 +17362,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-trigger-websocket"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@@ -17402,7 +17385,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-types"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags 2.9.4",
|
||||
@@ -17420,7 +17403,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-worker"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-once-cell",
|
||||
@@ -17526,7 +17509,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-worker-volumes"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -18126,9 +18109,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.7.15"
|
||||
version = "0.7.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
|
||||
checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
@@ -18409,18 +18392,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.42"
|
||||
version = "0.8.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
|
||||
checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.42"
|
||||
version = "0.8.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
|
||||
checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
+2
-4
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "windmill"
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
@@ -68,7 +68,6 @@ members = [
|
||||
"./parsers/windmill-parser-bash",
|
||||
"./parsers/windmill-parser-py",
|
||||
"./parsers/windmill-parser-py-imports",
|
||||
"./parsers/windmill-parser-wac",
|
||||
"./parsers/windmill-sql-datatype-parser-wasm",
|
||||
"./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu",
|
||||
"./windmill-worker-volumes",
|
||||
@@ -78,7 +77,7 @@ members = [
|
||||
exclude = ["./windmill-duckdb-ffi-internal"]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.652.0"
|
||||
version = "1.651.1"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
edition = "2021"
|
||||
|
||||
@@ -333,7 +332,6 @@ windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
|
||||
windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
|
||||
windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
|
||||
windmill-parser-php = { path = "./parsers/windmill-parser-php" }
|
||||
windmill-parser-wac = { path = "./parsers/windmill-parser-wac" }
|
||||
windmill-jseval = { path = "./windmill-jseval" }
|
||||
windmill-runtime-nativets = { path = "./windmill-runtime-nativets" }
|
||||
windmill-api-client = { path = "./windmill-api-client" }
|
||||
|
||||
@@ -1 +1 @@
|
||||
716b350bce1730b302c66ea69df618fa40f2f16b
|
||||
c3c543f4c60a8c4dfe0d912c79a051376fb091a9
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE worker_ping DROP COLUMN IF EXISTS uses_batch_http_pull;
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS uses_batch_http_pull BOOLEAN NOT NULL DEFAULT false;
|
||||
@@ -296,14 +296,11 @@ pub fn parse_python_signature(
|
||||
|
||||
// Check if main function was found
|
||||
if params.is_none() {
|
||||
let is_wac_v2 = (code.contains("@workflow") || code.contains("workflow("))
|
||||
&& (code.contains("@task") || code.contains("task("))
|
||||
&& (code.contains("import wmill") || code.contains("from wmill"));
|
||||
return Ok(MainArgSignature {
|
||||
star_args: false,
|
||||
star_kwargs: false,
|
||||
args: vec![],
|
||||
no_main_func: Some(!is_wac_v2),
|
||||
no_main_func: Some(true),
|
||||
has_preprocessor: Some(has_preprocessor),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -238,7 +238,7 @@ lazy_static::lazy_static! {
|
||||
|
||||
// used for `unsafe` sql interpolation
|
||||
// -- %%name%% (type) = default
|
||||
static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%[ \t]*([\w][\w \t\/]*)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
|
||||
static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%\s*([\s\w\/]+)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
|
||||
}
|
||||
|
||||
fn parsed_default(parsed_typ: &Typ, default: String) -> Option<serde_json::Value> {
|
||||
@@ -1547,36 +1547,4 @@ SELECT $1::integer;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_pgsql_safe_interpolated_args() -> anyhow::Result<()> {
|
||||
// There was a bug where enum would be "angrycreative"/"bishop"/"test SELECT x"
|
||||
let code = r#"
|
||||
-- %%table_name%% angrycreative/bishop/test
|
||||
SELECT x
|
||||
"#;
|
||||
assert_eq!(
|
||||
parse_pgsql_sig(code)?,
|
||||
MainArgSignature {
|
||||
star_args: false,
|
||||
star_kwargs: false,
|
||||
args: vec![Arg {
|
||||
otyp: Some("__sanitized_enum__".to_string()),
|
||||
name: "table_name".to_string(),
|
||||
typ: Typ::Str(Some(vec![
|
||||
"angrycreative".to_string(),
|
||||
"bishop".to_string(),
|
||||
"test".to_string()
|
||||
])),
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None,
|
||||
},],
|
||||
no_main_func: None,
|
||||
has_preprocessor: None
|
||||
}
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -261,9 +261,7 @@ pub fn parse_deno_signature(
|
||||
for specifier in &named_export.specifiers {
|
||||
if let swc_ecma_ast::ExportSpecifier::Named(spec) = specifier {
|
||||
let export_name = match &spec.exported {
|
||||
Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => {
|
||||
ident.sym.as_ref()
|
||||
}
|
||||
Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => ident.sym.as_ref(),
|
||||
Some(swc_ecma_ast::ModuleExportName::Str(s)) => s.value.as_ref(),
|
||||
None => match &spec.orig {
|
||||
swc_ecma_ast::ModuleExportName::Ident(ident) => ident.sym.as_ref(),
|
||||
@@ -317,11 +315,7 @@ pub fn parse_deno_signature(
|
||||
|
||||
let mut c: u16 = 0;
|
||||
|
||||
let is_wac_v2 = entrypoint_params.is_none()
|
||||
&& code.contains("workflow(")
|
||||
&& code.contains("task(")
|
||||
&& code.contains("windmill-client");
|
||||
let no_main_func = entrypoint_params.is_none() && !is_wac_v2;
|
||||
let no_main_func = entrypoint_params.is_none();
|
||||
let mut type_resolver = HashMap::new();
|
||||
let r = MainArgSignature {
|
||||
star_args: false,
|
||||
@@ -839,9 +833,7 @@ fn tstype_to_typ(
|
||||
false,
|
||||
),
|
||||
symbol @ _ if symbol.starts_with("DynMultiselect_") => (
|
||||
Typ::DynMultiselect(
|
||||
symbol.strip_prefix("DynMultiselect_").unwrap().to_string(),
|
||||
),
|
||||
Typ::DynMultiselect(symbol.strip_prefix("DynMultiselect_").unwrap().to_string()),
|
||||
false,
|
||||
),
|
||||
symbol @ _ => {
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
[package]
|
||||
name = "windmill-parser-wac"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "windmill_parser_wac"
|
||||
path = "./src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
rustpython-parser.workspace = true
|
||||
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
|
||||
swc_common.workspace = true
|
||||
swc_ecma_parser.workspace = true
|
||||
swc_ecma_ast.workspace = true
|
||||
swc_ecma_visit.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
anyhow.workspace = true
|
||||
sha2.workspace = true
|
||||
@@ -1,44 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct WorkflowDag {
|
||||
pub nodes: Vec<DagNode>,
|
||||
pub edges: Vec<DagEdge>,
|
||||
pub params: Vec<Param>,
|
||||
pub source_hash: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct Param {
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub typ: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct DagNode {
|
||||
pub id: String,
|
||||
pub node_type: DagNodeType,
|
||||
pub label: String,
|
||||
pub line: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum DagNodeType {
|
||||
Step { name: String, script: String },
|
||||
Branch { condition_source: String },
|
||||
ParallelStart,
|
||||
ParallelEnd,
|
||||
LoopStart { iter_source: String },
|
||||
LoopEnd,
|
||||
Return,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct DagEdge {
|
||||
pub from: String,
|
||||
pub to: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub label: Option<String>,
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
pub mod dag;
|
||||
pub mod python;
|
||||
pub mod typescript;
|
||||
pub mod validation;
|
||||
|
||||
use dag::WorkflowDag;
|
||||
use validation::CompileError;
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ParseResult {
|
||||
#[serde(rename = "success")]
|
||||
Success(WorkflowDag),
|
||||
#[serde(rename = "error")]
|
||||
Error { errors: Vec<CompileError> },
|
||||
}
|
||||
|
||||
pub fn parse_workflow(code: &str, language: &str) -> ParseResult {
|
||||
let result = match language {
|
||||
"python" | "python3" | "py" => python::parse_python_workflow(code),
|
||||
"typescript" | "ts" | "deno" | "bun" => typescript::parse_ts_workflow(code),
|
||||
_ => Err(vec![CompileError {
|
||||
message: format!("Unsupported language: {language}"),
|
||||
line: 0,
|
||||
}]),
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(dag) => ParseResult::Success(dag),
|
||||
Err(errors) => ParseResult::Error { errors },
|
||||
}
|
||||
}
|
||||
@@ -1,717 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use rustpython_parser::{
|
||||
ast::{
|
||||
Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry,
|
||||
StmtTryStar, StmtWhile,
|
||||
},
|
||||
Parse,
|
||||
};
|
||||
|
||||
use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag};
|
||||
use crate::validation::{self, CompileError};
|
||||
|
||||
struct LineIndex {
|
||||
newline_offsets: Vec<usize>,
|
||||
}
|
||||
|
||||
impl LineIndex {
|
||||
fn new(source: &str) -> Self {
|
||||
let mut offsets = vec![0];
|
||||
for (i, c) in source.char_indices() {
|
||||
if c == '\n' {
|
||||
offsets.push(i + 1);
|
||||
}
|
||||
}
|
||||
Self { newline_offsets: offsets }
|
||||
}
|
||||
|
||||
fn line_of(&self, byte_offset: usize) -> usize {
|
||||
match self.newline_offsets.binary_search(&byte_offset) {
|
||||
Ok(line) => line + 1,
|
||||
Err(line) => line,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps task function name → optional external path (from `@task(path="...")`)
|
||||
type TaskFunctions = HashMap<String, Option<String>>;
|
||||
|
||||
/// First pass: scan top-level `@task async def foo(...)` declarations.
|
||||
fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
|
||||
let mut tasks = HashMap::new();
|
||||
for stmt in stmts {
|
||||
if let Stmt::AsyncFunctionDef(func) = stmt {
|
||||
for dec in &func.decorator_list {
|
||||
match dec {
|
||||
// @task (bare decorator)
|
||||
Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => {
|
||||
tasks.insert(func.name.to_string(), None);
|
||||
}
|
||||
// @task(path="...")
|
||||
Expr::Call(call) => {
|
||||
if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
|
||||
if id.as_str() == "task" {
|
||||
let path = extract_task_path_kwarg(call);
|
||||
tasks.insert(func.name.to_string(), path);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
tasks
|
||||
}
|
||||
|
||||
/// Extract the `path=` keyword argument from a `@task(path="...")` call.
|
||||
fn extract_task_path_kwarg(call: &ExprCall) -> Option<String> {
|
||||
for kw in &call.keywords {
|
||||
if let Some(ref arg) = kw.arg {
|
||||
if arg.as_str() == "path" {
|
||||
if let Expr::Constant(c) = &kw.value {
|
||||
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
|
||||
return Some(s.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
struct WacWalker {
|
||||
nodes: Vec<DagNode>,
|
||||
edges: Vec<DagEdge>,
|
||||
errors: Vec<CompileError>,
|
||||
node_counter: usize,
|
||||
line_index: LineIndex,
|
||||
task_functions: TaskFunctions,
|
||||
in_try: bool,
|
||||
in_while: bool,
|
||||
in_nested_func: bool,
|
||||
in_comprehension: bool,
|
||||
}
|
||||
|
||||
impl WacWalker {
|
||||
fn new(source: &str, task_functions: TaskFunctions) -> Self {
|
||||
Self {
|
||||
nodes: Vec::new(),
|
||||
edges: Vec::new(),
|
||||
errors: Vec::new(),
|
||||
node_counter: 0,
|
||||
line_index: LineIndex::new(source),
|
||||
task_functions,
|
||||
in_try: false,
|
||||
in_while: false,
|
||||
in_nested_func: false,
|
||||
in_comprehension: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn next_id(&mut self) -> String {
|
||||
let id = format!("step_{}", self.node_counter);
|
||||
self.node_counter += 1;
|
||||
id
|
||||
}
|
||||
|
||||
fn add_node(&mut self, node: DagNode) -> String {
|
||||
let id = node.id.clone();
|
||||
self.nodes.push(node);
|
||||
id
|
||||
}
|
||||
|
||||
fn add_edge(&mut self, from: &str, to: &str, label: Option<String>) {
|
||||
self.edges
|
||||
.push(DagEdge { from: from.to_string(), to: to.to_string(), label });
|
||||
}
|
||||
|
||||
fn line_of_expr(&self, expr: &Expr) -> usize {
|
||||
let offset = match expr {
|
||||
Expr::Call(c) => c.range.start().to_usize(),
|
||||
Expr::Await(a) => a.range.start().to_usize(),
|
||||
Expr::Attribute(a) => a.range.start().to_usize(),
|
||||
Expr::Name(n) => n.range.start().to_usize(),
|
||||
_ => 0,
|
||||
};
|
||||
self.line_index.line_of(offset)
|
||||
}
|
||||
|
||||
fn line_of_stmt(&self, stmt: &Stmt) -> usize {
|
||||
let offset = match stmt {
|
||||
Stmt::If(s) => s.range.start().to_usize(),
|
||||
Stmt::For(s) => s.range.start().to_usize(),
|
||||
Stmt::While(s) => s.range.start().to_usize(),
|
||||
Stmt::Return(s) => s.range.start().to_usize(),
|
||||
Stmt::Expr(s) => s.range.start().to_usize(),
|
||||
Stmt::Try(s) => s.range.start().to_usize(),
|
||||
Stmt::TryStar(s) => s.range.start().to_usize(),
|
||||
Stmt::Assign(s) => s.range.start().to_usize(),
|
||||
Stmt::AnnAssign(s) => s.range.start().to_usize(),
|
||||
Stmt::FunctionDef(s) => s.range.start().to_usize(),
|
||||
Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(),
|
||||
_ => 0,
|
||||
};
|
||||
self.line_index.line_of(offset)
|
||||
}
|
||||
|
||||
/// Check if an expression is a call to a known @task function
|
||||
fn is_task_fn_call(&self, expr: &Expr) -> bool {
|
||||
if let Expr::Call(call) = expr {
|
||||
if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
|
||||
return self.task_functions.contains_key(id.as_str());
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Check if an expression is `asyncio.gather(...)` call
|
||||
fn is_asyncio_gather_call(expr: &Expr) -> bool {
|
||||
if let Expr::Call(call) = expr {
|
||||
if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) =
|
||||
call.func.as_ref()
|
||||
{
|
||||
if attr.as_str() == "gather" {
|
||||
if let Expr::Name(ExprName { id, .. }) = value.as_ref() {
|
||||
return id.as_str() == "asyncio";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Extract step name and script from a task function call.
|
||||
/// Name = function name, script = task_path or function name.
|
||||
fn extract_step_info_from_task_call(&self, call: &ExprCall) -> Option<(String, String)> {
|
||||
if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
|
||||
let name = id.to_string();
|
||||
let script = self
|
||||
.task_functions
|
||||
.get(id.as_str())
|
||||
.and_then(|p| p.clone())
|
||||
.unwrap_or_else(|| name.clone());
|
||||
Some((name, script))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn expr_to_source(expr: &Expr) -> String {
|
||||
match expr {
|
||||
Expr::Compare(c) => {
|
||||
let left = Self::expr_to_source(&c.left);
|
||||
if let Some(comparator) = c.comparators.first() {
|
||||
let right = Self::expr_to_source(comparator);
|
||||
let op = match c.ops.first() {
|
||||
Some(rustpython_parser::ast::CmpOp::Gt) => ">",
|
||||
Some(rustpython_parser::ast::CmpOp::Lt) => "<",
|
||||
Some(rustpython_parser::ast::CmpOp::GtE) => ">=",
|
||||
Some(rustpython_parser::ast::CmpOp::LtE) => "<=",
|
||||
Some(rustpython_parser::ast::CmpOp::Eq) => "==",
|
||||
Some(rustpython_parser::ast::CmpOp::NotEq) => "!=",
|
||||
Some(rustpython_parser::ast::CmpOp::In) => "in",
|
||||
Some(rustpython_parser::ast::CmpOp::NotIn) => "not in",
|
||||
Some(rustpython_parser::ast::CmpOp::Is) => "is",
|
||||
Some(rustpython_parser::ast::CmpOp::IsNot) => "is not",
|
||||
None => "?",
|
||||
};
|
||||
format!("{left} {op} {right}")
|
||||
} else {
|
||||
left
|
||||
}
|
||||
}
|
||||
Expr::Subscript(s) => {
|
||||
let value = Self::expr_to_source(&s.value);
|
||||
let slice = Self::expr_to_source(&s.slice);
|
||||
format!("{value}[{slice}]")
|
||||
}
|
||||
Expr::Attribute(a) => {
|
||||
let value = Self::expr_to_source(&a.value);
|
||||
format!("{value}.{}", a.attr)
|
||||
}
|
||||
Expr::Name(n) => n.id.to_string(),
|
||||
Expr::Constant(c) => match &c.value {
|
||||
rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""),
|
||||
rustpython_parser::ast::Constant::Int(i) => i.to_string(),
|
||||
rustpython_parser::ast::Constant::Float(f) => f.to_string(),
|
||||
rustpython_parser::ast::Constant::Bool(b) => b.to_string(),
|
||||
rustpython_parser::ast::Constant::None => "None".to_string(),
|
||||
_ => "...".to_string(),
|
||||
},
|
||||
_ => "...".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a statement body contains any task function calls (recursively)
|
||||
fn body_contains_step(&self, body: &[Stmt]) -> bool {
|
||||
for stmt in body {
|
||||
if self.stmt_contains_step(stmt) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn stmt_contains_step(&self, stmt: &Stmt) -> bool {
|
||||
match stmt {
|
||||
Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value),
|
||||
Stmt::Assign(a) => self.expr_contains_step(&a.value),
|
||||
Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
|
||||
Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
|
||||
Stmt::While(s) => {
|
||||
self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
|
||||
}
|
||||
Stmt::Try(s) => {
|
||||
self.body_contains_step(&s.body)
|
||||
|| self.body_contains_step(&s.orelse)
|
||||
|| self.body_contains_step(&s.finalbody)
|
||||
|| s.handlers.iter().any(|h| match h {
|
||||
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
|
||||
self.body_contains_step(&eh.body)
|
||||
}
|
||||
})
|
||||
}
|
||||
Stmt::TryStar(s) => {
|
||||
self.body_contains_step(&s.body)
|
||||
|| self.body_contains_step(&s.orelse)
|
||||
|| self.body_contains_step(&s.finalbody)
|
||||
|| s.handlers.iter().any(|h| match h {
|
||||
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
|
||||
self.body_contains_step(&eh.body)
|
||||
}
|
||||
})
|
||||
}
|
||||
Stmt::Return(_) => false,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn expr_contains_step(&self, expr: &Expr) -> bool {
|
||||
if self.is_task_fn_call(expr) {
|
||||
return true;
|
||||
}
|
||||
match expr {
|
||||
Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value),
|
||||
Expr::Call(call) => {
|
||||
if self.is_task_fn_call(&Expr::Call(call.clone())) {
|
||||
return true;
|
||||
}
|
||||
if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) {
|
||||
return call.args.iter().any(|a| self.expr_contains_step(a));
|
||||
}
|
||||
false
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk a list of statements, returning (first_node_id, last_node_id)
|
||||
fn walk_body(&mut self, body: &[Stmt]) -> Option<(String, String)> {
|
||||
let mut first_id: Option<String> = None;
|
||||
let mut prev_id: Option<String> = None;
|
||||
|
||||
for stmt in body {
|
||||
if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) {
|
||||
if let Some(ref prev) = prev_id {
|
||||
self.add_edge(prev, &stmt_first, None);
|
||||
}
|
||||
if first_id.is_none() {
|
||||
first_id = Some(stmt_first);
|
||||
}
|
||||
prev_id = Some(stmt_last);
|
||||
}
|
||||
}
|
||||
|
||||
match (first_id, prev_id) {
|
||||
(Some(f), Some(l)) => Some((f, l)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> {
|
||||
match stmt {
|
||||
Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value),
|
||||
Stmt::Assign(a) => self.walk_expr_stmt(&a.value),
|
||||
Stmt::If(if_stmt) => self.walk_if(if_stmt),
|
||||
Stmt::For(for_stmt) => self.walk_for(for_stmt),
|
||||
Stmt::While(while_stmt) => self.walk_while(while_stmt),
|
||||
Stmt::Try(try_stmt) => self.walk_try(try_stmt),
|
||||
Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt),
|
||||
Stmt::Return(ret) => self.walk_return(ret),
|
||||
Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => {
|
||||
if self.stmt_contains_step(stmt) {
|
||||
self.errors.push(validation::error_step_in_nested_function(
|
||||
self.line_of_stmt(stmt),
|
||||
));
|
||||
}
|
||||
None
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> {
|
||||
// await task_fn(...)
|
||||
if let Expr::Await(ExprAwait { value, .. }) = expr {
|
||||
// await task_fn(...)
|
||||
if let Expr::Call(call) = value.as_ref() {
|
||||
if self.is_task_fn_call(&Expr::Call(call.clone())) {
|
||||
return self.emit_step(call, expr);
|
||||
}
|
||||
}
|
||||
// await asyncio.gather(task_fn(...), task_fn(...), ...)
|
||||
if Self::is_asyncio_gather_call(value) {
|
||||
if let Expr::Call(gather_call) = value.as_ref() {
|
||||
return self.emit_parallel(gather_call, expr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bare task_fn() without await — validation error
|
||||
if self.is_task_fn_call(expr) {
|
||||
self.errors
|
||||
.push(validation::error_missing_await(self.line_of_expr(expr)));
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn emit_step(&mut self, call: &ExprCall, expr: &Expr) -> Option<(String, String)> {
|
||||
if self.in_try {
|
||||
self.errors
|
||||
.push(validation::error_step_in_try(self.line_of_expr(expr)));
|
||||
return None;
|
||||
}
|
||||
if self.in_while {
|
||||
self.errors
|
||||
.push(validation::error_step_in_while(self.line_of_expr(expr)));
|
||||
return None;
|
||||
}
|
||||
if self.in_nested_func {
|
||||
self.errors.push(validation::error_step_in_nested_function(
|
||||
self.line_of_expr(expr),
|
||||
));
|
||||
return None;
|
||||
}
|
||||
if self.in_comprehension {
|
||||
self.errors.push(validation::error_step_in_comprehension(
|
||||
self.line_of_expr(expr),
|
||||
));
|
||||
return None;
|
||||
}
|
||||
|
||||
let (name, script) = self
|
||||
.extract_step_info_from_task_call(call)
|
||||
.unwrap_or(("unknown".into(), "unknown".into()));
|
||||
let id = self.next_id();
|
||||
let node_id = self.add_node(DagNode {
|
||||
id: id.clone(),
|
||||
node_type: DagNodeType::Step { name: name.clone(), script },
|
||||
label: name,
|
||||
line: self.line_of_expr(expr),
|
||||
});
|
||||
Some((node_id.clone(), node_id))
|
||||
}
|
||||
|
||||
fn emit_parallel(&mut self, gather_call: &ExprCall, expr: &Expr) -> Option<(String, String)> {
|
||||
if self.in_try {
|
||||
self.errors
|
||||
.push(validation::error_step_in_try(self.line_of_expr(expr)));
|
||||
return None;
|
||||
}
|
||||
if self.in_while {
|
||||
self.errors
|
||||
.push(validation::error_step_in_while(self.line_of_expr(expr)));
|
||||
return None;
|
||||
}
|
||||
|
||||
let line = self.line_of_expr(expr);
|
||||
let start_id = self.next_id();
|
||||
let start_node_id = self.add_node(DagNode {
|
||||
id: start_id.clone(),
|
||||
node_type: DagNodeType::ParallelStart,
|
||||
label: "parallel".to_string(),
|
||||
line,
|
||||
});
|
||||
|
||||
let mut step_ids = Vec::new();
|
||||
for arg in &gather_call.args {
|
||||
// Each arg should be task_fn(...)
|
||||
if let Expr::Call(call) = arg {
|
||||
if self.is_task_fn_call(&Expr::Call(call.clone())) {
|
||||
let (name, script) = self
|
||||
.extract_step_info_from_task_call(call)
|
||||
.unwrap_or(("unknown".into(), "unknown".into()));
|
||||
let step_id = self.next_id();
|
||||
let node_id = self.add_node(DagNode {
|
||||
id: step_id.clone(),
|
||||
node_type: DagNodeType::Step { name: name.clone(), script },
|
||||
label: name,
|
||||
line: self.line_of_expr(arg),
|
||||
});
|
||||
self.add_edge(&start_node_id, &node_id, None);
|
||||
step_ids.push(node_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let end_id = self.next_id();
|
||||
let end_node_id = self.add_node(DagNode {
|
||||
id: end_id.clone(),
|
||||
node_type: DagNodeType::ParallelEnd,
|
||||
label: "join".to_string(),
|
||||
line,
|
||||
});
|
||||
|
||||
for step_id in &step_ids {
|
||||
self.add_edge(step_id, &end_node_id, None);
|
||||
}
|
||||
|
||||
Some((start_node_id, end_node_id))
|
||||
}
|
||||
|
||||
fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> {
|
||||
let has_steps_in_body = self.body_contains_step(&if_stmt.body);
|
||||
let has_steps_in_else = self.body_contains_step(&if_stmt.orelse);
|
||||
|
||||
if !has_steps_in_body && !has_steps_in_else {
|
||||
return None;
|
||||
}
|
||||
|
||||
let line = self.line_index.line_of(if_stmt.range.start().to_usize());
|
||||
let condition_source = Self::expr_to_source(&if_stmt.test);
|
||||
|
||||
let branch_id = self.next_id();
|
||||
let branch_node_id = self.add_node(DagNode {
|
||||
id: branch_id.clone(),
|
||||
node_type: DagNodeType::Branch { condition_source },
|
||||
label: "if".to_string(),
|
||||
line,
|
||||
});
|
||||
|
||||
let merge_id = format!("{branch_id}_merge");
|
||||
|
||||
let mut last_ids = Vec::new();
|
||||
|
||||
if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) {
|
||||
self.add_edge(&branch_node_id, &true_first, Some("true".to_string()));
|
||||
last_ids.push(true_last);
|
||||
} else {
|
||||
last_ids.push(branch_node_id.clone());
|
||||
}
|
||||
|
||||
if !if_stmt.orelse.is_empty() {
|
||||
if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) {
|
||||
self.add_edge(&branch_node_id, &else_first, Some("false".to_string()));
|
||||
last_ids.push(else_last);
|
||||
} else {
|
||||
last_ids.push(branch_node_id.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if last_ids.len() == 1 {
|
||||
Some((branch_node_id, last_ids.into_iter().next().unwrap()))
|
||||
} else {
|
||||
Some((branch_node_id, merge_id))
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_for(&mut self, for_stmt: &StmtFor) -> Option<(String, String)> {
|
||||
if !self.body_contains_step(&for_stmt.body) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let line = self.line_index.line_of(for_stmt.range.start().to_usize());
|
||||
let iter_source = Self::expr_to_source(&for_stmt.iter);
|
||||
|
||||
let start_id = self.next_id();
|
||||
let start_node_id = self.add_node(DagNode {
|
||||
id: start_id.clone(),
|
||||
node_type: DagNodeType::LoopStart { iter_source },
|
||||
label: "for".to_string(),
|
||||
line,
|
||||
});
|
||||
|
||||
if let Some((body_first, body_last)) = self.walk_body(&for_stmt.body) {
|
||||
self.add_edge(&start_node_id, &body_first, None);
|
||||
self.add_edge(&body_last, &start_node_id, Some("next".to_string()));
|
||||
}
|
||||
|
||||
let end_id = self.next_id();
|
||||
let end_node_id = self.add_node(DagNode {
|
||||
id: end_id.clone(),
|
||||
node_type: DagNodeType::LoopEnd,
|
||||
label: "end for".to_string(),
|
||||
line,
|
||||
});
|
||||
self.add_edge(&start_node_id, &end_node_id, Some("done".to_string()));
|
||||
|
||||
Some((start_node_id, end_node_id))
|
||||
}
|
||||
|
||||
fn walk_while(&mut self, while_stmt: &StmtWhile) -> Option<(String, String)> {
|
||||
if self.body_contains_step(&while_stmt.body) {
|
||||
let line = self.line_index.line_of(while_stmt.range.start().to_usize());
|
||||
self.errors.push(validation::error_step_in_while(line));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn walk_try(&mut self, try_stmt: &StmtTry) -> Option<(String, String)> {
|
||||
let has_steps = self.body_contains_step(&try_stmt.body)
|
||||
|| self.body_contains_step(&try_stmt.orelse)
|
||||
|| self.body_contains_step(&try_stmt.finalbody)
|
||||
|| try_stmt.handlers.iter().any(|h| match h {
|
||||
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
|
||||
self.body_contains_step(&eh.body)
|
||||
}
|
||||
});
|
||||
|
||||
if has_steps {
|
||||
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
|
||||
self.errors.push(validation::error_step_in_try(line));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> {
|
||||
let has_steps = self.body_contains_step(&try_stmt.body)
|
||||
|| self.body_contains_step(&try_stmt.orelse)
|
||||
|| self.body_contains_step(&try_stmt.finalbody)
|
||||
|| try_stmt.handlers.iter().any(|h| match h {
|
||||
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
|
||||
self.body_contains_step(&eh.body)
|
||||
}
|
||||
});
|
||||
|
||||
if has_steps {
|
||||
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
|
||||
self.errors.push(validation::error_step_in_try(line));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> {
|
||||
let line = self.line_index.line_of(ret.range.start().to_usize());
|
||||
let id = self.next_id();
|
||||
let node_id = self.add_node(DagNode {
|
||||
id: id.clone(),
|
||||
node_type: DagNodeType::Return,
|
||||
label: "return".to_string(),
|
||||
line,
|
||||
});
|
||||
Some((node_id.clone(), node_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract workflow function parameters (no longer skips ctx)
|
||||
fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
|
||||
let mut params = Vec::new();
|
||||
for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) {
|
||||
let name = arg_with_default.def.arg.to_string();
|
||||
let typ = arg_with_default
|
||||
.def
|
||||
.annotation
|
||||
.as_ref()
|
||||
.map(|ann| WacWalker::expr_to_source(ann));
|
||||
params.push(Param { name, typ });
|
||||
}
|
||||
params
|
||||
}
|
||||
|
||||
pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError>> {
|
||||
let ast = rustpython_parser::ast::Suite::parse(code, "<workflow>")
|
||||
.map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?;
|
||||
|
||||
// First pass: collect @task functions
|
||||
let task_functions = collect_task_functions(&ast);
|
||||
|
||||
// Find the @workflow async def
|
||||
let workflow_fn = ast.iter().find_map(|stmt| {
|
||||
if let Stmt::AsyncFunctionDef(func) = stmt {
|
||||
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
|
||||
if let Expr::Name(ExprName { id, .. }) = dec {
|
||||
id.as_str() == "workflow"
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
if has_workflow_decorator {
|
||||
return Some(func);
|
||||
}
|
||||
}
|
||||
// Also check non-async for error reporting
|
||||
if let Stmt::FunctionDef(func) = stmt {
|
||||
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
|
||||
if let Expr::Name(ExprName { id, .. }) = dec {
|
||||
id.as_str() == "workflow"
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
if has_workflow_decorator {
|
||||
return None; // Will be reported as not-async below
|
||||
}
|
||||
}
|
||||
None
|
||||
});
|
||||
|
||||
// Check for non-async workflow function
|
||||
let non_async_workflow = ast.iter().find_map(|stmt| {
|
||||
if let Stmt::FunctionDef(func) = stmt {
|
||||
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
|
||||
if let Expr::Name(ExprName { id, .. }) = dec {
|
||||
id.as_str() == "workflow"
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
if has_workflow_decorator {
|
||||
let line_index = LineIndex::new(code);
|
||||
return Some(line_index.line_of(func.range.start().to_usize()));
|
||||
}
|
||||
}
|
||||
None
|
||||
});
|
||||
|
||||
if let Some(line) = non_async_workflow {
|
||||
if workflow_fn.is_none() {
|
||||
return Err(vec![validation::error_not_async(line)]);
|
||||
}
|
||||
}
|
||||
|
||||
let workflow_fn = workflow_fn.ok_or_else(|| {
|
||||
vec![CompileError { message: "No @workflow async function found.".to_string(), line: 0 }]
|
||||
})?;
|
||||
|
||||
let params = extract_params(&workflow_fn.args);
|
||||
let source_hash = compute_source_hash(code);
|
||||
|
||||
let mut walker = WacWalker::new(code, task_functions);
|
||||
walker.walk_body(&workflow_fn.body);
|
||||
|
||||
if !walker.errors.is_empty() {
|
||||
return Err(walker.errors);
|
||||
}
|
||||
|
||||
Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash })
|
||||
}
|
||||
|
||||
fn compute_source_hash(code: &str) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(code.as_bytes());
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
trait ToUsize {
|
||||
fn to_usize(self) -> usize;
|
||||
}
|
||||
|
||||
impl ToUsize for rustpython_parser::text_size::TextSize {
|
||||
fn to_usize(self) -> usize {
|
||||
u32::from(self) as usize
|
||||
}
|
||||
}
|
||||
@@ -1,739 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned};
|
||||
use swc_ecma_ast::*;
|
||||
use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax};
|
||||
|
||||
use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag};
|
||||
use crate::validation::{self, CompileError};
|
||||
|
||||
/// Maps task function name → optional external path (from `task("f/path", ...)`)
|
||||
type TaskFunctions = HashMap<String, Option<String>>;
|
||||
|
||||
/// First pass: scan top-level `const foo = task(async (...) => {})` or
|
||||
/// `const foo = task("f/path", async (...) => {})` declarations.
|
||||
fn collect_task_functions(module: &Module) -> TaskFunctions {
|
||||
let mut tasks = HashMap::new();
|
||||
for item in &module.body {
|
||||
// const foo = task(async (...) => { ... })
|
||||
// const foo = task("f/path", async (...) => { ... })
|
||||
if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item {
|
||||
for decl in &var_decl.decls {
|
||||
if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) {
|
||||
if let Some(path) = extract_task_call_info(init) {
|
||||
tasks.insert(name, path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// export const foo = task(...)
|
||||
if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(export)) = item {
|
||||
if let Decl::Var(var_decl) = &export.decl {
|
||||
for decl in &var_decl.decls {
|
||||
if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) {
|
||||
if let Some(path) = extract_task_call_info(init) {
|
||||
tasks.insert(name, path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
tasks
|
||||
}
|
||||
|
||||
/// Extract variable name from a pattern (simple ident case)
|
||||
fn extract_var_name(pat: &Pat) -> Option<String> {
|
||||
if let Pat::Ident(BindingIdent { id, .. }) = pat {
|
||||
Some(id.sym.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if expr is `task(async fn)` or `task("path", async fn)`.
|
||||
/// Returns Some(optional_path) if it is a task() call.
|
||||
fn extract_task_call_info(expr: &Expr) -> Option<Option<String>> {
|
||||
if let Expr::Call(call) = expr {
|
||||
if let Callee::Expr(callee) = &call.callee {
|
||||
if let Expr::Ident(ident) = callee.as_ref() {
|
||||
if ident.sym.as_ref() == "task" {
|
||||
// task("f/path", async fn) or task(async fn)
|
||||
if call.args.len() == 2 {
|
||||
// task("f/path", async fn)
|
||||
let path = extract_string_lit(&call.args[0].expr);
|
||||
return Some(path);
|
||||
} else if call.args.len() == 1 {
|
||||
// task(async fn)
|
||||
return Some(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
struct TsWacWalker {
|
||||
nodes: Vec<DagNode>,
|
||||
edges: Vec<DagEdge>,
|
||||
errors: Vec<CompileError>,
|
||||
node_counter: usize,
|
||||
cm: Lrc<SourceMap>,
|
||||
task_functions: TaskFunctions,
|
||||
in_try: bool,
|
||||
in_while: bool,
|
||||
in_nested_func: bool,
|
||||
}
|
||||
|
||||
impl TsWacWalker {
|
||||
fn new(cm: Lrc<SourceMap>, task_functions: TaskFunctions) -> Self {
|
||||
Self {
|
||||
nodes: Vec::new(),
|
||||
edges: Vec::new(),
|
||||
errors: Vec::new(),
|
||||
node_counter: 0,
|
||||
cm,
|
||||
task_functions,
|
||||
in_try: false,
|
||||
in_while: false,
|
||||
in_nested_func: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn next_id(&mut self) -> String {
|
||||
let id = format!("step_{}", self.node_counter);
|
||||
self.node_counter += 1;
|
||||
id
|
||||
}
|
||||
|
||||
fn add_node(&mut self, node: DagNode) -> String {
|
||||
let id = node.id.clone();
|
||||
self.nodes.push(node);
|
||||
id
|
||||
}
|
||||
|
||||
fn add_edge(&mut self, from: &str, to: &str, label: Option<String>) {
|
||||
self.edges
|
||||
.push(DagEdge { from: from.to_string(), to: to.to_string(), label });
|
||||
}
|
||||
|
||||
fn span_line(&self, span: swc_common::Span) -> usize {
|
||||
let loc = self.cm.lookup_char_pos(span.lo);
|
||||
loc.line
|
||||
}
|
||||
|
||||
/// Check if expr is a call to a known task function
|
||||
fn is_task_call(&self, expr: &Expr) -> bool {
|
||||
if let Expr::Call(call) = expr {
|
||||
if let Callee::Expr(callee) = &call.callee {
|
||||
if let Expr::Ident(ident) = callee.as_ref() {
|
||||
return self.task_functions.contains_key(ident.sym.as_ref());
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Check if expr is `Promise.all([...])`
|
||||
fn is_promise_all(expr: &Expr) -> bool {
|
||||
if let Expr::Call(call) = expr {
|
||||
if let Callee::Expr(callee) = &call.callee {
|
||||
if let Expr::Member(MemberExpr { obj, prop: MemberProp::Ident(prop), .. }) =
|
||||
callee.as_ref()
|
||||
{
|
||||
if prop.sym.as_ref() == "all" {
|
||||
if let Expr::Ident(ident) = obj.as_ref() {
|
||||
return ident.sym.as_ref() == "Promise";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Extract step name and script from a task function call.
|
||||
/// Name = function name, script = task_path or function name.
|
||||
fn extract_step_info_from_task_call(&self, call: &CallExpr) -> Option<(String, String)> {
|
||||
if let Callee::Expr(callee) = &call.callee {
|
||||
if let Expr::Ident(ident) = callee.as_ref() {
|
||||
let name = ident.sym.to_string();
|
||||
let script = self
|
||||
.task_functions
|
||||
.get(ident.sym.as_ref())
|
||||
.and_then(|p| p.clone())
|
||||
.unwrap_or_else(|| name.clone());
|
||||
return Some((name, script));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn expr_to_source(&self, expr: &Expr) -> String {
|
||||
let span = expr.span();
|
||||
self.cm
|
||||
.span_to_snippet(span)
|
||||
.unwrap_or_else(|_| "...".to_string())
|
||||
}
|
||||
|
||||
fn body_contains_step(&self, stmts: &[Stmt]) -> bool {
|
||||
stmts.iter().any(|s| self.stmt_contains_step(s))
|
||||
}
|
||||
|
||||
fn stmt_contains_step(&self, stmt: &Stmt) -> bool {
|
||||
match stmt {
|
||||
Stmt::Expr(expr_stmt) => self.expr_contains_step(&expr_stmt.expr),
|
||||
Stmt::Decl(Decl::Var(var_decl)) => var_decl.decls.iter().any(|d| {
|
||||
d.init
|
||||
.as_ref()
|
||||
.map_or(false, |init| self.expr_contains_step(init))
|
||||
}),
|
||||
Stmt::If(if_stmt) => {
|
||||
self.stmt_contains_step(&if_stmt.cons)
|
||||
|| if_stmt
|
||||
.alt
|
||||
.as_ref()
|
||||
.map_or(false, |alt| self.stmt_contains_step(alt))
|
||||
}
|
||||
Stmt::Block(block) => self.body_contains_step(&block.stmts),
|
||||
Stmt::For(for_stmt) => self.stmt_contains_step(&for_stmt.body),
|
||||
Stmt::ForIn(for_in) => self.stmt_contains_step(&for_in.body),
|
||||
Stmt::ForOf(for_of) => self.stmt_contains_step(&for_of.body),
|
||||
Stmt::While(while_stmt) => self.stmt_contains_step(&while_stmt.body),
|
||||
Stmt::Try(try_stmt) => {
|
||||
self.body_contains_step(&try_stmt.block.stmts)
|
||||
|| try_stmt
|
||||
.handler
|
||||
.as_ref()
|
||||
.map_or(false, |h| self.body_contains_step(&h.body.stmts))
|
||||
|| try_stmt
|
||||
.finalizer
|
||||
.as_ref()
|
||||
.map_or(false, |f| self.body_contains_step(&f.stmts))
|
||||
}
|
||||
Stmt::Return(ret) => ret
|
||||
.arg
|
||||
.as_ref()
|
||||
.map_or(false, |arg| self.expr_contains_step(arg)),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn expr_contains_step(&self, expr: &Expr) -> bool {
|
||||
if self.is_task_call(expr) {
|
||||
return true;
|
||||
}
|
||||
match expr {
|
||||
Expr::Await(await_expr) => self.expr_contains_step(&await_expr.arg),
|
||||
Expr::Call(call) => {
|
||||
if Self::is_promise_all(&Expr::Call(call.clone())) {
|
||||
return call.args.iter().any(|a| self.expr_contains_step(&a.expr));
|
||||
}
|
||||
false
|
||||
}
|
||||
Expr::Paren(p) => self.expr_contains_step(&p.expr),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_body(&mut self, stmts: &[Stmt]) -> Option<(String, String)> {
|
||||
let mut first_id: Option<String> = None;
|
||||
let mut prev_id: Option<String> = None;
|
||||
|
||||
for stmt in stmts {
|
||||
if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) {
|
||||
if let Some(ref prev) = prev_id {
|
||||
self.add_edge(prev, &stmt_first, None);
|
||||
}
|
||||
if first_id.is_none() {
|
||||
first_id = Some(stmt_first);
|
||||
}
|
||||
prev_id = Some(stmt_last);
|
||||
}
|
||||
}
|
||||
|
||||
match (first_id, prev_id) {
|
||||
(Some(f), Some(l)) => Some((f, l)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> {
|
||||
match stmt {
|
||||
Stmt::Expr(expr_stmt) => self.walk_expr_stmt(&expr_stmt.expr),
|
||||
Stmt::Decl(Decl::Var(var_decl)) => {
|
||||
// const result = await task_fn(...)
|
||||
for decl in &var_decl.decls {
|
||||
if let Some(init) = &decl.init {
|
||||
if let Some(result) = self.walk_expr_stmt(init) {
|
||||
return Some(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
Stmt::If(if_stmt) => self.walk_if(if_stmt),
|
||||
Stmt::For(for_stmt) => self.walk_for_stmt(for_stmt),
|
||||
Stmt::ForIn(for_in) => self.walk_for_in(for_in),
|
||||
Stmt::ForOf(for_of) => self.walk_for_of(for_of),
|
||||
Stmt::While(while_stmt) => self.walk_while(while_stmt),
|
||||
Stmt::Try(try_stmt) => self.walk_try(try_stmt),
|
||||
Stmt::Block(block) => self.walk_body(&block.stmts),
|
||||
Stmt::Return(ret) => self.walk_return(ret),
|
||||
Stmt::Decl(Decl::Fn(_)) => {
|
||||
if self.stmt_contains_step(stmt) {
|
||||
self.errors.push(validation::error_step_in_nested_function(
|
||||
self.span_line(stmt.span()),
|
||||
));
|
||||
}
|
||||
None
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> {
|
||||
// await task_fn(...)
|
||||
if let Expr::Await(await_expr) = expr {
|
||||
if let Expr::Call(call) = await_expr.arg.as_ref() {
|
||||
if self.is_task_call(&Expr::Call(call.clone())) {
|
||||
return self.emit_step(call, expr);
|
||||
}
|
||||
}
|
||||
// await Promise.all([task_fn(...), ...])
|
||||
if Self::is_promise_all(&await_expr.arg) {
|
||||
if let Expr::Call(promise_call) = await_expr.arg.as_ref() {
|
||||
return self.emit_parallel(promise_call, expr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bare task_fn() without await
|
||||
if self.is_task_call(expr) {
|
||||
self.errors
|
||||
.push(validation::error_missing_await(self.span_line(expr.span())));
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn emit_step(&mut self, call: &CallExpr, expr: &Expr) -> Option<(String, String)> {
|
||||
if self.in_try {
|
||||
self.errors
|
||||
.push(validation::error_step_in_catch(self.span_line(expr.span())));
|
||||
return None;
|
||||
}
|
||||
if self.in_while {
|
||||
self.errors
|
||||
.push(validation::error_step_in_while(self.span_line(expr.span())));
|
||||
return None;
|
||||
}
|
||||
if self.in_nested_func {
|
||||
self.errors.push(validation::error_step_in_nested_function(
|
||||
self.span_line(expr.span()),
|
||||
));
|
||||
return None;
|
||||
}
|
||||
|
||||
let (name, script) = self
|
||||
.extract_step_info_from_task_call(call)
|
||||
.unwrap_or(("unknown".into(), "unknown".into()));
|
||||
let id = self.next_id();
|
||||
let node_id = self.add_node(DagNode {
|
||||
id: id.clone(),
|
||||
node_type: DagNodeType::Step { name: name.clone(), script },
|
||||
label: name,
|
||||
line: self.span_line(expr.span()),
|
||||
});
|
||||
Some((node_id.clone(), node_id))
|
||||
}
|
||||
|
||||
fn emit_parallel(&mut self, promise_call: &CallExpr, expr: &Expr) -> Option<(String, String)> {
|
||||
if self.in_try {
|
||||
self.errors
|
||||
.push(validation::error_step_in_catch(self.span_line(expr.span())));
|
||||
return None;
|
||||
}
|
||||
if self.in_while {
|
||||
self.errors
|
||||
.push(validation::error_step_in_while(self.span_line(expr.span())));
|
||||
return None;
|
||||
}
|
||||
|
||||
let line = self.span_line(expr.span());
|
||||
let start_id = self.next_id();
|
||||
let start_node_id = self.add_node(DagNode {
|
||||
id: start_id.clone(),
|
||||
node_type: DagNodeType::ParallelStart,
|
||||
label: "parallel".to_string(),
|
||||
line,
|
||||
});
|
||||
|
||||
let mut step_ids = Vec::new();
|
||||
|
||||
// Promise.all takes an array as first argument
|
||||
if let Some(first_arg) = promise_call.args.first() {
|
||||
if let Expr::Array(ArrayLit { elems, .. }) = first_arg.expr.as_ref() {
|
||||
for elem in elems.iter().flatten() {
|
||||
if let Expr::Call(call) = elem.expr.as_ref() {
|
||||
if self.is_task_call(&Expr::Call(call.clone())) {
|
||||
let (name, script) = self
|
||||
.extract_step_info_from_task_call(call)
|
||||
.unwrap_or(("unknown".into(), "unknown".into()));
|
||||
let step_id = self.next_id();
|
||||
let node_id = self.add_node(DagNode {
|
||||
id: step_id.clone(),
|
||||
node_type: DagNodeType::Step { name: name.clone(), script },
|
||||
label: name,
|
||||
line: self.span_line(elem.expr.span()),
|
||||
});
|
||||
self.add_edge(&start_node_id, &node_id, None);
|
||||
step_ids.push(node_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let end_id = self.next_id();
|
||||
let end_node_id = self.add_node(DagNode {
|
||||
id: end_id.clone(),
|
||||
node_type: DagNodeType::ParallelEnd,
|
||||
label: "join".to_string(),
|
||||
line,
|
||||
});
|
||||
|
||||
for step_id in &step_ids {
|
||||
self.add_edge(step_id, &end_node_id, None);
|
||||
}
|
||||
|
||||
Some((start_node_id, end_node_id))
|
||||
}
|
||||
|
||||
fn walk_if(&mut self, if_stmt: &IfStmt) -> Option<(String, String)> {
|
||||
let has_steps_cons = self.stmt_contains_step(&if_stmt.cons);
|
||||
let has_steps_alt = if_stmt
|
||||
.alt
|
||||
.as_ref()
|
||||
.map_or(false, |a| self.stmt_contains_step(a));
|
||||
|
||||
if !has_steps_cons && !has_steps_alt {
|
||||
return None;
|
||||
}
|
||||
|
||||
let line = self.span_line(if_stmt.span);
|
||||
let condition_source = self.expr_to_source(&if_stmt.test);
|
||||
|
||||
let branch_id = self.next_id();
|
||||
let branch_node_id = self.add_node(DagNode {
|
||||
id: branch_id.clone(),
|
||||
node_type: DagNodeType::Branch { condition_source },
|
||||
label: "if".to_string(),
|
||||
line,
|
||||
});
|
||||
|
||||
let mut last_ids = Vec::new();
|
||||
|
||||
// True branch
|
||||
if let Some((true_first, true_last)) = self.walk_stmt(&if_stmt.cons) {
|
||||
self.add_edge(&branch_node_id, &true_first, Some("true".to_string()));
|
||||
last_ids.push(true_last);
|
||||
} else {
|
||||
last_ids.push(branch_node_id.clone());
|
||||
}
|
||||
|
||||
// False branch
|
||||
if let Some(alt) = &if_stmt.alt {
|
||||
if let Some((else_first, else_last)) = self.walk_stmt(alt) {
|
||||
self.add_edge(&branch_node_id, &else_first, Some("false".to_string()));
|
||||
last_ids.push(else_last);
|
||||
} else {
|
||||
last_ids.push(branch_node_id.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if last_ids.len() == 1 {
|
||||
Some((branch_node_id, last_ids.into_iter().next().unwrap()))
|
||||
} else {
|
||||
let merge_id = format!("{branch_id}_merge");
|
||||
Some((branch_node_id, merge_id))
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_for_stmt(&mut self, for_stmt: &ForStmt) -> Option<(String, String)> {
|
||||
if !self.stmt_contains_step(&for_stmt.body) {
|
||||
return None;
|
||||
}
|
||||
self.walk_loop_body(&for_stmt.body, for_stmt.span, "for")
|
||||
}
|
||||
|
||||
fn walk_for_in(&mut self, for_in: &ForInStmt) -> Option<(String, String)> {
|
||||
if !self.stmt_contains_step(&for_in.body) {
|
||||
return None;
|
||||
}
|
||||
let iter_source = self.expr_to_source(&for_in.right);
|
||||
self.walk_loop_body_with_iter(&for_in.body, for_in.span, &iter_source)
|
||||
}
|
||||
|
||||
fn walk_for_of(&mut self, for_of: &ForOfStmt) -> Option<(String, String)> {
|
||||
if !self.stmt_contains_step(&for_of.body) {
|
||||
return None;
|
||||
}
|
||||
let iter_source = self.expr_to_source(&for_of.right);
|
||||
self.walk_loop_body_with_iter(&for_of.body, for_of.span, &iter_source)
|
||||
}
|
||||
|
||||
fn walk_loop_body(
|
||||
&mut self,
|
||||
body: &Stmt,
|
||||
span: swc_common::Span,
|
||||
_label: &str,
|
||||
) -> Option<(String, String)> {
|
||||
self.walk_loop_body_with_iter(body, span, "...")
|
||||
}
|
||||
|
||||
fn walk_loop_body_with_iter(
|
||||
&mut self,
|
||||
body: &Stmt,
|
||||
span: swc_common::Span,
|
||||
iter_source: &str,
|
||||
) -> Option<(String, String)> {
|
||||
let line = self.span_line(span);
|
||||
let start_id = self.next_id();
|
||||
let start_node_id = self.add_node(DagNode {
|
||||
id: start_id.clone(),
|
||||
node_type: DagNodeType::LoopStart { iter_source: iter_source.to_string() },
|
||||
label: "for".to_string(),
|
||||
line,
|
||||
});
|
||||
|
||||
if let Some((body_first, body_last)) = self.walk_stmt(body) {
|
||||
self.add_edge(&start_node_id, &body_first, None);
|
||||
self.add_edge(&body_last, &start_node_id, Some("next".to_string()));
|
||||
}
|
||||
|
||||
let end_id = self.next_id();
|
||||
let end_node_id = self.add_node(DagNode {
|
||||
id: end_id.clone(),
|
||||
node_type: DagNodeType::LoopEnd,
|
||||
label: "end for".to_string(),
|
||||
line,
|
||||
});
|
||||
self.add_edge(&start_node_id, &end_node_id, Some("done".to_string()));
|
||||
|
||||
Some((start_node_id, end_node_id))
|
||||
}
|
||||
|
||||
fn walk_while(&mut self, while_stmt: &WhileStmt) -> Option<(String, String)> {
|
||||
if self.stmt_contains_step(&while_stmt.body) {
|
||||
self.errors.push(validation::error_step_in_while(
|
||||
self.span_line(while_stmt.span),
|
||||
));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn walk_try(&mut self, try_stmt: &TryStmt) -> Option<(String, String)> {
|
||||
let has_steps = self.body_contains_step(&try_stmt.block.stmts)
|
||||
|| try_stmt
|
||||
.handler
|
||||
.as_ref()
|
||||
.map_or(false, |h| self.body_contains_step(&h.body.stmts))
|
||||
|| try_stmt
|
||||
.finalizer
|
||||
.as_ref()
|
||||
.map_or(false, |f| self.body_contains_step(&f.stmts));
|
||||
|
||||
if has_steps {
|
||||
self.errors.push(validation::error_step_in_catch(
|
||||
self.span_line(try_stmt.span),
|
||||
));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn walk_return(&mut self, ret: &ReturnStmt) -> Option<(String, String)> {
|
||||
let line = self.span_line(ret.span);
|
||||
let id = self.next_id();
|
||||
let node_id = self.add_node(DagNode {
|
||||
id: id.clone(),
|
||||
node_type: DagNodeType::Return,
|
||||
label: "return".to_string(),
|
||||
line,
|
||||
});
|
||||
Some((node_id.clone(), node_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract workflow function params (no longer skips ctx)
|
||||
fn extract_ts_params(params: &[swc_ecma_ast::Param], cm: &Lrc<SourceMap>) -> Vec<Param> {
|
||||
let mut result = Vec::new();
|
||||
for param in params {
|
||||
let (name, typ) = match ¶m.pat {
|
||||
Pat::Ident(BindingIdent { id, type_ann, .. }) => {
|
||||
let name = id.sym.to_string();
|
||||
let typ = type_ann.as_ref().map(|ann| {
|
||||
cm.span_to_snippet(ann.type_ann.span())
|
||||
.unwrap_or_else(|_| "unknown".to_string())
|
||||
});
|
||||
(name, typ)
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
result.push(Param { name, typ });
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn parse_ts_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError>> {
|
||||
let cm: Lrc<SourceMap> = Default::default();
|
||||
let fm = cm.new_source_file(FileName::Custom("workflow.ts".into()).into(), code.into());
|
||||
let lexer = Lexer::new(
|
||||
Syntax::Typescript(TsSyntax::default()),
|
||||
Default::default(),
|
||||
StringInput::from(&*fm),
|
||||
None,
|
||||
);
|
||||
|
||||
let mut parser = Parser::new_from(lexer);
|
||||
let module = parser
|
||||
.parse_module()
|
||||
.map_err(|e| vec![CompileError { message: format!("Parse error: {e:?}"), line: 0 }])?;
|
||||
|
||||
// First pass: collect task functions
|
||||
let task_functions = collect_task_functions(&module);
|
||||
|
||||
// Find: export default workflow(async (...) => { ... })
|
||||
// or: export default workflow(async function(...) { ... })
|
||||
let mut workflow_body: Option<(&[Stmt], Vec<Param>)> = None;
|
||||
|
||||
for item in &module.body {
|
||||
// export default workflow(async (...) => { ... })
|
||||
if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultExpr(export)) = item {
|
||||
if let Some(result) = find_workflow_call(&export.expr, &cm) {
|
||||
workflow_body = Some(result);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// const wf = workflow(async (...) => { ... }); export default wf;
|
||||
if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultDecl(export)) = item {
|
||||
if let DefaultDecl::Fn(_) = &export.decl {
|
||||
// `export default async function(...) { ... }` — not wrapped in workflow(), skip
|
||||
}
|
||||
}
|
||||
if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item {
|
||||
for decl in &var_decl.decls {
|
||||
if let Some(init) = &decl.init {
|
||||
if let Some(result) = find_workflow_call(init, &cm) {
|
||||
workflow_body = Some(result);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (stmts, params) = workflow_body.ok_or_else(|| {
|
||||
vec![CompileError {
|
||||
message: "No workflow() wrapped async function found.".to_string(),
|
||||
line: 0,
|
||||
}]
|
||||
})?;
|
||||
|
||||
let source_hash = compute_source_hash(code);
|
||||
|
||||
let mut walker = TsWacWalker::new(cm, task_functions);
|
||||
walker.walk_body(stmts);
|
||||
|
||||
if !walker.errors.is_empty() {
|
||||
return Err(walker.errors);
|
||||
}
|
||||
|
||||
Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash })
|
||||
}
|
||||
|
||||
/// Find workflow(async (...) => { ... }) or workflow(async function(...) { ... })
|
||||
fn find_workflow_call<'a>(expr: &'a Expr, cm: &Lrc<SourceMap>) -> Option<(&'a [Stmt], Vec<Param>)> {
|
||||
if let Expr::Call(call) = expr {
|
||||
// Check if callee is `workflow`
|
||||
let is_workflow = match &call.callee {
|
||||
Callee::Expr(callee_expr) => {
|
||||
if let Expr::Ident(ident) = callee_expr.as_ref() {
|
||||
ident.sym.as_ref() == "workflow"
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
if is_workflow {
|
||||
if let Some(first_arg) = call.args.first() {
|
||||
return extract_async_fn_body(&first_arg.expr, cm);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn extract_async_fn_body<'a>(
|
||||
expr: &'a Expr,
|
||||
cm: &Lrc<SourceMap>,
|
||||
) -> Option<(&'a [Stmt], Vec<Param>)> {
|
||||
match expr {
|
||||
Expr::Arrow(arrow) if arrow.is_async => {
|
||||
let params = extract_arrow_params(&arrow.params, cm);
|
||||
match &*arrow.body {
|
||||
BlockStmtOrExpr::BlockStmt(block) => Some((&block.stmts, params)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
Expr::Fn(fn_expr) if fn_expr.function.is_async => {
|
||||
let params = extract_ts_params(&fn_expr.function.params, cm);
|
||||
fn_expr
|
||||
.function
|
||||
.body
|
||||
.as_ref()
|
||||
.map(|body| (body.stmts.as_slice(), params))
|
||||
}
|
||||
Expr::Paren(p) => extract_async_fn_body(&p.expr, cm),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract arrow function params (no longer skips ctx)
|
||||
fn extract_arrow_params(pats: &[Pat], cm: &Lrc<SourceMap>) -> Vec<Param> {
|
||||
let mut result = Vec::new();
|
||||
for pat in pats {
|
||||
match pat {
|
||||
Pat::Ident(BindingIdent { id, type_ann, .. }) => {
|
||||
let name = id.sym.to_string();
|
||||
let typ = type_ann.as_ref().map(|ann| {
|
||||
cm.span_to_snippet(ann.type_ann.span())
|
||||
.unwrap_or_else(|_| "unknown".to_string())
|
||||
});
|
||||
result.push(Param { name, typ });
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn extract_string_lit(expr: &Expr) -> Option<String> {
|
||||
match expr {
|
||||
Expr::Lit(Lit::Str(s)) => Some(s.value.to_string()),
|
||||
Expr::Tpl(tpl) if tpl.exprs.is_empty() && tpl.quasis.len() == 1 => {
|
||||
tpl.quasis.first().map(|q| q.raw.to_string())
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn compute_source_hash(code: &str) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(code.as_bytes());
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct CompileError {
|
||||
pub message: String,
|
||||
pub line: usize,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for CompileError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "line {}: {}", self.line, self.message)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn error_step_in_try(line: usize) -> CompileError {
|
||||
CompileError {
|
||||
message:
|
||||
"Task calls inside try/except are not allowed. Steps have built-in error handling."
|
||||
.to_string(),
|
||||
line,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn error_step_in_while(line: usize) -> CompileError {
|
||||
CompileError {
|
||||
message: "Task calls inside while loops are not allowed. Use for loops instead."
|
||||
.to_string(),
|
||||
line,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn error_step_in_nested_function(line: usize) -> CompileError {
|
||||
CompileError {
|
||||
message: "Task calls inside nested functions, closures, or lambdas are not allowed."
|
||||
.to_string(),
|
||||
line,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn error_step_in_comprehension(line: usize) -> CompileError {
|
||||
CompileError { message: "Task calls inside comprehensions are not allowed.".to_string(), line }
|
||||
}
|
||||
|
||||
pub fn error_not_async(line: usize) -> CompileError {
|
||||
CompileError { message: "Workflow function must be async.".to_string(), line }
|
||||
}
|
||||
|
||||
pub fn error_missing_await(line: usize) -> CompileError {
|
||||
CompileError {
|
||||
message:
|
||||
"Task calls must be awaited directly or used inside asyncio.gather()/Promise.all()."
|
||||
.to_string(),
|
||||
line,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn error_step_in_catch(line: usize) -> CompileError {
|
||||
CompileError {
|
||||
message:
|
||||
"Task calls inside catch blocks are not allowed. Steps have built-in error handling."
|
||||
.to_string(),
|
||||
line,
|
||||
}
|
||||
}
|
||||
@@ -1,266 +0,0 @@
|
||||
use windmill_parser_wac::dag::DagNodeType;
|
||||
use windmill_parser_wac::python::parse_python_workflow;
|
||||
|
||||
#[test]
|
||||
fn test_simple_sequential_workflow() {
|
||||
let code = r#"
|
||||
import asyncio
|
||||
from wmill import workflow, task
|
||||
|
||||
@task
|
||||
async def extract_data(url: str): ...
|
||||
@task
|
||||
async def load_data(data: list): ...
|
||||
|
||||
@workflow
|
||||
async def my_etl(url: str):
|
||||
raw = await extract_data(url=url)
|
||||
await load_data(data=raw)
|
||||
return {"status": "done"}
|
||||
"#;
|
||||
|
||||
let dag = parse_python_workflow(code).expect("should parse");
|
||||
assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return
|
||||
assert_eq!(dag.edges.len(), 2); // step0->step1, step1->return
|
||||
|
||||
// Check params (url — no ctx to skip)
|
||||
assert_eq!(dag.params.len(), 1);
|
||||
assert_eq!(dag.params[0].name, "url");
|
||||
assert_eq!(dag.params[0].typ.as_deref(), Some("str"));
|
||||
|
||||
// Check first step
|
||||
match &dag.nodes[0].node_type {
|
||||
DagNodeType::Step { name, script } => {
|
||||
assert_eq!(name, "extract_data");
|
||||
assert_eq!(script, "extract_data");
|
||||
}
|
||||
_ => panic!("expected Step node"),
|
||||
}
|
||||
|
||||
// Check second step
|
||||
match &dag.nodes[1].node_type {
|
||||
DagNodeType::Step { name, script } => {
|
||||
assert_eq!(name, "load_data");
|
||||
assert_eq!(script, "load_data");
|
||||
}
|
||||
_ => panic!("expected Step node"),
|
||||
}
|
||||
|
||||
// Check return
|
||||
assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return));
|
||||
|
||||
// Check source hash is non-empty
|
||||
assert!(!dag.source_hash.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parallel_workflow() {
|
||||
let code = r#"
|
||||
import asyncio
|
||||
from wmill import workflow, task
|
||||
|
||||
@task
|
||||
async def extract_data(url: str): ...
|
||||
@task
|
||||
async def clean_data(data: list): ...
|
||||
@task
|
||||
async def compute_stats(data: list): ...
|
||||
@task
|
||||
async def load_to_warehouse(rows: list): ...
|
||||
|
||||
@workflow
|
||||
async def my_etl(url: str):
|
||||
raw = await extract_data(url=url)
|
||||
cleaned, stats = await asyncio.gather(
|
||||
clean_data(data=raw),
|
||||
compute_stats(data=raw),
|
||||
)
|
||||
await load_to_warehouse(rows=cleaned)
|
||||
return {"status": "done"}
|
||||
"#;
|
||||
|
||||
let dag = parse_python_workflow(code).expect("should parse");
|
||||
|
||||
// extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7
|
||||
assert_eq!(dag.nodes.len(), 7);
|
||||
|
||||
assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. }));
|
||||
assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart));
|
||||
assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. }));
|
||||
assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. }));
|
||||
assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd));
|
||||
assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. }));
|
||||
assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_conditional_workflow() {
|
||||
let code = r#"
|
||||
import asyncio
|
||||
from wmill import workflow, task
|
||||
|
||||
@task
|
||||
async def send_alert(msg: str): ...
|
||||
@task
|
||||
async def load_data(): ...
|
||||
|
||||
@workflow
|
||||
async def my_etl(count: int):
|
||||
if count > 100:
|
||||
await send_alert(msg="large")
|
||||
await load_data()
|
||||
return {"done": True}
|
||||
"#;
|
||||
|
||||
let dag = parse_python_workflow(code).expect("should parse");
|
||||
// Branch, notify step, load step, return = 4
|
||||
assert_eq!(dag.nodes.len(), 4);
|
||||
assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. }));
|
||||
assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_for_loop_workflow() {
|
||||
let code = r#"
|
||||
import asyncio
|
||||
from wmill import workflow, task
|
||||
|
||||
@task
|
||||
async def process_item(item: str): ...
|
||||
|
||||
@workflow
|
||||
async def my_etl(items: list):
|
||||
for item in items:
|
||||
await process_item(item=item)
|
||||
return {"done": True}
|
||||
"#;
|
||||
|
||||
let dag = parse_python_workflow(code).expect("should parse");
|
||||
// LoopStart, step, LoopEnd, return = 4
|
||||
assert_eq!(dag.nodes.len(), 4);
|
||||
assert!(matches!(
|
||||
dag.nodes[0].node_type,
|
||||
DagNodeType::LoopStart { .. }
|
||||
));
|
||||
assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. }));
|
||||
assert!(matches!(dag.nodes[2].node_type, DagNodeType::LoopEnd));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_step_in_try() {
|
||||
let code = r#"
|
||||
import asyncio
|
||||
from wmill import workflow, task
|
||||
|
||||
@task
|
||||
async def extract_data(): ...
|
||||
|
||||
@workflow
|
||||
async def my_etl():
|
||||
try:
|
||||
await extract_data()
|
||||
except Exception:
|
||||
pass
|
||||
"#;
|
||||
|
||||
let result = parse_python_workflow(code);
|
||||
assert!(result.is_err());
|
||||
let errors = result.unwrap_err();
|
||||
assert!(errors[0].message.contains("try/except"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_step_in_while() {
|
||||
let code = r#"
|
||||
import asyncio
|
||||
from wmill import workflow, task
|
||||
|
||||
@task
|
||||
async def extract_data(): ...
|
||||
|
||||
@workflow
|
||||
async def my_etl():
|
||||
while True:
|
||||
await extract_data()
|
||||
"#;
|
||||
|
||||
let result = parse_python_workflow(code);
|
||||
assert!(result.is_err());
|
||||
let errors = result.unwrap_err();
|
||||
assert!(errors[0].message.contains("while"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_non_async() {
|
||||
let code = r#"
|
||||
from wmill import workflow
|
||||
|
||||
@workflow
|
||||
def my_etl():
|
||||
pass
|
||||
"#;
|
||||
|
||||
let result = parse_python_workflow(code);
|
||||
assert!(result.is_err());
|
||||
let errors = result.unwrap_err();
|
||||
assert!(errors[0].message.contains("async"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_missing_await() {
|
||||
let code = r#"
|
||||
import asyncio
|
||||
from wmill import workflow, task
|
||||
|
||||
@task
|
||||
async def extract_data(): ...
|
||||
|
||||
@workflow
|
||||
async def my_etl():
|
||||
extract_data()
|
||||
"#;
|
||||
|
||||
let result = parse_python_workflow(code);
|
||||
assert!(result.is_err());
|
||||
let errors = result.unwrap_err();
|
||||
assert!(errors[0].message.contains("awaited"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_workflow_function() {
|
||||
let code = r#"
|
||||
async def my_func():
|
||||
pass
|
||||
"#;
|
||||
|
||||
let result = parse_python_workflow(code);
|
||||
assert!(result.is_err());
|
||||
let errors = result.unwrap_err();
|
||||
assert!(errors[0].message.contains("No @workflow"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_task_with_external_path() {
|
||||
let code = r#"
|
||||
import asyncio
|
||||
from wmill import workflow, task
|
||||
|
||||
@task(path="f/external_script")
|
||||
async def run_external(x: int): ...
|
||||
|
||||
@workflow
|
||||
async def my_wf(x: int):
|
||||
result = await run_external(x=x)
|
||||
return result
|
||||
"#;
|
||||
|
||||
let dag = parse_python_workflow(code).expect("should parse");
|
||||
assert_eq!(dag.nodes.len(), 2); // 1 step + 1 return (bare `return` is not a step node but walk_return creates one)
|
||||
match &dag.nodes[0].node_type {
|
||||
DagNodeType::Step { name, script } => {
|
||||
assert_eq!(name, "run_external");
|
||||
assert_eq!(script, "f/external_script");
|
||||
}
|
||||
_ => panic!("expected Step node"),
|
||||
}
|
||||
}
|
||||
@@ -1,245 +0,0 @@
|
||||
use windmill_parser_wac::dag::DagNodeType;
|
||||
use windmill_parser_wac::typescript::parse_ts_workflow;
|
||||
|
||||
#[test]
|
||||
fn test_simple_sequential_ts_workflow() {
|
||||
let code = r#"
|
||||
import { workflow, task } from "windmill-client";
|
||||
|
||||
const extract_data = task(async (url: string) => {});
|
||||
const load_data = task(async (data: any) => {});
|
||||
|
||||
export default workflow(async (url: string) => {
|
||||
const raw = await extract_data(url);
|
||||
await load_data(raw);
|
||||
return { status: "done" };
|
||||
});
|
||||
"#;
|
||||
|
||||
let dag = parse_ts_workflow(code).expect("should parse");
|
||||
assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return
|
||||
assert_eq!(dag.edges.len(), 2);
|
||||
|
||||
// Check params (url — no ctx to skip)
|
||||
assert_eq!(dag.params.len(), 1);
|
||||
assert_eq!(dag.params[0].name, "url");
|
||||
assert_eq!(dag.params[0].typ.as_deref(), Some("string"));
|
||||
|
||||
match &dag.nodes[0].node_type {
|
||||
DagNodeType::Step { name, script } => {
|
||||
assert_eq!(name, "extract_data");
|
||||
assert_eq!(script, "extract_data");
|
||||
}
|
||||
_ => panic!("expected Step node"),
|
||||
}
|
||||
|
||||
match &dag.nodes[1].node_type {
|
||||
DagNodeType::Step { name, script } => {
|
||||
assert_eq!(name, "load_data");
|
||||
assert_eq!(script, "load_data");
|
||||
}
|
||||
_ => panic!("expected Step node"),
|
||||
}
|
||||
|
||||
assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return));
|
||||
assert!(!dag.source_hash.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parallel_ts_workflow() {
|
||||
let code = r#"
|
||||
import { workflow, task } from "windmill-client";
|
||||
|
||||
const extract_data = task(async (url: string) => {});
|
||||
const clean_data = task(async (data: any) => {});
|
||||
const compute_stats = task(async (data: any) => {});
|
||||
const load_to_warehouse = task(async (rows: any) => {});
|
||||
|
||||
export default workflow(async (url: string) => {
|
||||
const raw = await extract_data(url);
|
||||
|
||||
const [cleaned, stats] = await Promise.all([
|
||||
clean_data(raw),
|
||||
compute_stats(raw),
|
||||
]);
|
||||
|
||||
await load_to_warehouse(cleaned);
|
||||
return { status: "done", rows: stats.rowCount };
|
||||
});
|
||||
"#;
|
||||
|
||||
let dag = parse_ts_workflow(code).expect("should parse");
|
||||
// extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7
|
||||
assert_eq!(dag.nodes.len(), 7);
|
||||
|
||||
assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. }));
|
||||
assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart));
|
||||
assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. }));
|
||||
assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. }));
|
||||
assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd));
|
||||
assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. }));
|
||||
assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_conditional_ts_workflow() {
|
||||
let code = r#"
|
||||
import { workflow, task } from "windmill-client";
|
||||
|
||||
const send_alert = task(async (msg: string) => {});
|
||||
const load_data = task(async () => {});
|
||||
|
||||
export default workflow(async (count: number) => {
|
||||
if (count > 100) {
|
||||
await send_alert("large");
|
||||
}
|
||||
await load_data();
|
||||
return { done: true };
|
||||
});
|
||||
"#;
|
||||
|
||||
let dag = parse_ts_workflow(code).expect("should parse");
|
||||
// Branch, notify, load, return = 4
|
||||
assert_eq!(dag.nodes.len(), 4);
|
||||
assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_for_of_ts_workflow() {
|
||||
let code = r#"
|
||||
import { workflow, task } from "windmill-client";
|
||||
|
||||
const process_item = task(async (item: string) => {});
|
||||
|
||||
export default workflow(async (items: string[]) => {
|
||||
for (const item of items) {
|
||||
await process_item(item);
|
||||
}
|
||||
return { done: true };
|
||||
});
|
||||
"#;
|
||||
|
||||
let dag = parse_ts_workflow(code).expect("should parse");
|
||||
// LoopStart, step, LoopEnd, return = 4
|
||||
assert_eq!(dag.nodes.len(), 4);
|
||||
assert!(matches!(
|
||||
dag.nodes[0].node_type,
|
||||
DagNodeType::LoopStart { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_step_in_try_catch() {
|
||||
let code = r#"
|
||||
import { workflow, task } from "windmill-client";
|
||||
|
||||
const extract_data = task(async () => {});
|
||||
|
||||
export default workflow(async () => {
|
||||
try {
|
||||
await extract_data();
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
});
|
||||
"#;
|
||||
|
||||
let result = parse_ts_workflow(code);
|
||||
assert!(result.is_err());
|
||||
let errors = result.unwrap_err();
|
||||
assert!(errors[0].message.contains("catch"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_step_in_while_ts() {
|
||||
let code = r#"
|
||||
import { workflow, task } from "windmill-client";
|
||||
|
||||
const extract_data = task(async () => {});
|
||||
|
||||
export default workflow(async () => {
|
||||
while (true) {
|
||||
await extract_data();
|
||||
}
|
||||
});
|
||||
"#;
|
||||
|
||||
let result = parse_ts_workflow(code);
|
||||
assert!(result.is_err());
|
||||
let errors = result.unwrap_err();
|
||||
assert!(errors[0].message.contains("while"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_missing_await_ts() {
|
||||
let code = r#"
|
||||
import { workflow, task } from "windmill-client";
|
||||
|
||||
const extract_data = task(async () => {});
|
||||
|
||||
export default workflow(async () => {
|
||||
extract_data();
|
||||
});
|
||||
"#;
|
||||
|
||||
let result = parse_ts_workflow(code);
|
||||
assert!(result.is_err());
|
||||
let errors = result.unwrap_err();
|
||||
assert!(errors[0].message.contains("awaited"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_workflow_wrapper() {
|
||||
let code = r#"
|
||||
export default async function main(ctx: any) {
|
||||
return {};
|
||||
}
|
||||
"#;
|
||||
|
||||
let result = parse_ts_workflow(code);
|
||||
assert!(result.is_err());
|
||||
let errors = result.unwrap_err();
|
||||
assert!(errors[0].message.contains("No workflow()"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_variable_declaration_with_step() {
|
||||
let code = r#"
|
||||
import { workflow, task } from "windmill-client";
|
||||
|
||||
const compute = task(async () => {});
|
||||
|
||||
export default workflow(async () => {
|
||||
const result = await compute();
|
||||
return result;
|
||||
});
|
||||
"#;
|
||||
|
||||
let dag = parse_ts_workflow(code).expect("should parse");
|
||||
assert_eq!(dag.nodes.len(), 2); // step + return
|
||||
assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_task_with_external_path() {
|
||||
let code = r#"
|
||||
import { workflow, task } from "windmill-client";
|
||||
|
||||
const run_external = task("f/external_script", async (x: number) => {});
|
||||
|
||||
export default workflow(async (x: number) => {
|
||||
const result = await run_external(x);
|
||||
return result;
|
||||
});
|
||||
"#;
|
||||
|
||||
let dag = parse_ts_workflow(code).expect("should parse");
|
||||
assert_eq!(dag.nodes.len(), 2); // step + return
|
||||
match &dag.nodes[0].node_type {
|
||||
DagNodeType::Step { name, script } => {
|
||||
assert_eq!(name, "run_external");
|
||||
assert_eq!(script, "f/external_script");
|
||||
}
|
||||
_ => panic!("expected Step node"),
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,6 @@ csharp-parser = [ "dep:windmill-parser-csharp"]
|
||||
nu-parser = [ "dep:windmill-parser-nu"]
|
||||
java-parser = [ "dep:windmill-parser-java"]
|
||||
ruby-parser = [ "dep:windmill-parser-ruby"]
|
||||
wac-parser = [ "dep:windmill-parser-wac"]
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
@@ -56,7 +55,6 @@ windmill-parser-csharp = { workspace = true, optional = true }
|
||||
windmill-parser-nu = { workspace = true, optional = true }
|
||||
windmill-parser-java = { workspace = true, optional = true }
|
||||
windmill-parser-ruby = { workspace = true, optional = true }
|
||||
windmill-parser-wac = { workspace = true, optional = true }
|
||||
wasm-bindgen.workspace = true
|
||||
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -56,12 +56,6 @@ const targets = [
|
||||
features: "ruby-parser",
|
||||
env: "tree-sitter",
|
||||
},
|
||||
{
|
||||
ident: "wac",
|
||||
desc: "Workflow-as-Code",
|
||||
features: "wac-parser",
|
||||
env: "default",
|
||||
},
|
||||
# ^^^ Add new entry here ^^^
|
||||
];
|
||||
# NOTE: This is legacy command for building all, but it is not more used
|
||||
|
||||
@@ -223,11 +223,4 @@ pub fn parse_assets_ansible(code: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "wac-parser")]
|
||||
#[wasm_bindgen]
|
||||
pub fn parse_workflow_as_code(code: &str, language: &str) -> String {
|
||||
let result = windmill_parser_wac::parse_workflow(code, language);
|
||||
serde_json::to_string(&result).unwrap_or_else(|_| "{\"type\": \"error\"}".to_string())
|
||||
}
|
||||
|
||||
// for related places search: ADD_NEW_LANG
|
||||
|
||||
+80
-2
@@ -61,8 +61,9 @@ use windmill_common::{
|
||||
MODE_AND_ADDONS,
|
||||
},
|
||||
worker::{
|
||||
is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR,
|
||||
HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP,
|
||||
is_native_mode_from_env, reload_custom_tags_setting, Connection, HttpClient, HUB_CACHE_DIR,
|
||||
HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, USES_BATCH_HTTP_PULL, WINDMILL_DIR,
|
||||
WORKER_GROUP,
|
||||
},
|
||||
KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED,
|
||||
};
|
||||
@@ -920,6 +921,20 @@ Windmill Community Edition {GIT_VERSION}
|
||||
default_base_internal_url.clone()
|
||||
};
|
||||
|
||||
// BATCH_PULL_URL: explicit URL for native workers to pull jobs via HTTP.
|
||||
// In standalone mode (server_mode=true), defaults to the local server.
|
||||
let batch_pull_url: Option<String> = if is_native_mode_from_env() {
|
||||
if let Ok(url) = std::env::var("BATCH_PULL_URL") {
|
||||
Some(url)
|
||||
} else if server_mode {
|
||||
Some(default_base_internal_url.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
initial_load(
|
||||
&conn,
|
||||
killpill_tx.clone(),
|
||||
@@ -1130,6 +1145,30 @@ Windmill Community Edition {GIT_VERSION}
|
||||
)?;
|
||||
let mut workers = vec![];
|
||||
|
||||
// For native workers, create a self-signed JWT for batch pulling via HTTP.
|
||||
// Enabled when BATCH_PULL_URL is set (explicitly or auto-detected in standalone mode).
|
||||
let batch_pull_client = if let Some(ref pull_url) = batch_pull_url {
|
||||
match create_native_batch_pull_client(pull_url).await {
|
||||
Ok(client) => {
|
||||
tracing::info!(
|
||||
"Native batch pull client created for HTTP pull at {}",
|
||||
pull_url
|
||||
);
|
||||
USES_BATCH_HTTP_PULL
|
||||
.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
Some(client)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to create native batch pull client, falling back to SQL pull: {e:#}"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
for i in 0..num_workers {
|
||||
let suffix = if i == 0 && first_suffix.is_some() {
|
||||
first_suffix.as_ref().unwrap().clone()
|
||||
@@ -1153,6 +1192,7 @@ Windmill Community Edition {GIT_VERSION}
|
||||
WORKER_GROUP.as_str(),
|
||||
&suffix,
|
||||
),
|
||||
batch_pull_client: batch_pull_client.clone(),
|
||||
};
|
||||
workers.push(worker_conn);
|
||||
}
|
||||
@@ -1766,6 +1806,7 @@ fn display_config(envs: &[&str]) {
|
||||
pub struct WorkerConn {
|
||||
conn: Connection,
|
||||
worker_name: String,
|
||||
batch_pull_client: Option<HttpClient>,
|
||||
}
|
||||
|
||||
pub async fn run_workers(
|
||||
@@ -1836,6 +1877,7 @@ pub async fn run_workers(
|
||||
let wk_conf = &workers[i as usize - 1];
|
||||
let conn1 = wk_conf.conn.clone();
|
||||
let worker_name = wk_conf.worker_name.clone();
|
||||
let batch_pull_client = wk_conf.batch_pull_client.clone();
|
||||
WORKERS_NAMES.write().await.push(worker_name.clone());
|
||||
let ip = ip.clone();
|
||||
let rx = killpill_rxs.pop().unwrap();
|
||||
@@ -1858,6 +1900,7 @@ pub async fn run_workers(
|
||||
rx,
|
||||
tx,
|
||||
&base_internal_url,
|
||||
batch_pull_client.as_ref(),
|
||||
);
|
||||
|
||||
// #[cfg(tokio_unstable)]
|
||||
@@ -1876,6 +1919,41 @@ pub async fn run_workers(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create an HTTP client for native workers to pull jobs from the local server's batch buffer.
|
||||
/// Self-signs a JWT with native_mode=true using the same JWT secret the server uses.
|
||||
async fn create_native_batch_pull_client(base_internal_url: &str) -> anyhow::Result<HttpClient> {
|
||||
use windmill_common::agent_workers::{build_agent_http_client, AGENT_JWT_PREFIX};
|
||||
use windmill_common::jwt::encode_with_internal_secret;
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct NativeAgentAuth {
|
||||
worker_group: String,
|
||||
tags: Vec<String>,
|
||||
native_mode: Option<bool>,
|
||||
exp: usize,
|
||||
}
|
||||
|
||||
let worker_config = windmill_common::worker::WORKER_CONFIG.read().await;
|
||||
let tags = worker_config.worker_tags.clone();
|
||||
drop(worker_config);
|
||||
|
||||
// Token expires in 30 days — renewed on restart
|
||||
let exp = (chrono::Utc::now() + chrono::Duration::days(30)).timestamp() as usize;
|
||||
|
||||
let claims = NativeAgentAuth {
|
||||
worker_group: WORKER_GROUP.to_string(),
|
||||
tags,
|
||||
native_mode: Some(true),
|
||||
exp,
|
||||
};
|
||||
|
||||
let jwt = encode_with_internal_secret(claims).await?;
|
||||
let token = format!("{}{}", AGENT_JWT_PREFIX, jwt);
|
||||
|
||||
let suffix = create_default_worker_suffix(&HOSTNAME);
|
||||
Ok(build_agent_http_client(&suffix, &token, base_internal_url))
|
||||
}
|
||||
|
||||
async fn send_delayed_killpill(tx: &KillpillSender, mut max_delay_secs: u64, context: &str) {
|
||||
if max_delay_secs == 0 {
|
||||
max_delay_secs = 1;
|
||||
|
||||
@@ -174,7 +174,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work
|
||||
windmill_migrations: name(text), created_at(ts)
|
||||
worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint)
|
||||
FK: (workspace_id) -> workspace(id)
|
||||
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[])
|
||||
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), native_mode(bool), uses_batch_http_pull(bool)
|
||||
workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char)
|
||||
FK: (parent_workspace_id) -> workspace(id)
|
||||
workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts)
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# E2E test for WAC v2 workflow-as-code suspend/resume lifecycle
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:8070}"
|
||||
TOKEN="${WM_TOKEN:-}"
|
||||
WORKSPACE="dev"
|
||||
TIMEOUT=60 # seconds
|
||||
|
||||
# Get auth token if not set
|
||||
if [ -z "$TOKEN" ]; then
|
||||
TOKEN=$(curl -s "${BASE_URL}/api/auth/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"email":"admin@windmill.dev","password":"changeme"}' | tr -d '"')
|
||||
fi
|
||||
|
||||
echo "=== WAC v2 E2E Test ==="
|
||||
echo "Base URL: $BASE_URL"
|
||||
echo ""
|
||||
|
||||
WAC_CODE='import { task, workflow } from "windmill-client@1.999.19";
|
||||
|
||||
const double = task(async (x: number): Promise<number> => {
|
||||
console.log("[double] START at " + new Date().toISOString());
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
console.log("[double] END at " + new Date().toISOString());
|
||||
return x * 2;
|
||||
});
|
||||
|
||||
const increment = task(async (x: number): Promise<number> => {
|
||||
console.log("[increment] START at " + new Date().toISOString());
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
console.log("[increment] END at " + new Date().toISOString());
|
||||
return x + 1;
|
||||
});
|
||||
|
||||
export const main = workflow(async (x: number = 10) => {
|
||||
const [doubled, incremented] = await Promise.all([
|
||||
double(x),
|
||||
increment(x),
|
||||
]);
|
||||
const final_result = await double(incremented);
|
||||
return { doubled, incremented, final_result };
|
||||
});'
|
||||
|
||||
echo "Step 1: Submitting preview job..."
|
||||
JOB_ID=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs/run/preview" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-d "$(jq -n --arg code "$WAC_CODE" '{
|
||||
content: $code,
|
||||
language: "bun",
|
||||
args: {"x": 10}
|
||||
}')" | tr -d '"')
|
||||
|
||||
echo "Job ID: $JOB_ID"
|
||||
|
||||
if [ -z "$JOB_ID" ] || [ "$JOB_ID" = "null" ]; then
|
||||
echo "FAIL: Could not create job"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 2: Polling for completion (timeout: ${TIMEOUT}s)..."
|
||||
|
||||
START=$SECONDS
|
||||
LAST_STATUS=""
|
||||
while true; do
|
||||
ELAPSED=$((SECONDS - START))
|
||||
if [ $ELAPSED -gt $TIMEOUT ]; then
|
||||
echo "FAIL: Timed out after ${TIMEOUT}s"
|
||||
# Dump job state for debugging
|
||||
echo ""
|
||||
echo "=== Debug info ==="
|
||||
echo "Parent job queue state:"
|
||||
source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local
|
||||
psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until, canceled_by FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null
|
||||
echo "Child jobs:"
|
||||
psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at" 2>/dev/null
|
||||
echo "Completed children:"
|
||||
psql "$DATABASE_URL" -c "SELECT id FROM completed_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null
|
||||
echo "Checkpoint:"
|
||||
psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null
|
||||
echo "Total child count:"
|
||||
psql "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check completed job
|
||||
RESULT=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \
|
||||
-H "Authorization: Bearer $TOKEN" 2>/dev/null)
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \
|
||||
-H "Authorization: Bearer $TOKEN" 2>/dev/null)
|
||||
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
echo "Job completed in ${ELAPSED}s!"
|
||||
echo ""
|
||||
echo "Step 3: Checking result..."
|
||||
echo "Result: $RESULT"
|
||||
|
||||
# Validate
|
||||
DOUBLED=$(echo "$RESULT" | jq -r '.doubled // empty')
|
||||
INCREMENTED=$(echo "$RESULT" | jq -r '.incremented // empty')
|
||||
FINAL=$(echo "$RESULT" | jq -r '.final_result // empty')
|
||||
|
||||
PASS=true
|
||||
if [ "$DOUBLED" != "20" ]; then
|
||||
echo "FAIL: doubled = $DOUBLED, expected 20"
|
||||
PASS=false
|
||||
fi
|
||||
if [ "$INCREMENTED" != "11" ]; then
|
||||
echo "FAIL: incremented = $INCREMENTED, expected 11"
|
||||
PASS=false
|
||||
fi
|
||||
if [ "$FINAL" != "22" ]; then
|
||||
echo "FAIL: final_result = $FINAL, expected 22"
|
||||
PASS=false
|
||||
fi
|
||||
|
||||
if $PASS; then
|
||||
echo "PASS: All values correct!"
|
||||
# Check no excessive child jobs
|
||||
source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null
|
||||
CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null)
|
||||
echo "Total child jobs created: $CHILD_COUNT (expected: 3)"
|
||||
if [ "$CHILD_COUNT" -gt "3" ]; then
|
||||
echo "WARN: More children than expected ($CHILD_COUNT > 3)"
|
||||
fi
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Show progress
|
||||
STATUS=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/get/${JOB_ID}" \
|
||||
-H "Authorization: Bearer $TOKEN" 2>/dev/null | jq -r '.type // empty')
|
||||
if [ "$STATUS" != "$LAST_STATUS" ]; then
|
||||
echo " [${ELAPSED}s] Status: $STATUS"
|
||||
LAST_STATUS="$STATUS"
|
||||
fi
|
||||
|
||||
# Check for runaway child creation
|
||||
source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null
|
||||
CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null)
|
||||
if [ "$CHILD_COUNT" -gt "10" ]; then
|
||||
echo "FAIL: Runaway child creation detected! $CHILD_COUNT children (expected 3)"
|
||||
echo ""
|
||||
echo "=== Debug info ==="
|
||||
psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null
|
||||
psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at LIMIT 20" 2>/dev/null
|
||||
psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
@@ -241,6 +241,7 @@ fn spawn_workers(
|
||||
rx,
|
||||
tx2,
|
||||
&base_internal_url,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
};
|
||||
|
||||
@@ -3548,170 +3548,3 @@ async fn test_flow_substep_tag_availability_check(db: Pool<Postgres>) -> anyhow:
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "quickjs", feature = "python"))]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_stop_after_all_iters_if_bad_expr_parallel_branchall(
|
||||
db: Pool<Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
|
||||
let port = 123;
|
||||
let flow: FlowValue = serde_json::from_value(serde_json::json!({
|
||||
"modules": [
|
||||
{
|
||||
"id": "a",
|
||||
"value": {
|
||||
"branches": [
|
||||
{"modules": [{
|
||||
"id": "b",
|
||||
"value": {
|
||||
"input_transforms": { "n": { "type": "javascript", "expr": "flow_input.n" } },
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": "def main(n): return n",
|
||||
},
|
||||
}]}
|
||||
],
|
||||
"type": "branchall",
|
||||
"parallel": true,
|
||||
},
|
||||
"stop_after_all_iters_if": {
|
||||
"expr": "invalid!!!syntax",
|
||||
"skip_if_stopped": false,
|
||||
},
|
||||
},
|
||||
],
|
||||
}))
|
||||
.unwrap();
|
||||
let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
|
||||
|
||||
let cjob = RunJob::from(job)
|
||||
.arg("n", json!(42))
|
||||
.run_until_complete(&db, false, port)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
!cjob.success,
|
||||
"flow should fail when stop_after_all_iters_if has bad expression"
|
||||
);
|
||||
|
||||
let result = cjob.json_result().unwrap();
|
||||
let error_msg = result["error"]["message"].as_str().unwrap_or("");
|
||||
assert!(
|
||||
error_msg.contains("stop_after_all_iters_if"),
|
||||
"error should mention stop_after_all_iters_if, got: {error_msg}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "quickjs", feature = "python"))]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_stop_after_all_iters_if_bad_expr_parallel_forloop(
|
||||
db: Pool<Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
|
||||
let port = 123;
|
||||
let flow: FlowValue = serde_json::from_value(serde_json::json!({
|
||||
"modules": [
|
||||
{
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "forloopflow",
|
||||
"iterator": { "type": "javascript", "expr": "result.items" },
|
||||
"skip_failures": false,
|
||||
"parallel": true,
|
||||
"modules": [{
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"n": { "type": "javascript", "expr": "flow_input.iter.value" },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": "def main(n): return n",
|
||||
},
|
||||
}],
|
||||
},
|
||||
"stop_after_all_iters_if": {
|
||||
"expr": "invalid!!!syntax",
|
||||
"skip_if_stopped": false,
|
||||
},
|
||||
},
|
||||
],
|
||||
}))
|
||||
.unwrap();
|
||||
let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
|
||||
|
||||
let cjob = RunJob::from(job)
|
||||
.arg("items", json!([1, 2, 3]))
|
||||
.run_until_complete(&db, false, port)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
!cjob.success,
|
||||
"flow should fail when stop_after_all_iters_if has bad expression"
|
||||
);
|
||||
|
||||
let result = cjob.json_result().unwrap();
|
||||
let error_msg = result["error"]["message"].as_str().unwrap_or("");
|
||||
assert!(
|
||||
error_msg.contains("stop_after_all_iters_if"),
|
||||
"error should mention stop_after_all_iters_if, got: {error_msg}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "quickjs", feature = "python"))]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_results_length_in_input_transform(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
// Step a returns a list, step b accesses results.a.length via input transform.
|
||||
// This tests that the handle_full_regex fast path falls through to QuickJS
|
||||
// when the SQL JSON path operator can't resolve JS properties like .length.
|
||||
let flow: FlowValue = serde_json::from_value(json!({
|
||||
"modules": [
|
||||
{
|
||||
"id": "a",
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": "def main(): return [10, 20, 30]",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "b",
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"v": { "type": "javascript", "expr": "results.a.length" },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": "def main(v): return v",
|
||||
},
|
||||
},
|
||||
],
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let result =
|
||||
RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None })
|
||||
.run_until_complete(&db, false, port)
|
||||
.await
|
||||
.json_result()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
result,
|
||||
json!(3),
|
||||
"results.a.length should resolve to 3, not null"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -19,7 +19,10 @@ use windmill_common::DB;
|
||||
use axum::Router;
|
||||
|
||||
#[cfg(not(feature = "private"))]
|
||||
pub fn global_service(_job_completed_tx: windmill_worker::JobCompletedSender) -> Router {
|
||||
pub fn global_service(
|
||||
_job_completed_tx: windmill_worker::JobCompletedSender,
|
||||
_batch_buffer: Option<()>,
|
||||
) -> Router {
|
||||
Router::new()
|
||||
}
|
||||
|
||||
@@ -31,6 +34,7 @@ pub fn workspaced_service(
|
||||
Router,
|
||||
Vec<tokio::task::JoinHandle<()>>,
|
||||
Option<windmill_worker::JobCompletedSender>,
|
||||
Option<()>,
|
||||
) {
|
||||
use windmill_common::worker::Connection;
|
||||
use windmill_worker::JobCompletedSender;
|
||||
@@ -40,7 +44,7 @@ pub fn workspaced_service(
|
||||
|
||||
let router = Router::new();
|
||||
|
||||
(router, vec![], Some(job_completed_tx))
|
||||
(router, vec![], Some(job_completed_tx), None)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "private"))]
|
||||
|
||||
@@ -316,11 +316,7 @@ pub async fn set_global_setting_internal(
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
"Set global setting {} to {}",
|
||||
key,
|
||||
instance_config::format_setting_value(&key, &v)
|
||||
);
|
||||
tracing::info!("Set global setting {} to {}", key, v);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -174,7 +174,6 @@ aws-sdk-bedrock = { workspace = true, optional = true }
|
||||
aws-sdk-bedrockruntime = { workspace = true, optional = true }
|
||||
aws-smithy-types = { workspace = true, optional = true }
|
||||
async-trait.workspace = true
|
||||
eventsource-stream.workspace = true
|
||||
windmill-jseval.workspace = true
|
||||
tar.workspace = true
|
||||
flate2.workspace = true
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: "3.0.3"
|
||||
|
||||
info:
|
||||
version: 1.652.0
|
||||
version: 1.651.1
|
||||
title: Windmill API
|
||||
|
||||
contact:
|
||||
@@ -18803,6 +18803,7 @@ components:
|
||||
required:
|
||||
- path
|
||||
- summary
|
||||
- description
|
||||
- content
|
||||
- language
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ const AI_TIMEOUT_MAX_SECS: u64 = 86400; // 24 hours
|
||||
const AI_TIMEOUT_DEFAULT_SECS: u64 = 3600; // 1 hour
|
||||
const HTTP_POOL_MAX_IDLE_PER_HOST: usize = 10;
|
||||
const HTTP_POOL_IDLE_TIMEOUT_SECS: u64 = 90;
|
||||
pub(crate) const KEEPALIVE_INTERVAL_SECS: u64 = 15;
|
||||
const KEEPALIVE_INTERVAL_SECS: u64 = 15;
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
/// AI request timeout in seconds.
|
||||
@@ -87,7 +87,7 @@ lazy_static::lazy_static! {
|
||||
}
|
||||
};
|
||||
|
||||
pub(crate) static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new()
|
||||
static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new()
|
||||
.timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS))
|
||||
.pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST)
|
||||
.pool_idle_timeout(Some(std::time::Duration::from_secs(HTTP_POOL_IDLE_TIMEOUT_SECS)))
|
||||
@@ -378,7 +378,12 @@ impl AIRequestConfig {
|
||||
let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some();
|
||||
let is_google_ai = matches!(provider, AIProvider::GoogleAI);
|
||||
|
||||
let base_url = base_url.to_string();
|
||||
// GoogleAI uses OpenAI-compatible endpoint in the proxy (for the chat), but not for the ai agent
|
||||
let base_url = if is_google_ai {
|
||||
format!("{}/openai", base_url)
|
||||
} else {
|
||||
base_url.to_string()
|
||||
};
|
||||
let base_url = base_url.as_str();
|
||||
|
||||
// Build URL based on provider
|
||||
@@ -423,9 +428,6 @@ impl AIRequestConfig {
|
||||
if let Some(api_key) = self.api_key {
|
||||
if is_azure {
|
||||
request = request.header("api-key", api_key.clone())
|
||||
} else if is_google_ai {
|
||||
// Native Gemini API uses x-goog-api-key, not Authorization: Bearer
|
||||
request = request.header("x-goog-api-key", api_key.clone())
|
||||
} else {
|
||||
request = request.header("authorization", format!("Bearer {}", api_key.clone()))
|
||||
}
|
||||
@@ -609,7 +611,7 @@ fn is_sse_response(headers: &HeaderMap) -> bool {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub(crate) fn inject_keepalives<S>(
|
||||
fn inject_keepalives<S>(
|
||||
upstream: S,
|
||||
interval: Duration,
|
||||
) -> impl futures::Stream<Item = std::result::Result<Bytes, reqwest::Error>>
|
||||
@@ -828,36 +830,6 @@ async fn proxy(
|
||||
ai_path = chat_path;
|
||||
}
|
||||
|
||||
// Handle GoogleAI (Gemini) using the native Gemini API
|
||||
if matches!(provider, AIProvider::GoogleAI) {
|
||||
let api_key = request_config.api_key.as_deref().unwrap_or("");
|
||||
let base_url = request_config.base_url.trim_end_matches('/');
|
||||
|
||||
let mut tx = db.begin().await?;
|
||||
audit_log(
|
||||
&mut *tx,
|
||||
&authed,
|
||||
"ai.request",
|
||||
ActionKind::Execute,
|
||||
&w_id,
|
||||
Some(&authed.email),
|
||||
Some([("ai_config_path", &format!("{:?}", ai_path)[..])].into()),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
return match ai_path.as_str() {
|
||||
"chat/completions" => {
|
||||
crate::google::handle_google_ai_chat(&body, api_key, base_url).await
|
||||
}
|
||||
"models" => crate::google::handle_google_ai_models(api_key, base_url).await,
|
||||
_ => Err(Error::BadRequest(format!(
|
||||
"Unsupported Google AI path: {}",
|
||||
ai_path
|
||||
))),
|
||||
};
|
||||
}
|
||||
|
||||
// Handle Bedrock-specific logic when the feature is enabled
|
||||
#[cfg(feature = "bedrock")]
|
||||
{
|
||||
|
||||
@@ -284,9 +284,6 @@ pub async fn migrate(
|
||||
20260207000004,
|
||||
];
|
||||
for m in migrator.migrations.iter() {
|
||||
if m.migration_type.is_down_migration() {
|
||||
continue;
|
||||
}
|
||||
if potentially_stale.contains(&m.version) {
|
||||
if let Err(err) =
|
||||
sqlx::query("DELETE FROM _sqlx_migrations WHERE version = $1 AND checksum != $2")
|
||||
|
||||
@@ -1,306 +0,0 @@
|
||||
//! Google AI (Gemini API) handler for the AI chat proxy.
|
||||
//!
|
||||
//! Handles POST `chat/completions` requests using the native Gemini API,
|
||||
//! converting from/to OpenAI format so the existing frontend parsers continue to work.
|
||||
//!
|
||||
//! Used by `windmill-api/src/ai.rs` when the provider is `GoogleAI`.
|
||||
//! Shared conversion logic lives in `windmill_common::ai_google`.
|
||||
|
||||
use axum::body::Body;
|
||||
use bytes::Bytes;
|
||||
use eventsource_stream::Eventsource;
|
||||
use futures::StreamExt;
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use windmill_common::{
|
||||
ai_google::{
|
||||
gemini_event_to_openai_sse_chunks, gemini_response_to_openai, openai_messages_to_gemini,
|
||||
parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google,
|
||||
GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiTextRequest, GeminiTool,
|
||||
},
|
||||
ai_types::OpenAIMessage,
|
||||
error::{Error, Result},
|
||||
};
|
||||
|
||||
use crate::ai::{inject_keepalives, HTTP_CLIENT, KEEPALIVE_INTERVAL_SECS};
|
||||
|
||||
// ============================================================================
|
||||
// Request type (OpenAI format received from the frontend)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct ChatRequest {
|
||||
model: String,
|
||||
messages: Vec<OpenAIMessage>,
|
||||
#[serde(default)]
|
||||
stream: bool,
|
||||
#[serde(default)]
|
||||
temperature: Option<f32>,
|
||||
#[serde(default)]
|
||||
max_tokens: Option<u32>,
|
||||
#[serde(default)]
|
||||
tools: Option<Vec<ChatRequestTool>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct ChatRequestTool {
|
||||
function: ChatRequestToolFunction,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct ChatRequestToolFunction {
|
||||
name: String,
|
||||
#[serde(default)]
|
||||
description: Option<String>,
|
||||
#[serde(default)]
|
||||
parameters: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Public handler
|
||||
// ============================================================================
|
||||
|
||||
/// Handle a `chat/completions` POST request using the native Gemini API.
|
||||
///
|
||||
/// Converts the incoming OpenAI-format body to a `GeminiTextRequest`, sends it
|
||||
/// to the appropriate Gemini endpoint, and converts the response back to the
|
||||
/// OpenAI SSE or JSON format that the frontend expects.
|
||||
pub async fn handle_google_ai_chat(
|
||||
body: &Bytes,
|
||||
api_key: &str,
|
||||
base_url: &str,
|
||||
) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
|
||||
let request: ChatRequest = serde_json::from_slice(body)
|
||||
.map_err(|e| Error::BadRequest(format!("Failed to parse request body: {}", e)))?;
|
||||
|
||||
let (contents, system_instruction) = openai_messages_to_gemini(&request.messages);
|
||||
|
||||
let generation_config =
|
||||
if request.temperature.is_some() || request.max_tokens.is_some() {
|
||||
Some(GeminiGenerationConfig {
|
||||
temperature: request.temperature,
|
||||
max_output_tokens: request.max_tokens,
|
||||
response_mime_type: None,
|
||||
response_schema: None,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let gemini_tools = request.tools.as_ref().map(|tools| {
|
||||
let declarations: Vec<GeminiFunctionDeclaration> = tools
|
||||
.iter()
|
||||
.map(|t| {
|
||||
let mut params = t.function.parameters.clone().unwrap_or(json!({}));
|
||||
sanitize_schema_for_google(&mut params);
|
||||
GeminiFunctionDeclaration {
|
||||
name: t.function.name.clone(),
|
||||
description: t.function.description.clone(),
|
||||
parameters: params,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
vec![GeminiTool {
|
||||
function_declarations: Some(declarations),
|
||||
google_search: None,
|
||||
}]
|
||||
});
|
||||
|
||||
let gemini_request = GeminiTextRequest {
|
||||
contents,
|
||||
tools: gemini_tools,
|
||||
tool_config: None,
|
||||
system_instruction,
|
||||
generation_config,
|
||||
};
|
||||
|
||||
let request_body = serde_json::to_string(&gemini_request)
|
||||
.map_err(|e| Error::internal_err(format!("Failed to serialize Gemini request: {}", e)))?;
|
||||
|
||||
let base_url = base_url.trim_end_matches('/');
|
||||
|
||||
if request.stream {
|
||||
handle_streaming(&request.model, request_body, api_key, base_url).await
|
||||
} else {
|
||||
handle_non_streaming(&request.model, request_body, api_key, base_url).await
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Streaming path
|
||||
// ============================================================================
|
||||
|
||||
async fn handle_streaming(
|
||||
model: &str,
|
||||
request_body: String,
|
||||
api_key: &str,
|
||||
base_url: &str,
|
||||
) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
|
||||
let endpoint = format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model);
|
||||
|
||||
let response = HTTP_CLIENT
|
||||
.post(&endpoint)
|
||||
.header("content-type", "application/json")
|
||||
.header("x-goog-api-key", api_key)
|
||||
.body(request_body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
Error::internal_err(format!("Failed to send request to Gemini API: {}", e))
|
||||
})?;
|
||||
|
||||
if let Err(e) = response.error_for_status_ref() {
|
||||
let status = e.status().map(|s| s.to_string()).unwrap_or_default();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(Error::AIError(format!("{}: {}", status, body)));
|
||||
}
|
||||
|
||||
let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple());
|
||||
let model_str = model.to_string();
|
||||
|
||||
let gemini_sse_stream = response.bytes_stream().eventsource();
|
||||
let openai_sse_stream = async_stream::stream! {
|
||||
tokio::pin!(gemini_sse_stream);
|
||||
let mut tool_call_index: usize = 0;
|
||||
while let Some(event) = gemini_sse_stream.next().await {
|
||||
match event {
|
||||
Ok(event) => match parse_gemini_sse_event(&event.data) {
|
||||
Ok(Some(parsed)) => {
|
||||
for chunk in gemini_event_to_openai_sse_chunks(
|
||||
&parsed, &id, &model_str, &mut tool_call_index,
|
||||
) {
|
||||
yield Ok::<Bytes, reqwest::Error>(Bytes::from(chunk));
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e),
|
||||
},
|
||||
Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e),
|
||||
}
|
||||
}
|
||||
yield Ok::<Bytes, reqwest::Error>(Bytes::from("data: [DONE]\n\n"));
|
||||
};
|
||||
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("content-type", "text/event-stream".parse().unwrap());
|
||||
headers.insert("cache-control", "no-cache".parse().unwrap());
|
||||
headers.insert("connection", "keep-alive".parse().unwrap());
|
||||
|
||||
Ok((
|
||||
http::StatusCode::OK,
|
||||
headers,
|
||||
Body::from_stream(inject_keepalives(
|
||||
Box::pin(openai_sse_stream),
|
||||
std::time::Duration::from_secs(KEEPALIVE_INTERVAL_SECS),
|
||||
)),
|
||||
))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Model listing
|
||||
// ============================================================================
|
||||
|
||||
/// List available Gemini models and convert to OpenAI format.
|
||||
///
|
||||
/// Gemini returns `{ models: [{ name: "models/gemini-2.5-flash", displayName, ... }] }`.
|
||||
/// The frontend expects OpenAI format `{ data: [{ id: "models/gemini-2.5-flash", ... }] }`.
|
||||
pub async fn handle_google_ai_models(
|
||||
api_key: &str,
|
||||
base_url: &str,
|
||||
) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
|
||||
#[derive(Deserialize)]
|
||||
struct GeminiModel {
|
||||
name: String,
|
||||
#[serde(rename = "displayName", default)]
|
||||
display_name: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GeminiModelsResponse {
|
||||
#[serde(default)]
|
||||
models: Vec<GeminiModel>,
|
||||
}
|
||||
|
||||
let endpoint = format!("{}/models", base_url.trim_end_matches('/'));
|
||||
let response = HTTP_CLIENT
|
||||
.get(&endpoint)
|
||||
.header("x-goog-api-key", api_key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::internal_err(format!("Failed to fetch Gemini models: {}", e)))?;
|
||||
|
||||
if let Err(e) = response.error_for_status_ref() {
|
||||
let status = e.status().map(|s| s.to_string()).unwrap_or_default();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(Error::AIError(format!("{}: {}", status, body)));
|
||||
}
|
||||
|
||||
let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| {
|
||||
Error::internal_err(format!("Failed to parse Gemini models response: {}", e))
|
||||
})?;
|
||||
|
||||
let data: Vec<serde_json::Value> = gemini_resp
|
||||
.models
|
||||
.into_iter()
|
||||
.map(|m| {
|
||||
json!({
|
||||
"id": m.name,
|
||||
"object": "model",
|
||||
"display_name": m.display_name,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
let body_bytes = serde_json::to_vec(&json!({ "data": data }))
|
||||
.map_err(|e| Error::internal_err(format!("Failed to serialize models: {}", e)))?;
|
||||
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("content-type", "application/json".parse().unwrap());
|
||||
|
||||
Ok((http::StatusCode::OK, headers, Body::from(body_bytes)))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Non-streaming path
|
||||
// ============================================================================
|
||||
|
||||
async fn handle_non_streaming(
|
||||
model: &str,
|
||||
request_body: String,
|
||||
api_key: &str,
|
||||
base_url: &str,
|
||||
) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
|
||||
let endpoint = format!("{}/models/{}:generateContent", base_url, model);
|
||||
|
||||
let response = HTTP_CLIENT
|
||||
.post(&endpoint)
|
||||
.header("content-type", "application/json")
|
||||
.header("x-goog-api-key", api_key)
|
||||
.body(request_body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
Error::internal_err(format!("Failed to send request to Gemini API: {}", e))
|
||||
})?;
|
||||
|
||||
if let Err(e) = response.error_for_status_ref() {
|
||||
let status = e.status().map(|s| s.to_string()).unwrap_or_default();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(Error::AIError(format!("{}: {}", status, body)));
|
||||
}
|
||||
|
||||
let body = response.bytes().await.map_err(|e| {
|
||||
Error::internal_err(format!("Failed to read Gemini response body: {}", e))
|
||||
})?;
|
||||
|
||||
let parsed = parse_gemini_response(&body)?;
|
||||
let openai_response = gemini_response_to_openai(&parsed, model);
|
||||
|
||||
let body_bytes = serde_json::to_vec(&openai_response)
|
||||
.map_err(|e| Error::internal_err(format!("Failed to serialize response: {}", e)))?;
|
||||
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("content-type", "application/json".parse().unwrap());
|
||||
|
||||
Ok((http::StatusCode::OK, headers, Body::from(body_bytes)))
|
||||
}
|
||||
@@ -2255,13 +2255,12 @@ async fn resume_suspended_job_internal(
|
||||
let value = value.unwrap_or(serde_json::Value::Null);
|
||||
verify_suspended_secret(&w_id, &db, job_id, resume_id, &approver, secret).await?;
|
||||
|
||||
// Get flow info - works for step-level, flow-level, and WAC approval
|
||||
let (flow_info, is_flow_level, is_wac) = get_flow_info_for_resume(job_id, &db).await?;
|
||||
// Get flow info - works for both step-level (job_id is a step) and flow-level (job_id is the flow)
|
||||
let (flow_info, is_flow_level) = get_flow_info_for_resume(job_id, &db).await?;
|
||||
|
||||
// For step-level resumes, verify user auth and flow status
|
||||
// For flow-level resumes (pre-approvals), the flow might not be at a suspended step yet
|
||||
// For WAC approvals, skip flow status checks (there is no flow)
|
||||
if !is_flow_level && !is_wac {
|
||||
if !is_flow_level {
|
||||
let parent_flow = GetQuery::new()
|
||||
.without_logs()
|
||||
.without_code()
|
||||
@@ -2323,16 +2322,6 @@ async fn resume_suspended_job_internal(
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
} else if is_wac {
|
||||
// WAC approval: decrement suspend counter directly on the WAC parent job
|
||||
if flow_info.suspend > 0 {
|
||||
sqlx::query!(
|
||||
"UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1",
|
||||
flow_info.id,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
} else if is_flow_level {
|
||||
// For flow-level resumes, decrement the suspend counter if the flow is currently suspended
|
||||
// The approval will be matched when the worker checks for resumes (both step-level and flow-level)
|
||||
@@ -2490,15 +2479,10 @@ struct FlowInfo {
|
||||
email: Option<String>,
|
||||
}
|
||||
|
||||
/// Get flow info from either a step job (by looking up its parent), a flow job directly,
|
||||
/// or a WAC workflow job (self-suspended for approval).
|
||||
/// Returns (FlowInfo, is_flow_level, is_wac) where:
|
||||
/// - is_flow_level: job_id was a flow job (pre-approval)
|
||||
/// - is_wac: job_id is a WAC workflow suspended for approval (target is itself)
|
||||
async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool, bool)> {
|
||||
// Single query that determines if job_id is a flow, step, or WAC job,
|
||||
// and fetches the appropriate suspended job info.
|
||||
// For WAC jobs (no parent, not a flow), the job itself is the suspended target.
|
||||
/// Get flow info from either a step job (by looking up its parent) or a flow job directly.
|
||||
/// Returns (FlowInfo, is_flow_level) where is_flow_level indicates if job_id was a flow job.
|
||||
async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool)> {
|
||||
// Single query that determines if job_id is a flow or step, and fetches the appropriate flow info
|
||||
let result = sqlx::query!(
|
||||
r#"
|
||||
WITH job_info AS (
|
||||
@@ -2512,15 +2496,14 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI
|
||||
q.suspend AS "suspend!",
|
||||
j.runnable_path AS script_path,
|
||||
j.permissioned_as_email AS email,
|
||||
(ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!",
|
||||
(ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS "is_wac!"
|
||||
(ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!"
|
||||
FROM job_info ji
|
||||
JOIN v2_job_queue q ON q.id = CASE
|
||||
WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id
|
||||
ELSE COALESCE(ji.parent_job, ji.id)
|
||||
ELSE ji.parent_job
|
||||
END
|
||||
JOIN v2_job j ON j.id = q.id
|
||||
LEFT JOIN v2_job_status s ON s.id = q.id
|
||||
JOIN v2_job_status s ON s.id = q.id
|
||||
FOR UPDATE OF q
|
||||
"#,
|
||||
job_id,
|
||||
@@ -2537,7 +2520,7 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI
|
||||
email: Some(result.email),
|
||||
};
|
||||
|
||||
Ok((flow_info, result.is_flow_level, result.is_wac))
|
||||
Ok((flow_info, result.is_flow_level))
|
||||
}
|
||||
|
||||
async fn get_suspended_flow_info<'c>(
|
||||
@@ -5433,12 +5416,12 @@ async fn add_batch_jobs(
|
||||
if dedicated_worker && path.is_some() {
|
||||
windmill_common::worker::dedicated_worker_tag(&w_id, &path.clone().unwrap())
|
||||
} else {
|
||||
format!("{}", language.as_str())
|
||||
language.as_worker_tag(false).to_string()
|
||||
}
|
||||
} else if let Some(tag) = batch_info.tag {
|
||||
tag
|
||||
} else {
|
||||
format!("{}", language.as_str())
|
||||
language.as_worker_tag(false).to_string()
|
||||
};
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
@@ -64,7 +64,6 @@ use crate::scim_oss::has_scim_token;
|
||||
use windmill_common::error::AppError;
|
||||
|
||||
mod ai;
|
||||
mod google;
|
||||
mod apps;
|
||||
pub mod args;
|
||||
mod audit;
|
||||
@@ -494,12 +493,16 @@ pub async fn run_server(
|
||||
};
|
||||
|
||||
#[cfg(feature = "agent_worker_server")]
|
||||
let (agent_workers_router, agent_workers_bg_processor, agent_workers_job_completed_tx) =
|
||||
if server_mode {
|
||||
windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
|
||||
} else {
|
||||
(Router::new(), vec![], None)
|
||||
};
|
||||
let (
|
||||
agent_workers_router,
|
||||
agent_workers_bg_processor,
|
||||
agent_workers_job_completed_tx,
|
||||
batch_buffer,
|
||||
) = if server_mode {
|
||||
windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
|
||||
} else {
|
||||
(Router::new(), vec![], None, None)
|
||||
};
|
||||
|
||||
#[cfg(feature = "agent_worker_server")]
|
||||
let agent_cache = Arc::new(AgentCache::new());
|
||||
@@ -685,6 +688,7 @@ pub async fn run_server(
|
||||
{
|
||||
windmill_api_agent_workers::global_service(
|
||||
agent_workers_job_completed_tx,
|
||||
batch_buffer.clone(),
|
||||
)
|
||||
.layer(Extension(agent_cache.clone()))
|
||||
} else {
|
||||
|
||||
@@ -1,726 +0,0 @@
|
||||
//! Shared Google AI (Gemini API) types and conversion utilities.
|
||||
//!
|
||||
//! This module provides:
|
||||
//! - Gemini request/response types
|
||||
//! - OpenAI → Gemini message conversion
|
||||
//! - Gemini SSE event parsing
|
||||
//!
|
||||
//! Used by both windmill-api (chat proxy) and windmill-worker (AI agent).
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::ai_types::{ContentPart, ExtraContent, GoogleExtraContent, OpenAIContent, OpenAIMessage, ToolDef, UrlCitation};
|
||||
use crate::error::Error;
|
||||
|
||||
// ============================================================================
|
||||
// Request / Content Types
|
||||
// ============================================================================
|
||||
|
||||
/// Inline data for binary content (images).
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct GeminiInlineData {
|
||||
#[serde(rename = "mimeType")]
|
||||
pub mime_type: String,
|
||||
pub data: String,
|
||||
}
|
||||
|
||||
/// A part of content — text, inline data, function call, or function response.
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
#[serde(untagged)]
|
||||
pub enum GeminiPart {
|
||||
Text {
|
||||
text: String,
|
||||
},
|
||||
InlineData {
|
||||
#[serde(rename = "inlineData")]
|
||||
inline_data: GeminiInlineData,
|
||||
},
|
||||
FunctionCall {
|
||||
#[serde(rename = "functionCall")]
|
||||
function_call: GeminiFunctionCall,
|
||||
/// Thought signature for Gemini 3+ models — required when replaying function calls.
|
||||
#[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")]
|
||||
thought_signature: Option<String>,
|
||||
},
|
||||
FunctionResponse {
|
||||
#[serde(rename = "functionResponse")]
|
||||
function_response: GeminiFunctionResponse,
|
||||
},
|
||||
}
|
||||
|
||||
/// A function call from the model.
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct GeminiFunctionCall {
|
||||
pub name: String,
|
||||
pub args: serde_json::Value,
|
||||
}
|
||||
|
||||
/// A function response sent back to the model.
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct GeminiFunctionResponse {
|
||||
pub name: String,
|
||||
pub response: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Content message with an optional role and a list of parts.
|
||||
#[derive(Serialize, Clone, Debug)]
|
||||
pub struct GeminiContentMessage {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
pub parts: Vec<GeminiPart>,
|
||||
}
|
||||
|
||||
/// Main request body for `generateContent` / `streamGenerateContent`.
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiTextRequest {
|
||||
pub contents: Vec<GeminiContentMessage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tools: Option<Vec<GeminiTool>>,
|
||||
#[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")]
|
||||
pub tool_config: Option<GeminiToolConfig>,
|
||||
#[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")]
|
||||
pub system_instruction: Option<GeminiContentMessage>,
|
||||
#[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")]
|
||||
pub generation_config: Option<GeminiGenerationConfig>,
|
||||
}
|
||||
|
||||
/// Tool definition — function declarations and/or Google Search grounding.
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiTool {
|
||||
#[serde(rename = "functionDeclarations", skip_serializing_if = "Option::is_none")]
|
||||
pub function_declarations: Option<Vec<GeminiFunctionDeclaration>>,
|
||||
#[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")]
|
||||
pub google_search: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// A single function declaration.
|
||||
///
|
||||
/// `parameters` holds a pre-serialized (and, for the worker, pre-sanitized) JSON Schema.
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiFunctionDeclaration {
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
pub parameters: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Tool configuration controlling when and how functions are called.
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiToolConfig {
|
||||
#[serde(rename = "functionCallingConfig")]
|
||||
pub function_calling_config: GeminiFunctionCallingConfig,
|
||||
}
|
||||
|
||||
/// Function calling mode and optional allow-list.
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiFunctionCallingConfig {
|
||||
pub mode: String,
|
||||
#[serde(rename = "allowedFunctionNames", skip_serializing_if = "Option::is_none")]
|
||||
pub allowed_function_names: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
/// Generation parameters (temperature, token limits, structured output).
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiGenerationConfig {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f32>,
|
||||
#[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")]
|
||||
pub max_output_tokens: Option<u32>,
|
||||
#[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")]
|
||||
pub response_mime_type: Option<String>,
|
||||
#[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")]
|
||||
pub response_schema: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Image Generation Types
|
||||
// ============================================================================
|
||||
|
||||
/// Request body for Imagen / Gemini image generation.
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiImageRequest {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub contents: Option<Vec<GeminiImageContent>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub instances: Option<Vec<GeminiPredictContent>>,
|
||||
}
|
||||
|
||||
/// Content wrapper used in `generateContent` image requests.
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiImageContent {
|
||||
pub parts: Vec<GeminiPart>,
|
||||
}
|
||||
|
||||
/// Prompt wrapper for Imagen `predict` endpoint.
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiPredictContent {
|
||||
pub prompt: String,
|
||||
}
|
||||
|
||||
/// Top-level response from Gemini/Imagen image generation.
|
||||
#[derive(Deserialize)]
|
||||
pub struct GeminiImageResponse {
|
||||
pub candidates: Option<Vec<GeminiImageCandidate>>,
|
||||
pub predictions: Option<Vec<GeminiPredictCandidate>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct GeminiImageCandidate {
|
||||
pub content: GeminiImageCandidateContent,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct GeminiImageCandidateContent {
|
||||
pub parts: Vec<GeminiImageCandidatePart>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct GeminiImageCandidatePart {
|
||||
#[serde(rename = "inlineData")]
|
||||
pub inline_data: Option<GeminiInlineData>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct GeminiPredictCandidate {
|
||||
#[serde(rename = "bytesBase64Encoded")]
|
||||
pub bytes_base64_encoded: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SSE Response Types
|
||||
// ============================================================================
|
||||
|
||||
/// One part inside a streaming candidate — text, function call, or thought signature.
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiSSEPart {
|
||||
#[serde(default)]
|
||||
pub text: Option<String>,
|
||||
#[serde(rename = "functionCall")]
|
||||
pub function_call: Option<GeminiSSEFunctionCall>,
|
||||
/// Thought signature for Gemini 3+ models.
|
||||
#[serde(rename = "thoughtSignature")]
|
||||
pub thought_signature: Option<String>,
|
||||
}
|
||||
|
||||
/// Function call contained in a streaming part.
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiSSEFunctionCall {
|
||||
pub name: String,
|
||||
pub args: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Content block inside a streaming candidate.
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiSSEContent {
|
||||
pub parts: Option<Vec<GeminiSSEPart>>,
|
||||
}
|
||||
|
||||
/// Web source from a Gemini grounding chunk.
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiGroundingChunkWeb {
|
||||
pub uri: String,
|
||||
#[serde(default)]
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
/// One grounding chunk (search result) from Gemini web search.
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiGroundingChunk {
|
||||
pub web: Option<GeminiGroundingChunkWeb>,
|
||||
}
|
||||
|
||||
/// Grounding metadata attached to a streaming candidate.
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiGroundingMetadata {
|
||||
#[serde(rename = "groundingChunks", default)]
|
||||
pub grounding_chunks: Vec<GeminiGroundingChunk>,
|
||||
#[serde(rename = "webSearchQueries", default)]
|
||||
pub web_search_queries: Vec<String>,
|
||||
}
|
||||
|
||||
/// One candidate inside a streaming Gemini response.
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiSSECandidate {
|
||||
pub content: Option<GeminiSSEContent>,
|
||||
#[serde(rename = "finishReason")]
|
||||
pub finish_reason: Option<String>,
|
||||
#[serde(rename = "groundingMetadata")]
|
||||
pub grounding_metadata: Option<GeminiGroundingMetadata>,
|
||||
}
|
||||
|
||||
/// Token usage from the `usageMetadata` field of a Gemini SSE event.
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct GeminiUsageMetadata {
|
||||
#[serde(rename = "promptTokenCount", default)]
|
||||
pub prompt_token_count: Option<i32>,
|
||||
#[serde(rename = "candidatesTokenCount", default)]
|
||||
pub candidates_token_count: Option<i32>,
|
||||
#[serde(rename = "totalTokenCount", default)]
|
||||
pub total_token_count: Option<i32>,
|
||||
}
|
||||
|
||||
/// Top-level structure of one Gemini SSE event.
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiSSEEvent {
|
||||
pub candidates: Option<Vec<GeminiSSECandidate>>,
|
||||
#[serde(rename = "usageMetadata")]
|
||||
pub usage_metadata: Option<GeminiUsageMetadata>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Parsed Event Result
|
||||
// ============================================================================
|
||||
|
||||
/// A single function call extracted from a Gemini SSE event.
|
||||
#[derive(Debug)]
|
||||
pub struct GeminiToolCallEvent {
|
||||
pub name: String,
|
||||
pub args: serde_json::Value,
|
||||
pub thought_signature: Option<String>,
|
||||
}
|
||||
|
||||
impl GeminiToolCallEvent {
|
||||
/// Convert the thought signature (if present) into an [`ExtraContent`].
|
||||
pub fn to_extra_content(&self) -> Option<ExtraContent> {
|
||||
self.thought_signature.as_ref().map(|sig| ExtraContent {
|
||||
google: Some(GoogleExtraContent { thought_signature: Some(sig.clone()) }),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Structured result of parsing a Gemini response (streaming SSE event or non-streaming body).
|
||||
#[derive(Debug, Default)]
|
||||
pub struct GeminiParsedEvent {
|
||||
pub text: Option<String>,
|
||||
pub tool_calls: Vec<GeminiToolCallEvent>,
|
||||
pub annotations: Vec<UrlCitation>,
|
||||
pub used_websearch: bool,
|
||||
pub usage: Option<GeminiUsageMetadata>,
|
||||
pub finish_reason: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Parse a data URL into `(mime_type, base64_data)`.
|
||||
///
|
||||
/// Expected format: `data:<mime_type>;base64,<data>`.
|
||||
pub fn parse_data_url(url: &str) -> Option<(String, String)> {
|
||||
let rest = url.strip_prefix("data:")?;
|
||||
let (header, data) = rest.split_once(',')?;
|
||||
let media_type = header.strip_suffix(";base64")?;
|
||||
Some((media_type.to_string(), data.to_string()))
|
||||
}
|
||||
|
||||
/// Find the function name associated with a `tool_call_id` by scanning prior messages.
|
||||
pub fn find_gemini_function_name(messages: &[OpenAIMessage], tool_call_id: &str) -> String {
|
||||
messages
|
||||
.iter()
|
||||
.filter_map(|msg| msg.tool_calls.as_ref())
|
||||
.flatten()
|
||||
.find(|tc| tc.id == tool_call_id)
|
||||
.map(|tc| tc.function.name.clone())
|
||||
.unwrap_or_else(|| "unknown_function".to_string())
|
||||
}
|
||||
|
||||
/// Convert an [`OpenAIContent`] value to a list of [`GeminiPart`]s.
|
||||
///
|
||||
/// Handles text and `image_url` (data URLs). `S3Object` variants are skipped here;
|
||||
/// the worker handles them by downloading and injecting inline data beforehand.
|
||||
pub fn convert_content_to_gemini_parts(content: &OpenAIContent) -> Vec<GeminiPart> {
|
||||
match content {
|
||||
OpenAIContent::Text(text) if !text.is_empty() => {
|
||||
vec![GeminiPart::Text { text: text.clone() }]
|
||||
}
|
||||
OpenAIContent::Text(_) => vec![],
|
||||
OpenAIContent::Parts(parts) => parts
|
||||
.iter()
|
||||
.filter_map(|part| match part {
|
||||
ContentPart::Text { text } if !text.is_empty() => {
|
||||
Some(GeminiPart::Text { text: text.clone() })
|
||||
}
|
||||
ContentPart::ImageUrl { image_url } => {
|
||||
parse_data_url(&image_url.url).map(|(mime_type, data)| {
|
||||
GeminiPart::InlineData {
|
||||
inline_data: GeminiInlineData { mime_type, data },
|
||||
}
|
||||
})
|
||||
}
|
||||
// S3Objects are handled by the worker
|
||||
_ => None,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert OpenAI-format messages to Gemini `contents` and an optional `systemInstruction`.
|
||||
///
|
||||
/// Returns `(contents, system_instruction)`.
|
||||
///
|
||||
/// `S3Object` images in content parts are skipped (the worker pre-converts them).
|
||||
/// Tool call history is preserved correctly for multi-turn agent conversations.
|
||||
pub fn openai_messages_to_gemini(
|
||||
messages: &[OpenAIMessage],
|
||||
) -> (Vec<GeminiContentMessage>, Option<GeminiContentMessage>) {
|
||||
let mut contents: Vec<GeminiContentMessage> = Vec::new();
|
||||
let mut system_instruction: Option<GeminiContentMessage> = None;
|
||||
|
||||
for msg in messages {
|
||||
match msg.role.as_str() {
|
||||
"system" => {
|
||||
if let Some(content) = &msg.content {
|
||||
let parts = convert_content_to_gemini_parts(content);
|
||||
if !parts.is_empty() {
|
||||
system_instruction =
|
||||
Some(GeminiContentMessage { role: None, parts });
|
||||
}
|
||||
}
|
||||
}
|
||||
"tool" => {
|
||||
if let (Some(tool_call_id), Some(content)) =
|
||||
(&msg.tool_call_id, &msg.content)
|
||||
{
|
||||
let func_name = find_gemini_function_name(messages, tool_call_id);
|
||||
let response_text = match content {
|
||||
OpenAIContent::Text(text) => text.clone(),
|
||||
OpenAIContent::Parts(parts) => parts
|
||||
.iter()
|
||||
.filter_map(|p| {
|
||||
if let ContentPart::Text { text } = p {
|
||||
Some(text.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
};
|
||||
contents.push(GeminiContentMessage {
|
||||
role: Some("user".to_string()),
|
||||
parts: vec![GeminiPart::FunctionResponse {
|
||||
function_response: GeminiFunctionResponse {
|
||||
name: func_name,
|
||||
response: serde_json::json!({ "result": response_text }),
|
||||
},
|
||||
}],
|
||||
});
|
||||
}
|
||||
}
|
||||
role => {
|
||||
let gemini_role = if role == "assistant" { "model" } else { "user" };
|
||||
let mut parts: Vec<GeminiPart> = Vec::new();
|
||||
|
||||
if let Some(content) = &msg.content {
|
||||
parts.extend(convert_content_to_gemini_parts(content));
|
||||
}
|
||||
|
||||
if let Some(tool_calls) = &msg.tool_calls {
|
||||
for tc in tool_calls {
|
||||
let args: serde_json::Value =
|
||||
serde_json::from_str(&tc.function.arguments).unwrap_or_default();
|
||||
let thought_signature = tc
|
||||
.extra_content
|
||||
.as_ref()
|
||||
.and_then(|ec| ec.google.as_ref())
|
||||
.and_then(|g| g.thought_signature.clone());
|
||||
parts.push(GeminiPart::FunctionCall {
|
||||
function_call: GeminiFunctionCall {
|
||||
name: tc.function.name.clone(),
|
||||
args,
|
||||
},
|
||||
thought_signature,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if !parts.is_empty() {
|
||||
contents.push(GeminiContentMessage {
|
||||
role: Some(gemini_role.to_string()),
|
||||
parts,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(contents, system_instruction)
|
||||
}
|
||||
|
||||
/// Convert OpenAI tool definitions to Gemini format.
|
||||
///
|
||||
/// `tool_params` must be pre-serialized (and, for the worker, pre-sanitized for Google)
|
||||
/// JSON schema values, one per entry in `tools` in the same order.
|
||||
pub fn openai_tools_to_gemini(
|
||||
tools: &[ToolDef],
|
||||
tool_params: &[serde_json::Value],
|
||||
has_websearch: bool,
|
||||
) -> Option<Vec<GeminiTool>> {
|
||||
let mut gemini_tools: Vec<GeminiTool> = Vec::new();
|
||||
|
||||
let declarations: Vec<GeminiFunctionDeclaration> = tools
|
||||
.iter()
|
||||
.zip(tool_params.iter())
|
||||
.map(|(t, params)| GeminiFunctionDeclaration {
|
||||
name: t.function.name.clone(),
|
||||
description: t.function.description.clone(),
|
||||
parameters: params.clone(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
if !declarations.is_empty() {
|
||||
gemini_tools.push(GeminiTool {
|
||||
function_declarations: Some(declarations),
|
||||
google_search: None,
|
||||
});
|
||||
}
|
||||
|
||||
if has_websearch {
|
||||
gemini_tools.push(GeminiTool {
|
||||
function_declarations: None,
|
||||
google_search: Some(serde_json::json!({})),
|
||||
});
|
||||
}
|
||||
|
||||
if gemini_tools.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(gemini_tools)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse one Gemini SSE data line into a [`GeminiParsedEvent`].
|
||||
///
|
||||
/// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`).
|
||||
/// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating.
|
||||
pub fn parse_gemini_sse_event(data: &str) -> Result<Option<GeminiParsedEvent>, Error> {
|
||||
if data.is_empty() || data == "[DONE]" {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let event: GeminiSSEEvent = match serde_json::from_str(data) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to parse Gemini SSE event {}: {}", data, e);
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
|
||||
let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() };
|
||||
|
||||
let Some(candidates) = event.candidates else {
|
||||
return Ok(Some(parsed));
|
||||
};
|
||||
|
||||
extract_candidates_into(&candidates, &mut parsed);
|
||||
|
||||
Ok(Some(parsed))
|
||||
}
|
||||
|
||||
/// Parse a non-streaming Gemini `generateContent` response body.
|
||||
pub fn parse_gemini_response(data: &[u8]) -> Result<GeminiParsedEvent, Error> {
|
||||
let event: GeminiSSEEvent = serde_json::from_slice(data)
|
||||
.map_err(|e| Error::internal_err(format!("Failed to parse Gemini response: {}", e)))?;
|
||||
|
||||
let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() };
|
||||
|
||||
if let Some(candidates) = event.candidates {
|
||||
extract_candidates_into(&candidates, &mut parsed);
|
||||
}
|
||||
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Gemini → OpenAI Format Conversion
|
||||
// ============================================================================
|
||||
|
||||
/// Convert a `GeminiParsedEvent` from a non-streaming response to an OpenAI chat completion JSON.
|
||||
pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> serde_json::Value {
|
||||
let content = parsed.text.as_deref().unwrap_or_default();
|
||||
|
||||
let tool_calls: Vec<serde_json::Value> = parsed
|
||||
.tool_calls
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, tc)| {
|
||||
serde_json::json!({
|
||||
"index": i,
|
||||
"id": format!("call_{}", uuid::Uuid::new_v4().simple()),
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tc.name,
|
||||
"arguments": serde_json::to_string(&tc.args).unwrap_or_default()
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
let finish_reason = parsed
|
||||
.finish_reason
|
||||
.as_deref()
|
||||
.map(|r| r.to_lowercase())
|
||||
.unwrap_or_else(|| "stop".to_string());
|
||||
|
||||
let usage = parsed.usage.as_ref().map(|u| {
|
||||
serde_json::json!({
|
||||
"prompt_tokens": u.prompt_token_count.unwrap_or(0),
|
||||
"completion_tokens": u.candidates_token_count.unwrap_or(0),
|
||||
"total_tokens": u.total_token_count.unwrap_or(0),
|
||||
})
|
||||
});
|
||||
|
||||
let mut message = serde_json::json!({
|
||||
"role": "assistant",
|
||||
"content": content,
|
||||
});
|
||||
if !tool_calls.is_empty() {
|
||||
message["tool_calls"] = serde_json::json!(tool_calls);
|
||||
}
|
||||
|
||||
serde_json::json!({
|
||||
"id": format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()),
|
||||
"object": "chat.completion",
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": message,
|
||||
"finish_reason": finish_reason,
|
||||
}],
|
||||
"usage": usage,
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert a `GeminiParsedEvent` from a streaming SSE event into OpenAI-format SSE lines.
|
||||
///
|
||||
/// Returns the serialized `"data: {...}\n\n"` lines ready to be written to the response stream.
|
||||
/// `tool_call_index` is mutated to track the running index across multiple SSE events.
|
||||
pub fn gemini_event_to_openai_sse_chunks(
|
||||
parsed: &GeminiParsedEvent,
|
||||
id: &str,
|
||||
model: &str,
|
||||
tool_call_index: &mut usize,
|
||||
) -> Vec<String> {
|
||||
let mut chunks = Vec::new();
|
||||
|
||||
if let Some(text) = &parsed.text {
|
||||
let chunk = serde_json::json!({
|
||||
"id": id,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": { "content": text },
|
||||
"finish_reason": null,
|
||||
}]
|
||||
});
|
||||
chunks.push(format!("data: {}\n\n", chunk));
|
||||
}
|
||||
|
||||
for tc in &parsed.tool_calls {
|
||||
let args_str = serde_json::to_string(&tc.args).unwrap_or_default();
|
||||
let call_id = format!("call_{}", uuid::Uuid::new_v4().simple());
|
||||
let chunk = serde_json::json!({
|
||||
"id": id,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": model,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"tool_calls": [{
|
||||
"index": *tool_call_index,
|
||||
"id": call_id,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tc.name,
|
||||
"arguments": args_str,
|
||||
}
|
||||
}]
|
||||
},
|
||||
"finish_reason": null,
|
||||
}]
|
||||
});
|
||||
chunks.push(format!("data: {}\n\n", chunk));
|
||||
*tool_call_index += 1;
|
||||
}
|
||||
|
||||
chunks
|
||||
}
|
||||
|
||||
/// Recursively remove JSON Schema fields unsupported by the Gemini API.
|
||||
pub fn sanitize_schema_for_google(value: &mut serde_json::Value) {
|
||||
const UNSUPPORTED: &[&str] = &[
|
||||
"additionalProperties",
|
||||
"strict",
|
||||
"$schema",
|
||||
"default",
|
||||
"exclusiveMinimum",
|
||||
"exclusiveMaximum",
|
||||
"const",
|
||||
"multipleOf",
|
||||
];
|
||||
|
||||
if let Some(obj) = value.as_object_mut() {
|
||||
for field in UNSUPPORTED {
|
||||
obj.remove(*field);
|
||||
}
|
||||
for v in obj.values_mut() {
|
||||
sanitize_schema_for_google(v);
|
||||
}
|
||||
} else if let Some(arr) = value.as_array_mut() {
|
||||
for v in arr.iter_mut() {
|
||||
sanitize_schema_for_google(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Internal Helpers
|
||||
// ============================================================================
|
||||
|
||||
fn extract_candidates_into(candidates: &[GeminiSSECandidate], parsed: &mut GeminiParsedEvent) {
|
||||
for candidate in candidates {
|
||||
if let Some(content) = &candidate.content {
|
||||
if let Some(parts) = &content.parts {
|
||||
for part in parts {
|
||||
if let Some(text) = &part.text {
|
||||
if !text.is_empty() {
|
||||
match parsed.text.as_mut() {
|
||||
Some(existing) => existing.push_str(text),
|
||||
None => parsed.text = Some(text.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(function_call) = &part.function_call {
|
||||
parsed.tool_calls.push(GeminiToolCallEvent {
|
||||
name: function_call.name.clone(),
|
||||
args: function_call.args.clone(),
|
||||
thought_signature: part.thought_signature.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if candidate.finish_reason.is_some() {
|
||||
parsed.finish_reason = candidate.finish_reason.clone();
|
||||
}
|
||||
|
||||
if let Some(grounding) = &candidate.grounding_metadata {
|
||||
if !grounding.web_search_queries.is_empty() || !grounding.grounding_chunks.is_empty() {
|
||||
parsed.used_websearch = true;
|
||||
}
|
||||
for chunk in &grounding.grounding_chunks {
|
||||
if let Some(web) = &chunk.web {
|
||||
parsed.annotations.push(UrlCitation {
|
||||
start_index: 0,
|
||||
end_index: 0,
|
||||
url: web.uri.clone(),
|
||||
title: web.title.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -78,8 +78,6 @@ pub enum Error {
|
||||
AIError(String),
|
||||
#[error("{0}")]
|
||||
AlreadyCompleted(String),
|
||||
#[error("WAC job suspended: {0}")]
|
||||
WacSuspended(String),
|
||||
#[error("Find python error: {0}")]
|
||||
FindPythonError(String),
|
||||
#[error("Problem with arguments: {0}")]
|
||||
@@ -110,7 +108,6 @@ impl Error {
|
||||
Self::JsonErr(_) => "JsonErr",
|
||||
Self::AIError(_) => "AIError",
|
||||
Self::AlreadyCompleted(_) => "AlreadyCompleted",
|
||||
Self::WacSuspended(_) => "WacSuspended",
|
||||
Self::FindPythonError(_) => "FindPythonError",
|
||||
Self::ArgumentErr(_) => "ArgumentErr",
|
||||
Self::Generic(_, _) => "Generic",
|
||||
|
||||
@@ -44,7 +44,7 @@ pub struct EnvRefWrapper {
|
||||
///
|
||||
/// `Literal` serializes back to a plain JSON string, preserving backwards
|
||||
/// compatibility with existing consumers.
|
||||
#[derive(Deserialize, Serialize, Clone)]
|
||||
#[derive(Deserialize, Serialize, Clone, Debug)]
|
||||
#[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))]
|
||||
#[serde(untagged)]
|
||||
pub enum StringOrSecretRef {
|
||||
@@ -53,16 +53,6 @@ pub enum StringOrSecretRef {
|
||||
EnvRef(EnvRefWrapper),
|
||||
}
|
||||
|
||||
impl fmt::Debug for StringOrSecretRef {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Literal(_) => f.write_str("Literal(****)"),
|
||||
Self::SecretRef(w) => f.debug_tuple("SecretRef").field(w).finish(),
|
||||
Self::EnvRef(w) => f.debug_tuple("EnvRef").field(w).finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StringOrSecretRef {
|
||||
/// Returns the literal string value, or `None` if this is an unresolved ref.
|
||||
pub fn as_literal(&self) -> Option<&str> {
|
||||
@@ -265,25 +255,25 @@ pub struct GlobalSettings {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub instance_python_version: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pip_index_url: Option<StringOrSecretRef>,
|
||||
pub pip_index_url: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pip_extra_index_url: Option<StringOrSecretRef>,
|
||||
pub pip_extra_index_url: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub npm_config_registry: Option<StringOrSecretRef>,
|
||||
pub npm_config_registry: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bunfig_install_scopes: Option<StringOrSecretRef>,
|
||||
pub bunfig_install_scopes: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub npmrc: Option<StringOrSecretRef>,
|
||||
pub npmrc: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub nuget_config: Option<StringOrSecretRef>,
|
||||
pub nuget_config: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub maven_repos: Option<StringOrSecretRef>,
|
||||
pub maven_repos: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ruby_repos: Option<StringOrSecretRef>,
|
||||
pub ruby_repos: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub powershell_repo_url: Option<StringOrSecretRef>,
|
||||
pub powershell_repo_url: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub powershell_repo_pat: Option<StringOrSecretRef>,
|
||||
pub powershell_repo_pat: Option<String>,
|
||||
|
||||
// Array settings
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -934,7 +924,7 @@ fn redact_string(s: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn format_setting_value(key: &str, value: &serde_json::Value) -> String {
|
||||
fn format_setting_value(key: &str, value: &serde_json::Value) -> String {
|
||||
if SENSITIVE_SETTINGS.contains(&key) {
|
||||
return match value {
|
||||
serde_json::Value::String(s) => format!("\"{}\"", redact_string(s)),
|
||||
@@ -2219,25 +2209,6 @@ mod tests {
|
||||
assert_eq!(v, *"world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string_or_secret_ref_debug_masks_literal() {
|
||||
let v = StringOrSecretRef::Literal("super-secret-value".to_string());
|
||||
let debug = format!("{v:?}");
|
||||
assert_eq!(debug, "Literal(****)");
|
||||
assert!(!debug.contains("super-secret-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_setting_value_redacts_oauth_secrets() {
|
||||
let val = serde_json::json!({
|
||||
"google": {"id": "client-id", "secret": "my-super-secret-12345"}
|
||||
});
|
||||
let formatted = format_setting_value("oauths", &val);
|
||||
assert!(!formatted.contains("my-super-secret-12345"));
|
||||
assert!(formatted.contains("client-id"));
|
||||
assert!(formatted.contains("****"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "literal_value() called on unresolved secret ref")]
|
||||
fn string_or_secret_ref_literal_value_panics_on_ref() {
|
||||
|
||||
@@ -29,7 +29,6 @@ use sqlx::{Acquire, Postgres};
|
||||
pub mod agent_workers;
|
||||
#[cfg(feature = "bedrock")]
|
||||
pub mod ai_bedrock;
|
||||
pub mod ai_google;
|
||||
pub mod ai_providers;
|
||||
pub mod ai_types;
|
||||
pub mod apps;
|
||||
|
||||
@@ -288,6 +288,10 @@ pub fn is_native_mode_from_env() -> bool {
|
||||
/// Use this for hot-path checks (e.g. per-job dispatch) to avoid read-locking WORKER_CONFIG.
|
||||
pub static NATIVE_MODE_RESOLVED: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
/// Whether this worker uses HTTP batch pull (set at startup in main.rs).
|
||||
/// Reported in worker_ping so the server knows which native workers to batch-pull for.
|
||||
pub static USES_BATCH_HTTP_PULL: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false);
|
||||
#[derive(Clone)]
|
||||
pub struct HttpClient {
|
||||
@@ -516,6 +520,62 @@ pub fn make_pull_query(tags: &[String]) -> String {
|
||||
query
|
||||
}
|
||||
|
||||
pub fn make_batch_pull_query(tags: &[String], limit: u32) -> String {
|
||||
format_batch_pull_query(format!(
|
||||
"SELECT id
|
||||
FROM v2_job_queue
|
||||
WHERE running = false AND tag IN ({}) AND scheduled_for <= now()
|
||||
ORDER BY priority DESC NULLS LAST, scheduled_for
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT {limit}",
|
||||
tags.iter().map(|x| format!("'{x}'")).join(", ")
|
||||
))
|
||||
}
|
||||
|
||||
fn format_batch_pull_query(peek: String) -> String {
|
||||
// Optimizations vs single-row format_pull_query:
|
||||
// 1. ANY(ARRAY(SELECT ...)) instead of IN (SELECT ...) — forces PG to materialize IDs
|
||||
// into an array, enabling Bitmap Index Scan instead of Hash Semi Join / Nested Loop
|
||||
// 2. r CTE chains off q (not peek) — only updates runtime for actually-locked rows,
|
||||
// avoids re-scanning peek
|
||||
// 3. No separate j CTE — join v2_job directly in final SELECT off q's IDs
|
||||
format!(
|
||||
"WITH peek AS (
|
||||
{}
|
||||
), q AS NOT MATERIALIZED (
|
||||
UPDATE v2_job_queue SET
|
||||
running = true,
|
||||
started_at = coalesce(started_at, now()),
|
||||
suspend_until = null,
|
||||
worker = $1
|
||||
WHERE id = ANY(ARRAY(SELECT id FROM peek))
|
||||
RETURNING
|
||||
id, started_at, scheduled_for,
|
||||
canceled_by, canceled_reason, worker, cache_ignore_s3_path, runnable_settings_handle
|
||||
), r AS NOT MATERIALIZED (
|
||||
UPDATE v2_job_runtime SET
|
||||
ping = now()
|
||||
WHERE id = ANY(ARRAY(SELECT id FROM q))
|
||||
) SELECT j.id, j.workspace_id, j.parent_job, j.created_by, q.started_at, q.scheduled_for,
|
||||
j.runnable_id, j.runnable_path, j.args, q.canceled_by,
|
||||
q.canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as,
|
||||
f.flow_status, j.script_lang,
|
||||
j.same_worker, j.pre_run_error, j.visible_to_owner,
|
||||
j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job,
|
||||
j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow,
|
||||
j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path,
|
||||
COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
|
||||
p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email
|
||||
FROM q
|
||||
JOIN v2_job j ON q.id = j.id
|
||||
LEFT JOIN v2_job_status f ON f.id = q.id
|
||||
LEFT JOIN job_perms p ON p.job_id = q.id
|
||||
LEFT JOIN v2_job pj ON j.parent_job = pj.id
|
||||
",
|
||||
peek
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn store_pull_query(wc: &WorkerConfig) {
|
||||
let mut queries = vec![];
|
||||
for tags in wc.priority_tags_sorted.iter() {
|
||||
@@ -1194,6 +1254,8 @@ pub struct Ping {
|
||||
pub occupancy_rate_30m: Option<f32>,
|
||||
pub job_isolation: Option<String>,
|
||||
pub native_mode: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub uses_batch_http_pull: Option<bool>,
|
||||
pub ping_type: PingType,
|
||||
}
|
||||
pub async fn update_ping_http(
|
||||
@@ -1218,6 +1280,7 @@ pub async fn update_ping_http(
|
||||
insert_ping.occupancy_rate_5m,
|
||||
insert_ping.occupancy_rate_30m,
|
||||
insert_ping.native_mode.unwrap_or(false),
|
||||
insert_ping.uses_batch_http_pull.unwrap_or(false),
|
||||
db,
|
||||
)
|
||||
.await?
|
||||
@@ -1245,6 +1308,7 @@ pub async fn update_ping_http(
|
||||
insert_ping.memory,
|
||||
insert_ping.job_isolation,
|
||||
insert_ping.native_mode.unwrap_or(false),
|
||||
insert_ping.uses_batch_http_pull.unwrap_or(false),
|
||||
db,
|
||||
)
|
||||
.await?;
|
||||
@@ -1377,11 +1441,12 @@ pub async fn insert_ping_query(
|
||||
memory: Option<i64>,
|
||||
job_isolation: Option<String>,
|
||||
native_mode: bool,
|
||||
uses_batch_http_pull: bool,
|
||||
db: &DB,
|
||||
) -> anyhow::Result<()> {
|
||||
sqlx::query!(
|
||||
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)
|
||||
DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode",
|
||||
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)
|
||||
DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
|
||||
worker_instance,
|
||||
worker_name,
|
||||
ip,
|
||||
@@ -1394,6 +1459,7 @@ pub async fn insert_ping_query(
|
||||
memory,
|
||||
job_isolation.as_deref(),
|
||||
native_mode,
|
||||
uses_batch_http_pull,
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
@@ -1485,12 +1551,13 @@ pub async fn update_worker_ping_main_loop_query(
|
||||
occupancy_rate_5m: Option<f32>,
|
||||
occupancy_rate_30m: Option<f32>,
|
||||
native_mode: bool,
|
||||
uses_batch_http_pull: bool,
|
||||
db: &DB,
|
||||
) -> anyhow::Result<()> {
|
||||
timeout(Duration::from_secs(10), sqlx::query!(
|
||||
"UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,
|
||||
occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),
|
||||
memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6",
|
||||
memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
|
||||
jobs_executed,
|
||||
tags,
|
||||
occupancy_rate,
|
||||
@@ -1503,6 +1570,7 @@ pub async fn update_worker_ping_main_loop_query(
|
||||
occupancy_rate_5m,
|
||||
occupancy_rate_30m,
|
||||
native_mode,
|
||||
uses_batch_http_pull,
|
||||
)
|
||||
.execute(db))
|
||||
.await??;
|
||||
|
||||
@@ -152,21 +152,6 @@ pub fn try_exact_property_access(
|
||||
None
|
||||
}
|
||||
|
||||
/// JS runtime properties (not methods) that cannot be resolved by PostgreSQL's
|
||||
/// #> JSON path operator. Function calls like .map(...) already don't match the
|
||||
/// RE_FULL regex due to parentheses, so only property accesses need listing here.
|
||||
const JS_ONLY_PROPERTIES: &[&str] = &["length"];
|
||||
|
||||
fn ends_with_js_only_property(rest: Option<&str>) -> bool {
|
||||
match rest {
|
||||
None => false,
|
||||
Some(rest) => {
|
||||
let last_segment = rest.rsplit('.').next().unwrap_or("");
|
||||
JS_ONLY_PROPERTIES.contains(&last_segment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle_full_regex(
|
||||
expr: &str,
|
||||
authed_client: &AuthedClient,
|
||||
@@ -177,13 +162,6 @@ pub async fn handle_full_regex(
|
||||
let obj_key = captures.get(2).unwrap().as_str();
|
||||
let idx_o = captures.get(3).map(|y| y.as_str());
|
||||
let rest = captures.get(4).map(|y| y.as_str());
|
||||
|
||||
// Skip the SQL fast path when the expression accesses a JS runtime
|
||||
// property (e.g. .length) that the PostgreSQL #> operator can't resolve.
|
||||
if ends_with_js_only_property(rest) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let query = if let Some(idx) = idx_o {
|
||||
match rest {
|
||||
Some(rest) => Some(format!("{}{}", idx, rest)),
|
||||
|
||||
@@ -94,7 +94,7 @@ pub struct OAuthConfig {
|
||||
}
|
||||
|
||||
/// OAuth client credentials
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct OAuthClient {
|
||||
#[serde(default = "empty_string")]
|
||||
pub id: String,
|
||||
@@ -110,21 +110,6 @@ pub struct OAuthClient {
|
||||
pub grant_types: Vec<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for OAuthClient {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("OAuthClient")
|
||||
.field("id", &self.id)
|
||||
.field("secret", &"***")
|
||||
.field("display_name", &self.display_name)
|
||||
.field("allowed_domains", &self.allowed_domains)
|
||||
.field("connect_config", &self.connect_config)
|
||||
.field("login_config", &self.login_config)
|
||||
.field("tenant", &self.tenant)
|
||||
.field("grant_types", &self.grant_types)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
fn empty_string() -> String {
|
||||
"".to_string()
|
||||
}
|
||||
@@ -623,18 +608,7 @@ pub async fn refresh_token<'c>(
|
||||
.await?;
|
||||
let account = windmill_common::utils::not_found_if_none(account, "Account", &id.to_string())?;
|
||||
|
||||
refresh_token_for_account(
|
||||
tx,
|
||||
path,
|
||||
w_id,
|
||||
id,
|
||||
db,
|
||||
account,
|
||||
oauth_clients,
|
||||
http_client,
|
||||
connect_configs_json,
|
||||
)
|
||||
.await
|
||||
refresh_token_for_account(tx, path, w_id, id, db, account, oauth_clients, http_client, connect_configs_json).await
|
||||
}
|
||||
|
||||
/// Refresh an OAuth token given pre-fetched account info (no additional SELECT).
|
||||
|
||||
@@ -818,7 +818,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
flow_is_done: bool,
|
||||
duration: Option<i64>,
|
||||
from_cache: bool,
|
||||
) -> Result<(Uuid, i64, Option<serde_json::Value>), Error> {
|
||||
) -> Result<(Uuid, i64), Error> {
|
||||
// tracing::error!("Start");
|
||||
// let start = tokio::time::Instant::now();
|
||||
|
||||
@@ -830,7 +830,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
}
|
||||
|
||||
let result_columns = result_columns.as_ref();
|
||||
let (opt_uuid, duration, _skip_downstream_error_handlers, wac_job_ids) = (|| {
|
||||
let (opt_uuid, duration, _skip_downstream_error_handlers) = (|| {
|
||||
commit_completed_job(
|
||||
db,
|
||||
completed_job,
|
||||
@@ -866,7 +866,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
|
||||
// if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout
|
||||
if let Some(job_id) = opt_uuid {
|
||||
return Ok((job_id, duration, None));
|
||||
return Ok((job_id, duration));
|
||||
}
|
||||
|
||||
#[cfg(feature = "cloud")]
|
||||
@@ -887,7 +887,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
|
||||
// tracing::error!("4 {:?}", start.elapsed());
|
||||
|
||||
Ok((completed_job.id, duration, wac_job_ids))
|
||||
Ok((completed_job.id, duration))
|
||||
}
|
||||
|
||||
async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
@@ -902,7 +902,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
flow_is_done: bool,
|
||||
duration: Option<i64>,
|
||||
from_cache: bool,
|
||||
) -> windmill_common::error::Result<(Option<Uuid>, i64, bool, Option<serde_json::Value>)> {
|
||||
) -> windmill_common::error::Result<(Option<Uuid>, i64, bool)> {
|
||||
// let start = std::time::Instant::now();
|
||||
|
||||
let mut tx = db.begin().warn_after_seconds(10).await?;
|
||||
@@ -1003,31 +1003,25 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
.map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?;
|
||||
}
|
||||
|
||||
let mut wac_job_ids: Option<serde_json::Value> = None;
|
||||
if !completed_job.is_flow_step() {
|
||||
if let Some(parent_job) = completed_job.parent_job {
|
||||
// Only update WAC parents (v1 or v2). The WHERE condition skips
|
||||
// non-WAC parents entirely (error handlers, run_script children, etc.).
|
||||
// Also returns pending_steps.job_ids so WAC v2 child completion
|
||||
// doesn't need a separate read.
|
||||
let row = sqlx::query_scalar!(
|
||||
r#"UPDATE v2_job_status SET
|
||||
let _ = sqlx::query_scalar!(
|
||||
"UPDATE v2_job_status SET
|
||||
workflow_as_code_status = jsonb_set(
|
||||
jsonb_set(
|
||||
workflow_as_code_status,
|
||||
COALESCE(workflow_as_code_status, '{}'::jsonb),
|
||||
array[$1],
|
||||
COALESCE(workflow_as_code_status->$1, '{}'::jsonb)
|
||||
),
|
||||
array[$1, 'duration_ms'],
|
||||
to_jsonb($2::bigint)
|
||||
)
|
||||
WHERE id = $3 AND workflow_as_code_status IS NOT NULL
|
||||
RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS "job_ids: serde_json::Value""#,
|
||||
WHERE id = $3",
|
||||
&completed_job.id.to_string(),
|
||||
duration,
|
||||
parent_job
|
||||
)
|
||||
.fetch_optional(&mut *tx)
|
||||
.execute(&mut *tx)
|
||||
.warn_after_seconds(10)
|
||||
.await
|
||||
.inspect_err(|e| {
|
||||
@@ -1035,10 +1029,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
"Could not update parent job `duration_ms` in workflow as code status: {}",
|
||||
e,
|
||||
)
|
||||
})
|
||||
.ok()
|
||||
.flatten();
|
||||
wac_job_ids = row.flatten();
|
||||
});
|
||||
}
|
||||
}
|
||||
// tracing::error!("Added completed job {:#?}", queued_job);
|
||||
@@ -1259,14 +1250,14 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
completed_job.id
|
||||
);
|
||||
// tracing::info!("completed job: {:?}", start.elapsed().as_micros());
|
||||
Ok((None, duration, _skip_downstream_error_handlers, wac_job_ids))
|
||||
Ok((None, duration, _skip_downstream_error_handlers))
|
||||
}
|
||||
|
||||
async fn check_result_size<T: ValidableJson>(
|
||||
db: &Pool<Postgres>,
|
||||
queued_job: &MiniCompletedJob,
|
||||
result: Json<&T>,
|
||||
) -> Option<Result<(Option<Uuid>, i64, bool, Option<serde_json::Value>), Error>> {
|
||||
) -> Option<Result<(Option<Uuid>, i64, bool), Error>> {
|
||||
let result_size = result.size() / 1024 / 1024;
|
||||
if result_size > 2 {
|
||||
if result_size > *MAX_RESULT_SIZE_MB {
|
||||
@@ -3443,6 +3434,38 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
|
||||
Ok(job_and_suspended)
|
||||
}
|
||||
|
||||
/// Batch-pull up to `limit` jobs in a single query, marking them all as running.
|
||||
/// The caller controls which tags are queried, so flow/dependency jobs are never
|
||||
/// pulled (they use distinct tags like "flow" / "dependency").
|
||||
pub async fn batch_pull(
|
||||
db: &Pool<Postgres>,
|
||||
worker_name: &str,
|
||||
tags: &[String],
|
||||
limit: u32,
|
||||
) -> windmill_common::error::Result<Vec<PulledJob>> {
|
||||
use windmill_common::worker::make_batch_pull_query;
|
||||
|
||||
if limit == 0 || tags.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let query = make_batch_pull_query(tags, limit);
|
||||
let jobs: Vec<PulledJob> = timeout(
|
||||
Duration::from_secs(15),
|
||||
sqlx::query_as::<_, PulledJob>(&query)
|
||||
.bind(worker_name)
|
||||
.fetch_all(db),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
windmill_common::error::Error::internal_err(
|
||||
"batch_pull query timed out after 15s".to_string(),
|
||||
)
|
||||
})??;
|
||||
|
||||
Ok(jobs)
|
||||
}
|
||||
|
||||
pub async fn custom_concurrency_key(
|
||||
db: &Pool<Postgres>,
|
||||
job_id: &Uuid,
|
||||
@@ -5382,15 +5405,7 @@ async fn push_inner<'c, 'd>(
|
||||
language
|
||||
.as_ref()
|
||||
.map(|x| {
|
||||
let tag_lang = if x == &ScriptLang::Bunnative {
|
||||
if job_kind == JobKind::Dependencies {
|
||||
ScriptLang::Bun.as_str()
|
||||
} else {
|
||||
ScriptLang::Nativets.as_str()
|
||||
}
|
||||
} else {
|
||||
x.as_str()
|
||||
};
|
||||
let tag_lang = x.as_worker_tag(job_kind == JobKind::Dependencies);
|
||||
if per_workspace {
|
||||
format!("{}-{}", tag_lang, workspace_id)
|
||||
} else {
|
||||
|
||||
@@ -421,6 +421,7 @@ pub fn spawn_test_worker(
|
||||
rx,
|
||||
tx2,
|
||||
&base_internal_url,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
};
|
||||
|
||||
@@ -88,6 +88,20 @@ impl ScriptLang {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the worker tag for this language.
|
||||
/// Bunnative scripts run on nativets workers (not bun), except dependency jobs which use bun.
|
||||
pub fn as_worker_tag(&self, is_dependency_job: bool) -> &'static str {
|
||||
if *self == ScriptLang::Bunnative {
|
||||
if is_dependency_job {
|
||||
ScriptLang::Bun.as_str()
|
||||
} else {
|
||||
ScriptLang::Nativets.as_str()
|
||||
}
|
||||
} else {
|
||||
self.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_dependencies_filename(&self) -> Option<String> {
|
||||
use ScriptLang::*;
|
||||
Some(
|
||||
@@ -459,7 +473,6 @@ pub struct NewScript {
|
||||
pub path: String,
|
||||
pub parent_hash: Option<ScriptHash>,
|
||||
pub summary: String,
|
||||
#[serde(default)]
|
||||
pub description: String,
|
||||
pub content: String,
|
||||
pub schema: Option<Schema>,
|
||||
|
||||
@@ -142,13 +142,6 @@ mount {
|
||||
rw: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/checkpoint.json"
|
||||
dst: "/tmp/{LANG}/checkpoint.json"
|
||||
is_bind: true
|
||||
mandatory: false
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/result.json"
|
||||
dst: "/tmp/{LANG}/result.json"
|
||||
|
||||
@@ -2,8 +2,8 @@ use base64::Engine;
|
||||
use futures;
|
||||
use ulid;
|
||||
use windmill_common::{client::AuthedClient, error::Error};
|
||||
use windmill_queue::MiniPulledJob;
|
||||
use windmill_types::s3::S3Object;
|
||||
use windmill_queue::MiniPulledJob;
|
||||
|
||||
use crate::ai::types::*;
|
||||
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::value::RawValue;
|
||||
use windmill_common::{
|
||||
ai_google::parse_data_url, ai_providers::AIProvider, client::AuthedClient, error::Error,
|
||||
};
|
||||
use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error};
|
||||
|
||||
use crate::ai::{
|
||||
image_handler::prepare_messages_for_api,
|
||||
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor},
|
||||
sse::{AnthropicSSEParser, SSEParser},
|
||||
types::*,
|
||||
utils::{extract_text_content, should_use_structured_output_tool},
|
||||
utils::{extract_text_content, parse_data_url, should_use_structured_output_tool},
|
||||
};
|
||||
|
||||
/// Anthropic API version for standard API
|
||||
|
||||
@@ -1,21 +1,215 @@
|
||||
use async_trait::async_trait;
|
||||
use windmill_common::{
|
||||
ai_google::{
|
||||
openai_messages_to_gemini, openai_tools_to_gemini, GeminiGenerationConfig,
|
||||
GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart,
|
||||
GeminiPredictContent, GeminiTextRequest, GeminiTool,
|
||||
},
|
||||
client::AuthedClient,
|
||||
error::Error,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use windmill_common::{client::AuthedClient, error::Error};
|
||||
|
||||
use crate::ai::{
|
||||
image_handler::{download_and_encode_s3_image, prepare_messages_for_api},
|
||||
image_handler::download_and_encode_s3_image,
|
||||
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor},
|
||||
sse::{GeminiSSEParser, SSEParser},
|
||||
types::*,
|
||||
utils::parse_data_url,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Gemini API Types - Shared between text and image
|
||||
// ============================================================================
|
||||
|
||||
/// Inline data for binary content (images)
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct GeminiInlineData {
|
||||
#[serde(rename = "mimeType")]
|
||||
pub mime_type: String,
|
||||
pub data: String,
|
||||
}
|
||||
|
||||
/// A part of content - can be text, inline data, function call, or function response
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
#[serde(untagged)]
|
||||
pub enum GeminiPart {
|
||||
Text {
|
||||
text: String,
|
||||
},
|
||||
InlineData {
|
||||
#[serde(rename = "inlineData")]
|
||||
inline_data: GeminiInlineData,
|
||||
},
|
||||
FunctionCall {
|
||||
#[serde(rename = "functionCall")]
|
||||
function_call: GeminiFunctionCall,
|
||||
/// Thought signature for Gemini 3+ models - required for function calling
|
||||
#[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")]
|
||||
thought_signature: Option<String>,
|
||||
},
|
||||
FunctionResponse {
|
||||
#[serde(rename = "functionResponse")]
|
||||
function_response: GeminiFunctionResponse,
|
||||
},
|
||||
}
|
||||
|
||||
/// A function call from the model
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct GeminiFunctionCall {
|
||||
pub name: String,
|
||||
pub args: serde_json::Value,
|
||||
}
|
||||
|
||||
/// A function response to send back to the model
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct GeminiFunctionResponse {
|
||||
pub name: String,
|
||||
pub response: serde_json::Value,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Gemini Text API Request Types
|
||||
// ============================================================================
|
||||
|
||||
/// Main request structure for Gemini generateContent
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiTextRequest {
|
||||
pub contents: Vec<GeminiContentMessage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tools: Option<Vec<GeminiTool>>,
|
||||
#[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")]
|
||||
pub tool_config: Option<GeminiToolConfig>,
|
||||
#[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")]
|
||||
pub system_instruction: Option<GeminiContentMessage>,
|
||||
#[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")]
|
||||
pub generation_config: Option<GeminiGenerationConfig>,
|
||||
}
|
||||
|
||||
/// Content message with role and parts
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiContentMessage {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
pub parts: Vec<GeminiPart>,
|
||||
}
|
||||
|
||||
/// Tool definition - either function declarations or Google Search
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiTool {
|
||||
#[serde(
|
||||
rename = "functionDeclarations",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub function_declarations: Option<Vec<GeminiFunctionDeclaration>>,
|
||||
#[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")]
|
||||
pub google_search: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Function declaration for tool use
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiFunctionDeclaration {
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
pub parameters: OpenAPISchema,
|
||||
}
|
||||
|
||||
/// Tool configuration for controlling function calling behavior
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiToolConfig {
|
||||
#[serde(rename = "functionCallingConfig")]
|
||||
pub function_calling_config: GeminiFunctionCallingConfig,
|
||||
}
|
||||
|
||||
/// Function calling configuration
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiFunctionCallingConfig {
|
||||
pub mode: String,
|
||||
#[serde(
|
||||
rename = "allowedFunctionNames",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub allowed_function_names: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
/// Generation configuration for output format
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiGenerationConfig {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f32>,
|
||||
#[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")]
|
||||
pub max_output_tokens: Option<u32>,
|
||||
#[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")]
|
||||
pub response_mime_type: Option<String>,
|
||||
#[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")]
|
||||
pub response_schema: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Gemini API Response Types
|
||||
// ============================================================================
|
||||
|
||||
/// Grounding metadata from Google Search
|
||||
#[derive(Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub struct GeminiGroundingMetadata {
|
||||
#[serde(rename = "webSearchQueries")]
|
||||
pub web_search_queries: Option<Vec<String>>,
|
||||
#[serde(rename = "groundingChunks")]
|
||||
pub grounding_chunks: Option<Vec<serde_json::Value>>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Gemini Image API Types (for Imagen models)
|
||||
// ============================================================================
|
||||
|
||||
/// Request for image generation (Imagen models)
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiImageRequest {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub contents: Option<Vec<GeminiImageContent>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub instances: Option<Vec<GeminiPredictContent>>,
|
||||
}
|
||||
|
||||
/// Content for image generation
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiImageContent {
|
||||
pub parts: Vec<GeminiPart>,
|
||||
}
|
||||
|
||||
/// Content for Imagen predict endpoint
|
||||
#[derive(Serialize)]
|
||||
pub struct GeminiPredictContent {
|
||||
pub prompt: String,
|
||||
}
|
||||
|
||||
/// Response for image generation
|
||||
#[derive(Deserialize)]
|
||||
pub struct GeminiImageResponse {
|
||||
pub candidates: Option<Vec<GeminiImageCandidate>>,
|
||||
pub predictions: Option<Vec<GeminiPredictCandidate>>,
|
||||
}
|
||||
|
||||
/// Image candidate from generateContent
|
||||
#[derive(Deserialize)]
|
||||
pub struct GeminiImageCandidate {
|
||||
pub content: GeminiImageCandidateContent,
|
||||
}
|
||||
|
||||
/// Content in image candidate
|
||||
#[derive(Deserialize)]
|
||||
pub struct GeminiImageCandidateContent {
|
||||
pub parts: Vec<GeminiImageCandidatePart>,
|
||||
}
|
||||
|
||||
/// Part of image candidate
|
||||
#[derive(Deserialize)]
|
||||
pub struct GeminiImageCandidatePart {
|
||||
#[serde(rename = "inlineData", skip_serializing_if = "Option::is_none")]
|
||||
pub inline_data: Option<GeminiInlineData>,
|
||||
}
|
||||
|
||||
/// Prediction candidate from Imagen
|
||||
#[derive(Deserialize)]
|
||||
pub struct GeminiPredictCandidate {
|
||||
#[serde(rename = "bytesBase64Encoded")]
|
||||
pub bytes_base64_encoded: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Query Builder Implementation
|
||||
// ============================================================================
|
||||
@@ -27,24 +221,34 @@ impl GoogleAIQueryBuilder {
|
||||
Self
|
||||
}
|
||||
|
||||
/// Build a text request using the native Gemini API format
|
||||
async fn build_text_request(
|
||||
&self,
|
||||
args: &BuildRequestArgs<'_>,
|
||||
client: &AuthedClient,
|
||||
workspace_id: &str,
|
||||
) -> Result<String, Error> {
|
||||
let prepared_messages =
|
||||
prepare_messages_for_api(args.messages, client, workspace_id).await?;
|
||||
let (contents, system_instruction) = openai_messages_to_gemini(&prepared_messages);
|
||||
// Convert messages to Gemini format
|
||||
let contents = self
|
||||
.convert_messages_to_gemini(args.messages, client, workspace_id)
|
||||
.await?;
|
||||
|
||||
// Build tools array
|
||||
let tools = self.convert_tools_to_gemini(args.tools, args.has_websearch);
|
||||
|
||||
// Build generation config
|
||||
let generation_config = self.build_generation_config(args);
|
||||
|
||||
// Build system instruction from system_prompt
|
||||
let system_instruction = args.system_prompt.map(|s| GeminiContentMessage {
|
||||
role: None,
|
||||
parts: vec![GeminiPart::Text { text: s.to_string() }],
|
||||
});
|
||||
|
||||
let request = GeminiTextRequest {
|
||||
contents,
|
||||
tools,
|
||||
tool_config: None,
|
||||
tool_config: None, // Use AUTO mode by default
|
||||
system_instruction,
|
||||
generation_config,
|
||||
};
|
||||
@@ -53,6 +257,7 @@ impl GoogleAIQueryBuilder {
|
||||
.map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e)))
|
||||
}
|
||||
|
||||
/// Build an image generation request
|
||||
async fn build_image_request(
|
||||
&self,
|
||||
args: &BuildRequestArgs<'_>,
|
||||
@@ -62,6 +267,7 @@ impl GoogleAIQueryBuilder {
|
||||
let is_imagen = args.model.contains("imagen");
|
||||
|
||||
let request = if is_imagen {
|
||||
// For Imagen models, use simple prompt format
|
||||
GeminiImageRequest {
|
||||
instances: Some(vec![GeminiPredictContent {
|
||||
prompt: args.user_message.trim().to_string(),
|
||||
@@ -69,6 +275,7 @@ impl GoogleAIQueryBuilder {
|
||||
contents: None,
|
||||
}
|
||||
} else {
|
||||
// For Gemini models with image generation, build parts
|
||||
let mut parts = vec![GeminiPart::Text { text: args.user_message.trim().to_string() }];
|
||||
|
||||
if let Some(system_prompt) = args.system_prompt {
|
||||
@@ -78,6 +285,7 @@ impl GoogleAIQueryBuilder {
|
||||
);
|
||||
}
|
||||
|
||||
// Add input images if provided
|
||||
if let Some(images) = args.images {
|
||||
for image in images.iter() {
|
||||
if !image.s3.is_empty() {
|
||||
@@ -100,39 +308,218 @@ impl GoogleAIQueryBuilder {
|
||||
.map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e)))
|
||||
}
|
||||
|
||||
/// Convert OpenAI tool definitions to Gemini format.
|
||||
///
|
||||
/// Sanitizes each tool's JSON schema for Google compatibility before delegating
|
||||
/// to the shared [`openai_tools_to_gemini`] function.
|
||||
/// Convert OpenAI-format messages to Gemini format
|
||||
async fn convert_messages_to_gemini(
|
||||
&self,
|
||||
messages: &[OpenAIMessage],
|
||||
client: &AuthedClient,
|
||||
workspace_id: &str,
|
||||
) -> Result<Vec<GeminiContentMessage>, Error> {
|
||||
let mut gemini_messages = Vec::new();
|
||||
|
||||
for msg in messages {
|
||||
match msg.role.as_str() {
|
||||
"system" => {
|
||||
// Skip - handled via args.system_prompt in build_text_request
|
||||
}
|
||||
"tool" => {
|
||||
// Handle tool responses
|
||||
if let (Some(tool_call_id), Some(content)) = (&msg.tool_call_id, &msg.content) {
|
||||
let func_name = self.find_function_name_by_id(messages, tool_call_id);
|
||||
let response_text = match content {
|
||||
OpenAIContent::Text(text) => text.clone(),
|
||||
OpenAIContent::Parts(parts) => parts
|
||||
.iter()
|
||||
.filter_map(|p| match p {
|
||||
ContentPart::Text { text } => Some(text.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
};
|
||||
|
||||
gemini_messages.push(GeminiContentMessage {
|
||||
role: Some("user".to_string()),
|
||||
parts: vec![GeminiPart::FunctionResponse {
|
||||
function_response: GeminiFunctionResponse {
|
||||
name: func_name,
|
||||
response: serde_json::json!({ "result": response_text }),
|
||||
},
|
||||
}],
|
||||
});
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Handle user/assistant messages
|
||||
let role = match msg.role.as_str() {
|
||||
"assistant" => "model",
|
||||
_ => "user",
|
||||
};
|
||||
|
||||
let mut parts = Vec::new();
|
||||
|
||||
// Handle regular content
|
||||
if let Some(content) = &msg.content {
|
||||
let content_parts = self
|
||||
.convert_content_to_parts(&Some(content.clone()), client, workspace_id)
|
||||
.await?;
|
||||
parts.extend(content_parts);
|
||||
}
|
||||
|
||||
// Handle tool calls from assistant
|
||||
if let Some(tool_calls) = &msg.tool_calls {
|
||||
for tc in tool_calls {
|
||||
let args: serde_json::Value =
|
||||
serde_json::from_str(&tc.function.arguments).unwrap_or_default();
|
||||
// Extract thought_signature from extra_content if present
|
||||
let thought_signature = tc
|
||||
.extra_content
|
||||
.as_ref()
|
||||
.and_then(|ec| ec.google.as_ref())
|
||||
.and_then(|g| g.thought_signature.clone());
|
||||
parts.push(GeminiPart::FunctionCall {
|
||||
function_call: GeminiFunctionCall {
|
||||
name: tc.function.name.clone(),
|
||||
args,
|
||||
},
|
||||
thought_signature,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if !parts.is_empty() {
|
||||
gemini_messages
|
||||
.push(GeminiContentMessage { role: Some(role.to_string()), parts });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(gemini_messages)
|
||||
}
|
||||
|
||||
/// Convert OpenAI content to Gemini parts
|
||||
async fn convert_content_to_parts(
|
||||
&self,
|
||||
content: &Option<OpenAIContent>,
|
||||
client: &AuthedClient,
|
||||
workspace_id: &str,
|
||||
) -> Result<Vec<GeminiPart>, Error> {
|
||||
let mut parts = Vec::new();
|
||||
|
||||
if let Some(content) = content {
|
||||
match content {
|
||||
OpenAIContent::Text(text) => {
|
||||
if !text.is_empty() {
|
||||
parts.push(GeminiPart::Text { text: text.clone() });
|
||||
}
|
||||
}
|
||||
OpenAIContent::Parts(content_parts) => {
|
||||
for part in content_parts {
|
||||
match part {
|
||||
ContentPart::Text { text } => {
|
||||
if !text.is_empty() {
|
||||
parts.push(GeminiPart::Text { text: text.clone() });
|
||||
}
|
||||
}
|
||||
ContentPart::ImageUrl { image_url } => {
|
||||
// Parse data URL format: data:mime_type;base64,data
|
||||
if let Some((mime_type, data)) = parse_data_url(&image_url.url) {
|
||||
parts.push(GeminiPart::InlineData {
|
||||
inline_data: GeminiInlineData { mime_type, data },
|
||||
});
|
||||
}
|
||||
}
|
||||
ContentPart::S3Object { s3_object } => {
|
||||
if !s3_object.s3.is_empty() {
|
||||
let (mime_type, data) = download_and_encode_s3_image(
|
||||
s3_object,
|
||||
client,
|
||||
workspace_id,
|
||||
)
|
||||
.await?;
|
||||
parts.push(GeminiPart::InlineData {
|
||||
inline_data: GeminiInlineData { mime_type, data },
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(parts)
|
||||
}
|
||||
|
||||
/// Find function name by tool call ID from previous messages
|
||||
fn find_function_name_by_id(&self, messages: &[OpenAIMessage], tool_call_id: &str) -> String {
|
||||
for msg in messages {
|
||||
if let Some(tool_calls) = &msg.tool_calls {
|
||||
for tc in tool_calls {
|
||||
if tc.id == tool_call_id {
|
||||
return tc.function.name.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"unknown_function".to_string()
|
||||
}
|
||||
|
||||
/// Convert OpenAI tools to Gemini format
|
||||
fn convert_tools_to_gemini(
|
||||
&self,
|
||||
tools: Option<&[ToolDef]>,
|
||||
has_websearch: bool,
|
||||
) -> Option<Vec<GeminiTool>> {
|
||||
let Some(tool_defs) = tools else {
|
||||
if has_websearch {
|
||||
return Some(vec![GeminiTool {
|
||||
function_declarations: None,
|
||||
google_search: Some(serde_json::json!({})),
|
||||
}]);
|
||||
let mut gemini_tools = Vec::new();
|
||||
|
||||
// Add function declarations
|
||||
if let Some(tool_defs) = tools {
|
||||
let declarations: Vec<GeminiFunctionDeclaration> = tool_defs
|
||||
.iter()
|
||||
.filter_map(|t| {
|
||||
// Deserialize RawValue into OpenAPISchema, sanitize, then use
|
||||
let mut schema: OpenAPISchema =
|
||||
serde_json::from_str(t.function.parameters.get()).ok()?;
|
||||
schema.sanitize_for_google();
|
||||
|
||||
Some(GeminiFunctionDeclaration {
|
||||
name: t.function.name.clone(),
|
||||
description: t.function.description.clone(),
|
||||
parameters: schema,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
if !declarations.is_empty() {
|
||||
gemini_tools.push(GeminiTool {
|
||||
function_declarations: Some(declarations),
|
||||
google_search: None,
|
||||
});
|
||||
}
|
||||
return None;
|
||||
};
|
||||
}
|
||||
|
||||
let tool_params: Vec<serde_json::Value> = tool_defs
|
||||
.iter()
|
||||
.map(|t| {
|
||||
let mut schema: OpenAPISchema =
|
||||
serde_json::from_str(t.function.parameters.get()).unwrap_or_default();
|
||||
schema.sanitize_for_google();
|
||||
serde_json::to_value(&schema).unwrap_or_default()
|
||||
})
|
||||
.collect();
|
||||
// Add Google Search tool if enabled
|
||||
if has_websearch {
|
||||
gemini_tools.push(GeminiTool {
|
||||
function_declarations: None,
|
||||
google_search: Some(serde_json::json!({})),
|
||||
});
|
||||
}
|
||||
|
||||
openai_tools_to_gemini(tool_defs, &tool_params, has_websearch)
|
||||
if gemini_tools.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(gemini_tools)
|
||||
}
|
||||
}
|
||||
|
||||
fn build_generation_config(&self, args: &BuildRequestArgs<'_>) -> Option<GeminiGenerationConfig> {
|
||||
/// Build generation config for structured output and other settings
|
||||
fn build_generation_config(
|
||||
&self,
|
||||
args: &BuildRequestArgs<'_>,
|
||||
) -> Option<GeminiGenerationConfig> {
|
||||
let has_output_schema = args
|
||||
.output_schema
|
||||
.and_then(|s| s.properties.as_ref())
|
||||
@@ -142,11 +529,15 @@ impl GoogleAIQueryBuilder {
|
||||
let (response_mime_type, response_schema) = if has_output_schema {
|
||||
let mut schema = args.output_schema.unwrap().clone();
|
||||
schema.sanitize_for_google();
|
||||
(Some("application/json".to_string()), serde_json::to_value(&schema).ok())
|
||||
(
|
||||
Some("application/json".to_string()),
|
||||
serde_json::to_value(&schema).ok(),
|
||||
)
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
// Only create config if there's something to configure
|
||||
if args.temperature.is_some() || args.max_tokens.is_some() || response_mime_type.is_some() {
|
||||
Some(GeminiGenerationConfig {
|
||||
temperature: args.temperature,
|
||||
@@ -163,6 +554,7 @@ impl GoogleAIQueryBuilder {
|
||||
#[async_trait]
|
||||
impl QueryBuilder for GoogleAIQueryBuilder {
|
||||
fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool {
|
||||
// Google AI supports tools only for text output
|
||||
matches!(output_type, OutputType::Text)
|
||||
}
|
||||
|
||||
@@ -186,6 +578,7 @@ impl QueryBuilder for GoogleAIQueryBuilder {
|
||||
Error::internal_err(format!("Failed to parse Gemini image response: {}", e))
|
||||
})?;
|
||||
|
||||
// First, check Gemini models (candidates -> content -> parts -> inline_data)
|
||||
let image_data_from_gemini = gemini_response.candidates.as_ref().and_then(|candidates| {
|
||||
candidates.iter().find_map(|candidate| {
|
||||
candidate
|
||||
@@ -196,11 +589,13 @@ impl QueryBuilder for GoogleAIQueryBuilder {
|
||||
})
|
||||
});
|
||||
|
||||
// Then, check Imagen models (predictions -> bytes_base64_encoded)
|
||||
let image_data_from_imagen = gemini_response
|
||||
.predictions
|
||||
.as_ref()
|
||||
.and_then(|predictions| predictions.first().map(|p| &p.bytes_base64_encoded));
|
||||
|
||||
// Image data, preferring Gemini first then Imagen models
|
||||
let image_data = image_data_from_gemini.or(image_data_from_imagen);
|
||||
|
||||
match image_data {
|
||||
@@ -232,6 +627,7 @@ impl QueryBuilder for GoogleAIQueryBuilder {
|
||||
..
|
||||
} = gemini_sse_parser;
|
||||
|
||||
// Send tool call arguments events for accumulated tool calls
|
||||
for tool_call in accumulated_tool_calls.values() {
|
||||
let event = StreamingEvent::ToolCallArguments {
|
||||
call_id: tool_call.id.clone(),
|
||||
@@ -241,6 +637,7 @@ impl QueryBuilder for GoogleAIQueryBuilder {
|
||||
stream_event_processor.send(event, &mut events_str).await?;
|
||||
}
|
||||
|
||||
// Convert Gemini usage metadata to TokenUsage
|
||||
let usage = gemini_usage.map(|u| {
|
||||
TokenUsage::new(
|
||||
u.prompt_token_count,
|
||||
@@ -250,7 +647,11 @@ impl QueryBuilder for GoogleAIQueryBuilder {
|
||||
});
|
||||
|
||||
Ok(ParsedResponse::Text {
|
||||
content: if accumulated_content.is_empty() { None } else { Some(accumulated_content) },
|
||||
content: if accumulated_content.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(accumulated_content)
|
||||
},
|
||||
tool_calls: accumulated_tool_calls.into_values().collect(),
|
||||
events_str: Some(events_str),
|
||||
annotations,
|
||||
@@ -262,11 +663,17 @@ impl QueryBuilder for GoogleAIQueryBuilder {
|
||||
fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String {
|
||||
match output_type {
|
||||
OutputType::Text => {
|
||||
format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model)
|
||||
format!(
|
||||
"{}/models/{}:streamGenerateContent?alt=sse",
|
||||
base_url, model
|
||||
)
|
||||
}
|
||||
OutputType::Image => {
|
||||
let url_suffix =
|
||||
if model.contains("imagen") { "predict" } else { "generateContent" };
|
||||
let url_suffix = if model.contains("imagen") {
|
||||
"predict"
|
||||
} else {
|
||||
"generateContent"
|
||||
};
|
||||
format!("{}/models/{}:{}", base_url, model, url_suffix)
|
||||
}
|
||||
}
|
||||
@@ -278,6 +685,7 @@ impl QueryBuilder for GoogleAIQueryBuilder {
|
||||
_base_url: &str,
|
||||
_output_type: &OutputType,
|
||||
) -> Vec<(&'static str, String)> {
|
||||
// Native Gemini API always uses x-goog-api-key
|
||||
vec![("x-goog-api-key", api_key.to_string())]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use async_trait::async_trait;
|
||||
use windmill_common::{client::AuthedClient, error::Error, worker::Connection};
|
||||
use windmill_common::{
|
||||
client::AuthedClient, error::Error, worker::Connection,
|
||||
};
|
||||
use windmill_queue::MiniPulledJob;
|
||||
use windmill_types::s3::S3Object;
|
||||
|
||||
|
||||
@@ -5,18 +5,15 @@ use reqwest::Response;
|
||||
use serde::Deserialize;
|
||||
use serde_json;
|
||||
use tokio_stream::StreamExt;
|
||||
use windmill_common::{
|
||||
ai_google::{parse_gemini_sse_event, GeminiUsageMetadata},
|
||||
ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall},
|
||||
error::Error,
|
||||
utils::rd_string,
|
||||
};
|
||||
use windmill_common::{error::Error, utils::rd_string};
|
||||
|
||||
use crate::ai::{
|
||||
query_builder::StreamEventProcessor,
|
||||
types::{StreamingEvent, UrlCitation},
|
||||
};
|
||||
|
||||
use windmill_common::ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct OpenAIChoiceDeltaToolCallFunction {
|
||||
pub name: Option<String>,
|
||||
@@ -460,19 +457,96 @@ impl SSEParser for AnthropicSSEParser {
|
||||
// Gemini SSE Parser
|
||||
// ============================================================================
|
||||
|
||||
/// Accumulates Gemini streaming events and converts them into the worker's
|
||||
/// internal [`OpenAIToolCall`] / [`StreamingEvent`] representation.
|
||||
///
|
||||
/// The actual SSE parsing is delegated to [`parse_gemini_sse_event`] from
|
||||
/// `windmill_common::ai_google` so the logic can be shared with the API proxy.
|
||||
/// Gemini streaming response part - can be text or function call
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiSSEPart {
|
||||
#[serde(default)]
|
||||
pub text: Option<String>,
|
||||
#[serde(rename = "functionCall")]
|
||||
pub function_call: Option<GeminiSSEFunctionCall>,
|
||||
/// Thought signature for Gemini 3+ models - required for function calling
|
||||
#[serde(rename = "thoughtSignature")]
|
||||
pub thought_signature: Option<String>,
|
||||
}
|
||||
|
||||
/// Function call in Gemini streaming response
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiSSEFunctionCall {
|
||||
pub name: String,
|
||||
pub args: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Content in Gemini streaming candidate
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiSSEContent {
|
||||
pub parts: Option<Vec<GeminiSSEPart>>,
|
||||
}
|
||||
|
||||
/// Web reference in Gemini grounding chunk
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiGroundingChunkWeb {
|
||||
pub uri: String,
|
||||
#[serde(default)]
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
/// Grounding chunk from Gemini web search
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiGroundingChunk {
|
||||
pub web: Option<GeminiGroundingChunkWeb>,
|
||||
}
|
||||
|
||||
/// Grounding metadata from Gemini web search
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiGroundingMetadata {
|
||||
#[serde(rename = "groundingChunks", default)]
|
||||
pub grounding_chunks: Vec<GeminiGroundingChunk>,
|
||||
#[serde(rename = "webSearchQueries", default)]
|
||||
pub web_search_queries: Vec<String>,
|
||||
}
|
||||
|
||||
/// Candidate in Gemini streaming response
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiSSECandidate {
|
||||
pub content: Option<GeminiSSEContent>,
|
||||
#[serde(rename = "finishReason")]
|
||||
#[allow(dead_code)]
|
||||
pub finish_reason: Option<String>,
|
||||
#[serde(rename = "groundingMetadata")]
|
||||
pub grounding_metadata: Option<GeminiGroundingMetadata>,
|
||||
}
|
||||
|
||||
/// Gemini usage metadata from SSE response
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct GeminiUsageMetadata {
|
||||
#[serde(rename = "promptTokenCount", default)]
|
||||
pub prompt_token_count: Option<i32>,
|
||||
#[serde(rename = "candidatesTokenCount", default)]
|
||||
pub candidates_token_count: Option<i32>,
|
||||
#[serde(rename = "totalTokenCount", default)]
|
||||
pub total_token_count: Option<i32>,
|
||||
}
|
||||
|
||||
/// Gemini SSE event structure
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GeminiSSEEvent {
|
||||
pub candidates: Option<Vec<GeminiSSECandidate>>,
|
||||
#[serde(rename = "usageMetadata")]
|
||||
pub usage_metadata: Option<GeminiUsageMetadata>,
|
||||
}
|
||||
|
||||
/// Gemini SSE Parser for streaming responses
|
||||
pub struct GeminiSSEParser {
|
||||
pub accumulated_content: String,
|
||||
pub accumulated_tool_calls: HashMap<i64, OpenAIToolCall>,
|
||||
pub events_str: String,
|
||||
pub stream_event_processor: StreamEventProcessor,
|
||||
tool_call_index: i64,
|
||||
/// Collected URL citation annotations from web search
|
||||
pub annotations: Vec<UrlCitation>,
|
||||
/// Whether web search was used in this response
|
||||
pub used_websearch: bool,
|
||||
/// Token usage from usageMetadata
|
||||
pub usage: Option<GeminiUsageMetadata>,
|
||||
}
|
||||
|
||||
@@ -493,57 +567,101 @@ impl GeminiSSEParser {
|
||||
|
||||
impl SSEParser for GeminiSSEParser {
|
||||
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
|
||||
let Some(parsed) = parse_gemini_sse_event(data)? else {
|
||||
return Ok(());
|
||||
};
|
||||
let event: Option<GeminiSSEEvent> = serde_json::from_str(data)
|
||||
.inspect_err(|e| {
|
||||
tracing::error!("Failed to parse SSE as a Gemini event {}: {}", data, e);
|
||||
})
|
||||
.ok();
|
||||
|
||||
if let Some(text) = parsed.text {
|
||||
self.accumulated_content.push_str(&text);
|
||||
self.stream_event_processor
|
||||
.send(StreamingEvent::TokenDelta { content: text }, &mut self.events_str)
|
||||
.await?;
|
||||
}
|
||||
if let Some(event) = event {
|
||||
if let Some(candidates) = event.candidates {
|
||||
for candidate in candidates {
|
||||
if let Some(content) = candidate.content {
|
||||
if let Some(parts) = content.parts {
|
||||
for part in parts {
|
||||
// Handle text content
|
||||
if let Some(text) = part.text {
|
||||
if !text.is_empty() {
|
||||
self.accumulated_content.push_str(&text);
|
||||
let event = StreamingEvent::TokenDelta { content: text };
|
||||
self.stream_event_processor
|
||||
.send(event, &mut self.events_str)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
for tool_call in parsed.tool_calls {
|
||||
let call_id = format!("call_{}", rd_string(24));
|
||||
let idx = self.tool_call_index;
|
||||
self.tool_call_index += 1;
|
||||
// Handle function calls
|
||||
if let Some(function_call) = part.function_call {
|
||||
let call_id = format!("call_{}", rd_string(24));
|
||||
let idx = self.tool_call_index;
|
||||
self.tool_call_index += 1;
|
||||
|
||||
self.stream_event_processor
|
||||
.send(
|
||||
StreamingEvent::ToolCall {
|
||||
call_id: call_id.clone(),
|
||||
function_name: tool_call.name.clone(),
|
||||
},
|
||||
&mut self.events_str,
|
||||
)
|
||||
.await?;
|
||||
// Send tool call start event
|
||||
let event = StreamingEvent::ToolCall {
|
||||
call_id: call_id.clone(),
|
||||
function_name: function_call.name.clone(),
|
||||
};
|
||||
self.stream_event_processor
|
||||
.send(event, &mut self.events_str)
|
||||
.await?;
|
||||
|
||||
let extra_content = tool_call.thought_signature.map(|sig| ExtraContent {
|
||||
google: Some(GoogleExtraContent { thought_signature: Some(sig) }),
|
||||
});
|
||||
// Build extra_content with thought_signature if present
|
||||
let extra_content =
|
||||
part.thought_signature.map(|sig| ExtraContent {
|
||||
google: Some(GoogleExtraContent {
|
||||
thought_signature: Some(sig),
|
||||
}),
|
||||
});
|
||||
|
||||
self.accumulated_tool_calls.insert(
|
||||
idx,
|
||||
OpenAIToolCall {
|
||||
id: call_id,
|
||||
function: OpenAIFunction {
|
||||
name: tool_call.name,
|
||||
arguments: serde_json::to_string(&tool_call.args)
|
||||
.unwrap_or_else(|_| "{}".to_string()),
|
||||
},
|
||||
r#type: "function".to_string(),
|
||||
extra_content,
|
||||
},
|
||||
);
|
||||
}
|
||||
// Store accumulated tool call
|
||||
self.accumulated_tool_calls.insert(
|
||||
idx,
|
||||
OpenAIToolCall {
|
||||
id: call_id,
|
||||
function: OpenAIFunction {
|
||||
name: function_call.name,
|
||||
arguments: serde_json::to_string(
|
||||
&function_call.args,
|
||||
)
|
||||
.unwrap_or_else(|_| "{}".to_string()),
|
||||
},
|
||||
r#type: "function".to_string(),
|
||||
extra_content,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.annotations.extend(parsed.annotations);
|
||||
if parsed.used_websearch {
|
||||
self.used_websearch = true;
|
||||
}
|
||||
if let Some(usage) = parsed.usage {
|
||||
self.usage = Some(usage);
|
||||
// Handle grounding metadata (web search results)
|
||||
if let Some(ref grounding_metadata) = candidate.grounding_metadata {
|
||||
// Set used_websearch if there are search queries or grounding chunks
|
||||
if !grounding_metadata.web_search_queries.is_empty()
|
||||
|| !grounding_metadata.grounding_chunks.is_empty()
|
||||
{
|
||||
self.used_websearch = true;
|
||||
}
|
||||
|
||||
// Extract citations from grounding chunks
|
||||
for chunk in &grounding_metadata.grounding_chunks {
|
||||
if let Some(ref web) = chunk.web {
|
||||
self.annotations.push(UrlCitation {
|
||||
start_index: 0, // Gemini doesn't provide character indices
|
||||
end_index: 0,
|
||||
url: web.uri.clone(),
|
||||
title: web.title.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract usage metadata
|
||||
if let Some(usage_metadata) = event.usage_metadata {
|
||||
self.usage = Some(usage_metadata);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -20,8 +20,8 @@ use windmill_common::{
|
||||
flow_status::AgentAction,
|
||||
flows::FlowModule,
|
||||
};
|
||||
use windmill_parser::Typ;
|
||||
use windmill_types::s3::S3Object;
|
||||
use windmill_parser::Typ;
|
||||
|
||||
// Re-export shared types from windmill_common::ai_types
|
||||
pub use windmill_common::ai_types::{
|
||||
@@ -1603,7 +1603,10 @@ mod tests {
|
||||
|
||||
schema.sanitize_for_google();
|
||||
|
||||
assert!(schema.multiple_of.is_none(), "multipleOf should be removed");
|
||||
assert!(
|
||||
schema.multiple_of.is_none(),
|
||||
"multipleOf should be removed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1636,10 +1639,7 @@ mod tests {
|
||||
assert!(schema.default.is_none());
|
||||
|
||||
let value_prop = schema.properties.as_ref().unwrap().get("value").unwrap();
|
||||
assert!(
|
||||
value_prop.default.is_none(),
|
||||
"nested default should be removed"
|
||||
);
|
||||
assert!(value_prop.default.is_none(), "nested default should be removed");
|
||||
assert!(
|
||||
value_prop.exclusive_minimum.is_none(),
|
||||
"nested exclusiveMinimum should be removed"
|
||||
@@ -1652,10 +1652,7 @@ mod tests {
|
||||
value_prop.multiple_of.is_none(),
|
||||
"nested multipleOf should be removed"
|
||||
);
|
||||
assert!(
|
||||
value_prop.r#const.is_none(),
|
||||
"nested const should be removed"
|
||||
);
|
||||
assert!(value_prop.r#const.is_none(), "nested const should be removed");
|
||||
|
||||
assert!(schema.properties.is_some());
|
||||
assert!(matches!(&schema.r#type, Some(SchemaType::Single(t)) if t == "object"));
|
||||
|
||||
@@ -731,3 +731,16 @@ pub fn extract_text_content(content: &OpenAIContent) -> String {
|
||||
.join(""),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a data URL to extract media type and base64 data
|
||||
/// Format: data:mime_type;base64,data
|
||||
/// Returns (media_type, data) tuple if successful
|
||||
pub fn parse_data_url(url: &str) -> Option<(String, String)> {
|
||||
if !url.starts_with("data:") {
|
||||
return None;
|
||||
}
|
||||
let rest = url.strip_prefix("data:")?;
|
||||
let (header, data) = rest.split_once(",")?;
|
||||
let media_type = header.strip_suffix(";base64")?;
|
||||
Some((media_type.to_string(), data.to_string()))
|
||||
}
|
||||
|
||||
@@ -215,10 +215,7 @@ exit $exit_status
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.envs(reserved_variables)
|
||||
.envs(
|
||||
get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn)
|
||||
.await?,
|
||||
)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.args(cmd_args)
|
||||
@@ -244,10 +241,7 @@ exit $exit_status
|
||||
.env_clear()
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(
|
||||
get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn)
|
||||
.await?,
|
||||
)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.env("HOME", HOME_ENV.as_str())
|
||||
|
||||
@@ -6,9 +6,9 @@ use reqwest::Client;
|
||||
use serde_json::{json, value::RawValue, Value};
|
||||
use windmill_common::client::AuthedClient;
|
||||
use windmill_common::error::to_anyhow;
|
||||
use windmill_object_store::convert_json_line_stream;
|
||||
use windmill_common::worker::{Connection, SqlResultCollectionStrategy};
|
||||
use windmill_common::{error::Error, worker::to_raw_value};
|
||||
use windmill_object_store::convert_json_line_stream;
|
||||
use windmill_parser_sql::{
|
||||
parse_bigquery_sig, parse_db_resource, parse_s3_mode, parse_sql_blocks,
|
||||
parse_sql_statement_named_params,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -563,6 +563,7 @@ pub async fn update_worker_ping_for_failed_init_script(
|
||||
wm_memory_usage: None,
|
||||
job_isolation: None,
|
||||
native_mode: None,
|
||||
uses_batch_http_pull: None,
|
||||
ping_type: PingType::InitScript,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -600,10 +600,7 @@ pub async fn handle_csharp_job(
|
||||
.env_clear()
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(
|
||||
get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn)
|
||||
.await?,
|
||||
)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("TZ", TZ_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
@@ -636,10 +633,7 @@ pub async fn handle_csharp_job(
|
||||
.env_clear()
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(
|
||||
get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn)
|
||||
.await?,
|
||||
)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("TZ", TZ_ENV.as_str())
|
||||
.env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR)
|
||||
|
||||
@@ -121,13 +121,11 @@ async fn get_common_deno_proc_envs(
|
||||
}
|
||||
|
||||
// Add proxy envs (including OTEL tracing proxy if enabled for deno)
|
||||
if let Some(conn) = conn {
|
||||
for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno, job_id, w_id, conn)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
{
|
||||
deno_envs.insert(k.to_string(), v);
|
||||
}
|
||||
for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
{
|
||||
deno_envs.insert(k.to_string(), v);
|
||||
}
|
||||
|
||||
return deno_envs;
|
||||
|
||||
@@ -354,7 +354,7 @@ func Run(req Req) (interface{{}}, error){{
|
||||
.env_clear()
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("TZ", TZ_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
@@ -375,7 +375,7 @@ func Run(req Req) (interface{{}}, error){{
|
||||
.env_clear()
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("TZ", TZ_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
|
||||
@@ -618,7 +618,6 @@ async fn run<'a>(
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(crate::get_otel_context_envs(&job.id))
|
||||
.args(vec![
|
||||
"--config",
|
||||
"run.config.proto",
|
||||
@@ -676,8 +675,7 @@ async fn run<'a>(
|
||||
.env("HOME", &*JAVA_HOME_DIR)
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(crate::get_otel_context_envs(&job.id));
|
||||
.envs(reserved_variables);
|
||||
if metadata(TRUST_STORE_PATH.clone()).await.is_ok() {
|
||||
cmd.args(&[
|
||||
&format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH),
|
||||
|
||||
@@ -73,7 +73,6 @@ mod universal_pkg_installer;
|
||||
#[cfg(feature = "private")]
|
||||
mod volume_ee;
|
||||
mod volume_oss;
|
||||
pub mod wac_executor;
|
||||
mod worker;
|
||||
mod worker_flow;
|
||||
mod worker_lockfiles;
|
||||
|
||||
@@ -264,7 +264,7 @@ async fn run<'a>(
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?)
|
||||
.args(vec![
|
||||
"--config",
|
||||
"run.config.proto",
|
||||
@@ -303,7 +303,7 @@ async fn run<'a>(
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?)
|
||||
// TODO(v1):
|
||||
// "--plugins",
|
||||
// &format!(
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::{
|
||||
get_reserved_variables, read_result, start_child_process, MaybeLock, OccupancyMetrics,
|
||||
},
|
||||
handle_child::handle_child,
|
||||
is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH,
|
||||
COMPOSER_CACHE_DIR, COMPOSER_PATH, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH,
|
||||
};
|
||||
use windmill_common::client::AuthedClient;
|
||||
|
||||
@@ -316,7 +316,6 @@ try {{
|
||||
.env_clear()
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(crate::get_otel_context_envs(&job.id))
|
||||
.env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR)
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.args(args)
|
||||
@@ -333,7 +332,6 @@ try {{
|
||||
.env_clear()
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(crate::get_otel_context_envs(&job.id))
|
||||
.env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR)
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.stdin(Stdio::null())
|
||||
|
||||
@@ -567,9 +567,6 @@ pub async fn handle_python_job(
|
||||
|
||||
let annotations = PythonAnnotations::parse(inner_content);
|
||||
|
||||
let is_wac_v2 = job.script_entrypoint_override.is_none()
|
||||
&& crate::wac_executor::is_wac_v2_py(inner_content);
|
||||
|
||||
if annotations.sandbox && NSJAIL_AVAILABLE.is_none() {
|
||||
return Err(Error::ExecutionErr(
|
||||
"Script has #sandbox annotation but nsjail is not available on this worker. \
|
||||
@@ -680,68 +677,8 @@ pub async fn handle_python_job(
|
||||
String::new()
|
||||
};
|
||||
let main_override = main_name.unwrap_or_else(|| "main".to_string());
|
||||
let wrapper_content: String = if is_wac_v2 {
|
||||
format!(
|
||||
r#"
|
||||
import os
|
||||
import json
|
||||
{import_loader}
|
||||
{import_base64}
|
||||
{import_datetime}
|
||||
import traceback
|
||||
import sys
|
||||
from {module_dir_dot} import {last} as inner_script
|
||||
from wmill.client import _run_workflow
|
||||
|
||||
with open("args.json") as f:
|
||||
kwargs = json.load(f, strict=False)
|
||||
args = {{}}
|
||||
{transforms}
|
||||
|
||||
with open("checkpoint.json") as f:
|
||||
checkpoint = json.load(f, strict=False)
|
||||
|
||||
result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json")
|
||||
|
||||
# Find the @workflow-decorated function
|
||||
workflow_fn = None
|
||||
for name in dir(inner_script):
|
||||
obj = getattr(inner_script, name)
|
||||
if callable(obj) and getattr(obj, '_is_workflow', False):
|
||||
workflow_fn = obj
|
||||
break
|
||||
|
||||
if workflow_fn is None:
|
||||
raise ValueError("No @workflow function found in script")
|
||||
|
||||
for k, v in list(args.items()):
|
||||
if v == '<function call>':
|
||||
del args[k]
|
||||
|
||||
try:
|
||||
output = _run_workflow(workflow_fn, checkpoint, args)
|
||||
output_json = json.dumps(output, separators=(',', ':'), default=str)
|
||||
with open(result_json, 'w') as f:
|
||||
f.write(output_json)
|
||||
except BaseException as e:
|
||||
exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||
tb = traceback.format_tb(exc_traceback)
|
||||
with open(result_json, 'w') as f:
|
||||
err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}
|
||||
extra = e.__dict__
|
||||
if extra and len(extra) > 0:
|
||||
err['extra'] = extra
|
||||
flow_node_id = os.environ.get('WM_FLOW_STEP_ID')
|
||||
if flow_node_id:
|
||||
err['step_id'] = flow_node_id
|
||||
err_json = json.dumps(err, separators=(',', ':'), default=str).replace('\n', '')
|
||||
f.write(err_json)
|
||||
sys.exit(1)
|
||||
"#,
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
r#"
|
||||
let wrapper_content: String = format!(
|
||||
r#"
|
||||
import os
|
||||
import json
|
||||
{import_loader}
|
||||
@@ -814,26 +751,9 @@ except BaseException as e:
|
||||
f.write(err_json)
|
||||
sys.exit(1)
|
||||
"#,
|
||||
)
|
||||
};
|
||||
);
|
||||
write_file(job_dir, "wrapper.py", &wrapper_content)?;
|
||||
|
||||
// For WAC v2, write checkpoint.json before python runs.
|
||||
if is_wac_v2 {
|
||||
if let Connection::Sql(db) = conn {
|
||||
let checkpoint = crate::wac_executor::load_checkpoint(db, &job.id).await?;
|
||||
let checkpoint =
|
||||
crate::wac_executor::prepare_checkpoint_for_resume(db, &job.id, checkpoint).await?;
|
||||
|
||||
let checkpoint_json = serde_json::to_string(&checkpoint).map_err(|e| {
|
||||
error::Error::internal_err(format!("Failed to serialize checkpoint: {e}"))
|
||||
})?;
|
||||
write_file(job_dir, "checkpoint.json", &checkpoint_json)?;
|
||||
} else {
|
||||
write_file(job_dir, "checkpoint.json", r#"{"completed_steps":{}}"#)?;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!("Finished writing wrapper");
|
||||
|
||||
let mut reserved_variables =
|
||||
@@ -921,10 +841,7 @@ mount {{
|
||||
.env_clear()
|
||||
// inject PYTHONPATH here - for some reason I had to do it in nsjail conf
|
||||
.envs(reserved_variables)
|
||||
.envs(
|
||||
get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn)
|
||||
.await?,
|
||||
)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("TZ", TZ_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
@@ -950,10 +867,7 @@ mount {{
|
||||
.env_clear()
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(
|
||||
get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn)
|
||||
.await?,
|
||||
)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("TZ", TZ_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
@@ -1016,15 +930,7 @@ mount {{
|
||||
*new_args = Some(args.clone());
|
||||
}
|
||||
|
||||
let result = read_result(job_dir, handle_result.result_stream).await?;
|
||||
|
||||
// WAC v2 post-execution: parse output and handle dispatch/suspend.
|
||||
// Box::pin to avoid bloating handle_python_job's async state machine (stack overflow).
|
||||
if is_wac_v2 {
|
||||
return Box::pin(crate::bun_executor::handle_wac_v2_output(result, job, conn)).await;
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
read_result(job_dir, handle_result.result_stream).await
|
||||
}
|
||||
|
||||
async fn prepare_wrapper(
|
||||
|
||||
@@ -665,7 +665,7 @@ pub async fn process_completed_job(
|
||||
|
||||
add_time!(bench, "pre add_completed_job");
|
||||
|
||||
let (_, duration, wac_job_ids) = add_completed_job(
|
||||
let (_, duration) = add_completed_job(
|
||||
db,
|
||||
&job,
|
||||
true,
|
||||
@@ -717,29 +717,6 @@ pub async fn process_completed_job(
|
||||
}
|
||||
return Ok(r);
|
||||
}
|
||||
} else if let Some(parent_job) = parent_job {
|
||||
// wac_job_ids is piggybacked from the duration write in
|
||||
// add_completed_job — no extra query needed.
|
||||
if let Some(job_ids) = wac_job_ids {
|
||||
if let Ok(Some(_)) = handle_wac_child_completion(
|
||||
db,
|
||||
&job_id,
|
||||
parent_job,
|
||||
&workspace_id,
|
||||
result,
|
||||
true,
|
||||
job_ids,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if let Some(done_tx) = done_tx {
|
||||
done_tx
|
||||
.send(())
|
||||
.expect("done receiver should still be alive");
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let result = add_completed_job_error(
|
||||
@@ -793,229 +770,11 @@ pub async fn process_completed_job(
|
||||
}
|
||||
return Ok(r);
|
||||
}
|
||||
} else if let Some(parent_job) = job.parent_job {
|
||||
// WAC child failed — query job_ids from parent (errors are rare,
|
||||
// so the extra read is acceptable here).
|
||||
let job_ids_json: Option<Option<Value>> = sqlx::query_scalar(
|
||||
"SELECT workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \
|
||||
FROM v2_job_status WHERE id = $1",
|
||||
)
|
||||
.bind(&parent_job)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
if let Some(Some(job_ids)) = job_ids_json {
|
||||
let err_result = Arc::new(serde_json::value::to_raw_value(&result).unwrap());
|
||||
if let Ok(Some(_)) = handle_wac_child_completion(
|
||||
db,
|
||||
&job.id,
|
||||
parent_job,
|
||||
&job.workspace_id,
|
||||
err_result,
|
||||
false,
|
||||
job_ids,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if let Some(done_tx) = done_tx {
|
||||
done_tx
|
||||
.send(())
|
||||
.expect("done receiver should still be alive");
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
/// Handle a WAC v2 child job completion.
|
||||
/// Returns Ok(Some(())) if the parent was a WAC job and was handled,
|
||||
/// Ok(None) if the parent is not a WAC job (caller should fall through).
|
||||
///
|
||||
/// CONCURRENCY: Multiple parallel children may complete simultaneously on
|
||||
/// different workers. We use atomic SQL operations throughout:
|
||||
/// - `completed_steps` is merged via `jsonb_set(... || jsonb_build_object(...))`
|
||||
/// — PostgreSQL serialises concurrent UPDATEs on the same row, so each
|
||||
/// worker sees the previous worker's writes.
|
||||
/// - The suspend counter (set to N at dispatch time) is decremented atomically
|
||||
/// with `RETURNING` to determine the "all done" condition.
|
||||
pub(crate) async fn handle_wac_child_completion(
|
||||
db: &DB,
|
||||
child_job_id: &Uuid,
|
||||
parent_job_id: Uuid,
|
||||
workspace_id: &str,
|
||||
result: Arc<Box<RawValue>>,
|
||||
success: bool,
|
||||
job_ids_value: Value,
|
||||
) -> error::Result<Option<()>> {
|
||||
let job_ids = match job_ids_value {
|
||||
Value::Object(m) => m,
|
||||
_ => return Ok(None), // Not a WAC parent or no pending steps
|
||||
};
|
||||
|
||||
let child_id_str = child_job_id.to_string();
|
||||
let step_key = job_ids.iter().find_map(|(key, val)| {
|
||||
if val.as_str() == Some(&child_id_str) {
|
||||
Some(key.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
let step_key = match step_key {
|
||||
Some(k) => k,
|
||||
None => {
|
||||
if !success {
|
||||
// No step key and failed — can't store error, fail parent immediately
|
||||
tracing::error!(
|
||||
parent_job = %parent_job_id,
|
||||
child_job = %child_job_id,
|
||||
"WAC v2 child job failed but no step key found, failing parent"
|
||||
);
|
||||
sqlx::query!(
|
||||
"UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1",
|
||||
parent_job_id,
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
let parent_mini = get_mini_completed_job(&parent_job_id, workspace_id, db).await?;
|
||||
if let Some(parent_mini) = parent_mini {
|
||||
let child_err: Value =
|
||||
serde_json::from_str(result.get()).unwrap_or(Value::Null);
|
||||
let err_value = json!({
|
||||
"message": format!("WAC child job {} failed (no step key)", child_job_id),
|
||||
"error": child_err,
|
||||
});
|
||||
let _ = windmill_queue::add_completed_job_error(
|
||||
db,
|
||||
&parent_mini,
|
||||
0,
|
||||
None,
|
||||
err_value,
|
||||
"wac_child_handler",
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
return Ok(Some(()));
|
||||
}
|
||||
tracing::warn!(
|
||||
parent_job = %parent_job_id,
|
||||
child_job = %child_job_id,
|
||||
"WAC v2 child completed but no matching step key found in checkpoint, decrementing suspend to avoid parent hang"
|
||||
);
|
||||
// Still decrement suspend so the parent doesn't hang indefinitely
|
||||
let _ = sqlx::query_scalar!(
|
||||
"UPDATE v2_job_queue \
|
||||
SET suspend = GREATEST(suspend - 1, 0) \
|
||||
WHERE id = $1 \
|
||||
RETURNING suspend",
|
||||
parent_job_id,
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
return Ok(Some(()));
|
||||
}
|
||||
};
|
||||
|
||||
// Build result — wrap errors with _error marker so workflow try/catch can handle them
|
||||
let result_value: Value = if success {
|
||||
serde_json::from_str(result.get()).unwrap_or(Value::Null)
|
||||
} else {
|
||||
let child_err: Value = serde_json::from_str(result.get()).unwrap_or(Value::Null);
|
||||
tracing::info!(
|
||||
parent_job = %parent_job_id,
|
||||
child_job = %child_job_id,
|
||||
step_key = %step_key,
|
||||
"WAC v2 child job failed, storing error for workflow try/catch"
|
||||
);
|
||||
json!({
|
||||
"__wmill_error": true,
|
||||
"message": format!("WAC task '{}' failed (child job {})", step_key, child_job_id),
|
||||
"child_job_id": child_job_id.to_string(),
|
||||
"step_key": step_key,
|
||||
"result": child_err,
|
||||
})
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
parent_job = %parent_job_id,
|
||||
child_job = %child_job_id,
|
||||
step_key = %step_key,
|
||||
success = success,
|
||||
"WAC v2 child job completed"
|
||||
);
|
||||
|
||||
// Use a transaction to ensure completed_steps merge + suspend decrement
|
||||
// are atomic. Without this, a crash between the two could strand the parent.
|
||||
let result_json = serde_json::to_value(&result_value)
|
||||
.map_err(|e| error::Error::InternalErr(format!("Failed to serialize step result: {e}")))?;
|
||||
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
// Merge the completed step into the checkpoint.
|
||||
// Uses `|| jsonb_build_object(key, value)` so concurrent children on
|
||||
// different workers don't overwrite each other — PostgreSQL serialises
|
||||
// concurrent UPDATEs on the same row and each sees the previous write.
|
||||
sqlx::query(
|
||||
"UPDATE v2_job_status SET workflow_as_code_status = jsonb_set(
|
||||
workflow_as_code_status,
|
||||
'{_checkpoint,completed_steps}',
|
||||
COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb)
|
||||
|| jsonb_build_object($2::text, $3::jsonb)
|
||||
) WHERE id = $1",
|
||||
)
|
||||
.bind(&parent_job_id)
|
||||
.bind(&step_key)
|
||||
.bind(&result_json)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| error::Error::InternalErr(format!("Failed to add WAC completed step: {e}")))?;
|
||||
|
||||
// Decrement the suspend counter. The counter was set to N (number of
|
||||
// children) at dispatch time. When it reaches 0 all children are done.
|
||||
// Keep suspend_until non-null so the suspended pull query
|
||||
// (`WHERE suspend_until IS NOT NULL AND suspend <= 0`) picks up the parent.
|
||||
let new_suspend: Option<i32> = sqlx::query_scalar!(
|
||||
"UPDATE v2_job_queue \
|
||||
SET suspend = GREATEST(suspend - 1, 0) \
|
||||
WHERE id = $1 \
|
||||
RETURNING suspend",
|
||||
parent_job_id,
|
||||
)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
|
||||
let all_done = new_suspend == Some(0);
|
||||
|
||||
if all_done {
|
||||
// Clear pending_steps from checkpoint since all children are complete.
|
||||
// This is cosmetic — the next replay will overwrite it anyway — but
|
||||
// keeps the checkpoint clean for frontend display.
|
||||
let _ = sqlx::query(
|
||||
"UPDATE v2_job_status SET workflow_as_code_status = \
|
||||
workflow_as_code_status #- '{_checkpoint,pending_steps}' \
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(&parent_job_id)
|
||||
.execute(&mut *tx)
|
||||
.await;
|
||||
}
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
if all_done {
|
||||
tracing::info!(
|
||||
parent_job = %parent_job_id,
|
||||
"WAC v2 all child jobs completed, unsuspending parent"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(Some(()))
|
||||
}
|
||||
|
||||
pub async fn handle_non_flow_job_error(
|
||||
db: &DB,
|
||||
job: &MiniCompletedJob,
|
||||
|
||||
@@ -812,10 +812,7 @@ mount {{
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(RUBY_PROXY_ENVS.clone())
|
||||
.envs(
|
||||
get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn)
|
||||
.await?,
|
||||
)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?)
|
||||
.args(vec![
|
||||
"--config",
|
||||
"run.config.proto",
|
||||
@@ -854,10 +851,7 @@ mount {{
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.envs(reserved_variables)
|
||||
.envs(RUBY_PROXY_ENVS.clone())
|
||||
.envs(
|
||||
get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn)
|
||||
.await?,
|
||||
)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?)
|
||||
.envs(envs);
|
||||
|
||||
cmd.stdin(Stdio::null())
|
||||
|
||||
@@ -700,10 +700,7 @@ pub async fn handle_rust_job(
|
||||
.env_clear()
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(
|
||||
get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn)
|
||||
.await?,
|
||||
)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("TZ", TZ_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
@@ -719,10 +716,7 @@ pub async fn handle_rust_job(
|
||||
.env_clear()
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(
|
||||
get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn)
|
||||
.await?,
|
||||
)
|
||||
.envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("TZ", TZ_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
|
||||
@@ -9,8 +9,8 @@ use serde_json::{json, value::RawValue, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use windmill_common::error::to_anyhow;
|
||||
use windmill_common::worker::{Connection, SqlResultCollectionStrategy};
|
||||
use windmill_object_store::convert_json_line_stream;
|
||||
use windmill_common::worker::{Connection, SqlResultCollectionStrategy};
|
||||
|
||||
use windmill_common::{error::Error, worker::to_raw_value};
|
||||
use windmill_parser_sql::{
|
||||
|
||||
@@ -1,339 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::value::RawValue;
|
||||
use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
use windmill_common::error::{self, Error};
|
||||
use windmill_common::DB;
|
||||
|
||||
/// Checkpoint state persisted across workflow invocations.
|
||||
#[derive(Debug, Serialize, Deserialize, Default, Clone)]
|
||||
pub struct WacCheckpoint {
|
||||
#[serde(default)]
|
||||
pub source_hash: String,
|
||||
#[serde(default)]
|
||||
pub completed_steps: serde_json::Map<String, Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pending_steps: Option<WacPendingSteps>,
|
||||
#[serde(default)]
|
||||
pub input_args: serde_json::Map<String, Value>,
|
||||
/// Accumulated map of step_key → child job UUID across all dispatch rounds.
|
||||
/// Unlike `pending_steps.job_ids` (cleared after completion), this persists
|
||||
/// so the frontend can always resolve step keys to child job names.
|
||||
#[serde(default)]
|
||||
pub job_ids: serde_json::Map<String, Value>,
|
||||
/// When set on a child job's checkpoint, indicates which step this child
|
||||
/// should execute directly (instead of dispatching).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[serde(default)]
|
||||
pub _executing_key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct WacPendingSteps {
|
||||
pub mode: String,
|
||||
pub keys: Vec<String>,
|
||||
pub job_ids: serde_json::Map<String, Value>,
|
||||
}
|
||||
|
||||
/// Output from a single WAC invocation (parsed from result.json).
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum WacOutput {
|
||||
#[serde(rename = "dispatch")]
|
||||
Dispatch { mode: String, steps: Vec<WacStepDispatch> },
|
||||
#[serde(rename = "complete")]
|
||||
Complete { result: Value },
|
||||
/// An inline step executed in the parent process — persist result to
|
||||
/// checkpoint and re-run immediately (no child job, no suspend).
|
||||
#[serde(rename = "inline_checkpoint")]
|
||||
InlineCheckpoint { key: String, result: Value },
|
||||
/// Suspend the workflow waiting for an external approval event.
|
||||
/// No child job is dispatched — the parent suspends directly and resumes
|
||||
/// when a user hits the resume/cancel endpoint.
|
||||
#[serde(rename = "approval")]
|
||||
Approval { key: String, timeout: Option<u32>, form: Option<Value> },
|
||||
/// Server-side sleep — suspend the workflow for a duration without holding a worker.
|
||||
#[serde(rename = "sleep")]
|
||||
Sleep { key: String, seconds: u32 },
|
||||
}
|
||||
|
||||
/// A step dispatched by the WAC SDK.
|
||||
///
|
||||
/// `dispatch_type` determines how the child job is created:
|
||||
/// - `"inline"` (default): re-runs the parent workflow with `_executing_key` set
|
||||
/// - `"script"`: runs a separate Windmill script resolved from `script` path
|
||||
/// - `"flow"`: runs a separate Windmill flow resolved from `script` path
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct WacStepDispatch {
|
||||
pub name: String,
|
||||
pub script: String,
|
||||
pub args: serde_json::Map<String, Value>,
|
||||
pub key: String,
|
||||
#[serde(default = "default_dispatch_type")]
|
||||
pub dispatch_type: String,
|
||||
// Per-task options forwarded to push()
|
||||
#[serde(default)]
|
||||
pub timeout: Option<i32>,
|
||||
#[serde(default)]
|
||||
pub tag: Option<String>,
|
||||
#[serde(default)]
|
||||
pub cache_ttl: Option<i32>,
|
||||
#[serde(default)]
|
||||
pub priority: Option<i16>,
|
||||
#[serde(default)]
|
||||
pub concurrent_limit: Option<i32>,
|
||||
#[serde(default)]
|
||||
pub concurrency_key: Option<String>,
|
||||
#[serde(default)]
|
||||
pub concurrency_time_window_s: Option<i32>,
|
||||
}
|
||||
|
||||
fn default_dispatch_type() -> String {
|
||||
"inline".to_string()
|
||||
}
|
||||
|
||||
/// Load the WAC checkpoint from `v2_job_status.workflow_as_code_status._checkpoint`.
|
||||
pub async fn load_checkpoint(db: &DB, job_id: &Uuid) -> error::Result<WacCheckpoint> {
|
||||
let row: Option<Option<Value>> = sqlx::query_scalar(
|
||||
"SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = $1",
|
||||
)
|
||||
.bind(job_id)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
match row {
|
||||
Some(Some(status)) => {
|
||||
let checkpoint: WacCheckpoint = match serde_json::from_value(status) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
job_id = %job_id,
|
||||
error = %e,
|
||||
"Failed to deserialize WAC checkpoint, resetting to empty"
|
||||
);
|
||||
WacCheckpoint::default()
|
||||
}
|
||||
};
|
||||
Ok(checkpoint)
|
||||
}
|
||||
_ => Ok(WacCheckpoint::default()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Save the WAC checkpoint to `v2_job_status.workflow_as_code_status._checkpoint`.
|
||||
/// The top level of workflow_as_code_status is reserved for per-child-job timeline data.
|
||||
pub async fn save_checkpoint(
|
||||
db: &DB,
|
||||
job_id: &Uuid,
|
||||
checkpoint: &WacCheckpoint,
|
||||
) -> error::Result<()> {
|
||||
let status_json = serde_json::to_value(checkpoint)
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to serialize checkpoint: {e}")))?;
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO v2_job_status (id, workflow_as_code_status)
|
||||
VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb))
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
workflow_as_code_status = jsonb_set(
|
||||
COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb),
|
||||
'{_checkpoint}',
|
||||
$2::jsonb
|
||||
)",
|
||||
)
|
||||
.bind(job_id)
|
||||
.bind(&status_json)
|
||||
.execute(db)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to save WAC checkpoint: {e}")))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parse the WAC result from result.json content.
|
||||
pub fn parse_wac_output(result: &RawValue) -> error::Result<WacOutput> {
|
||||
serde_json::from_str(result.get())
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to parse WAC output: {e}")))
|
||||
}
|
||||
|
||||
/// Process a "dispatch" result: update checkpoint with pending steps info.
|
||||
pub fn update_checkpoint_for_dispatch(
|
||||
checkpoint: &mut WacCheckpoint,
|
||||
steps: &[WacStepDispatch],
|
||||
mode: &str,
|
||||
job_ids: &[(String, Uuid)],
|
||||
) {
|
||||
let ids_map: serde_json::Map<String, Value> = job_ids
|
||||
.iter()
|
||||
.map(|(key, id)| (key.clone(), Value::String(id.to_string())))
|
||||
.collect();
|
||||
// Accumulate into persistent job_ids (survives pending_steps clearing)
|
||||
for (k, v) in ids_map.iter() {
|
||||
checkpoint.job_ids.insert(k.clone(), v.clone());
|
||||
}
|
||||
let pending = WacPendingSteps {
|
||||
mode: mode.to_string(),
|
||||
keys: steps.iter().map(|s| s.key.clone()).collect(),
|
||||
job_ids: ids_map,
|
||||
};
|
||||
checkpoint.pending_steps = Some(pending);
|
||||
}
|
||||
|
||||
/// Process a completed child job result: add to checkpoint's completed_steps.
|
||||
pub fn add_completed_step(checkpoint: &mut WacCheckpoint, step_key: &str, result: Value) {
|
||||
checkpoint
|
||||
.completed_steps
|
||||
.insert(step_key.to_string(), result);
|
||||
// If all pending steps are complete, clear pending
|
||||
if let Some(ref pending) = checkpoint.pending_steps {
|
||||
let all_done = pending
|
||||
.keys
|
||||
.iter()
|
||||
.all(|k| checkpoint.completed_steps.contains_key(k));
|
||||
if all_done {
|
||||
checkpoint.pending_steps = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if all pending parallel steps are complete.
|
||||
pub fn all_pending_complete(checkpoint: &WacCheckpoint) -> bool {
|
||||
match &checkpoint.pending_steps {
|
||||
None => true,
|
||||
Some(pending) => pending
|
||||
.keys
|
||||
.iter()
|
||||
.all(|k| checkpoint.completed_steps.contains_key(k)),
|
||||
}
|
||||
}
|
||||
|
||||
/// If the checkpoint has a pending approval or sleep, inject the resume result
|
||||
/// into `completed_steps` and save back to DB. Returns the (possibly modified) checkpoint.
|
||||
///
|
||||
/// Called by both bun and python executors before writing checkpoint.json to disk.
|
||||
pub async fn prepare_checkpoint_for_resume(
|
||||
db: &DB,
|
||||
job_id: &Uuid,
|
||||
mut checkpoint: WacCheckpoint,
|
||||
) -> error::Result<WacCheckpoint> {
|
||||
let pending_mode = checkpoint.pending_steps.as_ref().map(|p| p.mode.as_str());
|
||||
|
||||
match pending_mode {
|
||||
Some("approval") => {
|
||||
let approval_key = checkpoint
|
||||
.pending_steps
|
||||
.as_ref()
|
||||
.and_then(|p| p.keys.first().cloned())
|
||||
.unwrap_or_default();
|
||||
|
||||
let resume_row = sqlx::query_as::<_, (sqlx::types::Json<Box<serde_json::value::RawValue>>, Option<String>, bool)>(
|
||||
"SELECT value, approver, approved FROM resume_job WHERE job = $1 ORDER BY created_at ASC LIMIT 1",
|
||||
)
|
||||
.bind(job_id)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let approval_result = if let Some((value, approver, approved)) = resume_row {
|
||||
serde_json::json!({
|
||||
"value": serde_json::from_str::<Value>(value.get()).unwrap_or(Value::Null),
|
||||
"approver": approver.unwrap_or_else(|| "anonymous".to_string()),
|
||||
"approved": approved,
|
||||
})
|
||||
} else {
|
||||
serde_json::json!({
|
||||
"value": null,
|
||||
"approver": null,
|
||||
"approved": false,
|
||||
})
|
||||
};
|
||||
checkpoint
|
||||
.completed_steps
|
||||
.insert(approval_key.clone(), approval_result);
|
||||
checkpoint.pending_steps = None;
|
||||
save_checkpoint(db, job_id, &checkpoint).await?;
|
||||
|
||||
tracing::info!(
|
||||
job_id = %job_id,
|
||||
approval_key = %approval_key,
|
||||
"WAC v2 injected approval result into checkpoint"
|
||||
);
|
||||
}
|
||||
Some("sleep") => {
|
||||
let sleep_key = checkpoint
|
||||
.pending_steps
|
||||
.as_ref()
|
||||
.and_then(|p| p.keys.first().cloned())
|
||||
.unwrap_or_default();
|
||||
|
||||
checkpoint
|
||||
.completed_steps
|
||||
.insert(sleep_key.clone(), Value::Bool(true));
|
||||
checkpoint.pending_steps = None;
|
||||
save_checkpoint(db, job_id, &checkpoint).await?;
|
||||
|
||||
tracing::info!(
|
||||
job_id = %job_id,
|
||||
sleep_key = %sleep_key,
|
||||
"WAC v2 resumed from sleep"
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
Ok(checkpoint)
|
||||
}
|
||||
|
||||
/// Detect WAC v2 patterns in TypeScript/Bun code.
|
||||
/// Checks for `import ... from "windmill-client"` containing workflow/task,
|
||||
/// skipping comment lines.
|
||||
pub fn is_wac_v2_ts(code: &str) -> bool {
|
||||
let mut has_wac_import = false;
|
||||
let mut has_workflow = false;
|
||||
let mut has_task = false;
|
||||
for line in code.lines() {
|
||||
let trimmed = line.trim();
|
||||
if trimmed.starts_with("//") {
|
||||
continue;
|
||||
}
|
||||
if trimmed.contains("windmill-client")
|
||||
&& (trimmed.starts_with("import") || trimmed.starts_with("from"))
|
||||
{
|
||||
has_wac_import = true;
|
||||
if trimmed.contains("workflow") {
|
||||
has_workflow = true;
|
||||
}
|
||||
if trimmed.contains("task") {
|
||||
has_task = true;
|
||||
}
|
||||
}
|
||||
if trimmed.contains("export") && trimmed.contains("workflow(") {
|
||||
has_workflow = true;
|
||||
}
|
||||
}
|
||||
has_wac_import && has_workflow && has_task
|
||||
}
|
||||
|
||||
/// Detect WAC v2 patterns in Python code.
|
||||
/// Checks for `@workflow` decorator and `@task` decorator with wmill import,
|
||||
/// skipping comment lines.
|
||||
pub fn is_wac_v2_py(code: &str) -> bool {
|
||||
let mut has_wmill_import = false;
|
||||
let mut has_workflow_decorator = false;
|
||||
let mut has_task_decorator = false;
|
||||
for line in code.lines() {
|
||||
let trimmed = line.trim();
|
||||
if trimmed.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
if trimmed.starts_with("import wmill") || trimmed.starts_with("from wmill") {
|
||||
has_wmill_import = true;
|
||||
}
|
||||
if trimmed == "@workflow" || trimmed.starts_with("@workflow(") {
|
||||
has_workflow_decorator = true;
|
||||
}
|
||||
if trimmed == "@task" || trimmed.starts_with("@task(") {
|
||||
has_task_decorator = true;
|
||||
}
|
||||
}
|
||||
has_wmill_import && has_workflow_decorator && has_task_decorator
|
||||
}
|
||||
@@ -740,69 +740,26 @@ pub async fn is_otel_tracing_proxy_enabled_for_lang(lang: &ScriptLang) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get OTEL trace context environment variables for a job (TRACEPARENT, OTEL_TRACE_ID, OTEL_SPAN_ID).
|
||||
/// Returns an empty vec when OTEL tracing is not enabled or on non-enterprise builds.
|
||||
pub fn get_otel_context_envs(job_id: &uuid::Uuid) -> Vec<(&'static str, String)> {
|
||||
#[cfg(all(feature = "private", feature = "enterprise"))]
|
||||
if windmill_common::OTEL_TRACING_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
let trace_id = format!("{:032x}", job_id.as_u128());
|
||||
let span_id = format!("{:016x}", job_id.as_u64_pair().1);
|
||||
let traceparent = format!("00-{}-{}-01", trace_id, span_id);
|
||||
return vec![
|
||||
("TRACEPARENT", traceparent),
|
||||
("OTEL_TRACE_ID", trace_id),
|
||||
("OTEL_SPAN_ID", span_id),
|
||||
];
|
||||
}
|
||||
let _ = job_id;
|
||||
vec![]
|
||||
}
|
||||
|
||||
/// Get proxy environment variables for job execution for a specific language.
|
||||
/// When OTEL tracing proxy is enabled for this language, routes all traffic through the proxy.
|
||||
/// Otherwise, uses the standard HTTP_PROXY/HTTPS_PROXY from environment.
|
||||
pub async fn get_proxy_envs_for_lang(
|
||||
lang: &ScriptLang,
|
||||
job_id: &uuid::Uuid,
|
||||
w_id: &str,
|
||||
conn: &Connection,
|
||||
) -> anyhow::Result<Vec<(&'static str, String)>> {
|
||||
#[allow(unused_mut)]
|
||||
let mut envs;
|
||||
#[cfg(all(feature = "private", feature = "enterprise"))]
|
||||
if is_otel_tracing_proxy_enabled_for_lang(lang).await {
|
||||
envs = get_otel_tracing_proxy_envs(job_id, w_id, conn).await?;
|
||||
} else {
|
||||
envs = PROXY_ENVS.clone();
|
||||
return get_otel_tracing_proxy_envs().await;
|
||||
}
|
||||
#[cfg(not(all(feature = "private", feature = "enterprise")))]
|
||||
{
|
||||
let _ = (lang, w_id, conn);
|
||||
envs = PROXY_ENVS.clone();
|
||||
}
|
||||
envs.extend(get_otel_context_envs(job_id));
|
||||
Ok(envs)
|
||||
let _ = lang;
|
||||
Ok(PROXY_ENVS.clone())
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "private", feature = "enterprise"))]
|
||||
async fn get_otel_tracing_proxy_envs(
|
||||
job_id: &uuid::Uuid,
|
||||
w_id: &str,
|
||||
conn: &Connection,
|
||||
) -> anyhow::Result<Vec<(&'static str, String)>> {
|
||||
let port = match *crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT
|
||||
async fn get_otel_tracing_proxy_envs() -> anyhow::Result<Vec<(&'static str, String)>> {
|
||||
let port = crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT
|
||||
.read()
|
||||
.await
|
||||
{
|
||||
Some(p) => p,
|
||||
None => {
|
||||
let reason = "OTEL tracing proxy is enabled but not available (not initialized yet, or NUM_WORKERS > 1). \
|
||||
This job's HTTP requests will not be traced.";
|
||||
tracing::warn!("{}", reason);
|
||||
append_logs(job_id, w_id, format!("\n[warning] {reason}\n"), conn).await;
|
||||
return Ok(PROXY_ENVS.clone());
|
||||
}
|
||||
};
|
||||
.ok_or_else(|| anyhow::anyhow!("OTEL tracing proxy port not initialized"))?;
|
||||
let proxy_url = format!("http://127.0.0.1:{}", port);
|
||||
Ok(vec![
|
||||
("HTTP_PROXY", proxy_url.clone()),
|
||||
@@ -1411,6 +1368,7 @@ pub async fn run_worker(
|
||||
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
killpill_tx: KillpillSender,
|
||||
base_internal_url: &str,
|
||||
batch_pull_client: Option<&HttpClient>,
|
||||
) {
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
if is_sandboxing_enabled() {
|
||||
@@ -2111,135 +2069,149 @@ pub async fn run_worker(
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
match &conn {
|
||||
Connection::Sql(db) => {
|
||||
let pull_time = Instant::now();
|
||||
let likelihood_of_suspend = last_30jobs_suspended as f64 / 30.0;
|
||||
|
||||
let suspend_first = suspend_first_success
|
||||
|| rand::random::<f64>() < likelihood_of_suspend
|
||||
|| last_suspend_first.elapsed().as_secs_f64() > 5.0;
|
||||
|
||||
if suspend_first {
|
||||
last_suspend_first = Instant::now();
|
||||
}
|
||||
let mut job = match timeout(
|
||||
Duration::from_secs(30),
|
||||
pull(
|
||||
&db,
|
||||
suspend_first,
|
||||
&worker_name,
|
||||
None,
|
||||
#[cfg(feature = "benchmark")]
|
||||
&mut bench,
|
||||
)
|
||||
.warn_after_seconds(2),
|
||||
)
|
||||
// If batch_pull_client is set (native worker with co-located server),
|
||||
// use HTTP pull from batch buffer. Otherwise use direct SQL pull.
|
||||
if let Some(bpc) = batch_pull_client {
|
||||
crate::agent_workers::pull_job(bpc, None, None)
|
||||
.await
|
||||
{
|
||||
Ok(job) => job,
|
||||
Err(e) => {
|
||||
tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 20s, sleeping for 30s: {e:?}");
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
.map_err(|e| error::Error::InternalErr(e.to_string()))
|
||||
.map(|x| x.map(|y| NextJob::Http(y)))
|
||||
} else {
|
||||
match &conn {
|
||||
Connection::Sql(db) => {
|
||||
let pull_time = Instant::now();
|
||||
let likelihood_of_suspend = last_30jobs_suspended as f64 / 30.0;
|
||||
|
||||
// Preprocess pulled job result
|
||||
if let Ok(ref mut pulled_job_res) = job {
|
||||
if let Err(e) = timeout(
|
||||
// Will fail if longer than 10 seconds
|
||||
core::time::Duration::from_secs(10),
|
||||
pulled_job_res.maybe_apply_debouncing(db),
|
||||
)
|
||||
.warn_after_seconds(2)
|
||||
.await
|
||||
// Flatten result
|
||||
.map_err(error::Error::from)
|
||||
.and_then(|r| r)
|
||||
{
|
||||
pulled_job_res.error_while_preprocessing = Some(e.to_string());
|
||||
}
|
||||
}
|
||||
let suspend_first = suspend_first_success
|
||||
|| rand::random::<f64>() < likelihood_of_suspend
|
||||
|| last_suspend_first.elapsed().as_secs_f64() > 5.0;
|
||||
|
||||
add_time!(bench, "job pulled from DB");
|
||||
let duration_pull_s = pull_time.elapsed().as_secs_f64();
|
||||
let err_pull = job.is_ok();
|
||||
// let empty = job.as_ref().is_ok_and(|x| x.is_none());
|
||||
|
||||
if duration_pull_s > 0.5 {
|
||||
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
|
||||
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
|
||||
#[cfg(feature = "prometheus")]
|
||||
if empty {
|
||||
if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if let Some(wp) = worker_pull_over_500_counter.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if duration_pull_s > 0.1 {
|
||||
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
|
||||
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
|
||||
#[cfg(feature = "prometheus")]
|
||||
if empty {
|
||||
if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if let Some(wp) = worker_pull_over_100_counter.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(j) = job.as_ref() {
|
||||
let suspend_success = j.suspended;
|
||||
if suspend_first {
|
||||
if last_30jobs_suspended < 30 {
|
||||
last_30jobs_suspended += 1;
|
||||
}
|
||||
} else {
|
||||
last_30jobs_suspended -= 1;
|
||||
last_suspend_first = Instant::now();
|
||||
}
|
||||
suspend_first_success = suspend_first && suspend_success;
|
||||
#[cfg(feature = "prometheus")]
|
||||
if j.job.is_some() {
|
||||
if let Some(wp) = worker_pull_duration_counter.as_ref() {
|
||||
wp.inc_by(duration_pull_s);
|
||||
let mut job = match timeout(
|
||||
Duration::from_secs(30),
|
||||
pull(
|
||||
&db,
|
||||
suspend_first,
|
||||
&worker_name,
|
||||
None,
|
||||
#[cfg(feature = "benchmark")]
|
||||
&mut bench,
|
||||
)
|
||||
.warn_after_seconds(2),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(job) => job,
|
||||
Err(e) => {
|
||||
tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 20s, sleeping for 30s: {e:?}");
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
continue;
|
||||
}
|
||||
if let Some(wp) = worker_pull_duration.as_ref() {
|
||||
wp.observe(duration_pull_s);
|
||||
}
|
||||
} else {
|
||||
if let Some(wp) = worker_pull_duration_counter_empty.as_ref() {
|
||||
wp.inc_by(duration_pull_s);
|
||||
}
|
||||
if let Some(wp) = worker_pull_duration_empty.as_ref() {
|
||||
wp.observe(duration_pull_s);
|
||||
};
|
||||
|
||||
// Preprocess pulled job result
|
||||
if let Ok(ref mut pulled_job_res) = job {
|
||||
if let Err(e) = timeout(
|
||||
// Will fail if longer than 10 seconds
|
||||
core::time::Duration::from_secs(10),
|
||||
pulled_job_res.maybe_apply_debouncing(db),
|
||||
)
|
||||
.warn_after_seconds(2)
|
||||
.await
|
||||
// Flatten result
|
||||
.map_err(error::Error::from)
|
||||
.and_then(|r| r)
|
||||
{
|
||||
pulled_job_res.error_while_preprocessing = Some(e.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
match job {
|
||||
Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() {
|
||||
Ok(j) => Ok(j.map(|job| NextJob::Sql { flow_runners: None, job })),
|
||||
Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc))
|
||||
| Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(jc)) => {
|
||||
if let Err(err) = job_completed_tx.send_job(jc, true).await {
|
||||
tracing::error!(
|
||||
|
||||
add_time!(bench, "job pulled from DB");
|
||||
let duration_pull_s = pull_time.elapsed().as_secs_f64();
|
||||
let err_pull = job.is_ok();
|
||||
// let empty = job.as_ref().is_ok_and(|x| x.is_none());
|
||||
|
||||
if duration_pull_s > 0.5 {
|
||||
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
|
||||
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
|
||||
#[cfg(feature = "prometheus")]
|
||||
if empty {
|
||||
if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if let Some(wp) = worker_pull_over_500_counter.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if duration_pull_s > 0.1 {
|
||||
let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
|
||||
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
|
||||
#[cfg(feature = "prometheus")]
|
||||
if empty {
|
||||
if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
} else if let Some(wp) = worker_pull_over_100_counter.as_ref() {
|
||||
wp.inc();
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(j) = job.as_ref() {
|
||||
let suspend_success = j.suspended;
|
||||
if suspend_first {
|
||||
if last_30jobs_suspended < 30 {
|
||||
last_30jobs_suspended += 1;
|
||||
}
|
||||
} else {
|
||||
last_30jobs_suspended -= 1;
|
||||
}
|
||||
suspend_first_success = suspend_first && suspend_success;
|
||||
#[cfg(feature = "prometheus")]
|
||||
if j.job.is_some() {
|
||||
if let Some(wp) = worker_pull_duration_counter.as_ref() {
|
||||
wp.inc_by(duration_pull_s);
|
||||
}
|
||||
if let Some(wp) = worker_pull_duration.as_ref() {
|
||||
wp.observe(duration_pull_s);
|
||||
}
|
||||
} else {
|
||||
if let Some(wp) = worker_pull_duration_counter_empty.as_ref() {
|
||||
wp.inc_by(duration_pull_s);
|
||||
}
|
||||
if let Some(wp) = worker_pull_duration_empty.as_ref() {
|
||||
wp.observe(duration_pull_s);
|
||||
}
|
||||
}
|
||||
}
|
||||
match job {
|
||||
Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() {
|
||||
Ok(j) => {
|
||||
Ok(j.map(|job| NextJob::Sql { flow_runners: None, job }))
|
||||
}
|
||||
Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc))
|
||||
| Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(jc)) => {
|
||||
if let Err(err) = job_completed_tx.send_job(jc, true).await
|
||||
{
|
||||
tracing::error!(
|
||||
"An error occurred while sending job completed: {:#?}",
|
||||
err
|
||||
)
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
},
|
||||
Err(err) => Err(err),
|
||||
},
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
Connection::Http(client) => {
|
||||
crate::agent_workers::pull_job(&client, None, None)
|
||||
.await
|
||||
.map_err(|e| error::Error::InternalErr(e.to_string()))
|
||||
.map(|x| x.map(|y| NextJob::Http(y)))
|
||||
}
|
||||
}
|
||||
|
||||
Connection::Http(client) => crate::agent_workers::pull_job(&client, None, None)
|
||||
.await
|
||||
.map_err(|e| error::Error::InternalErr(e.to_string()))
|
||||
.map(|x| x.map(|y| NextJob::Http(y))),
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -3528,13 +3500,6 @@ pub async fn handle_queued_job(
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
if result
|
||||
.as_ref()
|
||||
.is_err_and(|err| matches!(err, &Error::WacSuspended(_)))
|
||||
{
|
||||
// WAC v2 job suspended while waiting for child jobs — don't complete it
|
||||
return Ok(true);
|
||||
}
|
||||
process_result(
|
||||
cjob,
|
||||
result.map(|x| Arc::new(x)),
|
||||
@@ -3933,7 +3898,7 @@ pub async fn run_language_executor(
|
||||
run_inline: bool,
|
||||
) -> error::Result<Box<RawValue>> {
|
||||
if language == Some(ScriptLang::Postgresql) {
|
||||
return Box::pin(do_postgresql(
|
||||
return do_postgresql(
|
||||
job,
|
||||
&client,
|
||||
&code,
|
||||
@@ -3945,7 +3910,7 @@ pub async fn run_language_executor(
|
||||
occupancy_metrics,
|
||||
parent_runnable_path,
|
||||
run_inline,
|
||||
))
|
||||
)
|
||||
.await;
|
||||
} else if language == Some(ScriptLang::Mysql) {
|
||||
#[cfg(not(feature = "mysql"))]
|
||||
@@ -3960,7 +3925,7 @@ pub async fn run_language_executor(
|
||||
"Inline execution is not yet supported for this language".to_string(),
|
||||
));
|
||||
}
|
||||
return Box::pin(do_mysql(
|
||||
return do_mysql(
|
||||
job,
|
||||
&client,
|
||||
&code,
|
||||
@@ -3971,7 +3936,7 @@ pub async fn run_language_executor(
|
||||
column_order,
|
||||
occupancy_metrics,
|
||||
parent_runnable_path,
|
||||
))
|
||||
)
|
||||
.await;
|
||||
}
|
||||
} else if language == Some(ScriptLang::Bigquery) {
|
||||
@@ -3997,7 +3962,7 @@ pub async fn run_language_executor(
|
||||
"Inline execution is not yet supported for this language".to_string(),
|
||||
));
|
||||
}
|
||||
return Box::pin(do_bigquery(
|
||||
return do_bigquery(
|
||||
job,
|
||||
&client,
|
||||
&code,
|
||||
@@ -4008,7 +3973,7 @@ pub async fn run_language_executor(
|
||||
column_order,
|
||||
occupancy_metrics,
|
||||
parent_runnable_path,
|
||||
))
|
||||
)
|
||||
.await;
|
||||
}
|
||||
} else if language == Some(ScriptLang::Snowflake) {
|
||||
@@ -4026,7 +3991,7 @@ pub async fn run_language_executor(
|
||||
"Inline execution is not yet supported for this language".to_string(),
|
||||
));
|
||||
}
|
||||
return Box::pin(do_snowflake(
|
||||
return do_snowflake(
|
||||
job,
|
||||
&client,
|
||||
&code,
|
||||
@@ -4037,7 +4002,7 @@ pub async fn run_language_executor(
|
||||
column_order,
|
||||
occupancy_metrics,
|
||||
parent_runnable_path,
|
||||
))
|
||||
)
|
||||
.await;
|
||||
}
|
||||
} else if language == Some(ScriptLang::Mssql) {
|
||||
@@ -4063,7 +4028,7 @@ pub async fn run_language_executor(
|
||||
"Inline execution is not yet supported for this language".to_string(),
|
||||
));
|
||||
}
|
||||
return Box::pin(do_mssql(
|
||||
return do_mssql(
|
||||
job,
|
||||
&client,
|
||||
&code,
|
||||
@@ -4074,7 +4039,7 @@ pub async fn run_language_executor(
|
||||
occupancy_metrics,
|
||||
job_dir,
|
||||
parent_runnable_path,
|
||||
))
|
||||
)
|
||||
.await;
|
||||
}
|
||||
} else if language == Some(ScriptLang::OracleDB) {
|
||||
@@ -4100,7 +4065,7 @@ pub async fn run_language_executor(
|
||||
"Inline execution is not yet supported for this language".to_string(),
|
||||
));
|
||||
}
|
||||
return Box::pin(do_oracledb(
|
||||
return do_oracledb(
|
||||
job,
|
||||
&client,
|
||||
&code,
|
||||
@@ -4111,7 +4076,7 @@ pub async fn run_language_executor(
|
||||
column_order,
|
||||
occupancy_metrics,
|
||||
parent_runnable_path,
|
||||
))
|
||||
)
|
||||
.await;
|
||||
}
|
||||
} else if language == Some(ScriptLang::DuckDb) {
|
||||
@@ -4125,7 +4090,7 @@ pub async fn run_language_executor(
|
||||
|
||||
#[cfg(feature = "duckdb")]
|
||||
{
|
||||
return Box::pin(do_duckdb(
|
||||
return do_duckdb(
|
||||
job,
|
||||
&client,
|
||||
&code,
|
||||
@@ -4137,7 +4102,7 @@ pub async fn run_language_executor(
|
||||
occupancy_metrics,
|
||||
parent_runnable_path,
|
||||
run_inline,
|
||||
))
|
||||
)
|
||||
.await;
|
||||
}
|
||||
} else if language == Some(ScriptLang::Graphql) {
|
||||
@@ -4146,7 +4111,7 @@ pub async fn run_language_executor(
|
||||
"Inline execution is not yet supported for this language".to_string(),
|
||||
));
|
||||
}
|
||||
return Box::pin(do_graphql(
|
||||
return do_graphql(
|
||||
job,
|
||||
&client,
|
||||
&code,
|
||||
@@ -4155,7 +4120,7 @@ pub async fn run_language_executor(
|
||||
canceled_by,
|
||||
worker_name,
|
||||
occupancy_metrics,
|
||||
))
|
||||
)
|
||||
.await;
|
||||
} else if language == Some(ScriptLang::Nativets) {
|
||||
if run_inline {
|
||||
@@ -4182,7 +4147,7 @@ pub async fn run_language_executor(
|
||||
.collect::<Vec<String>>()
|
||||
.join("\n"));
|
||||
|
||||
let result = Box::pin(do_nativets(
|
||||
let result = do_nativets(
|
||||
job,
|
||||
&client,
|
||||
env_code,
|
||||
@@ -4193,7 +4158,7 @@ pub async fn run_language_executor(
|
||||
worker_name,
|
||||
occupancy_metrics,
|
||||
has_stream,
|
||||
))
|
||||
)
|
||||
.await?;
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
@@ -865,7 +865,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
.and_then(|x| x.stop_after_all_iters_if.as_ref())
|
||||
{
|
||||
let args = from_result_to_args(args.as_ref().await.get_ref())?;
|
||||
if let Err(e) = evaluate_stop_after_all_iters_if(
|
||||
evaluate_stop_after_all_iters_if(
|
||||
db,
|
||||
stop_after_all_iters_if,
|
||||
module_status,
|
||||
@@ -879,16 +879,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
flow,
|
||||
&old_status,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::error!("error evaluating stop_after_all_iters_if: {e:#}");
|
||||
stop_early = true;
|
||||
skip_if_stop_early = false;
|
||||
stop_early_err_msg = Some(format!(
|
||||
"Error evaluating stop_after_all_iters_if expression `{}`: {e:#}",
|
||||
stop_after_all_iters_if.expr
|
||||
));
|
||||
}
|
||||
.await?;
|
||||
}
|
||||
|
||||
let new_status = if
|
||||
@@ -1083,7 +1074,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
{
|
||||
let args = from_result_to_args(args.as_ref().await.get_ref())?;
|
||||
|
||||
if let Err(e) = evaluate_stop_after_all_iters_if(
|
||||
evaluate_stop_after_all_iters_if(
|
||||
db,
|
||||
stop_after_all_iters_if,
|
||||
module_status,
|
||||
@@ -1097,15 +1088,7 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
flow,
|
||||
&old_status,
|
||||
)
|
||||
.await
|
||||
{
|
||||
stop_early = true;
|
||||
skip_if_stop_early = false;
|
||||
stop_early_err_msg = Some(format!(
|
||||
"Error evaluating stop_after_all_iters_if expression `{}`: {e:#}",
|
||||
stop_after_all_iters_if.expr
|
||||
));
|
||||
}
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1722,8 +1705,8 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
chat_ai_info.conversation_id,
|
||||
)
|
||||
.await?;
|
||||
let (duration, wac_job_ids) = if success {
|
||||
let (_, duration, wac_job_ids) = add_completed_job(
|
||||
let duration = if success {
|
||||
let (_, duration) = add_completed_job(
|
||||
db,
|
||||
&cflow_job,
|
||||
true,
|
||||
@@ -1737,9 +1720,9 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
(duration, wac_job_ids)
|
||||
duration
|
||||
} else {
|
||||
let (_, duration, wac_job_ids) = add_completed_job(
|
||||
let (_, duration) = add_completed_job(
|
||||
db,
|
||||
&cflow_job,
|
||||
false,
|
||||
@@ -1757,30 +1740,11 @@ pub async fn update_flow_status_after_job_completion_internal(
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
(duration, wac_job_ids)
|
||||
duration
|
||||
};
|
||||
flow_job_duration = flow_job
|
||||
.started_at
|
||||
.map(|x| FlowJobDuration { started_at: x, duration_ms: duration });
|
||||
|
||||
// If this flow is a WAC child (not a flow step, has parent),
|
||||
// notify the WAC parent of completion.
|
||||
if !flow_job.is_flow_step() {
|
||||
if let Some(parent_job) = flow_job.parent_job {
|
||||
if let Some(job_ids) = wac_job_ids {
|
||||
let _ = crate::result_processor::handle_wac_child_completion(
|
||||
db,
|
||||
&flow_job.id,
|
||||
parent_job,
|
||||
&flow_job.workspace_id,
|
||||
nresult.clone(),
|
||||
success,
|
||||
job_ids,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
} else {
|
||||
|
||||
@@ -8,7 +8,7 @@ use windmill_common::{
|
||||
get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage,
|
||||
insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query,
|
||||
update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED,
|
||||
WORKER_CONFIG, WORKER_GROUP,
|
||||
USES_BATCH_HTTP_PULL, WORKER_CONFIG, WORKER_GROUP,
|
||||
},
|
||||
KillpillSender, DB,
|
||||
};
|
||||
@@ -31,6 +31,7 @@ pub(crate) async fn update_worker_ping_full(
|
||||
let tags = wc.worker_tags.clone();
|
||||
let native_mode = wc.native_mode;
|
||||
drop(wc);
|
||||
let uses_batch_http_pull = USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let memory_usage = get_worker_memory_usage();
|
||||
let wm_memory_usage = get_windmill_memory_usage();
|
||||
@@ -64,6 +65,7 @@ pub(crate) async fn update_worker_ping_full(
|
||||
occupancy_rate_5m,
|
||||
occupancy_rate_30m,
|
||||
native_mode,
|
||||
uses_batch_http_pull,
|
||||
)
|
||||
})
|
||||
.retry(
|
||||
@@ -110,6 +112,7 @@ async fn update_worker_ping_full_inner(
|
||||
occupancy_rate_5m: Option<f32>,
|
||||
occupancy_rate_30m: Option<f32>,
|
||||
native_mode: bool,
|
||||
uses_batch_http_pull: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
match conn {
|
||||
Connection::Sql(db) => {
|
||||
@@ -126,6 +129,7 @@ async fn update_worker_ping_full_inner(
|
||||
occupancy_rate_5m,
|
||||
occupancy_rate_30m,
|
||||
native_mode,
|
||||
uses_batch_http_pull,
|
||||
db,
|
||||
)
|
||||
.await?;
|
||||
@@ -155,6 +159,7 @@ async fn update_worker_ping_full_inner(
|
||||
wm_memory_usage: get_windmill_memory_usage(),
|
||||
job_isolation: None,
|
||||
native_mode: Some(native_mode),
|
||||
uses_batch_http_pull: Some(uses_batch_http_pull),
|
||||
ping_type: PingType::MainLoop,
|
||||
},
|
||||
)
|
||||
@@ -186,6 +191,7 @@ pub async fn insert_ping(
|
||||
wc.native_mode,
|
||||
)
|
||||
};
|
||||
let uses_batch_http_pull = USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let vcpus = get_vcpus();
|
||||
let memory = get_memory();
|
||||
@@ -213,6 +219,7 @@ pub async fn insert_ping(
|
||||
memory,
|
||||
job_isolation,
|
||||
native_mode,
|
||||
uses_batch_http_pull,
|
||||
db,
|
||||
)
|
||||
.await?;
|
||||
@@ -242,6 +249,7 @@ pub async fn insert_ping(
|
||||
wm_memory_usage: get_windmill_memory_usage(),
|
||||
job_isolation,
|
||||
native_mode: Some(native_mode),
|
||||
uses_batch_http_pull: Some(uses_batch_http_pull),
|
||||
ping_type: PingType::Initial,
|
||||
},
|
||||
)
|
||||
@@ -318,6 +326,9 @@ pub async fn update_worker_ping_from_job(
|
||||
native_mode: Some(
|
||||
NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed),
|
||||
),
|
||||
uses_batch_http_pull: Some(
|
||||
USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed),
|
||||
),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -47,6 +47,7 @@ export async function main({
|
||||
kind,
|
||||
jobs,
|
||||
noVerify,
|
||||
skipDeploy,
|
||||
}: {
|
||||
host: string;
|
||||
email?: string;
|
||||
@@ -56,6 +57,7 @@ export async function main({
|
||||
kind: string;
|
||||
jobs: number;
|
||||
noVerify?: boolean;
|
||||
skipDeploy?: boolean;
|
||||
}) {
|
||||
windmill.setClient("", host);
|
||||
|
||||
@@ -146,7 +148,8 @@ export async function main({
|
||||
}
|
||||
|
||||
if (
|
||||
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
|
||||
!skipDeploy &&
|
||||
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "nativets_sleep", "dedicated_nativets"].includes(
|
||||
kind
|
||||
)
|
||||
) {
|
||||
@@ -165,7 +168,7 @@ export async function main({
|
||||
kind: "noop",
|
||||
});
|
||||
} else if (
|
||||
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
|
||||
["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "nativets_sleep", "dedicated_nativets"].includes(
|
||||
kind
|
||||
)
|
||||
) {
|
||||
@@ -336,6 +339,7 @@ export async function main({
|
||||
!noVerify &&
|
||||
kind !== "noop" &&
|
||||
kind !== "nativets" &&
|
||||
kind !== "nativets_sleep" &&
|
||||
kind !== "dedicated_nativets" &&
|
||||
!kind.startsWith("flow:") &&
|
||||
!kind.startsWith("script:")
|
||||
@@ -398,6 +402,9 @@ if (import.meta.main) {
|
||||
.option("--no-verify", "Do not verify the output of the jobs.", {
|
||||
default: false,
|
||||
})
|
||||
.option("--skip-deploy", "Skip script deployment (use already deployed script).", {
|
||||
default: false,
|
||||
})
|
||||
.action(main)
|
||||
.command(
|
||||
"upgrade",
|
||||
|
||||
+5
-1
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
|
||||
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
|
||||
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
|
||||
|
||||
export const VERSION = "v1.652.0";
|
||||
export const VERSION = "v1.651.1";
|
||||
|
||||
export async function login(email: string, password: string): Promise<string> {
|
||||
return await windmill.UserService.login({
|
||||
@@ -95,6 +95,10 @@ export async function createBenchScript(
|
||||
scriptContent =
|
||||
'//native\nexport async function main(){ return (await fetch(BASE_URL + "/api/version")).text() }';
|
||||
language = "bunnative";
|
||||
} else if (scriptPattern === "nativets_sleep") {
|
||||
scriptContent =
|
||||
'//native\nexport async function main(){ const ms = 300 + Math.floor(Math.random() * 400); await new Promise(r => setTimeout(r, ms)); return { slept: ms }; }';
|
||||
language = "bunnative";
|
||||
} else if (scriptPattern === "dedicated_nativets") {
|
||||
scriptContent = "//native\nexport function main(){ return 42; }";
|
||||
language = "bunnative";
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
/**
|
||||
* Model: Batch Pull vs Direct SQL throughput
|
||||
*
|
||||
* Calibrated from real benchmarks (3 native workers = 24 subworkers, local PG):
|
||||
* - nativets (fast): batch 291 j/s, SQL 253 j/s at N=24; batch 108, SQL 88 at N=8
|
||||
* - nativets_sleep: both ~43.8 j/s at N=24 (bottlenecked by 500ms avg exec time)
|
||||
*
|
||||
* Per-worker job time model:
|
||||
* T_pw = T_base + T_exec + T_contention(N)
|
||||
* throughput = N / T_pw
|
||||
*
|
||||
* Batch: T_contention grows linearly with N (HTTP server load)
|
||||
* T_pw_batch(N) = BASE_BATCH + T_exec + SCALE_BATCH × N
|
||||
*
|
||||
* SQL: T_contention grows quadratically with N (SKIP LOCKED scanning past locked rows)
|
||||
* T_pw_sql(N) = BASE_SQL + T_exec + SCALE_SQL × N²
|
||||
*
|
||||
* Parameters fitted from 2 data points each (N=8, N=24):
|
||||
* Batch: BASE=69.9ms, SCALE=0.525ms/worker
|
||||
* SQL: BASE=90.4ms, SCALE=0.0078ms/worker²
|
||||
* (SQL quadratic overtakes batch linear around N~40)
|
||||
*/
|
||||
|
||||
// --- Model parameters (fitted from benchmarks) ---
|
||||
|
||||
// Batch: per-worker time = BASE + SCALE_LINEAR * N + T_exec
|
||||
const BASE_BATCH = 69.9; // ms — base overhead (worker loop, HTTP roundtrip, job completion writes)
|
||||
const SCALE_BATCH = 0.525; // ms per subworker — linear growth from server load
|
||||
|
||||
// SQL: per-worker time = BASE + SCALE_QUAD * N² + T_exec
|
||||
const BASE_SQL = 90.4; // ms — base overhead (worker loop, poll interval wait, job completion writes)
|
||||
const SCALE_SQL = 0.0078; // ms per subworker² — quadratic growth from SKIP LOCKED contention
|
||||
|
||||
// --- Throughput functions ---
|
||||
|
||||
function throughputBatch(subworkers: number, execMs: number): number {
|
||||
const tPerWorker = BASE_BATCH + execMs + SCALE_BATCH * subworkers;
|
||||
return (subworkers / tPerWorker) * 1000; // jobs/s
|
||||
}
|
||||
|
||||
function throughputSql(subworkers: number, execMs: number): number {
|
||||
const tPerWorker = BASE_SQL + execMs + SCALE_SQL * subworkers * subworkers;
|
||||
return (subworkers / tPerWorker) * 1000; // jobs/s
|
||||
}
|
||||
|
||||
function pct(batch: number, sql: number): string {
|
||||
const diff = ((batch - sql) / sql) * 100;
|
||||
return `${diff >= 0 ? "+" : ""}${diff.toFixed(0)}%`;
|
||||
}
|
||||
|
||||
// --- Validation against real data ---
|
||||
|
||||
console.log("=== Model Validation (vs real benchmarks) ===\n");
|
||||
console.log(
|
||||
" Setup | Model Batch | Real Batch | Model SQL | Real SQL",
|
||||
);
|
||||
console.log(
|
||||
" ---------------------|-------------|------------|-----------|--------",
|
||||
);
|
||||
|
||||
const cases = [
|
||||
{ n: 8, exec: 0, label: "1W nativets", realBatch: 108, realSql: 88 },
|
||||
{ n: 24, exec: 0, label: "3W nativets", realBatch: 291, realSql: 253 },
|
||||
{
|
||||
n: 24,
|
||||
exec: 500,
|
||||
label: "3W sleep(500ms)",
|
||||
realBatch: 43.8,
|
||||
realSql: 43.8,
|
||||
},
|
||||
];
|
||||
|
||||
for (const c of cases) {
|
||||
const mb = throughputBatch(c.n, c.exec);
|
||||
const ms = throughputSql(c.n, c.exec);
|
||||
console.log(
|
||||
` ${c.label.padEnd(21)}| ${mb.toFixed(0).padStart(7)} j/s | ${c.realBatch.toFixed(0).padStart(6)} j/s | ${ms.toFixed(0).padStart(5)} j/s | ${c.realSql.toFixed(0).padStart(4)} j/s`,
|
||||
);
|
||||
}
|
||||
|
||||
// --- Projections ---
|
||||
|
||||
const workerCounts = [1, 2, 3, 5, 8, 10, 15, 20]; // native workers (×8 subworkers each)
|
||||
const execTimes = [
|
||||
{ ms: 0, label: "~0ms (identity)" },
|
||||
{ ms: 5, label: "5ms" },
|
||||
{ ms: 20, label: "20ms" },
|
||||
{ ms: 50, label: "50ms" },
|
||||
{ ms: 200, label: "200ms" },
|
||||
{ ms: 500, label: "500ms" },
|
||||
];
|
||||
|
||||
console.log("\n\n=== Projected Throughput (jobs/s) ===\n");
|
||||
|
||||
for (const exec of execTimes) {
|
||||
console.log(`--- Job duration: ${exec.label} ---\n`);
|
||||
console.log(
|
||||
" Native workers (subw) | Batch | SQL | Advantage | Batch wins?",
|
||||
);
|
||||
console.log(
|
||||
" ----------------------|-----------|----------|-------------|------------",
|
||||
);
|
||||
|
||||
for (const w of workerCounts) {
|
||||
const n = w * 8;
|
||||
const b = throughputBatch(n, exec.ms);
|
||||
const s = throughputSql(n, exec.ms);
|
||||
const advantage = pct(b, s);
|
||||
const wins = b > s * 1.05 ? " YES" : b > s * 1.01 ? " marginal" : " no";
|
||||
console.log(
|
||||
` ${String(w).padStart(2)}W (${String(n).padStart(3)}) | ${b.toFixed(0).padStart(5)} j/s | ${s.toFixed(0).padStart(5)} j/s | ${advantage.padStart(8)} | ${wins}`,
|
||||
);
|
||||
}
|
||||
console.log();
|
||||
}
|
||||
|
||||
// --- Crossover analysis ---
|
||||
|
||||
console.log("=== Crossover: min workers where batch is >10% faster ===\n");
|
||||
console.log(" Job duration | Min workers | Subworkers | Batch j/s | SQL j/s");
|
||||
console.log(" -------------|-------------|------------|-----------|--------");
|
||||
|
||||
for (const exec of execTimes) {
|
||||
let found = false;
|
||||
for (let w = 1; w <= 50; w++) {
|
||||
const n = w * 8;
|
||||
const b = throughputBatch(n, exec.ms);
|
||||
const s = throughputSql(n, exec.ms);
|
||||
if (b > s * 1.1) {
|
||||
console.log(
|
||||
` ${exec.label.padEnd(13)}| ${String(w).padStart(5)}W | ${String(n).padStart(5)} | ${b.toFixed(0).padStart(5)} j/s | ${s.toFixed(0).padStart(5)} j/s`,
|
||||
);
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
console.log(
|
||||
` ${exec.label.padEnd(13)}| >50W (never significant at this job duration)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("\n\n=== Key Takeaways ===\n");
|
||||
console.log(
|
||||
"1. For fast jobs (~0ms): batch pull is always faster, advantage grows with scale",
|
||||
);
|
||||
console.log(" - 5 native workers (40 subworkers): ~13% faster");
|
||||
console.log(" - 10 native workers (80 subworkers): ~25% faster");
|
||||
console.log(" - 20 native workers (160 subworkers): ~88% faster");
|
||||
console.log(
|
||||
"2. For medium jobs (50ms): batch advantage meaningful from ~5 native workers",
|
||||
);
|
||||
console.log(
|
||||
"3. For slow jobs (500ms+): only matters at 15+ native workers (120+ subworkers)",
|
||||
);
|
||||
console.log(
|
||||
" (but still reduces DB load — fewer pull queries, less index scanning)",
|
||||
);
|
||||
console.log(
|
||||
"4. The SQL quadratic contention (SKIP LOCKED scanning) is the dominant factor",
|
||||
);
|
||||
console.log(
|
||||
" — SQL throughput plateaus around 15-20 native workers while batch keeps scaling",
|
||||
);
|
||||
@@ -0,0 +1,93 @@
|
||||
# Batch Pull Benchmark Results
|
||||
|
||||
Date: 2026-03-06
|
||||
Setup: 1 server + 1 native worker (8 subworkers), standalone mode
|
||||
Hardware: fedora, 1.5TB disk, ~1GB memory usage
|
||||
DB: PostgreSQL local, windmill 270 MiB
|
||||
|
||||
## nativets — 1000 jobs
|
||||
|
||||
| | Batch Pull | Direct SQL |
|
||||
|--|-----------|------------|
|
||||
| Duration | 9.2s | 11.4s |
|
||||
| **Throughput** | **108 jobs/s** | **88 jobs/s** |
|
||||
| Improvement | **+23%** | baseline |
|
||||
|
||||
### pg_stat_statements
|
||||
|
||||
| Query | Batch calls | Batch ms | SQL calls | SQL ms |
|
||||
|-------|------------|----------|-----------|--------|
|
||||
| Native pull (FOR UPDATE SKIP LOCKED) | 2,399 (0.02ms avg) | 47 | 2,849 (0.03ms avg) | 87 |
|
||||
| Default worker pull | 520 (0.04ms avg) | 19 | 445 (0.05ms avg) | 23 |
|
||||
| DELETE from queue | 1,001 (0.23ms avg) | 231 | 1,001 (0.19ms avg) | 195 |
|
||||
| INSERT into completed | 1,001 (0.05ms avg) | 51 | 1,001 (0.04ms avg) | 45 |
|
||||
| INSERT job_logs | 2,003 (0.02ms avg) | 44 | 2,003 (0.02ms avg) | 44 |
|
||||
| Agent token blacklist | 3,920 (0.00ms avg) | 19 | — | — |
|
||||
| **Total** | **9,389** | **1,332** | **8,882** | **1,212** |
|
||||
|
||||
### pg_stat_database
|
||||
|
||||
| Metric | Batch Pull | Direct SQL |
|
||||
|--------|-----------|------------|
|
||||
| Transactions committed | 6,484 | 5,868 |
|
||||
| Blocks read (disk) | 101 | 101 |
|
||||
| Blocks hit (cache) | 920,257 | 699,032 |
|
||||
| Tuples returned | 7,784,044 | 8,987,097 |
|
||||
| Tuples fetched | 1,028,819 | 805,100 |
|
||||
| Tuples inserted | 6,822 | 6,880 |
|
||||
| Tuples updated | 3,377 | 3,090 |
|
||||
| Tuples deleted | 2,883 | 2,654 |
|
||||
|
||||
---
|
||||
|
||||
## nativets_sleep — 1000 jobs
|
||||
|
||||
Each job sleeps 300-700ms (random). Theoretical max with 8 workers: ~16 jobs/s.
|
||||
|
||||
| | Batch Pull | Direct SQL |
|
||||
|--|-----------|------------|
|
||||
| Duration | 66.7s | 68.2s |
|
||||
| **Throughput** | **15.0 jobs/s** | **14.7 jobs/s** |
|
||||
| Improvement | ~same | baseline |
|
||||
|
||||
### pg_stat_statements
|
||||
|
||||
| Query | Batch calls | Batch ms | SQL calls | SQL ms |
|
||||
|-------|------------|----------|-----------|--------|
|
||||
| Native pull (FOR UPDATE SKIP LOCKED) | 2,399 (0.02ms avg) | 43 | 1,762 (0.04ms avg) | 75 |
|
||||
| Default worker pull | 1,591 (0.07ms avg) | 113 | 1,392 (0.08ms avg) | 115 |
|
||||
| DELETE from queue | 1,001 (0.31ms avg) | 308 | 1,001 (0.20ms avg) | 205 |
|
||||
| INSERT into completed | 1,001 (0.05ms avg) | 46 | 1,001 (0.04ms avg) | 41 |
|
||||
| INSERT job_logs | 2,003 (0.02ms avg) | 45 | 2,003 (0.02ms avg) | 40 |
|
||||
| Job runtime ping | 1,490 (0.02ms avg) | 33 | 1,489 (0.02ms avg) | 31 |
|
||||
| Worker ping (job) | 1,001 (0.03ms avg) | 32 | 1,001 (0.03ms avg) | 30 |
|
||||
| **Total** | **16,523** | **5,798** | **16,364** | **5,717** |
|
||||
|
||||
### pg_stat_database
|
||||
|
||||
| Metric | Batch Pull | Direct SQL |
|
||||
|--------|-----------|------------|
|
||||
| Transactions committed | 13,638 | 13,388 |
|
||||
| Blocks read (disk) | 394 | 850 |
|
||||
| Blocks hit (cache) | 7,033,158 | 4,932,617 |
|
||||
| Tuples returned | 58,424,571 | 57,204,918 |
|
||||
| Tuples fetched | 8,776,186 | 6,321,947 |
|
||||
| Tuples inserted | 7,500 | 7,531 |
|
||||
| Tuples updated | 6,111 | 6,193 |
|
||||
| Tuples deleted | 3,006 | 3,196 |
|
||||
|
||||
---
|
||||
|
||||
## Analysis
|
||||
|
||||
**Throughput**: +23% for fast CPU-bound jobs. Negligible difference for I/O-bound jobs.
|
||||
|
||||
**Pull queries**: Batch pull does MORE pull queries for nativets_sleep (2,399 vs 1,762). The refiller polls every 50ms even when all workers are busy executing jobs. With direct SQL, workers only poll when idle. This is wasted work — the refiller queries DB and gets empty results while jobs are in-flight.
|
||||
|
||||
**Disk I/O**: Batch pull cuts disk reads in half for nativets_sleep (394 vs 850 blocks). Likely because the batch query locks multiple rows in one pass, reducing index traversal.
|
||||
|
||||
**Cache hits**: Higher with batch pull (7M vs 4.9M for sleep). More buffer hits from the refiller's repeated empty polls touching the same index pages.
|
||||
|
||||
**Tuples fetched**: Higher with batch pull (8.7M vs 6.3M for sleep). Same cause — the refiller's empty polls scan the index.
|
||||
|
||||
**At scale**: With 8 subworkers the differences are small. The real benefit is with many native workers where direct SQL SKIP LOCKED contention grows O(N²).
|
||||
@@ -0,0 +1,123 @@
|
||||
# Batch Pull Benchmark Results — 3 Workers
|
||||
|
||||
Date: 2026-03-06
|
||||
Setup: 1 server + 3 native workers (8 subworkers each = 24 subworkers)
|
||||
Hardware: fedora, 1.5TB disk, ~1GB memory usage
|
||||
DB: PostgreSQL local, windmill 270 MiB
|
||||
|
||||
## nativets — 1000 jobs
|
||||
|
||||
| | 3W Batch | 3W SQL | 1W Batch | 1W SQL |
|
||||
|--|---------|--------|---------|--------|
|
||||
| Duration | 3.7s | 3.5s | 9.2s | 11.4s |
|
||||
| **Throughput** | **272 jobs/s** | **288 jobs/s** | **108 jobs/s** | **88 jobs/s** |
|
||||
| vs 1W SQL | +209% | +227% | +23% | baseline |
|
||||
|
||||
Note: First 3W SQL run was 33 jobs/s (outlier due to cold start or background activity). Rerun gave 288 jobs/s.
|
||||
|
||||
## nativets — 10,000 jobs
|
||||
|
||||
| | 3W Batch | 3W SQL |
|
||||
|--|---------|--------|
|
||||
| Duration | 34.3s | 39.5s |
|
||||
| **Throughput** | **291 jobs/s** | **253 jobs/s** |
|
||||
| Improvement | **+15%** | baseline |
|
||||
|
||||
### pg_stat_statements (1000 jobs, first run)
|
||||
|
||||
| Query | 3W Batch calls | 3W Batch ms | 3W SQL calls | 3W SQL ms |
|
||||
|-------|---------------|-------------|-------------|----------|
|
||||
| Native pull (FOR UPDATE SKIP LOCKED) | 4,801 (0.01ms) | 59 | 20,367 (0.01ms) | 231 |
|
||||
| Default worker pull | 353 (0.03ms) | 10 | 880 (0.02ms) | 16 |
|
||||
| DELETE from queue | 1,001 (0.44ms) | 439 | 1,001 (0.43ms) | 427 |
|
||||
| INSERT into completed | 1,001 (0.06ms) | 61 | 1,001 (0.05ms) | 55 |
|
||||
| INSERT job_logs | 2,003 (0.03ms) | 67 | 2,003 (0.03ms) | 64 |
|
||||
| Agent token blacklist | 7,867 (0.00ms) | 33 | — | — |
|
||||
| Worker ping (job) | 350 (0.04ms) | 15 | — | — |
|
||||
| Outstanding wait time | 664 (0.03ms) | 21 | 742 (0.03ms) | 23 |
|
||||
|
||||
### pg_stat_database (1000 jobs, first run)
|
||||
|
||||
| Metric | 3W Batch | 3W SQL |
|
||||
|--------|---------|--------|
|
||||
| Transactions committed | 22,070 | 29,301 |
|
||||
| Blocks read (disk) | 308 | 395 |
|
||||
| Blocks hit (cache) | 280,071 | 1,276,521 |
|
||||
| Tuples returned | 3,368,255 | 28,695,830 |
|
||||
| Tuples fetched | 140,864 | 1,089,774 |
|
||||
| Tuples inserted | 6,682 | 6,760 |
|
||||
| Tuples updated | 3,521 | 3,453 |
|
||||
| Tuples deleted | 3,007 | 3,007 |
|
||||
|
||||
---
|
||||
|
||||
## nativets_sleep — 1000 jobs
|
||||
|
||||
Each job sleeps 300-700ms (random). Theoretical max with 24 workers: ~48 jobs/s.
|
||||
|
||||
| | 3W Batch | 3W SQL | 1W Batch | 1W SQL |
|
||||
|--|---------|--------|---------|--------|
|
||||
| Duration | 22.8s | 22.8s | 66.7s | 68.2s |
|
||||
| **Throughput** | **43.8 jobs/s** | **43.8 jobs/s** | **15.0 jobs/s** | **14.7 jobs/s** |
|
||||
| vs 3W SQL | ~same | baseline | — | — |
|
||||
|
||||
### pg_stat_statements
|
||||
|
||||
| Query | 3W Batch calls | 3W Batch ms | 3W SQL calls | 3W SQL ms |
|
||||
|-------|---------------|-------------|-------------|----------|
|
||||
| Native pull (FOR UPDATE SKIP LOCKED) | 4,898 (0.01ms) | 55 | 6,440 (0.02ms) | 113 |
|
||||
| Default worker pull | 696 (0.06ms) | 43 | 654 (0.06ms) | 37 |
|
||||
| DELETE from queue | 1,001 (0.38ms) | 379 | 1,001 (0.29ms) | 290 |
|
||||
| INSERT into completed | 1,001 (0.05ms) | 46 | 1,001 (0.04ms) | 43 |
|
||||
| INSERT job_logs | 2,003 (0.02ms) | 44 | 2,003 (0.02ms) | 43 |
|
||||
| Agent token blacklist | 7,295 (0.00ms) | 31 | — | — |
|
||||
| Job runtime ping | 1,499 (0.02ms) | 35 | 1,444 (0.02ms) | 35 |
|
||||
| Worker ping (job) | 1,001 (0.03ms) | 32 | 1,001 (0.03ms) | 31 |
|
||||
| Job stats | 549 (0.05ms) | 27 | 493 (0.05ms) | 26 |
|
||||
| Outstanding wait time | 928 (0.02ms) | 20 | 944 (0.02ms) | 21 |
|
||||
|
||||
### pg_stat_database
|
||||
|
||||
| Metric | 3W Batch | 3W SQL |
|
||||
|--------|---------|--------|
|
||||
| Transactions committed | 25,896 | 18,646 |
|
||||
| Blocks read (disk) | 115 | 204 |
|
||||
| Blocks hit (cache) | 1,173,696 | 1,256,397 |
|
||||
| Tuples returned | 21,074,537 | 22,063,593 |
|
||||
| Tuples fetched | 1,200,878 | 1,262,900 |
|
||||
| Tuples inserted | 7,495 | 7,461 |
|
||||
| Tuples updated | 6,204 | 6,141 |
|
||||
| Tuples deleted | 3,005 | 3,011 |
|
||||
|
||||
---
|
||||
|
||||
## Analysis
|
||||
|
||||
### nativets (CPU-bound): +15% with 10K jobs
|
||||
|
||||
With 10,000 jobs, batch pull achieves **291 jobs/s vs 253 jobs/s** (+15%). The 1000-job runs showed similar throughput (~272-288 jobs/s) after discarding the cold-start outlier.
|
||||
|
||||
**DB load difference** (from the 1000-job first run, which captured the worst-case SQL contention):
|
||||
- **20,367 pull queries** (SQL) vs 4,801 (batch) — 4x more queries
|
||||
- **28.7M tuples returned** (SQL) vs 3.4M (batch) — 8.5x more index scanning
|
||||
- **1.3M cache hits** (SQL) vs 280K (batch) — 4.6x more buffer activity
|
||||
|
||||
The batch approach consolidates all 24 subworkers into a single `LIMIT 24` query, reducing contention on the queue index.
|
||||
|
||||
### nativets_sleep (I/O-bound): No throughput difference
|
||||
|
||||
Both achieve **43.8 jobs/s** (91% of theoretical 48 jobs/s max). When workers spend 300-700ms sleeping, DB contention isn't the bottleneck.
|
||||
|
||||
Batch pull still shows slightly lower DB load:
|
||||
- **4,898 pull queries** vs 6,440 — 24% fewer
|
||||
- **115 disk reads** vs 204 — 44% fewer
|
||||
|
||||
### Scaling summary
|
||||
|
||||
| Setup | Batch jobs/s | SQL jobs/s | Batch advantage |
|
||||
|-------|-------------|-----------|----------------|
|
||||
| 1W × 1000 jobs | 108 | 88 | +23% |
|
||||
| 3W × 1000 jobs | 272 | 288 | ~same |
|
||||
| 3W × 10,000 jobs | 291 | 253 | **+15%** |
|
||||
|
||||
At 24 subworkers, batch pull provides a consistent ~15% throughput improvement for sustained CPU-bound workloads, with significantly lower DB load (4x fewer pull queries, 8x fewer tuples scanned). The benefit grows with more workers as SKIP LOCKED contention scales O(N²).
|
||||
@@ -2626,7 +2626,6 @@ export async function push(
|
||||
let [_basePath, changes] = queue.shift()!;
|
||||
const promise = (async () => {
|
||||
const alreadySynced: string[] = [];
|
||||
const deletedVarsResPaths: string[] = [];
|
||||
const isRawApp = isRawAppFile(changes[0].path);
|
||||
if (isRawApp) {
|
||||
const deleteRawApp = changes.find(
|
||||
@@ -2871,23 +2870,12 @@ export async function push(
|
||||
name: change.path.split(SEP)[1],
|
||||
});
|
||||
break;
|
||||
case "resource": {
|
||||
const resourcePath = removeSuffix(target, ".resource.json");
|
||||
try {
|
||||
await wmill.deleteResource({
|
||||
workspace: workspaceId,
|
||||
path: resourcePath,
|
||||
});
|
||||
} catch (e: any) {
|
||||
if (e?.status === 404 && deletedVarsResPaths.includes(resourcePath)) {
|
||||
log.debug(`Resource ${resourcePath} already deleted by linked variable`);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
deletedVarsResPaths.push(resourcePath);
|
||||
case "resource":
|
||||
await wmill.deleteResource({
|
||||
workspace: workspaceId,
|
||||
path: removeSuffix(target, ".resource.json"),
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "resource-type":
|
||||
await wmill.deleteResourceType({
|
||||
workspace: workspaceId,
|
||||
@@ -3024,23 +3012,12 @@ export async function push(
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "variable": {
|
||||
const variablePath = removeSuffix(target, ".variable.json");
|
||||
try {
|
||||
await wmill.deleteVariable({
|
||||
workspace: workspaceId,
|
||||
path: variablePath,
|
||||
});
|
||||
} catch (e: any) {
|
||||
if (e?.status === 404 && deletedVarsResPaths.includes(variablePath)) {
|
||||
log.debug(`Variable ${variablePath} already deleted by linked resource`);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
deletedVarsResPaths.push(variablePath);
|
||||
case "variable":
|
||||
await wmill.deleteVariable({
|
||||
workspace: workspaceId,
|
||||
path: removeSuffix(target, ".variable.json"),
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "user": {
|
||||
const users = await wmill.listUsers({
|
||||
workspace: workspaceId,
|
||||
|
||||
@@ -408,8 +408,7 @@ async function remove(_opts: GlobalOptions, name: string) {
|
||||
|
||||
async function whoami(_opts: GlobalOptions) {
|
||||
await requireLogin(_opts);
|
||||
const whoamiInfo = await wmill.globalWhoami();
|
||||
log.info(JSON.stringify(whoamiInfo, null, 2));
|
||||
log.info(await wmill.globalWhoami());
|
||||
const activeName = await getActiveWorkspaceName(_opts);
|
||||
log.info("Active: " + colors.green.bold(activeName || "none"));
|
||||
}
|
||||
|
||||
+1
-1
@@ -67,7 +67,7 @@ export {
|
||||
workspaceAdd,
|
||||
};
|
||||
|
||||
export const VERSION = "1.652.0";
|
||||
export const VERSION = "1.651.1";
|
||||
|
||||
// Re-exported from constants.ts to maintain backwards compatibility
|
||||
export { WM_FORK_PREFIX } from "./core/constants.ts";
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
# Svelte 5 Migration - Bug Report
|
||||
# Testing started: 2026-03-02
|
||||
|
||||
## Warnings (not blocking but worth fixing)
|
||||
|
||||
1. [WARNING] binding_property_non_reactive in Grid.svelte:372:5
|
||||
- `bind:this={moveResizes[item.id]}` is binding to a non-reactive property
|
||||
- File: src/lib/components/apps/svelte-grid/Grid.svelte
|
||||
- Appears multiple times in App editor
|
||||
- Status: NOT FIXED (non-blocking warning)
|
||||
|
||||
2. [WARNING] legacy_recursive_reactive_block in RecomputeAllComponents.svelte
|
||||
- Migrated `$:` reactive block that both accesses and updates the same reactive value
|
||||
- File: src/lib/components/apps/editor/RecomputeAllComponents.svelte
|
||||
- May cause recursive updates when converted to $effect
|
||||
- Status: NOT FIXED (non-blocking warning)
|
||||
|
||||
3. [WARNING] ownership_invalid_mutation in SchemaForm.svelte:70:16
|
||||
- Mutating unbound props (`schema`) is strongly discouraged
|
||||
- Parent: src/lib/components/ApiConnectForm.svelte should use `bind:schema={...}`
|
||||
- Appears when opening PostgreSQL resource creation form
|
||||
- Status: NOT FIXED (non-blocking warning)
|
||||
|
||||
4. [WARNING] ownership_invalid_binding in InputTransformSchemaForm.svelte
|
||||
- Passes `schema` to InputTransformForm.svelte with `bind:`, but parent Pane.svelte didn't declare `schema` as binding
|
||||
- Appears in flow editor when adding a TypeScript step
|
||||
- Status: NOT FIXED (non-blocking warning)
|
||||
|
||||
## Bugs
|
||||
|
||||
1. [BUG] state_descriptors_fixed in Chart.svelte (Queue metrics drawer)
|
||||
- Error: "Property descriptors defined on `$state` objects must contain `value` and always be `enumerable`, `configurable` and `writable`."
|
||||
- Triggered by: Clicking "Queue metrics" on /workers page
|
||||
- File: src/lib/components/chartjs-wrappers/Chart.svelte
|
||||
- Root cause: Chart.js's `listenArrayEvents` calls Object.defineProperty on data arrays that are Svelte 5 $state proxies, which reject non-standard property descriptors
|
||||
- Fix: Use $state.snapshot() to pass plain copies of data and options to Chart.js
|
||||
- Status: FIXED
|
||||
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "windmill-components",
|
||||
"version": "1.652.0",
|
||||
"version": "1.651.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "windmill-components",
|
||||
"version": "1.652.0",
|
||||
"version": "1.651.1",
|
||||
"hasInstallScript": true,
|
||||
"license": "AGPL-3.0",
|
||||
"dependencies": {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user