chore(main): release 1.652.0 (#8247)

* chore(main): release 1.652.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>

.webmux.yaml

@@ -0,0 +1,65 @@
+# Project display name in the dashboard
+name: Windmill
+
+workspace:
+  mainBranch: main
+  worktreeRoot: ../windmill__worktrees
+  defaultAgent: claude
+
+startupEnvs:
+  CARGO_FEATURES: "quickjs"
+  WM_CLONE_DB: false
+  USE_RUST_PLUGIN: false
+
+lifecycleHooks:
+  postCreate: bash ./scripts/post-create.sh
+  preRemove: bash ./scripts/pre-remove.sh
+
+auto_name:
+  model: gemini-2.5-flash-lite
+
+# Each service defines a port env var that webmux injects into pane and agent
+# process environments when creating a worktree. Ports are auto-assigned:
+# base + (slot x step).
+services:
+  - name: backend
+    portEnv: BACKEND_PORT
+    portStart: 8000
+    portStep: 10
+  - name: frontend
+    portEnv: FRONTEND_PORT
+    portStart: 3000
+    portStep: 10
+
+profiles:
+  default:
+    runtime: host
+    yolo: true
+    envPassthrough: []
+    systemPrompt: >
+      You are running inside a tmux session with other panes running services.
+      Pane layout (current window):
+      - Pane 0: this pane (claude agent)
+      - Pane 1: backend (cargo watch -x run)
+      - Pane 2: frontend (npm run dev)
+      To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (backend) or \`tmux capture-pane -t .2 -p -S -50\` (frontend).
+      When restarting backend or frontend, make sure to use ${BACKEND_PORT} and ${FRONTEND_PORT}.
+      Because we are running backend with cargo watch, to verify your changes, just check the logs in the backend pane. No need for cargo check.
+    panes:
+      - id: agent
+        kind: agent
+        focus: true
+      - id: backend
+        kind: command
+        split: right
+        command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/backend" && PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}"
+      - id: frontend
+        kind: command
+        split: bottom
+        command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/frontend" && npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0
+
+integrations:
+  github:
+    linkedRepos: []
+  linear:
+    enabled: true

.wmdev.yaml

@@ -1,113 +0,0 @@
-name: Windmill
-
-startupEnvs:
-  CARGO_FEATURES: "quickjs"
-  WM_CLONE_DB: false
-  USE_RUST_PLUGIN: false
-
-services:
-  - name: BE
-    portEnv: BACKEND_PORT
-  - name: FE
-    portEnv: FRONTEND_PORT
-
-profiles:
-  default:
-    name: default
-
-  sandbox:
-    name: sandbox
-    image: windmill-sandbox
-    envPassthrough:
-      - AWS_ACCESS_KEY_ID
-      - AWS_SECRET_ACCESS_KEY
-      - R2_ENDPOINT
-      - R2_BUCKET
-      - R2_PUBLIC_URL
-    extraMounts:
-      - hostPath: ~/.ssh
-        guestPath: /root/.ssh
-        writable: true
-      - hostPath: ~/.codex
-        guestPath: /root/.codex
-        writable: true
-      - hostPath: ~/windmill-ee-private
-        writable: true
-      - hostPath: ~/windmill-ee-private__worktrees
-        writable: true
-    systemPrompt: >
-      You are running inside a sandboxed container with full permissions.
-      This worktree is configured with the following ports:
-
-      - Backend: port ${BACKEND_PORT}.
-        Start with: cd backend && PORT=${BACKEND_PORT}
-        DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill
-        cargo watch -x run
-
-      - Frontend: port ${FRONTEND_PORT}.
-        Start with: cd frontend && REMOTE=http://localhost:${BACKEND_PORT}
-        npm run dev -- --port ${FRONTEND_PORT} --host 0.0.0.0
-
-      --- Screenshots ---
-      You can take screenshots of the frontend UI and upload them to R2
-      for use in PR descriptions.
-      1) Take a screenshot:
-         bunx playwright screenshot --browser chromium
-         http://localhost:${FRONTEND_PORT}/path/to/page /tmp/screenshot.png
-      2) Upload to R2:
-         aws s3 cp /tmp/screenshot.png
-         "s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/screenshot.png"
-         --endpoint-url "$(printenv R2_ENDPOINT)"
-      3) The public URL will be:
-         $(printenv R2_PUBLIC_URL)/<branch>/screenshot.png
-      4) Include in PR descriptions using markdown image syntax.
-
-      --- Terminal Recordings (asciinema) ---
-      You can record terminal sessions and upload them for sharing.
-      asciinema is available on PATH.
-
-      1) Write a shell script with the commands to demo. Add sleep
-          delays for readable pacing:
-          - 0.5s after printing a "$ command" line (lets viewer read it)
-          - 1.5-2s after command output (lets viewer absorb the result)
-          - Set GIT_PAGER=cat and PAGER=cat to prevent pager hangs
-
-      2) Record headlessly:
-          asciinema rec --headless --overwrite \
-            -c "bash /tmp/demo.sh" \
-            --window-size 120x50 \
-            --title "Description of demo" \
-            /tmp/demo.cast
-
-      3) Upload to asciinema.org:
-          XDG_DATA_HOME=/tmp/.local/share \
-            asciinema upload --server-url https://asciinema.org /tmp/demo.cast
-
-      --- Mermaid Diagrams ---
-      You can render Mermaid diagrams to SVG using the pre-installed mmdc CLI.
-      The puppeteer config (no-sandbox + Chromium path) is at /root/.puppeteerrc.json.
-
-      1) Write a .mmd file with your diagram:
-          cat > /tmp/diagram.mmd << 'EOF'
-          graph TD
-            A[Start] --> B[End]
-          EOF
-
-      2) Render to SVG (the -p flag is required):
-          mmdc -i /tmp/diagram.mmd -o /tmp/diagram.svg -p /root/.puppeteerrc.json
-
-      3) Upload to R2:
-          aws s3 cp /tmp/diagram.svg
-          "s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/diagram.svg"
-          --endpoint-url "$(printenv R2_ENDPOINT)"
-
-      4) The public URL will be:
-          $(printenv R2_PUBLIC_URL)/<branch>/diagram.svg
-
-      5) Include in PR descriptions using markdown image syntax.
-
-      IMPORTANT: Read docs/autonomous-mode.md before starting any work.
-
-linkedRepos:
-  - repo: windmill-labs/windmill-ee-private
-    alias: ee

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## [1.652.0](https://github.com/windmill-labs/windmill/compare/v1.651.1...v1.652.0) (2026-03-09)
+
+
+### Features
+
+* add secretKeyRef support for package registry and storage credentials ([#8275](https://github.com/windmill-labs/windmill/issues/8275)) ([73d27e9](https://github.com/windmill-labs/windmill/commit/73d27e92dd6ced1602f6328f245fec0fa96860e1))
+* expose OTEL trace context as env vars in job execution ([#8277](https://github.com/windmill-labs/windmill/issues/8277)) ([93f75ad](https://github.com/windmill-labs/windmill/commit/93f75ada5e49036f0d998e3d3d53de4dc2c2e83f))
+* workflow-as-code (WAC) v2 ([#8172](https://github.com/windmill-labs/windmill/issues/8172)) ([a6d4390](https://github.com/windmill-labs/windmill/commit/a6d4390790d21d535df1e9d525bffd577c50d8dc))
+
+
+### Bug Fixes
+
+* cli: support deleting linked resources-variables without throwing ([#8248](https://github.com/windmill-labs/windmill/issues/8248)) ([7859bca](https://github.com/windmill-labs/windmill/commit/7859bca6ae80d32a73a46910960afc6812e64115))
+* Database studio fixes ([#8251](https://github.com/windmill-labs/windmill/issues/8251)) ([1d78589](https://github.com/windmill-labs/windmill/commit/1d785899404e8636a206cda9a2914df32a1a5269))
+* **frontend:** unsaved changes dialog when flow already saved ([#8259](https://github.com/windmill-labs/windmill/issues/8259)) ([0330993](https://github.com/windmill-labs/windmill/commit/0330993cb66cdabffcd6e552a0f85a9a3931c62d))
+* gracefully handle uninitialized OTEL tracing proxy port ([#8274](https://github.com/windmill-labs/windmill/issues/8274)) ([8b1fe8f](https://github.com/windmill-labs/windmill/commit/8b1fe8f9de7b0c03655558d0c46cfff71a4b2047))
+* guard iteration picker VirtualList against empty items array ([#8273](https://github.com/windmill-labs/windmill/issues/8273)) ([c97cf60](https://github.com/windmill-labs/windmill/commit/c97cf604ab4a902d89fe873b90dbeb9dabc940eb)), closes [#8272](https://github.com/windmill-labs/windmill/issues/8272)
+* mask secrets in OAuth config debug/log output ([#8269](https://github.com/windmill-labs/windmill/issues/8269)) ([e75763d](https://github.com/windmill-labs/windmill/commit/e75763dbe5ffe08e6cde082203596d510c2c3b29))
+* parallel branchall hang on bad stop_after_all_iters_if + results.x.length null ([#8276](https://github.com/windmill-labs/windmill/issues/8276)) ([41e523f](https://github.com/windmill-labs/windmill/commit/41e523f827c4e3d5db525a1f14e24936b0b8af46))
+* redact secrets in set_global_setting log line ([#8270](https://github.com/windmill-labs/windmill/issues/8270)) ([6a0473c](https://github.com/windmill-labs/windmill/commit/6a0473c5783dc0fef2ae82dc5345a5f0596f124d))
+* remove $bindable() fallback values causing props_invalid_value error in oauth settings ([#8265](https://github.com/windmill-labs/windmill/issues/8265)) ([037035e](https://github.com/windmill-labs/windmill/commit/037035e094937827305dad29bd76a495d78bc46f))
+* skip down migrations in potentially_stale checksum comparison ([#8271](https://github.com/windmill-labs/windmill/issues/8271)) ([5ba4029](https://github.com/windmill-labs/windmill/commit/5ba4029d8692b2e6054fca7f45ed4cfded4738ef))
+* sql input horizontal scroll missing after switching flow steps ([#8249](https://github.com/windmill-labs/windmill/issues/8249)) ([ce8ac9c](https://github.com/windmill-labs/windmill/commit/ce8ac9cf52dc17061673b9b72556279c48c26f8e))
+* wmill workspace whoami output ([#8246](https://github.com/windmill-labs/windmill/issues/8246)) ([1ac391a](https://github.com/windmill-labs/windmill/commit/1ac391a795585747fe5911ac41b157556569fedb))
+
 ## [1.651.1](https://github.com/windmill-labs/windmill/compare/v1.651.0...v1.651.1) (2026-03-05)
 
 

CLAUDE.md

@@ -26,6 +26,27 @@ Open-source platform for internal tools, workflows, API integrations, background
 - **Login**: `admin@windmill.dev` / `changeme`
 - **Instance settings**: navigate to `/#superadmin-settings`
 
+## Banned Patterns
+
+### `$bindable(default_value)` on optional props
+
+Using `$bindable(default_value)` on props that can be `undefined` is **banned**. This pattern causes subtle bugs because the default value masks the `undefined` state.
+
+**Bad:**
+```svelte
+let { my_prop = $bindable(default_value) }: { my_prop?: string } = $props()
+```
+
+**Correct alternatives:**
+
+1. **Use `$derived` with nullish coalescing** — handle the potential `undefined` at the usage site:
+   ```svelte
+   let { my_prop = $bindable() }: { my_prop?: string } = $props()
+   let effective_value = $derived(my_prop ?? default_value)
+   ```
+
+2. **Create a `useMyPropState()` helper** — encapsulate the undefined-handling logic in a reusable function and call it higher in the component tree, so the child component always receives a defined value.
+
 ## Core Principles
 
 - Search for existing code to reuse before writing new code

backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE v2_job_queue SET canceled_by = $2, canceled_reason = $3 WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Varchar",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699"
+}

backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6"
+}

backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json

@@ -15,7 +15,7 @@
       ]
     },
     "nullable": [
-      false
+      true
     ]
   },
   "hash": "2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91"

backend/.sqlx/query-3e8afd021088a99a24f27fa6f0a1b7f3edba3e9b834c814b464305bc2eb6ba80.json

@@ -1,26 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)\n        DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "TextArray",
-        "Varchar",
-        "Varchar",
-        "TextArray",
-        "Varchar",
-        "Int8",
-        "Int8",
-        "Text",
-        "Bool",
-        "Bool"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "3e8afd021088a99a24f27fa6f0a1b7f3edba3e9b834c814b464305bc2eb6ba80"
-}

backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE v2_job_status SET\n                        workflow_as_code_status = jsonb_set(\n                            jsonb_set(\n                                COALESCE(workflow_as_code_status, '{}'::jsonb),\n                                array[$1],\n                                COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n                            ),\n                            array[$1, 'duration_ms'],\n                            to_jsonb($2::bigint)\n                        )\n                    WHERE id = $3",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Int8",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5"
-}

backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json

@@ -15,7 +15,7 @@
       ]
     },
     "nullable": [
-      true
+      null
     ]
   },
   "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

backend/.sqlx/query-6cd099d458ac380d5da27b9e69da035755496ea50f2b78fb9b1cd3a2eb7e7625.json

@@ -1,26 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n         occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n         memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "TextArray",
-        "Float4",
-        "Int8",
-        "Int8",
-        "Text",
-        "Int8",
-        "Int8",
-        "Float4",
-        "Float4",
-        "Float4",
-        "Bool",
-        "Bool"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "6cd099d458ac380d5da27b9e69da035755496ea50f2b78fb9b1cd3a2eb7e7625"
-}

backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4"
+}

backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5"
+}

backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e"
+}

backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1 RETURNING suspend",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "suspend",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a"
+}

backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE v2_job_status SET\n                        workflow_as_code_status = jsonb_set(\n                            jsonb_set(\n                                workflow_as_code_status,\n                                array[$1],\n                                COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n                            ),\n                            array[$1, 'duration_ms'],\n                            to_jsonb($2::bigint)\n                        )\n                    WHERE id = $3 AND workflow_as_code_status IS NOT NULL\n                    RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS \"job_ids: serde_json::Value\"",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "job_ids: serde_json::Value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340"
+}

backend/.sqlx/query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        WITH job_info AS (\n            SELECT id, kind::text AS kind, parent_job\n            FROM v2_job\n            WHERE id = $1\n        )\n        SELECT\n            q.id AS \"id!\",\n            s.flow_status,\n            q.suspend AS \"suspend!\",\n            j.runnable_path AS script_path,\n            j.permissioned_as_email AS email,\n            (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\"\n        FROM job_info ji\n        JOIN v2_job_queue q ON q.id = CASE\n            WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n            ELSE ji.parent_job\n        END\n        JOIN v2_job j ON j.id = q.id\n        JOIN v2_job_status s ON s.id = q.id\n        FOR UPDATE OF q\n        ",
+  "query": "\n        WITH job_info AS (\n            SELECT id, kind::text AS kind, parent_job\n            FROM v2_job\n            WHERE id = $1\n        )\n        SELECT\n            q.id AS \"id!\",\n            s.flow_status,\n            q.suspend AS \"suspend!\",\n            j.runnable_path AS script_path,\n            j.permissioned_as_email AS email,\n            (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\",\n            (ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS \"is_wac!\"\n        FROM job_info ji\n        JOIN v2_job_queue q ON q.id = CASE\n            WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n            ELSE COALESCE(ji.parent_job, ji.id)\n        END\n        JOIN v2_job j ON j.id = q.id\n        LEFT JOIN v2_job_status s ON s.id = q.id\n        FOR UPDATE OF q\n        ",
   "describe": {
     "columns": [
       {
@@ -32,6 +32,11 @@
         "ordinal": 5,
         "name": "is_flow_level!",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "is_wac!",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -45,8 +50,9 @@
       false,
       true,
       false,
+      null,
       null
     ]
   },
-  "hash": "1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82"
+  "hash": "dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353"
 }

backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json

@@ -15,7 +15,7 @@
       ]
     },
     "nullable": [
-      false
+      true
     ]
   },
   "hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06"

backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Float8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20"
+}

backend/Cargo.lock

@@ -7103,7 +7103,7 @@ dependencies = [
  "libc",
  "percent-encoding",
  "pin-project-lite",
- "socket2 0.6.2",
+ "socket2 0.6.3",
  "system-configuration",
  "tokio",
  "tower-service",
@@ -7974,9 +7974,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.182"
+version = "0.2.183"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
+checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
 
 [[package]]
 name = "libffi"
@@ -10664,7 +10664,7 @@ dependencies = [
  "quinn-udp",
  "rustc-hash 2.1.1",
  "rustls 0.23.35",
- "socket2 0.6.2",
+ "socket2 0.6.3",
  "thiserror 2.0.18",
  "tokio",
  "tracing",
@@ -10673,9 +10673,9 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.13"
+version = "0.11.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31"
+checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
 dependencies = [
  "aws-lc-rs",
  "bytes",
@@ -10702,7 +10702,7 @@ dependencies = [
  "cfg_aliases 0.2.1",
  "libc",
  "once_cell",
- "socket2 0.6.2",
+ "socket2 0.6.3",
  "tracing",
  "windows-sys 0.60.2",
 ]
@@ -12677,12 +12677,12 @@ dependencies = [
 
 [[package]]
 name = "socket2"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0"
+checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
 dependencies = [
  "libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -14462,7 +14462,7 @@ dependencies = [
  "indexmap 2.11.1",
  "toml_datetime 0.7.0",
  "toml_parser",
- "winnow 0.7.14",
+ "winnow 0.7.15",
 ]
 
 [[package]]
@@ -14471,7 +14471,7 @@ version = "1.0.9+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4"
 dependencies = [
- "winnow 0.7.14",
+ "winnow 0.7.15",
 ]
 
 [[package]]
@@ -15741,7 +15741,7 @@ dependencies = [
 
 [[package]]
 name = "windmill"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -15808,7 +15808,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-alerting"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -15821,7 +15821,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "argon2",
@@ -15849,6 +15849,7 @@ dependencies = [
  "dashmap 6.1.0",
  "datafusion",
  "ed25519-dalek",
+ "eventsource-stream",
  "flate2",
  "futures",
  "git-version",
@@ -15960,7 +15961,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-agent-workers"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -15983,7 +15984,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-assets"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -15996,7 +15997,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-auth"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "axum 0.7.9",
@@ -16022,7 +16023,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-client"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "reqwest 0.12.28",
  "serde",
@@ -16032,7 +16033,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-configs"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -16049,7 +16050,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-debug"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "base64 0.22.1",
@@ -16072,7 +16073,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-embeddings"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "axum 0.7.9",
@@ -16095,7 +16096,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-flow-conversations"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -16111,7 +16112,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-flows"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -16131,7 +16132,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-groups"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -16151,7 +16152,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-inputs"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -16165,7 +16166,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-integration-tests"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -16192,7 +16193,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-jobs"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "axum 0.7.9",
@@ -16217,7 +16218,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-npm-proxy"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "flate2",
@@ -16235,7 +16236,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-openapi"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "axum 0.7.9",
@@ -16256,7 +16257,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-schedule"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -16276,7 +16277,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-scripts"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -16306,7 +16307,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-settings"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "axum 0.7.9",
@@ -16333,7 +16334,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-sse"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "lazy_static",
  "serde",
@@ -16345,7 +16346,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-users"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "argon2",
  "axum 0.7.9",
@@ -16368,7 +16369,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-workers"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -16382,7 +16383,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-workspaces"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "axum 0.7.9",
  "chrono",
@@ -16413,7 +16414,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-audit"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "chrono",
  "lazy_static",
@@ -16427,7 +16428,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-autoscaling"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "axum 0.7.9",
@@ -16446,7 +16447,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-common"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -16545,7 +16546,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-dep-map"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "chrono",
  "itertools 0.14.0",
@@ -16564,7 +16565,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-git-sync"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "regex",
  "serde",
@@ -16579,7 +16580,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-indexer"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "astral-tokio-tar",
@@ -16603,7 +16604,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-jseval"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "futures",
@@ -16620,7 +16621,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-macros"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "itertools 0.14.0",
  "lazy_static",
@@ -16636,7 +16637,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-mcp"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16657,7 +16658,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-native-triggers"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16688,7 +16689,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-oauth"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-oauth2",
@@ -16712,7 +16713,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-object-store"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -16746,7 +16747,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-operator"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "futures",
@@ -16764,7 +16765,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "convert_case 0.6.0",
  "serde",
@@ -16773,7 +16774,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16785,7 +16786,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-csharp"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -16797,7 +16798,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "gosyn",
@@ -16809,7 +16810,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16821,7 +16822,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-java"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -16833,7 +16834,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-nu"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "nu-parser",
@@ -16844,7 +16845,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-php"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -16855,7 +16856,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -16868,7 +16869,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -16892,7 +16893,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ruby"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16906,7 +16907,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-rust"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "convert_case 0.6.0",
@@ -16923,7 +16924,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16938,7 +16939,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16955,9 +16956,25 @@ dependencies = [
  "windmill-parser-sql",
 ]
 
+[[package]]
+name = "windmill-parser-wac"
+version = "1.652.0"
+dependencies = [
+ "anyhow",
+ "rustpython-ast",
+ "rustpython-parser",
+ "serde",
+ "serde_json",
+ "sha2 0.10.9",
+ "swc_common",
+ "swc_ecma_ast",
+ "swc_ecma_parser",
+ "swc_ecma_visit",
+]
+
 [[package]]
 name = "windmill-parser-yaml"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -16968,7 +16985,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-queue"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -17005,7 +17022,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-runtime-nativets"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "const_format",
@@ -17043,7 +17060,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-sql-datatype-parser-wasm"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "getrandom 0.3.4",
  "wasm-bindgen",
@@ -17054,7 +17071,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-store"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -17083,7 +17100,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-test-utils"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "axum 0.7.9",
@@ -17106,7 +17123,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17139,7 +17156,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-email"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17159,7 +17176,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-gcp"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17193,7 +17210,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-http"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17228,7 +17245,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-kafka"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17251,7 +17268,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-mqtt"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17275,7 +17292,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-nats"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -17299,7 +17316,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-postgres"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17334,7 +17351,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-sqs"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17362,7 +17379,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-websocket"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17385,7 +17402,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-types"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "bitflags 2.9.4",
@@ -17403,7 +17420,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "anyhow",
  "async-once-cell",
@@ -17509,7 +17526,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker-volumes"
-version = "1.651.1"
+version = "1.652.0"
 dependencies = [
  "bytes",
  "futures",
@@ -18109,9 +18126,9 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.7.14"
+version = "0.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
+checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
 dependencies = [
  "memchr",
 ]
@@ -18392,18 +18409,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.40"
+version = "0.8.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5"
+checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.40"
+version = "0.8.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953"
+checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
 dependencies = [
  "proc-macro2",
  "quote",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.651.1"
+version = "1.652.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -68,6 +68,7 @@ members = [
     "./parsers/windmill-parser-bash",
     "./parsers/windmill-parser-py",
     "./parsers/windmill-parser-py-imports",
+    "./parsers/windmill-parser-wac",
     "./parsers/windmill-sql-datatype-parser-wasm",
     "./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu",
     "./windmill-worker-volumes",
@@ -77,7 +78,7 @@ members = [
 exclude = ["./windmill-duckdb-ffi-internal"]
 
 [workspace.package]
-version = "1.651.1"
+version = "1.652.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -332,6 +333,7 @@ windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
 windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
 windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
 windmill-parser-php = { path = "./parsers/windmill-parser-php" }
+windmill-parser-wac = { path = "./parsers/windmill-parser-wac" }
 windmill-jseval = { path = "./windmill-jseval" }
 windmill-runtime-nativets = { path = "./windmill-runtime-nativets" }
 windmill-api-client = { path = "./windmill-api-client" }

backend/ee-repo-ref.txt

@@ -1 +1 @@
-c3c543f4c60a8c4dfe0d912c79a051376fb091a9
+716b350bce1730b302c66ea69df618fa40f2f16b

backend/migrations/20260305000000_worker_ping_batch_pull.down.sql

@@ -1 +0,0 @@
-ALTER TABLE worker_ping DROP COLUMN IF EXISTS uses_batch_http_pull;

backend/migrations/20260305000000_worker_ping_batch_pull.up.sql

@@ -1 +0,0 @@
-ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS uses_batch_http_pull BOOLEAN NOT NULL DEFAULT false;

backend/parsers/windmill-parser-py/src/lib.rs

@@ -296,11 +296,14 @@ pub fn parse_python_signature(
 
     // Check if main function was found
     if params.is_none() {
+        let is_wac_v2 = (code.contains("@workflow") || code.contains("workflow("))
+            && (code.contains("@task") || code.contains("task("))
+            && (code.contains("import wmill") || code.contains("from wmill"));
         return Ok(MainArgSignature {
             star_args: false,
             star_kwargs: false,
             args: vec![],
-            no_main_func: Some(true),
+            no_main_func: Some(!is_wac_v2),
             has_preprocessor: Some(has_preprocessor),
         });
     }

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -238,7 +238,7 @@ lazy_static::lazy_static! {
 
     // used for `unsafe` sql interpolation
     // -- %%name%% (type) = default
-    static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%\s*([\s\w\/]+)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
+    static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%[ \t]*([\w][\w \t\/]*)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
 }
 
 fn parsed_default(parsed_typ: &Typ, default: String) -> Option<serde_json::Value> {
@@ -1547,4 +1547,36 @@ SELECT $1::integer;
 
         Ok(())
     }
+
+    #[test]
+    fn test_parse_pgsql_safe_interpolated_args() -> anyhow::Result<()> {
+        // There was a bug where enum would be "angrycreative"/"bishop"/"test SELECT x"
+        let code = r#"
+-- %%table_name%% angrycreative/bishop/test
+SELECT x
+"#;
+        assert_eq!(
+            parse_pgsql_sig(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![Arg {
+                    otyp: Some("__sanitized_enum__".to_string()),
+                    name: "table_name".to_string(),
+                    typ: Typ::Str(Some(vec![
+                        "angrycreative".to_string(),
+                        "bishop".to_string(),
+                        "test".to_string()
+                    ])),
+                    default: None,
+                    has_default: false,
+                    oidx: None,
+                },],
+                no_main_func: None,
+                has_preprocessor: None
+            }
+        );
+
+        Ok(())
+    }
 }

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -261,7 +261,9 @@ pub fn parse_deno_signature(
                 for specifier in &named_export.specifiers {
                     if let swc_ecma_ast::ExportSpecifier::Named(spec) = specifier {
                         let export_name = match &spec.exported {
-                            Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => ident.sym.as_ref(),
+                            Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => {
+                                ident.sym.as_ref()
+                            }
                             Some(swc_ecma_ast::ModuleExportName::Str(s)) => s.value.as_ref(),
                             None => match &spec.orig {
                                 swc_ecma_ast::ModuleExportName::Ident(ident) => ident.sym.as_ref(),
@@ -315,7 +317,11 @@ pub fn parse_deno_signature(
 
     let mut c: u16 = 0;
 
-    let no_main_func = entrypoint_params.is_none();
+    let is_wac_v2 = entrypoint_params.is_none()
+        && code.contains("workflow(")
+        && code.contains("task(")
+        && code.contains("windmill-client");
+    let no_main_func = entrypoint_params.is_none() && !is_wac_v2;
     let mut type_resolver = HashMap::new();
     let r = MainArgSignature {
         star_args: false,
@@ -833,7 +839,9 @@ fn tstype_to_typ(
                     false,
                 ),
                 symbol @ _ if symbol.starts_with("DynMultiselect_") => (
-                    Typ::DynMultiselect(symbol.strip_prefix("DynMultiselect_").unwrap().to_string()),
+                    Typ::DynMultiselect(
+                        symbol.strip_prefix("DynMultiselect_").unwrap().to_string(),
+                    ),
                     false,
                 ),
                 symbol @ _ => {

backend/parsers/windmill-parser-wac/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "windmill-parser-wac"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+
+[lib]
+name = "windmill_parser_wac"
+path = "./src/lib.rs"
+
+[dependencies]
+rustpython-parser.workspace = true
+rustpython-ast = { version = "0.4.0", features = ["visitor"] }
+swc_common.workspace = true
+swc_ecma_parser.workspace = true
+swc_ecma_ast.workspace = true
+swc_ecma_visit.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+anyhow.workspace = true
+sha2.workspace = true

backend/parsers/windmill-parser-wac/src/dag.rs

@@ -0,0 +1,44 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct WorkflowDag {
+    pub nodes: Vec<DagNode>,
+    pub edges: Vec<DagEdge>,
+    pub params: Vec<Param>,
+    pub source_hash: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct Param {
+    pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub typ: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct DagNode {
+    pub id: String,
+    pub node_type: DagNodeType,
+    pub label: String,
+    pub line: usize,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(tag = "type")]
+pub enum DagNodeType {
+    Step { name: String, script: String },
+    Branch { condition_source: String },
+    ParallelStart,
+    ParallelEnd,
+    LoopStart { iter_source: String },
+    LoopEnd,
+    Return,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct DagEdge {
+    pub from: String,
+    pub to: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub label: Option<String>,
+}

backend/parsers/windmill-parser-wac/src/lib.rs

@@ -0,0 +1,32 @@
+pub mod dag;
+pub mod python;
+pub mod typescript;
+pub mod validation;
+
+use dag::WorkflowDag;
+use validation::CompileError;
+
+#[derive(Debug, serde::Serialize)]
+#[serde(tag = "type")]
+pub enum ParseResult {
+    #[serde(rename = "success")]
+    Success(WorkflowDag),
+    #[serde(rename = "error")]
+    Error { errors: Vec<CompileError> },
+}
+
+pub fn parse_workflow(code: &str, language: &str) -> ParseResult {
+    let result = match language {
+        "python" | "python3" | "py" => python::parse_python_workflow(code),
+        "typescript" | "ts" | "deno" | "bun" => typescript::parse_ts_workflow(code),
+        _ => Err(vec![CompileError {
+            message: format!("Unsupported language: {language}"),
+            line: 0,
+        }]),
+    };
+
+    match result {
+        Ok(dag) => ParseResult::Success(dag),
+        Err(errors) => ParseResult::Error { errors },
+    }
+}

backend/parsers/windmill-parser-wac/src/python.rs

@@ -0,0 +1,717 @@
+use std::collections::HashMap;
+
+use rustpython_parser::{
+    ast::{
+        Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry,
+        StmtTryStar, StmtWhile,
+    },
+    Parse,
+};
+
+use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag};
+use crate::validation::{self, CompileError};
+
+struct LineIndex {
+    newline_offsets: Vec<usize>,
+}
+
+impl LineIndex {
+    fn new(source: &str) -> Self {
+        let mut offsets = vec![0];
+        for (i, c) in source.char_indices() {
+            if c == '\n' {
+                offsets.push(i + 1);
+            }
+        }
+        Self { newline_offsets: offsets }
+    }
+
+    fn line_of(&self, byte_offset: usize) -> usize {
+        match self.newline_offsets.binary_search(&byte_offset) {
+            Ok(line) => line + 1,
+            Err(line) => line,
+        }
+    }
+}
+
+/// Maps task function name → optional external path (from `@task(path="...")`)
+type TaskFunctions = HashMap<String, Option<String>>;
+
+/// First pass: scan top-level `@task async def foo(...)` declarations.
+fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
+    let mut tasks = HashMap::new();
+    for stmt in stmts {
+        if let Stmt::AsyncFunctionDef(func) = stmt {
+            for dec in &func.decorator_list {
+                match dec {
+                    // @task  (bare decorator)
+                    Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => {
+                        tasks.insert(func.name.to_string(), None);
+                    }
+                    // @task(path="...")
+                    Expr::Call(call) => {
+                        if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
+                            if id.as_str() == "task" {
+                                let path = extract_task_path_kwarg(call);
+                                tasks.insert(func.name.to_string(), path);
+                            }
+                        }
+                    }
+                    _ => {}
+                }
+            }
+        }
+    }
+    tasks
+}
+
+/// Extract the `path=` keyword argument from a `@task(path="...")` call.
+fn extract_task_path_kwarg(call: &ExprCall) -> Option<String> {
+    for kw in &call.keywords {
+        if let Some(ref arg) = kw.arg {
+            if arg.as_str() == "path" {
+                if let Expr::Constant(c) = &kw.value {
+                    if let rustpython_parser::ast::Constant::Str(s) = &c.value {
+                        return Some(s.to_string());
+                    }
+                }
+            }
+        }
+    }
+    None
+}
+
+struct WacWalker {
+    nodes: Vec<DagNode>,
+    edges: Vec<DagEdge>,
+    errors: Vec<CompileError>,
+    node_counter: usize,
+    line_index: LineIndex,
+    task_functions: TaskFunctions,
+    in_try: bool,
+    in_while: bool,
+    in_nested_func: bool,
+    in_comprehension: bool,
+}
+
+impl WacWalker {
+    fn new(source: &str, task_functions: TaskFunctions) -> Self {
+        Self {
+            nodes: Vec::new(),
+            edges: Vec::new(),
+            errors: Vec::new(),
+            node_counter: 0,
+            line_index: LineIndex::new(source),
+            task_functions,
+            in_try: false,
+            in_while: false,
+            in_nested_func: false,
+            in_comprehension: false,
+        }
+    }
+
+    fn next_id(&mut self) -> String {
+        let id = format!("step_{}", self.node_counter);
+        self.node_counter += 1;
+        id
+    }
+
+    fn add_node(&mut self, node: DagNode) -> String {
+        let id = node.id.clone();
+        self.nodes.push(node);
+        id
+    }
+
+    fn add_edge(&mut self, from: &str, to: &str, label: Option<String>) {
+        self.edges
+            .push(DagEdge { from: from.to_string(), to: to.to_string(), label });
+    }
+
+    fn line_of_expr(&self, expr: &Expr) -> usize {
+        let offset = match expr {
+            Expr::Call(c) => c.range.start().to_usize(),
+            Expr::Await(a) => a.range.start().to_usize(),
+            Expr::Attribute(a) => a.range.start().to_usize(),
+            Expr::Name(n) => n.range.start().to_usize(),
+            _ => 0,
+        };
+        self.line_index.line_of(offset)
+    }
+
+    fn line_of_stmt(&self, stmt: &Stmt) -> usize {
+        let offset = match stmt {
+            Stmt::If(s) => s.range.start().to_usize(),
+            Stmt::For(s) => s.range.start().to_usize(),
+            Stmt::While(s) => s.range.start().to_usize(),
+            Stmt::Return(s) => s.range.start().to_usize(),
+            Stmt::Expr(s) => s.range.start().to_usize(),
+            Stmt::Try(s) => s.range.start().to_usize(),
+            Stmt::TryStar(s) => s.range.start().to_usize(),
+            Stmt::Assign(s) => s.range.start().to_usize(),
+            Stmt::AnnAssign(s) => s.range.start().to_usize(),
+            Stmt::FunctionDef(s) => s.range.start().to_usize(),
+            Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(),
+            _ => 0,
+        };
+        self.line_index.line_of(offset)
+    }
+
+    /// Check if an expression is a call to a known @task function
+    fn is_task_fn_call(&self, expr: &Expr) -> bool {
+        if let Expr::Call(call) = expr {
+            if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
+                return self.task_functions.contains_key(id.as_str());
+            }
+        }
+        false
+    }
+
+    /// Check if an expression is `asyncio.gather(...)` call
+    fn is_asyncio_gather_call(expr: &Expr) -> bool {
+        if let Expr::Call(call) = expr {
+            if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) =
+                call.func.as_ref()
+            {
+                if attr.as_str() == "gather" {
+                    if let Expr::Name(ExprName { id, .. }) = value.as_ref() {
+                        return id.as_str() == "asyncio";
+                    }
+                }
+            }
+        }
+        false
+    }
+
+    /// Extract step name and script from a task function call.
+    /// Name = function name, script = task_path or function name.
+    fn extract_step_info_from_task_call(&self, call: &ExprCall) -> Option<(String, String)> {
+        if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
+            let name = id.to_string();
+            let script = self
+                .task_functions
+                .get(id.as_str())
+                .and_then(|p| p.clone())
+                .unwrap_or_else(|| name.clone());
+            Some((name, script))
+        } else {
+            None
+        }
+    }
+
+    fn expr_to_source(expr: &Expr) -> String {
+        match expr {
+            Expr::Compare(c) => {
+                let left = Self::expr_to_source(&c.left);
+                if let Some(comparator) = c.comparators.first() {
+                    let right = Self::expr_to_source(comparator);
+                    let op = match c.ops.first() {
+                        Some(rustpython_parser::ast::CmpOp::Gt) => ">",
+                        Some(rustpython_parser::ast::CmpOp::Lt) => "<",
+                        Some(rustpython_parser::ast::CmpOp::GtE) => ">=",
+                        Some(rustpython_parser::ast::CmpOp::LtE) => "<=",
+                        Some(rustpython_parser::ast::CmpOp::Eq) => "==",
+                        Some(rustpython_parser::ast::CmpOp::NotEq) => "!=",
+                        Some(rustpython_parser::ast::CmpOp::In) => "in",
+                        Some(rustpython_parser::ast::CmpOp::NotIn) => "not in",
+                        Some(rustpython_parser::ast::CmpOp::Is) => "is",
+                        Some(rustpython_parser::ast::CmpOp::IsNot) => "is not",
+                        None => "?",
+                    };
+                    format!("{left} {op} {right}")
+                } else {
+                    left
+                }
+            }
+            Expr::Subscript(s) => {
+                let value = Self::expr_to_source(&s.value);
+                let slice = Self::expr_to_source(&s.slice);
+                format!("{value}[{slice}]")
+            }
+            Expr::Attribute(a) => {
+                let value = Self::expr_to_source(&a.value);
+                format!("{value}.{}", a.attr)
+            }
+            Expr::Name(n) => n.id.to_string(),
+            Expr::Constant(c) => match &c.value {
+                rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""),
+                rustpython_parser::ast::Constant::Int(i) => i.to_string(),
+                rustpython_parser::ast::Constant::Float(f) => f.to_string(),
+                rustpython_parser::ast::Constant::Bool(b) => b.to_string(),
+                rustpython_parser::ast::Constant::None => "None".to_string(),
+                _ => "...".to_string(),
+            },
+            _ => "...".to_string(),
+        }
+    }
+
+    /// Check if a statement body contains any task function calls (recursively)
+    fn body_contains_step(&self, body: &[Stmt]) -> bool {
+        for stmt in body {
+            if self.stmt_contains_step(stmt) {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn stmt_contains_step(&self, stmt: &Stmt) -> bool {
+        match stmt {
+            Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value),
+            Stmt::Assign(a) => self.expr_contains_step(&a.value),
+            Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
+            Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
+            Stmt::While(s) => {
+                self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
+            }
+            Stmt::Try(s) => {
+                self.body_contains_step(&s.body)
+                    || self.body_contains_step(&s.orelse)
+                    || self.body_contains_step(&s.finalbody)
+                    || s.handlers.iter().any(|h| match h {
+                        rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
+                            self.body_contains_step(&eh.body)
+                        }
+                    })
+            }
+            Stmt::TryStar(s) => {
+                self.body_contains_step(&s.body)
+                    || self.body_contains_step(&s.orelse)
+                    || self.body_contains_step(&s.finalbody)
+                    || s.handlers.iter().any(|h| match h {
+                        rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
+                            self.body_contains_step(&eh.body)
+                        }
+                    })
+            }
+            Stmt::Return(_) => false,
+            _ => false,
+        }
+    }
+
+    fn expr_contains_step(&self, expr: &Expr) -> bool {
+        if self.is_task_fn_call(expr) {
+            return true;
+        }
+        match expr {
+            Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value),
+            Expr::Call(call) => {
+                if self.is_task_fn_call(&Expr::Call(call.clone())) {
+                    return true;
+                }
+                if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) {
+                    return call.args.iter().any(|a| self.expr_contains_step(a));
+                }
+                false
+            }
+            _ => false,
+        }
+    }
+
+    /// Walk a list of statements, returning (first_node_id, last_node_id)
+    fn walk_body(&mut self, body: &[Stmt]) -> Option<(String, String)> {
+        let mut first_id: Option<String> = None;
+        let mut prev_id: Option<String> = None;
+
+        for stmt in body {
+            if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) {
+                if let Some(ref prev) = prev_id {
+                    self.add_edge(prev, &stmt_first, None);
+                }
+                if first_id.is_none() {
+                    first_id = Some(stmt_first);
+                }
+                prev_id = Some(stmt_last);
+            }
+        }
+
+        match (first_id, prev_id) {
+            (Some(f), Some(l)) => Some((f, l)),
+            _ => None,
+        }
+    }
+
+    fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> {
+        match stmt {
+            Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value),
+            Stmt::Assign(a) => self.walk_expr_stmt(&a.value),
+            Stmt::If(if_stmt) => self.walk_if(if_stmt),
+            Stmt::For(for_stmt) => self.walk_for(for_stmt),
+            Stmt::While(while_stmt) => self.walk_while(while_stmt),
+            Stmt::Try(try_stmt) => self.walk_try(try_stmt),
+            Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt),
+            Stmt::Return(ret) => self.walk_return(ret),
+            Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => {
+                if self.stmt_contains_step(stmt) {
+                    self.errors.push(validation::error_step_in_nested_function(
+                        self.line_of_stmt(stmt),
+                    ));
+                }
+                None
+            }
+            _ => None,
+        }
+    }
+
+    fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> {
+        // await task_fn(...)
+        if let Expr::Await(ExprAwait { value, .. }) = expr {
+            // await task_fn(...)
+            if let Expr::Call(call) = value.as_ref() {
+                if self.is_task_fn_call(&Expr::Call(call.clone())) {
+                    return self.emit_step(call, expr);
+                }
+            }
+            // await asyncio.gather(task_fn(...), task_fn(...), ...)
+            if Self::is_asyncio_gather_call(value) {
+                if let Expr::Call(gather_call) = value.as_ref() {
+                    return self.emit_parallel(gather_call, expr);
+                }
+            }
+        }
+
+        // Bare task_fn() without await — validation error
+        if self.is_task_fn_call(expr) {
+            self.errors
+                .push(validation::error_missing_await(self.line_of_expr(expr)));
+        }
+
+        None
+    }
+
+    fn emit_step(&mut self, call: &ExprCall, expr: &Expr) -> Option<(String, String)> {
+        if self.in_try {
+            self.errors
+                .push(validation::error_step_in_try(self.line_of_expr(expr)));
+            return None;
+        }
+        if self.in_while {
+            self.errors
+                .push(validation::error_step_in_while(self.line_of_expr(expr)));
+            return None;
+        }
+        if self.in_nested_func {
+            self.errors.push(validation::error_step_in_nested_function(
+                self.line_of_expr(expr),
+            ));
+            return None;
+        }
+        if self.in_comprehension {
+            self.errors.push(validation::error_step_in_comprehension(
+                self.line_of_expr(expr),
+            ));
+            return None;
+        }
+
+        let (name, script) = self
+            .extract_step_info_from_task_call(call)
+            .unwrap_or(("unknown".into(), "unknown".into()));
+        let id = self.next_id();
+        let node_id = self.add_node(DagNode {
+            id: id.clone(),
+            node_type: DagNodeType::Step { name: name.clone(), script },
+            label: name,
+            line: self.line_of_expr(expr),
+        });
+        Some((node_id.clone(), node_id))
+    }
+
+    fn emit_parallel(&mut self, gather_call: &ExprCall, expr: &Expr) -> Option<(String, String)> {
+        if self.in_try {
+            self.errors
+                .push(validation::error_step_in_try(self.line_of_expr(expr)));
+            return None;
+        }
+        if self.in_while {
+            self.errors
+                .push(validation::error_step_in_while(self.line_of_expr(expr)));
+            return None;
+        }
+
+        let line = self.line_of_expr(expr);
+        let start_id = self.next_id();
+        let start_node_id = self.add_node(DagNode {
+            id: start_id.clone(),
+            node_type: DagNodeType::ParallelStart,
+            label: "parallel".to_string(),
+            line,
+        });
+
+        let mut step_ids = Vec::new();
+        for arg in &gather_call.args {
+            // Each arg should be task_fn(...)
+            if let Expr::Call(call) = arg {
+                if self.is_task_fn_call(&Expr::Call(call.clone())) {
+                    let (name, script) = self
+                        .extract_step_info_from_task_call(call)
+                        .unwrap_or(("unknown".into(), "unknown".into()));
+                    let step_id = self.next_id();
+                    let node_id = self.add_node(DagNode {
+                        id: step_id.clone(),
+                        node_type: DagNodeType::Step { name: name.clone(), script },
+                        label: name,
+                        line: self.line_of_expr(arg),
+                    });
+                    self.add_edge(&start_node_id, &node_id, None);
+                    step_ids.push(node_id);
+                }
+            }
+        }
+
+        let end_id = self.next_id();
+        let end_node_id = self.add_node(DagNode {
+            id: end_id.clone(),
+            node_type: DagNodeType::ParallelEnd,
+            label: "join".to_string(),
+            line,
+        });
+
+        for step_id in &step_ids {
+            self.add_edge(step_id, &end_node_id, None);
+        }
+
+        Some((start_node_id, end_node_id))
+    }
+
+    fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> {
+        let has_steps_in_body = self.body_contains_step(&if_stmt.body);
+        let has_steps_in_else = self.body_contains_step(&if_stmt.orelse);
+
+        if !has_steps_in_body && !has_steps_in_else {
+            return None;
+        }
+
+        let line = self.line_index.line_of(if_stmt.range.start().to_usize());
+        let condition_source = Self::expr_to_source(&if_stmt.test);
+
+        let branch_id = self.next_id();
+        let branch_node_id = self.add_node(DagNode {
+            id: branch_id.clone(),
+            node_type: DagNodeType::Branch { condition_source },
+            label: "if".to_string(),
+            line,
+        });
+
+        let merge_id = format!("{branch_id}_merge");
+
+        let mut last_ids = Vec::new();
+
+        if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) {
+            self.add_edge(&branch_node_id, &true_first, Some("true".to_string()));
+            last_ids.push(true_last);
+        } else {
+            last_ids.push(branch_node_id.clone());
+        }
+
+        if !if_stmt.orelse.is_empty() {
+            if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) {
+                self.add_edge(&branch_node_id, &else_first, Some("false".to_string()));
+                last_ids.push(else_last);
+            } else {
+                last_ids.push(branch_node_id.clone());
+            }
+        }
+
+        if last_ids.len() == 1 {
+            Some((branch_node_id, last_ids.into_iter().next().unwrap()))
+        } else {
+            Some((branch_node_id, merge_id))
+        }
+    }
+
+    fn walk_for(&mut self, for_stmt: &StmtFor) -> Option<(String, String)> {
+        if !self.body_contains_step(&for_stmt.body) {
+            return None;
+        }
+
+        let line = self.line_index.line_of(for_stmt.range.start().to_usize());
+        let iter_source = Self::expr_to_source(&for_stmt.iter);
+
+        let start_id = self.next_id();
+        let start_node_id = self.add_node(DagNode {
+            id: start_id.clone(),
+            node_type: DagNodeType::LoopStart { iter_source },
+            label: "for".to_string(),
+            line,
+        });
+
+        if let Some((body_first, body_last)) = self.walk_body(&for_stmt.body) {
+            self.add_edge(&start_node_id, &body_first, None);
+            self.add_edge(&body_last, &start_node_id, Some("next".to_string()));
+        }
+
+        let end_id = self.next_id();
+        let end_node_id = self.add_node(DagNode {
+            id: end_id.clone(),
+            node_type: DagNodeType::LoopEnd,
+            label: "end for".to_string(),
+            line,
+        });
+        self.add_edge(&start_node_id, &end_node_id, Some("done".to_string()));
+
+        Some((start_node_id, end_node_id))
+    }
+
+    fn walk_while(&mut self, while_stmt: &StmtWhile) -> Option<(String, String)> {
+        if self.body_contains_step(&while_stmt.body) {
+            let line = self.line_index.line_of(while_stmt.range.start().to_usize());
+            self.errors.push(validation::error_step_in_while(line));
+        }
+        None
+    }
+
+    fn walk_try(&mut self, try_stmt: &StmtTry) -> Option<(String, String)> {
+        let has_steps = self.body_contains_step(&try_stmt.body)
+            || self.body_contains_step(&try_stmt.orelse)
+            || self.body_contains_step(&try_stmt.finalbody)
+            || try_stmt.handlers.iter().any(|h| match h {
+                rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
+                    self.body_contains_step(&eh.body)
+                }
+            });
+
+        if has_steps {
+            let line = self.line_index.line_of(try_stmt.range.start().to_usize());
+            self.errors.push(validation::error_step_in_try(line));
+        }
+        None
+    }
+
+    fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> {
+        let has_steps = self.body_contains_step(&try_stmt.body)
+            || self.body_contains_step(&try_stmt.orelse)
+            || self.body_contains_step(&try_stmt.finalbody)
+            || try_stmt.handlers.iter().any(|h| match h {
+                rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
+                    self.body_contains_step(&eh.body)
+                }
+            });
+
+        if has_steps {
+            let line = self.line_index.line_of(try_stmt.range.start().to_usize());
+            self.errors.push(validation::error_step_in_try(line));
+        }
+        None
+    }
+
+    fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> {
+        let line = self.line_index.line_of(ret.range.start().to_usize());
+        let id = self.next_id();
+        let node_id = self.add_node(DagNode {
+            id: id.clone(),
+            node_type: DagNodeType::Return,
+            label: "return".to_string(),
+            line,
+        });
+        Some((node_id.clone(), node_id))
+    }
+}
+
+/// Extract workflow function parameters (no longer skips ctx)
+fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
+    let mut params = Vec::new();
+    for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) {
+        let name = arg_with_default.def.arg.to_string();
+        let typ = arg_with_default
+            .def
+            .annotation
+            .as_ref()
+            .map(|ann| WacWalker::expr_to_source(ann));
+        params.push(Param { name, typ });
+    }
+    params
+}
+
+pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError>> {
+    let ast = rustpython_parser::ast::Suite::parse(code, "<workflow>")
+        .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?;
+
+    // First pass: collect @task functions
+    let task_functions = collect_task_functions(&ast);
+
+    // Find the @workflow async def
+    let workflow_fn = ast.iter().find_map(|stmt| {
+        if let Stmt::AsyncFunctionDef(func) = stmt {
+            let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
+                if let Expr::Name(ExprName { id, .. }) = dec {
+                    id.as_str() == "workflow"
+                } else {
+                    false
+                }
+            });
+            if has_workflow_decorator {
+                return Some(func);
+            }
+        }
+        // Also check non-async for error reporting
+        if let Stmt::FunctionDef(func) = stmt {
+            let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
+                if let Expr::Name(ExprName { id, .. }) = dec {
+                    id.as_str() == "workflow"
+                } else {
+                    false
+                }
+            });
+            if has_workflow_decorator {
+                return None; // Will be reported as not-async below
+            }
+        }
+        None
+    });
+
+    // Check for non-async workflow function
+    let non_async_workflow = ast.iter().find_map(|stmt| {
+        if let Stmt::FunctionDef(func) = stmt {
+            let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
+                if let Expr::Name(ExprName { id, .. }) = dec {
+                    id.as_str() == "workflow"
+                } else {
+                    false
+                }
+            });
+            if has_workflow_decorator {
+                let line_index = LineIndex::new(code);
+                return Some(line_index.line_of(func.range.start().to_usize()));
+            }
+        }
+        None
+    });
+
+    if let Some(line) = non_async_workflow {
+        if workflow_fn.is_none() {
+            return Err(vec![validation::error_not_async(line)]);
+        }
+    }
+
+    let workflow_fn = workflow_fn.ok_or_else(|| {
+        vec![CompileError { message: "No @workflow async function found.".to_string(), line: 0 }]
+    })?;
+
+    let params = extract_params(&workflow_fn.args);
+    let source_hash = compute_source_hash(code);
+
+    let mut walker = WacWalker::new(code, task_functions);
+    walker.walk_body(&workflow_fn.body);
+
+    if !walker.errors.is_empty() {
+        return Err(walker.errors);
+    }
+
+    Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash })
+}
+
+fn compute_source_hash(code: &str) -> String {
+    use sha2::{Digest, Sha256};
+    let mut hasher = Sha256::new();
+    hasher.update(code.as_bytes());
+    format!("{:x}", hasher.finalize())
+}
+
+trait ToUsize {
+    fn to_usize(self) -> usize;
+}
+
+impl ToUsize for rustpython_parser::text_size::TextSize {
+    fn to_usize(self) -> usize {
+        u32::from(self) as usize
+    }
+}

backend/parsers/windmill-parser-wac/src/typescript.rs

@@ -0,0 +1,739 @@
+use std::collections::HashMap;
+
+use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned};
+use swc_ecma_ast::*;
+use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax};
+
+use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag};
+use crate::validation::{self, CompileError};
+
+/// Maps task function name → optional external path (from `task("f/path", ...)`)
+type TaskFunctions = HashMap<String, Option<String>>;
+
+/// First pass: scan top-level `const foo = task(async (...) => {})` or
+/// `const foo = task("f/path", async (...) => {})` declarations.
+fn collect_task_functions(module: &Module) -> TaskFunctions {
+    let mut tasks = HashMap::new();
+    for item in &module.body {
+        // const foo = task(async (...) => { ... })
+        // const foo = task("f/path", async (...) => { ... })
+        if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item {
+            for decl in &var_decl.decls {
+                if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) {
+                    if let Some(path) = extract_task_call_info(init) {
+                        tasks.insert(name, path);
+                    }
+                }
+            }
+        }
+        // export const foo = task(...)
+        if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(export)) = item {
+            if let Decl::Var(var_decl) = &export.decl {
+                for decl in &var_decl.decls {
+                    if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) {
+                        if let Some(path) = extract_task_call_info(init) {
+                            tasks.insert(name, path);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    tasks
+}
+
+/// Extract variable name from a pattern (simple ident case)
+fn extract_var_name(pat: &Pat) -> Option<String> {
+    if let Pat::Ident(BindingIdent { id, .. }) = pat {
+        Some(id.sym.to_string())
+    } else {
+        None
+    }
+}
+
+/// Check if expr is `task(async fn)` or `task("path", async fn)`.
+/// Returns Some(optional_path) if it is a task() call.
+fn extract_task_call_info(expr: &Expr) -> Option<Option<String>> {
+    if let Expr::Call(call) = expr {
+        if let Callee::Expr(callee) = &call.callee {
+            if let Expr::Ident(ident) = callee.as_ref() {
+                if ident.sym.as_ref() == "task" {
+                    // task("f/path", async fn) or task(async fn)
+                    if call.args.len() == 2 {
+                        // task("f/path", async fn)
+                        let path = extract_string_lit(&call.args[0].expr);
+                        return Some(path);
+                    } else if call.args.len() == 1 {
+                        // task(async fn)
+                        return Some(None);
+                    }
+                }
+            }
+        }
+    }
+    None
+}
+
+struct TsWacWalker {
+    nodes: Vec<DagNode>,
+    edges: Vec<DagEdge>,
+    errors: Vec<CompileError>,
+    node_counter: usize,
+    cm: Lrc<SourceMap>,
+    task_functions: TaskFunctions,
+    in_try: bool,
+    in_while: bool,
+    in_nested_func: bool,
+}
+
+impl TsWacWalker {
+    fn new(cm: Lrc<SourceMap>, task_functions: TaskFunctions) -> Self {
+        Self {
+            nodes: Vec::new(),
+            edges: Vec::new(),
+            errors: Vec::new(),
+            node_counter: 0,
+            cm,
+            task_functions,
+            in_try: false,
+            in_while: false,
+            in_nested_func: false,
+        }
+    }
+
+    fn next_id(&mut self) -> String {
+        let id = format!("step_{}", self.node_counter);
+        self.node_counter += 1;
+        id
+    }
+
+    fn add_node(&mut self, node: DagNode) -> String {
+        let id = node.id.clone();
+        self.nodes.push(node);
+        id
+    }
+
+    fn add_edge(&mut self, from: &str, to: &str, label: Option<String>) {
+        self.edges
+            .push(DagEdge { from: from.to_string(), to: to.to_string(), label });
+    }
+
+    fn span_line(&self, span: swc_common::Span) -> usize {
+        let loc = self.cm.lookup_char_pos(span.lo);
+        loc.line
+    }
+
+    /// Check if expr is a call to a known task function
+    fn is_task_call(&self, expr: &Expr) -> bool {
+        if let Expr::Call(call) = expr {
+            if let Callee::Expr(callee) = &call.callee {
+                if let Expr::Ident(ident) = callee.as_ref() {
+                    return self.task_functions.contains_key(ident.sym.as_ref());
+                }
+            }
+        }
+        false
+    }
+
+    /// Check if expr is `Promise.all([...])`
+    fn is_promise_all(expr: &Expr) -> bool {
+        if let Expr::Call(call) = expr {
+            if let Callee::Expr(callee) = &call.callee {
+                if let Expr::Member(MemberExpr { obj, prop: MemberProp::Ident(prop), .. }) =
+                    callee.as_ref()
+                {
+                    if prop.sym.as_ref() == "all" {
+                        if let Expr::Ident(ident) = obj.as_ref() {
+                            return ident.sym.as_ref() == "Promise";
+                        }
+                    }
+                }
+            }
+        }
+        false
+    }
+
+    /// Extract step name and script from a task function call.
+    /// Name = function name, script = task_path or function name.
+    fn extract_step_info_from_task_call(&self, call: &CallExpr) -> Option<(String, String)> {
+        if let Callee::Expr(callee) = &call.callee {
+            if let Expr::Ident(ident) = callee.as_ref() {
+                let name = ident.sym.to_string();
+                let script = self
+                    .task_functions
+                    .get(ident.sym.as_ref())
+                    .and_then(|p| p.clone())
+                    .unwrap_or_else(|| name.clone());
+                return Some((name, script));
+            }
+        }
+        None
+    }
+
+    fn expr_to_source(&self, expr: &Expr) -> String {
+        let span = expr.span();
+        self.cm
+            .span_to_snippet(span)
+            .unwrap_or_else(|_| "...".to_string())
+    }
+
+    fn body_contains_step(&self, stmts: &[Stmt]) -> bool {
+        stmts.iter().any(|s| self.stmt_contains_step(s))
+    }
+
+    fn stmt_contains_step(&self, stmt: &Stmt) -> bool {
+        match stmt {
+            Stmt::Expr(expr_stmt) => self.expr_contains_step(&expr_stmt.expr),
+            Stmt::Decl(Decl::Var(var_decl)) => var_decl.decls.iter().any(|d| {
+                d.init
+                    .as_ref()
+                    .map_or(false, |init| self.expr_contains_step(init))
+            }),
+            Stmt::If(if_stmt) => {
+                self.stmt_contains_step(&if_stmt.cons)
+                    || if_stmt
+                        .alt
+                        .as_ref()
+                        .map_or(false, |alt| self.stmt_contains_step(alt))
+            }
+            Stmt::Block(block) => self.body_contains_step(&block.stmts),
+            Stmt::For(for_stmt) => self.stmt_contains_step(&for_stmt.body),
+            Stmt::ForIn(for_in) => self.stmt_contains_step(&for_in.body),
+            Stmt::ForOf(for_of) => self.stmt_contains_step(&for_of.body),
+            Stmt::While(while_stmt) => self.stmt_contains_step(&while_stmt.body),
+            Stmt::Try(try_stmt) => {
+                self.body_contains_step(&try_stmt.block.stmts)
+                    || try_stmt
+                        .handler
+                        .as_ref()
+                        .map_or(false, |h| self.body_contains_step(&h.body.stmts))
+                    || try_stmt
+                        .finalizer
+                        .as_ref()
+                        .map_or(false, |f| self.body_contains_step(&f.stmts))
+            }
+            Stmt::Return(ret) => ret
+                .arg
+                .as_ref()
+                .map_or(false, |arg| self.expr_contains_step(arg)),
+            _ => false,
+        }
+    }
+
+    fn expr_contains_step(&self, expr: &Expr) -> bool {
+        if self.is_task_call(expr) {
+            return true;
+        }
+        match expr {
+            Expr::Await(await_expr) => self.expr_contains_step(&await_expr.arg),
+            Expr::Call(call) => {
+                if Self::is_promise_all(&Expr::Call(call.clone())) {
+                    return call.args.iter().any(|a| self.expr_contains_step(&a.expr));
+                }
+                false
+            }
+            Expr::Paren(p) => self.expr_contains_step(&p.expr),
+            _ => false,
+        }
+    }
+
+    fn walk_body(&mut self, stmts: &[Stmt]) -> Option<(String, String)> {
+        let mut first_id: Option<String> = None;
+        let mut prev_id: Option<String> = None;
+
+        for stmt in stmts {
+            if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) {
+                if let Some(ref prev) = prev_id {
+                    self.add_edge(prev, &stmt_first, None);
+                }
+                if first_id.is_none() {
+                    first_id = Some(stmt_first);
+                }
+                prev_id = Some(stmt_last);
+            }
+        }
+
+        match (first_id, prev_id) {
+            (Some(f), Some(l)) => Some((f, l)),
+            _ => None,
+        }
+    }
+
+    fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> {
+        match stmt {
+            Stmt::Expr(expr_stmt) => self.walk_expr_stmt(&expr_stmt.expr),
+            Stmt::Decl(Decl::Var(var_decl)) => {
+                // const result = await task_fn(...)
+                for decl in &var_decl.decls {
+                    if let Some(init) = &decl.init {
+                        if let Some(result) = self.walk_expr_stmt(init) {
+                            return Some(result);
+                        }
+                    }
+                }
+                None
+            }
+            Stmt::If(if_stmt) => self.walk_if(if_stmt),
+            Stmt::For(for_stmt) => self.walk_for_stmt(for_stmt),
+            Stmt::ForIn(for_in) => self.walk_for_in(for_in),
+            Stmt::ForOf(for_of) => self.walk_for_of(for_of),
+            Stmt::While(while_stmt) => self.walk_while(while_stmt),
+            Stmt::Try(try_stmt) => self.walk_try(try_stmt),
+            Stmt::Block(block) => self.walk_body(&block.stmts),
+            Stmt::Return(ret) => self.walk_return(ret),
+            Stmt::Decl(Decl::Fn(_)) => {
+                if self.stmt_contains_step(stmt) {
+                    self.errors.push(validation::error_step_in_nested_function(
+                        self.span_line(stmt.span()),
+                    ));
+                }
+                None
+            }
+            _ => None,
+        }
+    }
+
+    fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> {
+        // await task_fn(...)
+        if let Expr::Await(await_expr) = expr {
+            if let Expr::Call(call) = await_expr.arg.as_ref() {
+                if self.is_task_call(&Expr::Call(call.clone())) {
+                    return self.emit_step(call, expr);
+                }
+            }
+            // await Promise.all([task_fn(...), ...])
+            if Self::is_promise_all(&await_expr.arg) {
+                if let Expr::Call(promise_call) = await_expr.arg.as_ref() {
+                    return self.emit_parallel(promise_call, expr);
+                }
+            }
+        }
+
+        // Bare task_fn() without await
+        if self.is_task_call(expr) {
+            self.errors
+                .push(validation::error_missing_await(self.span_line(expr.span())));
+        }
+
+        None
+    }
+
+    fn emit_step(&mut self, call: &CallExpr, expr: &Expr) -> Option<(String, String)> {
+        if self.in_try {
+            self.errors
+                .push(validation::error_step_in_catch(self.span_line(expr.span())));
+            return None;
+        }
+        if self.in_while {
+            self.errors
+                .push(validation::error_step_in_while(self.span_line(expr.span())));
+            return None;
+        }
+        if self.in_nested_func {
+            self.errors.push(validation::error_step_in_nested_function(
+                self.span_line(expr.span()),
+            ));
+            return None;
+        }
+
+        let (name, script) = self
+            .extract_step_info_from_task_call(call)
+            .unwrap_or(("unknown".into(), "unknown".into()));
+        let id = self.next_id();
+        let node_id = self.add_node(DagNode {
+            id: id.clone(),
+            node_type: DagNodeType::Step { name: name.clone(), script },
+            label: name,
+            line: self.span_line(expr.span()),
+        });
+        Some((node_id.clone(), node_id))
+    }
+
+    fn emit_parallel(&mut self, promise_call: &CallExpr, expr: &Expr) -> Option<(String, String)> {
+        if self.in_try {
+            self.errors
+                .push(validation::error_step_in_catch(self.span_line(expr.span())));
+            return None;
+        }
+        if self.in_while {
+            self.errors
+                .push(validation::error_step_in_while(self.span_line(expr.span())));
+            return None;
+        }
+
+        let line = self.span_line(expr.span());
+        let start_id = self.next_id();
+        let start_node_id = self.add_node(DagNode {
+            id: start_id.clone(),
+            node_type: DagNodeType::ParallelStart,
+            label: "parallel".to_string(),
+            line,
+        });
+
+        let mut step_ids = Vec::new();
+
+        // Promise.all takes an array as first argument
+        if let Some(first_arg) = promise_call.args.first() {
+            if let Expr::Array(ArrayLit { elems, .. }) = first_arg.expr.as_ref() {
+                for elem in elems.iter().flatten() {
+                    if let Expr::Call(call) = elem.expr.as_ref() {
+                        if self.is_task_call(&Expr::Call(call.clone())) {
+                            let (name, script) = self
+                                .extract_step_info_from_task_call(call)
+                                .unwrap_or(("unknown".into(), "unknown".into()));
+                            let step_id = self.next_id();
+                            let node_id = self.add_node(DagNode {
+                                id: step_id.clone(),
+                                node_type: DagNodeType::Step { name: name.clone(), script },
+                                label: name,
+                                line: self.span_line(elem.expr.span()),
+                            });
+                            self.add_edge(&start_node_id, &node_id, None);
+                            step_ids.push(node_id);
+                        }
+                    }
+                }
+            }
+        }
+
+        let end_id = self.next_id();
+        let end_node_id = self.add_node(DagNode {
+            id: end_id.clone(),
+            node_type: DagNodeType::ParallelEnd,
+            label: "join".to_string(),
+            line,
+        });
+
+        for step_id in &step_ids {
+            self.add_edge(step_id, &end_node_id, None);
+        }
+
+        Some((start_node_id, end_node_id))
+    }
+
+    fn walk_if(&mut self, if_stmt: &IfStmt) -> Option<(String, String)> {
+        let has_steps_cons = self.stmt_contains_step(&if_stmt.cons);
+        let has_steps_alt = if_stmt
+            .alt
+            .as_ref()
+            .map_or(false, |a| self.stmt_contains_step(a));
+
+        if !has_steps_cons && !has_steps_alt {
+            return None;
+        }
+
+        let line = self.span_line(if_stmt.span);
+        let condition_source = self.expr_to_source(&if_stmt.test);
+
+        let branch_id = self.next_id();
+        let branch_node_id = self.add_node(DagNode {
+            id: branch_id.clone(),
+            node_type: DagNodeType::Branch { condition_source },
+            label: "if".to_string(),
+            line,
+        });
+
+        let mut last_ids = Vec::new();
+
+        // True branch
+        if let Some((true_first, true_last)) = self.walk_stmt(&if_stmt.cons) {
+            self.add_edge(&branch_node_id, &true_first, Some("true".to_string()));
+            last_ids.push(true_last);
+        } else {
+            last_ids.push(branch_node_id.clone());
+        }
+
+        // False branch
+        if let Some(alt) = &if_stmt.alt {
+            if let Some((else_first, else_last)) = self.walk_stmt(alt) {
+                self.add_edge(&branch_node_id, &else_first, Some("false".to_string()));
+                last_ids.push(else_last);
+            } else {
+                last_ids.push(branch_node_id.clone());
+            }
+        }
+
+        if last_ids.len() == 1 {
+            Some((branch_node_id, last_ids.into_iter().next().unwrap()))
+        } else {
+            let merge_id = format!("{branch_id}_merge");
+            Some((branch_node_id, merge_id))
+        }
+    }
+
+    fn walk_for_stmt(&mut self, for_stmt: &ForStmt) -> Option<(String, String)> {
+        if !self.stmt_contains_step(&for_stmt.body) {
+            return None;
+        }
+        self.walk_loop_body(&for_stmt.body, for_stmt.span, "for")
+    }
+
+    fn walk_for_in(&mut self, for_in: &ForInStmt) -> Option<(String, String)> {
+        if !self.stmt_contains_step(&for_in.body) {
+            return None;
+        }
+        let iter_source = self.expr_to_source(&for_in.right);
+        self.walk_loop_body_with_iter(&for_in.body, for_in.span, &iter_source)
+    }
+
+    fn walk_for_of(&mut self, for_of: &ForOfStmt) -> Option<(String, String)> {
+        if !self.stmt_contains_step(&for_of.body) {
+            return None;
+        }
+        let iter_source = self.expr_to_source(&for_of.right);
+        self.walk_loop_body_with_iter(&for_of.body, for_of.span, &iter_source)
+    }
+
+    fn walk_loop_body(
+        &mut self,
+        body: &Stmt,
+        span: swc_common::Span,
+        _label: &str,
+    ) -> Option<(String, String)> {
+        self.walk_loop_body_with_iter(body, span, "...")
+    }
+
+    fn walk_loop_body_with_iter(
+        &mut self,
+        body: &Stmt,
+        span: swc_common::Span,
+        iter_source: &str,
+    ) -> Option<(String, String)> {
+        let line = self.span_line(span);
+        let start_id = self.next_id();
+        let start_node_id = self.add_node(DagNode {
+            id: start_id.clone(),
+            node_type: DagNodeType::LoopStart { iter_source: iter_source.to_string() },
+            label: "for".to_string(),
+            line,
+        });
+
+        if let Some((body_first, body_last)) = self.walk_stmt(body) {
+            self.add_edge(&start_node_id, &body_first, None);
+            self.add_edge(&body_last, &start_node_id, Some("next".to_string()));
+        }
+
+        let end_id = self.next_id();
+        let end_node_id = self.add_node(DagNode {
+            id: end_id.clone(),
+            node_type: DagNodeType::LoopEnd,
+            label: "end for".to_string(),
+            line,
+        });
+        self.add_edge(&start_node_id, &end_node_id, Some("done".to_string()));
+
+        Some((start_node_id, end_node_id))
+    }
+
+    fn walk_while(&mut self, while_stmt: &WhileStmt) -> Option<(String, String)> {
+        if self.stmt_contains_step(&while_stmt.body) {
+            self.errors.push(validation::error_step_in_while(
+                self.span_line(while_stmt.span),
+            ));
+        }
+        None
+    }
+
+    fn walk_try(&mut self, try_stmt: &TryStmt) -> Option<(String, String)> {
+        let has_steps = self.body_contains_step(&try_stmt.block.stmts)
+            || try_stmt
+                .handler
+                .as_ref()
+                .map_or(false, |h| self.body_contains_step(&h.body.stmts))
+            || try_stmt
+                .finalizer
+                .as_ref()
+                .map_or(false, |f| self.body_contains_step(&f.stmts));
+
+        if has_steps {
+            self.errors.push(validation::error_step_in_catch(
+                self.span_line(try_stmt.span),
+            ));
+        }
+        None
+    }
+
+    fn walk_return(&mut self, ret: &ReturnStmt) -> Option<(String, String)> {
+        let line = self.span_line(ret.span);
+        let id = self.next_id();
+        let node_id = self.add_node(DagNode {
+            id: id.clone(),
+            node_type: DagNodeType::Return,
+            label: "return".to_string(),
+            line,
+        });
+        Some((node_id.clone(), node_id))
+    }
+}
+
+/// Extract workflow function params (no longer skips ctx)
+fn extract_ts_params(params: &[swc_ecma_ast::Param], cm: &Lrc<SourceMap>) -> Vec<Param> {
+    let mut result = Vec::new();
+    for param in params {
+        let (name, typ) = match &param.pat {
+            Pat::Ident(BindingIdent { id, type_ann, .. }) => {
+                let name = id.sym.to_string();
+                let typ = type_ann.as_ref().map(|ann| {
+                    cm.span_to_snippet(ann.type_ann.span())
+                        .unwrap_or_else(|_| "unknown".to_string())
+                });
+                (name, typ)
+            }
+            _ => continue,
+        };
+        result.push(Param { name, typ });
+    }
+    result
+}
+
+pub fn parse_ts_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError>> {
+    let cm: Lrc<SourceMap> = Default::default();
+    let fm = cm.new_source_file(FileName::Custom("workflow.ts".into()).into(), code.into());
+    let lexer = Lexer::new(
+        Syntax::Typescript(TsSyntax::default()),
+        Default::default(),
+        StringInput::from(&*fm),
+        None,
+    );
+
+    let mut parser = Parser::new_from(lexer);
+    let module = parser
+        .parse_module()
+        .map_err(|e| vec![CompileError { message: format!("Parse error: {e:?}"), line: 0 }])?;
+
+    // First pass: collect task functions
+    let task_functions = collect_task_functions(&module);
+
+    // Find: export default workflow(async (...) => { ... })
+    // or: export default workflow(async function(...) { ... })
+    let mut workflow_body: Option<(&[Stmt], Vec<Param>)> = None;
+
+    for item in &module.body {
+        // export default workflow(async (...) => { ... })
+        if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultExpr(export)) = item {
+            if let Some(result) = find_workflow_call(&export.expr, &cm) {
+                workflow_body = Some(result);
+                break;
+            }
+        }
+        // const wf = workflow(async (...) => { ... }); export default wf;
+        if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultDecl(export)) = item {
+            if let DefaultDecl::Fn(_) = &export.decl {
+                // `export default async function(...) { ... }` — not wrapped in workflow(), skip
+            }
+        }
+        if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item {
+            for decl in &var_decl.decls {
+                if let Some(init) = &decl.init {
+                    if let Some(result) = find_workflow_call(init, &cm) {
+                        workflow_body = Some(result);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    let (stmts, params) = workflow_body.ok_or_else(|| {
+        vec![CompileError {
+            message: "No workflow() wrapped async function found.".to_string(),
+            line: 0,
+        }]
+    })?;
+
+    let source_hash = compute_source_hash(code);
+
+    let mut walker = TsWacWalker::new(cm, task_functions);
+    walker.walk_body(stmts);
+
+    if !walker.errors.is_empty() {
+        return Err(walker.errors);
+    }
+
+    Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash })
+}
+
+/// Find workflow(async (...) => { ... }) or workflow(async function(...) { ... })
+fn find_workflow_call<'a>(expr: &'a Expr, cm: &Lrc<SourceMap>) -> Option<(&'a [Stmt], Vec<Param>)> {
+    if let Expr::Call(call) = expr {
+        // Check if callee is `workflow`
+        let is_workflow = match &call.callee {
+            Callee::Expr(callee_expr) => {
+                if let Expr::Ident(ident) = callee_expr.as_ref() {
+                    ident.sym.as_ref() == "workflow"
+                } else {
+                    false
+                }
+            }
+            _ => false,
+        };
+
+        if is_workflow {
+            if let Some(first_arg) = call.args.first() {
+                return extract_async_fn_body(&first_arg.expr, cm);
+            }
+        }
+    }
+    None
+}
+
+fn extract_async_fn_body<'a>(
+    expr: &'a Expr,
+    cm: &Lrc<SourceMap>,
+) -> Option<(&'a [Stmt], Vec<Param>)> {
+    match expr {
+        Expr::Arrow(arrow) if arrow.is_async => {
+            let params = extract_arrow_params(&arrow.params, cm);
+            match &*arrow.body {
+                BlockStmtOrExpr::BlockStmt(block) => Some((&block.stmts, params)),
+                _ => None,
+            }
+        }
+        Expr::Fn(fn_expr) if fn_expr.function.is_async => {
+            let params = extract_ts_params(&fn_expr.function.params, cm);
+            fn_expr
+                .function
+                .body
+                .as_ref()
+                .map(|body| (body.stmts.as_slice(), params))
+        }
+        Expr::Paren(p) => extract_async_fn_body(&p.expr, cm),
+        _ => None,
+    }
+}
+
+/// Extract arrow function params (no longer skips ctx)
+fn extract_arrow_params(pats: &[Pat], cm: &Lrc<SourceMap>) -> Vec<Param> {
+    let mut result = Vec::new();
+    for pat in pats {
+        match pat {
+            Pat::Ident(BindingIdent { id, type_ann, .. }) => {
+                let name = id.sym.to_string();
+                let typ = type_ann.as_ref().map(|ann| {
+                    cm.span_to_snippet(ann.type_ann.span())
+                        .unwrap_or_else(|_| "unknown".to_string())
+                });
+                result.push(Param { name, typ });
+            }
+            _ => {}
+        }
+    }
+    result
+}
+
+fn extract_string_lit(expr: &Expr) -> Option<String> {
+    match expr {
+        Expr::Lit(Lit::Str(s)) => Some(s.value.to_string()),
+        Expr::Tpl(tpl) if tpl.exprs.is_empty() && tpl.quasis.len() == 1 => {
+            tpl.quasis.first().map(|q| q.raw.to_string())
+        }
+        _ => None,
+    }
+}
+
+fn compute_source_hash(code: &str) -> String {
+    use sha2::{Digest, Sha256};
+    let mut hasher = Sha256::new();
+    hasher.update(code.as_bytes());
+    format!("{:x}", hasher.finalize())
+}

backend/parsers/windmill-parser-wac/src/validation.rs

@@ -0,0 +1,64 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct CompileError {
+    pub message: String,
+    pub line: usize,
+}
+
+impl std::fmt::Display for CompileError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "line {}: {}", self.line, self.message)
+    }
+}
+
+pub fn error_step_in_try(line: usize) -> CompileError {
+    CompileError {
+        message:
+            "Task calls inside try/except are not allowed. Steps have built-in error handling."
+                .to_string(),
+        line,
+    }
+}
+
+pub fn error_step_in_while(line: usize) -> CompileError {
+    CompileError {
+        message: "Task calls inside while loops are not allowed. Use for loops instead."
+            .to_string(),
+        line,
+    }
+}
+
+pub fn error_step_in_nested_function(line: usize) -> CompileError {
+    CompileError {
+        message: "Task calls inside nested functions, closures, or lambdas are not allowed."
+            .to_string(),
+        line,
+    }
+}
+
+pub fn error_step_in_comprehension(line: usize) -> CompileError {
+    CompileError { message: "Task calls inside comprehensions are not allowed.".to_string(), line }
+}
+
+pub fn error_not_async(line: usize) -> CompileError {
+    CompileError { message: "Workflow function must be async.".to_string(), line }
+}
+
+pub fn error_missing_await(line: usize) -> CompileError {
+    CompileError {
+        message:
+            "Task calls must be awaited directly or used inside asyncio.gather()/Promise.all()."
+                .to_string(),
+        line,
+    }
+}
+
+pub fn error_step_in_catch(line: usize) -> CompileError {
+    CompileError {
+        message:
+            "Task calls inside catch blocks are not allowed. Steps have built-in error handling."
+                .to_string(),
+        line,
+    }
+}

backend/parsers/windmill-parser-wac/tests/python_tests.rs

@@ -0,0 +1,266 @@
+use windmill_parser_wac::dag::DagNodeType;
+use windmill_parser_wac::python::parse_python_workflow;
+
+#[test]
+fn test_simple_sequential_workflow() {
+    let code = r#"
+import asyncio
+from wmill import workflow, task
+
+@task
+async def extract_data(url: str): ...
+@task
+async def load_data(data: list): ...
+
+@workflow
+async def my_etl(url: str):
+    raw = await extract_data(url=url)
+    await load_data(data=raw)
+    return {"status": "done"}
+"#;
+
+    let dag = parse_python_workflow(code).expect("should parse");
+    assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return
+    assert_eq!(dag.edges.len(), 2); // step0->step1, step1->return
+
+    // Check params (url — no ctx to skip)
+    assert_eq!(dag.params.len(), 1);
+    assert_eq!(dag.params[0].name, "url");
+    assert_eq!(dag.params[0].typ.as_deref(), Some("str"));
+
+    // Check first step
+    match &dag.nodes[0].node_type {
+        DagNodeType::Step { name, script } => {
+            assert_eq!(name, "extract_data");
+            assert_eq!(script, "extract_data");
+        }
+        _ => panic!("expected Step node"),
+    }
+
+    // Check second step
+    match &dag.nodes[1].node_type {
+        DagNodeType::Step { name, script } => {
+            assert_eq!(name, "load_data");
+            assert_eq!(script, "load_data");
+        }
+        _ => panic!("expected Step node"),
+    }
+
+    // Check return
+    assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return));
+
+    // Check source hash is non-empty
+    assert!(!dag.source_hash.is_empty());
+}
+
+#[test]
+fn test_parallel_workflow() {
+    let code = r#"
+import asyncio
+from wmill import workflow, task
+
+@task
+async def extract_data(url: str): ...
+@task
+async def clean_data(data: list): ...
+@task
+async def compute_stats(data: list): ...
+@task
+async def load_to_warehouse(rows: list): ...
+
+@workflow
+async def my_etl(url: str):
+    raw = await extract_data(url=url)
+    cleaned, stats = await asyncio.gather(
+        clean_data(data=raw),
+        compute_stats(data=raw),
+    )
+    await load_to_warehouse(rows=cleaned)
+    return {"status": "done"}
+"#;
+
+    let dag = parse_python_workflow(code).expect("should parse");
+
+    // extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7
+    assert_eq!(dag.nodes.len(), 7);
+
+    assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. }));
+    assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart));
+    assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. }));
+    assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. }));
+    assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd));
+    assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. }));
+    assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return));
+}
+
+#[test]
+fn test_conditional_workflow() {
+    let code = r#"
+import asyncio
+from wmill import workflow, task
+
+@task
+async def send_alert(msg: str): ...
+@task
+async def load_data(): ...
+
+@workflow
+async def my_etl(count: int):
+    if count > 100:
+        await send_alert(msg="large")
+    await load_data()
+    return {"done": True}
+"#;
+
+    let dag = parse_python_workflow(code).expect("should parse");
+    // Branch, notify step, load step, return = 4
+    assert_eq!(dag.nodes.len(), 4);
+    assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. }));
+    assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. }));
+}
+
+#[test]
+fn test_for_loop_workflow() {
+    let code = r#"
+import asyncio
+from wmill import workflow, task
+
+@task
+async def process_item(item: str): ...
+
+@workflow
+async def my_etl(items: list):
+    for item in items:
+        await process_item(item=item)
+    return {"done": True}
+"#;
+
+    let dag = parse_python_workflow(code).expect("should parse");
+    // LoopStart, step, LoopEnd, return = 4
+    assert_eq!(dag.nodes.len(), 4);
+    assert!(matches!(
+        dag.nodes[0].node_type,
+        DagNodeType::LoopStart { .. }
+    ));
+    assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. }));
+    assert!(matches!(dag.nodes[2].node_type, DagNodeType::LoopEnd));
+}
+
+#[test]
+fn test_reject_step_in_try() {
+    let code = r#"
+import asyncio
+from wmill import workflow, task
+
+@task
+async def extract_data(): ...
+
+@workflow
+async def my_etl():
+    try:
+        await extract_data()
+    except Exception:
+        pass
+"#;
+
+    let result = parse_python_workflow(code);
+    assert!(result.is_err());
+    let errors = result.unwrap_err();
+    assert!(errors[0].message.contains("try/except"));
+}
+
+#[test]
+fn test_reject_step_in_while() {
+    let code = r#"
+import asyncio
+from wmill import workflow, task
+
+@task
+async def extract_data(): ...
+
+@workflow
+async def my_etl():
+    while True:
+        await extract_data()
+"#;
+
+    let result = parse_python_workflow(code);
+    assert!(result.is_err());
+    let errors = result.unwrap_err();
+    assert!(errors[0].message.contains("while"));
+}
+
+#[test]
+fn test_reject_non_async() {
+    let code = r#"
+from wmill import workflow
+
+@workflow
+def my_etl():
+    pass
+"#;
+
+    let result = parse_python_workflow(code);
+    assert!(result.is_err());
+    let errors = result.unwrap_err();
+    assert!(errors[0].message.contains("async"));
+}
+
+#[test]
+fn test_reject_missing_await() {
+    let code = r#"
+import asyncio
+from wmill import workflow, task
+
+@task
+async def extract_data(): ...
+
+@workflow
+async def my_etl():
+    extract_data()
+"#;
+
+    let result = parse_python_workflow(code);
+    assert!(result.is_err());
+    let errors = result.unwrap_err();
+    assert!(errors[0].message.contains("awaited"));
+}
+
+#[test]
+fn test_no_workflow_function() {
+    let code = r#"
+async def my_func():
+    pass
+"#;
+
+    let result = parse_python_workflow(code);
+    assert!(result.is_err());
+    let errors = result.unwrap_err();
+    assert!(errors[0].message.contains("No @workflow"));
+}
+
+#[test]
+fn test_task_with_external_path() {
+    let code = r#"
+import asyncio
+from wmill import workflow, task
+
+@task(path="f/external_script")
+async def run_external(x: int): ...
+
+@workflow
+async def my_wf(x: int):
+    result = await run_external(x=x)
+    return result
+"#;
+
+    let dag = parse_python_workflow(code).expect("should parse");
+    assert_eq!(dag.nodes.len(), 2); // 1 step + 1 return (bare `return` is not a step node but walk_return creates one)
+    match &dag.nodes[0].node_type {
+        DagNodeType::Step { name, script } => {
+            assert_eq!(name, "run_external");
+            assert_eq!(script, "f/external_script");
+        }
+        _ => panic!("expected Step node"),
+    }
+}

backend/parsers/windmill-parser-wac/tests/ts_tests.rs

@@ -0,0 +1,245 @@
+use windmill_parser_wac::dag::DagNodeType;
+use windmill_parser_wac::typescript::parse_ts_workflow;
+
+#[test]
+fn test_simple_sequential_ts_workflow() {
+    let code = r#"
+import { workflow, task } from "windmill-client";
+
+const extract_data = task(async (url: string) => {});
+const load_data = task(async (data: any) => {});
+
+export default workflow(async (url: string) => {
+  const raw = await extract_data(url);
+  await load_data(raw);
+  return { status: "done" };
+});
+"#;
+
+    let dag = parse_ts_workflow(code).expect("should parse");
+    assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return
+    assert_eq!(dag.edges.len(), 2);
+
+    // Check params (url — no ctx to skip)
+    assert_eq!(dag.params.len(), 1);
+    assert_eq!(dag.params[0].name, "url");
+    assert_eq!(dag.params[0].typ.as_deref(), Some("string"));
+
+    match &dag.nodes[0].node_type {
+        DagNodeType::Step { name, script } => {
+            assert_eq!(name, "extract_data");
+            assert_eq!(script, "extract_data");
+        }
+        _ => panic!("expected Step node"),
+    }
+
+    match &dag.nodes[1].node_type {
+        DagNodeType::Step { name, script } => {
+            assert_eq!(name, "load_data");
+            assert_eq!(script, "load_data");
+        }
+        _ => panic!("expected Step node"),
+    }
+
+    assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return));
+    assert!(!dag.source_hash.is_empty());
+}
+
+#[test]
+fn test_parallel_ts_workflow() {
+    let code = r#"
+import { workflow, task } from "windmill-client";
+
+const extract_data = task(async (url: string) => {});
+const clean_data = task(async (data: any) => {});
+const compute_stats = task(async (data: any) => {});
+const load_to_warehouse = task(async (rows: any) => {});
+
+export default workflow(async (url: string) => {
+  const raw = await extract_data(url);
+
+  const [cleaned, stats] = await Promise.all([
+    clean_data(raw),
+    compute_stats(raw),
+  ]);
+
+  await load_to_warehouse(cleaned);
+  return { status: "done", rows: stats.rowCount };
+});
+"#;
+
+    let dag = parse_ts_workflow(code).expect("should parse");
+    // extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7
+    assert_eq!(dag.nodes.len(), 7);
+
+    assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. }));
+    assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart));
+    assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. }));
+    assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. }));
+    assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd));
+    assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. }));
+    assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return));
+}
+
+#[test]
+fn test_conditional_ts_workflow() {
+    let code = r#"
+import { workflow, task } from "windmill-client";
+
+const send_alert = task(async (msg: string) => {});
+const load_data = task(async () => {});
+
+export default workflow(async (count: number) => {
+  if (count > 100) {
+    await send_alert("large");
+  }
+  await load_data();
+  return { done: true };
+});
+"#;
+
+    let dag = parse_ts_workflow(code).expect("should parse");
+    // Branch, notify, load, return = 4
+    assert_eq!(dag.nodes.len(), 4);
+    assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. }));
+}
+
+#[test]
+fn test_for_of_ts_workflow() {
+    let code = r#"
+import { workflow, task } from "windmill-client";
+
+const process_item = task(async (item: string) => {});
+
+export default workflow(async (items: string[]) => {
+  for (const item of items) {
+    await process_item(item);
+  }
+  return { done: true };
+});
+"#;
+
+    let dag = parse_ts_workflow(code).expect("should parse");
+    // LoopStart, step, LoopEnd, return = 4
+    assert_eq!(dag.nodes.len(), 4);
+    assert!(matches!(
+        dag.nodes[0].node_type,
+        DagNodeType::LoopStart { .. }
+    ));
+}
+
+#[test]
+fn test_reject_step_in_try_catch() {
+    let code = r#"
+import { workflow, task } from "windmill-client";
+
+const extract_data = task(async () => {});
+
+export default workflow(async () => {
+  try {
+    await extract_data();
+  } catch (e) {
+    console.log(e);
+  }
+});
+"#;
+
+    let result = parse_ts_workflow(code);
+    assert!(result.is_err());
+    let errors = result.unwrap_err();
+    assert!(errors[0].message.contains("catch"));
+}
+
+#[test]
+fn test_reject_step_in_while_ts() {
+    let code = r#"
+import { workflow, task } from "windmill-client";
+
+const extract_data = task(async () => {});
+
+export default workflow(async () => {
+  while (true) {
+    await extract_data();
+  }
+});
+"#;
+
+    let result = parse_ts_workflow(code);
+    assert!(result.is_err());
+    let errors = result.unwrap_err();
+    assert!(errors[0].message.contains("while"));
+}
+
+#[test]
+fn test_reject_missing_await_ts() {
+    let code = r#"
+import { workflow, task } from "windmill-client";
+
+const extract_data = task(async () => {});
+
+export default workflow(async () => {
+  extract_data();
+});
+"#;
+
+    let result = parse_ts_workflow(code);
+    assert!(result.is_err());
+    let errors = result.unwrap_err();
+    assert!(errors[0].message.contains("awaited"));
+}
+
+#[test]
+fn test_no_workflow_wrapper() {
+    let code = r#"
+export default async function main(ctx: any) {
+  return {};
+}
+"#;
+
+    let result = parse_ts_workflow(code);
+    assert!(result.is_err());
+    let errors = result.unwrap_err();
+    assert!(errors[0].message.contains("No workflow()"));
+}
+
+#[test]
+fn test_variable_declaration_with_step() {
+    let code = r#"
+import { workflow, task } from "windmill-client";
+
+const compute = task(async () => {});
+
+export default workflow(async () => {
+  const result = await compute();
+  return result;
+});
+"#;
+
+    let dag = parse_ts_workflow(code).expect("should parse");
+    assert_eq!(dag.nodes.len(), 2); // step + return
+    assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. }));
+}
+
+#[test]
+fn test_task_with_external_path() {
+    let code = r#"
+import { workflow, task } from "windmill-client";
+
+const run_external = task("f/external_script", async (x: number) => {});
+
+export default workflow(async (x: number) => {
+  const result = await run_external(x);
+  return result;
+});
+"#;
+
+    let dag = parse_ts_workflow(code).expect("should parse");
+    assert_eq!(dag.nodes.len(), 2); // step + return
+    match &dag.nodes[0].node_type {
+        DagNodeType::Step { name, script } => {
+            assert_eq!(name, "run_external");
+            assert_eq!(script, "f/external_script");
+        }
+        _ => panic!("expected Step node"),
+    }
+}

backend/parsers/windmill-parser-wasm/Cargo.toml

@@ -38,6 +38,7 @@ csharp-parser = [ "dep:windmill-parser-csharp"]
 nu-parser = [ "dep:windmill-parser-nu"]
 java-parser = [ "dep:windmill-parser-java"]
 ruby-parser = [ "dep:windmill-parser-ruby"]
+wac-parser = [ "dep:windmill-parser-wac"]
 
 [dependencies]
 anyhow.workspace = true
@@ -55,6 +56,7 @@ windmill-parser-csharp = { workspace = true, optional = true }
 windmill-parser-nu = { workspace = true, optional = true }
 windmill-parser-java = { workspace = true, optional = true }
 windmill-parser-ruby = { workspace = true, optional = true }
+windmill-parser-wac = { workspace = true, optional = true }
 wasm-bindgen.workspace = true
 
 serde_json.workspace = true

backend/parsers/windmill-parser-wasm/build.nu

@@ -56,6 +56,12 @@ const targets = [
     features: "ruby-parser",
     env: "tree-sitter",
   },
+  {
+    ident: "wac",
+    desc: "Workflow-as-Code",
+    features: "wac-parser",
+    env: "default",
+  },
   # ^^^ Add new entry here ^^^
 ];
 # NOTE: This is legacy command for building all, but it is not more used

backend/parsers/windmill-parser-wasm/src/lib.rs

@@ -223,4 +223,11 @@ pub fn parse_assets_ansible(code: &str) -> String {
     }
 }
 
+#[cfg(feature = "wac-parser")]
+#[wasm_bindgen]
+pub fn parse_workflow_as_code(code: &str, language: &str) -> String {
+    let result = windmill_parser_wac::parse_workflow(code, language);
+    serde_json::to_string(&result).unwrap_or_else(|_| "{\"type\": \"error\"}".to_string())
+}
+
 // for related places search: ADD_NEW_LANG

backend/src/main.rs

@@ -61,9 +61,8 @@ use windmill_common::{
         MODE_AND_ADDONS,
     },
     worker::{
-        is_native_mode_from_env, reload_custom_tags_setting, Connection, HttpClient, HUB_CACHE_DIR,
-        HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, USES_BATCH_HTTP_PULL, WINDMILL_DIR,
-        WORKER_GROUP,
+        is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR,
+        HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP,
     },
     KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED,
 };
@@ -921,20 +920,6 @@ Windmill Community Edition {GIT_VERSION}
             default_base_internal_url.clone()
         };
 
-        // BATCH_PULL_URL: explicit URL for native workers to pull jobs via HTTP.
-        // In standalone mode (server_mode=true), defaults to the local server.
-        let batch_pull_url: Option<String> = if is_native_mode_from_env() {
-            if let Ok(url) = std::env::var("BATCH_PULL_URL") {
-                Some(url)
-            } else if server_mode {
-                Some(default_base_internal_url.clone())
-            } else {
-                None
-            }
-        } else {
-            None
-        };
-
         initial_load(
             &conn,
             killpill_tx.clone(),
@@ -1145,30 +1130,6 @@ Windmill Community Edition {GIT_VERSION}
                     )?;
                     let mut workers = vec![];
 
-                    // For native workers, create a self-signed JWT for batch pulling via HTTP.
-                    // Enabled when BATCH_PULL_URL is set (explicitly or auto-detected in standalone mode).
-                    let batch_pull_client = if let Some(ref pull_url) = batch_pull_url {
-                        match create_native_batch_pull_client(pull_url).await {
-                            Ok(client) => {
-                                tracing::info!(
-                                    "Native batch pull client created for HTTP pull at {}",
-                                    pull_url
-                                );
-                                USES_BATCH_HTTP_PULL
-                                    .store(true, std::sync::atomic::Ordering::Relaxed);
-                                Some(client)
-                            }
-                            Err(e) => {
-                                tracing::warn!(
-                                    "Failed to create native batch pull client, falling back to SQL pull: {e:#}"
-                                );
-                                None
-                            }
-                        }
-                    } else {
-                        None
-                    };
-
                     for i in 0..num_workers {
                         let suffix = if i == 0 && first_suffix.is_some() {
                             first_suffix.as_ref().unwrap().clone()
@@ -1192,7 +1153,6 @@ Windmill Community Edition {GIT_VERSION}
                                 WORKER_GROUP.as_str(),
                                 &suffix,
                             ),
-                            batch_pull_client: batch_pull_client.clone(),
                         };
                         workers.push(worker_conn);
                     }
@@ -1806,7 +1766,6 @@ fn display_config(envs: &[&str]) {
 pub struct WorkerConn {
     conn: Connection,
     worker_name: String,
-    batch_pull_client: Option<HttpClient>,
 }
 
 pub async fn run_workers(
@@ -1877,7 +1836,6 @@ pub async fn run_workers(
         let wk_conf = &workers[i as usize - 1];
         let conn1 = wk_conf.conn.clone();
         let worker_name = wk_conf.worker_name.clone();
-        let batch_pull_client = wk_conf.batch_pull_client.clone();
         WORKERS_NAMES.write().await.push(worker_name.clone());
         let ip = ip.clone();
         let rx = killpill_rxs.pop().unwrap();
@@ -1900,7 +1858,6 @@ pub async fn run_workers(
                 rx,
                 tx,
                 &base_internal_url,
-                batch_pull_client.as_ref(),
             );
 
             // #[cfg(tokio_unstable)]
@@ -1919,41 +1876,6 @@ pub async fn run_workers(
     Ok(())
 }
 
-/// Create an HTTP client for native workers to pull jobs from the local server's batch buffer.
-/// Self-signs a JWT with native_mode=true using the same JWT secret the server uses.
-async fn create_native_batch_pull_client(base_internal_url: &str) -> anyhow::Result<HttpClient> {
-    use windmill_common::agent_workers::{build_agent_http_client, AGENT_JWT_PREFIX};
-    use windmill_common::jwt::encode_with_internal_secret;
-
-    #[derive(serde::Serialize)]
-    struct NativeAgentAuth {
-        worker_group: String,
-        tags: Vec<String>,
-        native_mode: Option<bool>,
-        exp: usize,
-    }
-
-    let worker_config = windmill_common::worker::WORKER_CONFIG.read().await;
-    let tags = worker_config.worker_tags.clone();
-    drop(worker_config);
-
-    // Token expires in 30 days — renewed on restart
-    let exp = (chrono::Utc::now() + chrono::Duration::days(30)).timestamp() as usize;
-
-    let claims = NativeAgentAuth {
-        worker_group: WORKER_GROUP.to_string(),
-        tags,
-        native_mode: Some(true),
-        exp,
-    };
-
-    let jwt = encode_with_internal_secret(claims).await?;
-    let token = format!("{}{}", AGENT_JWT_PREFIX, jwt);
-
-    let suffix = create_default_worker_suffix(&HOSTNAME);
-    Ok(build_agent_http_client(&suffix, &token, base_internal_url))
-}
-
 async fn send_delayed_killpill(tx: &KillpillSender, mut max_delay_secs: u64, context: &str) {
     if max_delay_secs == 0 {
         max_delay_secs = 1;

backend/summarized_schema.txt

@@ -174,7 +174,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work
 windmill_migrations: name(text), created_at(ts)
 worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint)
   FK: (workspace_id) -> workspace(id)
-worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), native_mode(bool), uses_batch_http_pull(bool)
+worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[])
 workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char)
   FK: (parent_workspace_id) -> workspace(id)
 workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts)

backend/test_wac_e2e.sh

@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+# E2E test for WAC v2 workflow-as-code suspend/resume lifecycle
+set -euo pipefail
+
+BASE_URL="${BASE_URL:-http://localhost:8070}"
+TOKEN="${WM_TOKEN:-}"
+WORKSPACE="dev"
+TIMEOUT=60  # seconds
+
+# Get auth token if not set
+if [ -z "$TOKEN" ]; then
+    TOKEN=$(curl -s "${BASE_URL}/api/auth/login" \
+        -H "Content-Type: application/json" \
+        -d '{"email":"admin@windmill.dev","password":"changeme"}' | tr -d '"')
+fi
+
+echo "=== WAC v2 E2E Test ==="
+echo "Base URL: $BASE_URL"
+echo ""
+
+WAC_CODE='import { task, workflow } from "windmill-client@1.999.19";
+
+const double = task(async (x: number): Promise<number> => {
+  console.log("[double] START at " + new Date().toISOString());
+  await new Promise(r => setTimeout(r, 2000));
+  console.log("[double] END at " + new Date().toISOString());
+  return x * 2;
+});
+
+const increment = task(async (x: number): Promise<number> => {
+  console.log("[increment] START at " + new Date().toISOString());
+  await new Promise(r => setTimeout(r, 2000));
+  console.log("[increment] END at " + new Date().toISOString());
+  return x + 1;
+});
+
+export const main = workflow(async (x: number = 10) => {
+  const [doubled, incremented] = await Promise.all([
+    double(x),
+    increment(x),
+  ]);
+  const final_result = await double(incremented);
+  return { doubled, incremented, final_result };
+});'
+
+echo "Step 1: Submitting preview job..."
+JOB_ID=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs/run/preview" \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer $TOKEN" \
+    -d "$(jq -n --arg code "$WAC_CODE" '{
+        content: $code,
+        language: "bun",
+        args: {"x": 10}
+    }')" | tr -d '"')
+
+echo "Job ID: $JOB_ID"
+
+if [ -z "$JOB_ID" ] || [ "$JOB_ID" = "null" ]; then
+    echo "FAIL: Could not create job"
+    exit 1
+fi
+
+echo ""
+echo "Step 2: Polling for completion (timeout: ${TIMEOUT}s)..."
+
+START=$SECONDS
+LAST_STATUS=""
+while true; do
+    ELAPSED=$((SECONDS - START))
+    if [ $ELAPSED -gt $TIMEOUT ]; then
+        echo "FAIL: Timed out after ${TIMEOUT}s"
+        # Dump job state for debugging
+        echo ""
+        echo "=== Debug info ==="
+        echo "Parent job queue state:"
+        source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local
+        psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until, canceled_by FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null
+        echo "Child jobs:"
+        psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at" 2>/dev/null
+        echo "Completed children:"
+        psql "$DATABASE_URL" -c "SELECT id FROM completed_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null
+        echo "Checkpoint:"
+        psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null
+        echo "Total child count:"
+        psql "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null
+        exit 1
+    fi
+
+    # Check completed job
+    RESULT=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \
+        -H "Authorization: Bearer $TOKEN" 2>/dev/null)
+    HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \
+        -H "Authorization: Bearer $TOKEN" 2>/dev/null)
+
+    if [ "$HTTP_CODE" = "200" ]; then
+        echo "Job completed in ${ELAPSED}s!"
+        echo ""
+        echo "Step 3: Checking result..."
+        echo "Result: $RESULT"
+
+        # Validate
+        DOUBLED=$(echo "$RESULT" | jq -r '.doubled // empty')
+        INCREMENTED=$(echo "$RESULT" | jq -r '.incremented // empty')
+        FINAL=$(echo "$RESULT" | jq -r '.final_result // empty')
+
+        PASS=true
+        if [ "$DOUBLED" != "20" ]; then
+            echo "FAIL: doubled = $DOUBLED, expected 20"
+            PASS=false
+        fi
+        if [ "$INCREMENTED" != "11" ]; then
+            echo "FAIL: incremented = $INCREMENTED, expected 11"
+            PASS=false
+        fi
+        if [ "$FINAL" != "22" ]; then
+            echo "FAIL: final_result = $FINAL, expected 22"
+            PASS=false
+        fi
+
+        if $PASS; then
+            echo "PASS: All values correct!"
+            # Check no excessive child jobs
+            source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null
+            CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null)
+            echo "Total child jobs created: $CHILD_COUNT (expected: 3)"
+            if [ "$CHILD_COUNT" -gt "3" ]; then
+                echo "WARN: More children than expected ($CHILD_COUNT > 3)"
+            fi
+            exit 0
+        else
+            exit 1
+        fi
+    fi
+
+    # Show progress
+    STATUS=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/get/${JOB_ID}" \
+        -H "Authorization: Bearer $TOKEN" 2>/dev/null | jq -r '.type // empty')
+    if [ "$STATUS" != "$LAST_STATUS" ]; then
+        echo "  [${ELAPSED}s] Status: $STATUS"
+        LAST_STATUS="$STATUS"
+    fi
+
+    # Check for runaway child creation
+    source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null
+    CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null)
+    if [ "$CHILD_COUNT" -gt "10" ]; then
+        echo "FAIL: Runaway child creation detected! $CHILD_COUNT children (expected 3)"
+        echo ""
+        echo "=== Debug info ==="
+        psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null
+        psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at LIMIT 20" 2>/dev/null
+        psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null
+        exit 1
+    fi
+
+    sleep 1
+done

backend/tests/nativets_stress.rs

@@ -241,7 +241,6 @@ fn spawn_workers(
                 rx,
                 tx2,
                 &base_internal_url,
-                None,
             )
             .await;
         };

backend/tests/worker.rs

@@ -3548,3 +3548,170 @@ async fn test_flow_substep_tag_availability_check(db: Pool<Postgres>) -> anyhow:
 
     Ok(())
 }
+
+#[cfg(all(feature = "quickjs", feature = "python"))]
+#[sqlx::test(fixtures("base"))]
+async fn test_stop_after_all_iters_if_bad_expr_parallel_branchall(
+    db: Pool<Postgres>,
+) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let port = 123;
+    let flow: FlowValue = serde_json::from_value(serde_json::json!({
+        "modules": [
+            {
+                "id": "a",
+                "value": {
+                    "branches": [
+                        {"modules": [{
+                            "id": "b",
+                            "value": {
+                                "input_transforms": { "n": { "type": "javascript", "expr": "flow_input.n" } },
+                                "type": "rawscript",
+                                "language": "python3",
+                                "content": "def main(n): return n",
+                            },
+                        }]}
+                    ],
+                    "type": "branchall",
+                    "parallel": true,
+                },
+                "stop_after_all_iters_if": {
+                    "expr": "invalid!!!syntax",
+                    "skip_if_stopped": false,
+                },
+            },
+        ],
+    }))
+    .unwrap();
+    let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
+
+    let cjob = RunJob::from(job)
+        .arg("n", json!(42))
+        .run_until_complete(&db, false, port)
+        .await;
+
+    assert!(
+        !cjob.success,
+        "flow should fail when stop_after_all_iters_if has bad expression"
+    );
+
+    let result = cjob.json_result().unwrap();
+    let error_msg = result["error"]["message"].as_str().unwrap_or("");
+    assert!(
+        error_msg.contains("stop_after_all_iters_if"),
+        "error should mention stop_after_all_iters_if, got: {error_msg}"
+    );
+
+    Ok(())
+}
+
+#[cfg(all(feature = "quickjs", feature = "python"))]
+#[sqlx::test(fixtures("base"))]
+async fn test_stop_after_all_iters_if_bad_expr_parallel_forloop(
+    db: Pool<Postgres>,
+) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let port = 123;
+    let flow: FlowValue = serde_json::from_value(serde_json::json!({
+        "modules": [
+            {
+                "id": "a",
+                "value": {
+                    "type": "forloopflow",
+                    "iterator": { "type": "javascript", "expr": "result.items" },
+                    "skip_failures": false,
+                    "parallel": true,
+                    "modules": [{
+                        "value": {
+                            "input_transforms": {
+                                "n": { "type": "javascript", "expr": "flow_input.iter.value" },
+                            },
+                            "type": "rawscript",
+                            "language": "python3",
+                            "content": "def main(n): return n",
+                        },
+                    }],
+                },
+                "stop_after_all_iters_if": {
+                    "expr": "invalid!!!syntax",
+                    "skip_if_stopped": false,
+                },
+            },
+        ],
+    }))
+    .unwrap();
+    let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
+
+    let cjob = RunJob::from(job)
+        .arg("items", json!([1, 2, 3]))
+        .run_until_complete(&db, false, port)
+        .await;
+
+    assert!(
+        !cjob.success,
+        "flow should fail when stop_after_all_iters_if has bad expression"
+    );
+
+    let result = cjob.json_result().unwrap();
+    let error_msg = result["error"]["message"].as_str().unwrap_or("");
+    assert!(
+        error_msg.contains("stop_after_all_iters_if"),
+        "error should mention stop_after_all_iters_if, got: {error_msg}"
+    );
+
+    Ok(())
+}
+
+#[cfg(all(feature = "quickjs", feature = "python"))]
+#[sqlx::test(fixtures("base"))]
+async fn test_results_length_in_input_transform(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    // Step a returns a list, step b accesses results.a.length via input transform.
+    // This tests that the handle_full_regex fast path falls through to QuickJS
+    // when the SQL JSON path operator can't resolve JS properties like .length.
+    let flow: FlowValue = serde_json::from_value(json!({
+        "modules": [
+            {
+                "id": "a",
+                "value": {
+                    "type": "rawscript",
+                    "language": "python3",
+                    "content": "def main(): return [10, 20, 30]",
+                },
+            },
+            {
+                "id": "b",
+                "value": {
+                    "input_transforms": {
+                        "v": { "type": "javascript", "expr": "results.a.length" },
+                    },
+                    "type": "rawscript",
+                    "language": "python3",
+                    "content": "def main(v): return v",
+                },
+            },
+        ],
+    }))
+    .unwrap();
+
+    let result =
+        RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None })
+            .run_until_complete(&db, false, port)
+            .await
+            .json_result()
+            .unwrap();
+
+    assert_eq!(
+        result,
+        json!(3),
+        "results.a.length should resolve to 3, not null"
+    );
+
+    Ok(())
+}

backend/windmill-api-agent-workers/src/lib.rs

@@ -19,10 +19,7 @@ use windmill_common::DB;
 use axum::Router;
 
 #[cfg(not(feature = "private"))]
-pub fn global_service(
-    _job_completed_tx: windmill_worker::JobCompletedSender,
-    _batch_buffer: Option<()>,
-) -> Router {
+pub fn global_service(_job_completed_tx: windmill_worker::JobCompletedSender) -> Router {
     Router::new()
 }
 
@@ -34,7 +31,6 @@ pub fn workspaced_service(
     Router,
     Vec<tokio::task::JoinHandle<()>>,
     Option<windmill_worker::JobCompletedSender>,
-    Option<()>,
 ) {
     use windmill_common::worker::Connection;
     use windmill_worker::JobCompletedSender;
@@ -44,7 +40,7 @@ pub fn workspaced_service(
 
     let router = Router::new();
 
-    (router, vec![], Some(job_completed_tx), None)
+    (router, vec![], Some(job_completed_tx))
 }
 
 #[cfg(not(feature = "private"))]

backend/windmill-api-settings/src/lib.rs

@@ -316,7 +316,11 @@ pub async fn set_global_setting_internal(
              )
              .execute(db)
              .await?;
-            tracing::info!("Set global setting {} to {}", key, v);
+            tracing::info!(
+                "Set global setting {} to {}",
+                key,
+                instance_config::format_setting_value(&key, &v)
+            );
         }
     };
 

backend/windmill-api/Cargo.toml

@@ -174,6 +174,7 @@ aws-sdk-bedrock = { workspace = true, optional = true }
 aws-sdk-bedrockruntime = { workspace = true, optional = true }
 aws-smithy-types = { workspace = true, optional = true }
 async-trait.workspace = true
+eventsource-stream.workspace = true
 windmill-jseval.workspace = true
 tar.workspace = true
 flate2.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.651.1
+  version: 1.652.0
   title: Windmill API
 
   contact:
@@ -18803,7 +18803,6 @@ components:
       required:
         - path
         - summary
-        - description
         - content
         - language
 

backend/windmill-api/src/ai.rs

@@ -29,7 +29,7 @@ const AI_TIMEOUT_MAX_SECS: u64 = 86400; // 24 hours
 const AI_TIMEOUT_DEFAULT_SECS: u64 = 3600; // 1 hour
 const HTTP_POOL_MAX_IDLE_PER_HOST: usize = 10;
 const HTTP_POOL_IDLE_TIMEOUT_SECS: u64 = 90;
-const KEEPALIVE_INTERVAL_SECS: u64 = 15;
+pub(crate) const KEEPALIVE_INTERVAL_SECS: u64 = 15;
 
 lazy_static::lazy_static! {
     /// AI request timeout in seconds.
@@ -87,7 +87,7 @@ lazy_static::lazy_static! {
         }
     };
 
-    static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new()
+    pub(crate) static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new()
         .timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS))
         .pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST)
         .pool_idle_timeout(Some(std::time::Duration::from_secs(HTTP_POOL_IDLE_TIMEOUT_SECS)))
@@ -378,12 +378,7 @@ impl AIRequestConfig {
         let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some();
         let is_google_ai = matches!(provider, AIProvider::GoogleAI);
 
-        // GoogleAI uses OpenAI-compatible endpoint in the proxy (for the chat), but not for the ai agent
-        let base_url = if is_google_ai {
-            format!("{}/openai", base_url)
-        } else {
-            base_url.to_string()
-        };
+        let base_url = base_url.to_string();
         let base_url = base_url.as_str();
 
         // Build URL based on provider
@@ -428,6 +423,9 @@ impl AIRequestConfig {
         if let Some(api_key) = self.api_key {
             if is_azure {
                 request = request.header("api-key", api_key.clone())
+            } else if is_google_ai {
+                // Native Gemini API uses x-goog-api-key, not Authorization: Bearer
+                request = request.header("x-goog-api-key", api_key.clone())
             } else {
                 request = request.header("authorization", format!("Bearer {}", api_key.clone()))
             }
@@ -611,7 +609,7 @@ fn is_sse_response(headers: &HeaderMap) -> bool {
         .unwrap_or(false)
 }
 
-fn inject_keepalives<S>(
+pub(crate) fn inject_keepalives<S>(
     upstream: S,
     interval: Duration,
 ) -> impl futures::Stream<Item = std::result::Result<Bytes, reqwest::Error>>
@@ -830,6 +828,36 @@ async fn proxy(
         ai_path = chat_path;
     }
 
+    // Handle GoogleAI (Gemini) using the native Gemini API
+    if matches!(provider, AIProvider::GoogleAI) {
+        let api_key = request_config.api_key.as_deref().unwrap_or("");
+        let base_url = request_config.base_url.trim_end_matches('/');
+
+        let mut tx = db.begin().await?;
+        audit_log(
+            &mut *tx,
+            &authed,
+            "ai.request",
+            ActionKind::Execute,
+            &w_id,
+            Some(&authed.email),
+            Some([("ai_config_path", &format!("{:?}", ai_path)[..])].into()),
+        )
+        .await?;
+        tx.commit().await?;
+
+        return match ai_path.as_str() {
+            "chat/completions" => {
+                crate::google::handle_google_ai_chat(&body, api_key, base_url).await
+            }
+            "models" => crate::google::handle_google_ai_models(api_key, base_url).await,
+            _ => Err(Error::BadRequest(format!(
+                "Unsupported Google AI path: {}",
+                ai_path
+            ))),
+        };
+    }
+
     // Handle Bedrock-specific logic when the feature is enabled
     #[cfg(feature = "bedrock")]
     {

backend/windmill-api/src/db.rs

@@ -284,6 +284,9 @@ pub async fn migrate(
         20260207000004,
     ];
     for m in migrator.migrations.iter() {
+        if m.migration_type.is_down_migration() {
+            continue;
+        }
         if potentially_stale.contains(&m.version) {
             if let Err(err) =
                 sqlx::query("DELETE FROM _sqlx_migrations WHERE version = $1 AND checksum != $2")

backend/windmill-api/src/google.rs

@@ -0,0 +1,306 @@
+//! Google AI (Gemini API) handler for the AI chat proxy.
+//!
+//! Handles POST `chat/completions` requests using the native Gemini API,
+//! converting from/to OpenAI format so the existing frontend parsers continue to work.
+//!
+//! Used by `windmill-api/src/ai.rs` when the provider is `GoogleAI`.
+//! Shared conversion logic lives in `windmill_common::ai_google`.
+
+use axum::body::Body;
+use bytes::Bytes;
+use eventsource_stream::Eventsource;
+use futures::StreamExt;
+use serde::Deserialize;
+use serde_json::json;
+use windmill_common::{
+    ai_google::{
+        gemini_event_to_openai_sse_chunks, gemini_response_to_openai, openai_messages_to_gemini,
+        parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google,
+        GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiTextRequest, GeminiTool,
+    },
+    ai_types::OpenAIMessage,
+    error::{Error, Result},
+};
+
+use crate::ai::{inject_keepalives, HTTP_CLIENT, KEEPALIVE_INTERVAL_SECS};
+
+// ============================================================================
+// Request type (OpenAI format received from the frontend)
+// ============================================================================
+
+#[derive(Deserialize, Debug)]
+struct ChatRequest {
+    model: String,
+    messages: Vec<OpenAIMessage>,
+    #[serde(default)]
+    stream: bool,
+    #[serde(default)]
+    temperature: Option<f32>,
+    #[serde(default)]
+    max_tokens: Option<u32>,
+    #[serde(default)]
+    tools: Option<Vec<ChatRequestTool>>,
+}
+
+#[derive(Deserialize, Debug)]
+struct ChatRequestTool {
+    function: ChatRequestToolFunction,
+}
+
+#[derive(Deserialize, Debug)]
+struct ChatRequestToolFunction {
+    name: String,
+    #[serde(default)]
+    description: Option<String>,
+    #[serde(default)]
+    parameters: Option<serde_json::Value>,
+}
+
+// ============================================================================
+// Public handler
+// ============================================================================
+
+/// Handle a `chat/completions` POST request using the native Gemini API.
+///
+/// Converts the incoming OpenAI-format body to a `GeminiTextRequest`, sends it
+/// to the appropriate Gemini endpoint, and converts the response back to the
+/// OpenAI SSE or JSON format that the frontend expects.
+pub async fn handle_google_ai_chat(
+    body: &Bytes,
+    api_key: &str,
+    base_url: &str,
+) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
+    let request: ChatRequest = serde_json::from_slice(body)
+        .map_err(|e| Error::BadRequest(format!("Failed to parse request body: {}", e)))?;
+
+    let (contents, system_instruction) = openai_messages_to_gemini(&request.messages);
+
+    let generation_config =
+        if request.temperature.is_some() || request.max_tokens.is_some() {
+            Some(GeminiGenerationConfig {
+                temperature: request.temperature,
+                max_output_tokens: request.max_tokens,
+                response_mime_type: None,
+                response_schema: None,
+            })
+        } else {
+            None
+        };
+
+    let gemini_tools = request.tools.as_ref().map(|tools| {
+        let declarations: Vec<GeminiFunctionDeclaration> = tools
+            .iter()
+            .map(|t| {
+                let mut params = t.function.parameters.clone().unwrap_or(json!({}));
+                sanitize_schema_for_google(&mut params);
+                GeminiFunctionDeclaration {
+                    name: t.function.name.clone(),
+                    description: t.function.description.clone(),
+                    parameters: params,
+                }
+            })
+            .collect();
+        vec![GeminiTool {
+            function_declarations: Some(declarations),
+            google_search: None,
+        }]
+    });
+
+    let gemini_request = GeminiTextRequest {
+        contents,
+        tools: gemini_tools,
+        tool_config: None,
+        system_instruction,
+        generation_config,
+    };
+
+    let request_body = serde_json::to_string(&gemini_request)
+        .map_err(|e| Error::internal_err(format!("Failed to serialize Gemini request: {}", e)))?;
+
+    let base_url = base_url.trim_end_matches('/');
+
+    if request.stream {
+        handle_streaming(&request.model, request_body, api_key, base_url).await
+    } else {
+        handle_non_streaming(&request.model, request_body, api_key, base_url).await
+    }
+}
+
+// ============================================================================
+// Streaming path
+// ============================================================================
+
+async fn handle_streaming(
+    model: &str,
+    request_body: String,
+    api_key: &str,
+    base_url: &str,
+) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
+    let endpoint = format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model);
+
+    let response = HTTP_CLIENT
+        .post(&endpoint)
+        .header("content-type", "application/json")
+        .header("x-goog-api-key", api_key)
+        .body(request_body)
+        .send()
+        .await
+        .map_err(|e| {
+            Error::internal_err(format!("Failed to send request to Gemini API: {}", e))
+        })?;
+
+    if let Err(e) = response.error_for_status_ref() {
+        let status = e.status().map(|s| s.to_string()).unwrap_or_default();
+        let body = response.text().await.unwrap_or_default();
+        return Err(Error::AIError(format!("{}: {}", status, body)));
+    }
+
+    let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple());
+    let model_str = model.to_string();
+
+    let gemini_sse_stream = response.bytes_stream().eventsource();
+    let openai_sse_stream = async_stream::stream! {
+        tokio::pin!(gemini_sse_stream);
+        let mut tool_call_index: usize = 0;
+        while let Some(event) = gemini_sse_stream.next().await {
+            match event {
+                Ok(event) => match parse_gemini_sse_event(&event.data) {
+                    Ok(Some(parsed)) => {
+                        for chunk in gemini_event_to_openai_sse_chunks(
+                            &parsed, &id, &model_str, &mut tool_call_index,
+                        ) {
+                            yield Ok::<Bytes, reqwest::Error>(Bytes::from(chunk));
+                        }
+                    }
+                    Ok(None) => {}
+                    Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e),
+                },
+                Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e),
+            }
+        }
+        yield Ok::<Bytes, reqwest::Error>(Bytes::from("data: [DONE]\n\n"));
+    };
+
+    let mut headers = http::HeaderMap::new();
+    headers.insert("content-type", "text/event-stream".parse().unwrap());
+    headers.insert("cache-control", "no-cache".parse().unwrap());
+    headers.insert("connection", "keep-alive".parse().unwrap());
+
+    Ok((
+        http::StatusCode::OK,
+        headers,
+        Body::from_stream(inject_keepalives(
+            Box::pin(openai_sse_stream),
+            std::time::Duration::from_secs(KEEPALIVE_INTERVAL_SECS),
+        )),
+    ))
+}
+
+// ============================================================================
+// Model listing
+// ============================================================================
+
+/// List available Gemini models and convert to OpenAI format.
+///
+/// Gemini returns `{ models: [{ name: "models/gemini-2.5-flash", displayName, ... }] }`.
+/// The frontend expects OpenAI format `{ data: [{ id: "models/gemini-2.5-flash", ... }] }`.
+pub async fn handle_google_ai_models(
+    api_key: &str,
+    base_url: &str,
+) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
+    #[derive(Deserialize)]
+    struct GeminiModel {
+        name: String,
+        #[serde(rename = "displayName", default)]
+        display_name: String,
+    }
+
+    #[derive(Deserialize)]
+    struct GeminiModelsResponse {
+        #[serde(default)]
+        models: Vec<GeminiModel>,
+    }
+
+    let endpoint = format!("{}/models", base_url.trim_end_matches('/'));
+    let response = HTTP_CLIENT
+        .get(&endpoint)
+        .header("x-goog-api-key", api_key)
+        .send()
+        .await
+        .map_err(|e| Error::internal_err(format!("Failed to fetch Gemini models: {}", e)))?;
+
+    if let Err(e) = response.error_for_status_ref() {
+        let status = e.status().map(|s| s.to_string()).unwrap_or_default();
+        let body = response.text().await.unwrap_or_default();
+        return Err(Error::AIError(format!("{}: {}", status, body)));
+    }
+
+    let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| {
+        Error::internal_err(format!("Failed to parse Gemini models response: {}", e))
+    })?;
+
+    let data: Vec<serde_json::Value> = gemini_resp
+        .models
+        .into_iter()
+        .map(|m| {
+            json!({
+                "id": m.name,
+                "object": "model",
+                "display_name": m.display_name,
+            })
+        })
+        .collect();
+
+    let body_bytes = serde_json::to_vec(&json!({ "data": data }))
+        .map_err(|e| Error::internal_err(format!("Failed to serialize models: {}", e)))?;
+
+    let mut headers = http::HeaderMap::new();
+    headers.insert("content-type", "application/json".parse().unwrap());
+
+    Ok((http::StatusCode::OK, headers, Body::from(body_bytes)))
+}
+
+// ============================================================================
+// Non-streaming path
+// ============================================================================
+
+async fn handle_non_streaming(
+    model: &str,
+    request_body: String,
+    api_key: &str,
+    base_url: &str,
+) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
+    let endpoint = format!("{}/models/{}:generateContent", base_url, model);
+
+    let response = HTTP_CLIENT
+        .post(&endpoint)
+        .header("content-type", "application/json")
+        .header("x-goog-api-key", api_key)
+        .body(request_body)
+        .send()
+        .await
+        .map_err(|e| {
+            Error::internal_err(format!("Failed to send request to Gemini API: {}", e))
+        })?;
+
+    if let Err(e) = response.error_for_status_ref() {
+        let status = e.status().map(|s| s.to_string()).unwrap_or_default();
+        let body = response.text().await.unwrap_or_default();
+        return Err(Error::AIError(format!("{}: {}", status, body)));
+    }
+
+    let body = response.bytes().await.map_err(|e| {
+        Error::internal_err(format!("Failed to read Gemini response body: {}", e))
+    })?;
+
+    let parsed = parse_gemini_response(&body)?;
+    let openai_response = gemini_response_to_openai(&parsed, model);
+
+    let body_bytes = serde_json::to_vec(&openai_response)
+        .map_err(|e| Error::internal_err(format!("Failed to serialize response: {}", e)))?;
+
+    let mut headers = http::HeaderMap::new();
+    headers.insert("content-type", "application/json".parse().unwrap());
+
+    Ok((http::StatusCode::OK, headers, Body::from(body_bytes)))
+}

backend/windmill-api/src/jobs.rs

@@ -2255,12 +2255,13 @@ async fn resume_suspended_job_internal(
     let value = value.unwrap_or(serde_json::Value::Null);
     verify_suspended_secret(&w_id, &db, job_id, resume_id, &approver, secret).await?;
 
-    // Get flow info - works for both step-level (job_id is a step) and flow-level (job_id is the flow)
-    let (flow_info, is_flow_level) = get_flow_info_for_resume(job_id, &db).await?;
+    // Get flow info - works for step-level, flow-level, and WAC approval
+    let (flow_info, is_flow_level, is_wac) = get_flow_info_for_resume(job_id, &db).await?;
 
     // For step-level resumes, verify user auth and flow status
     // For flow-level resumes (pre-approvals), the flow might not be at a suspended step yet
-    if !is_flow_level {
+    // For WAC approvals, skip flow status checks (there is no flow)
+    if !is_flow_level && !is_wac {
         let parent_flow = GetQuery::new()
             .without_logs()
             .without_code()
@@ -2322,6 +2323,16 @@ async fn resume_suspended_job_internal(
         )
         .execute(&mut *tx)
         .await?;
+    } else if is_wac {
+        // WAC approval: decrement suspend counter directly on the WAC parent job
+        if flow_info.suspend > 0 {
+            sqlx::query!(
+                "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1",
+                flow_info.id,
+            )
+            .execute(&mut *tx)
+            .await?;
+        }
     } else if is_flow_level {
         // For flow-level resumes, decrement the suspend counter if the flow is currently suspended
         // The approval will be matched when the worker checks for resumes (both step-level and flow-level)
@@ -2479,10 +2490,15 @@ struct FlowInfo {
     email: Option<String>,
 }
 
-/// Get flow info from either a step job (by looking up its parent) or a flow job directly.
-/// Returns (FlowInfo, is_flow_level) where is_flow_level indicates if job_id was a flow job.
-async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool)> {
-    // Single query that determines if job_id is a flow or step, and fetches the appropriate flow info
+/// Get flow info from either a step job (by looking up its parent), a flow job directly,
+/// or a WAC workflow job (self-suspended for approval).
+/// Returns (FlowInfo, is_flow_level, is_wac) where:
+/// - is_flow_level: job_id was a flow job (pre-approval)
+/// - is_wac: job_id is a WAC workflow suspended for approval (target is itself)
+async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool, bool)> {
+    // Single query that determines if job_id is a flow, step, or WAC job,
+    // and fetches the appropriate suspended job info.
+    // For WAC jobs (no parent, not a flow), the job itself is the suspended target.
     let result = sqlx::query!(
         r#"
         WITH job_info AS (
@@ -2496,14 +2512,15 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI
             q.suspend AS "suspend!",
             j.runnable_path AS script_path,
             j.permissioned_as_email AS email,
-            (ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!"
+            (ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!",
+            (ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS "is_wac!"
         FROM job_info ji
         JOIN v2_job_queue q ON q.id = CASE
             WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id
-            ELSE ji.parent_job
+            ELSE COALESCE(ji.parent_job, ji.id)
         END
         JOIN v2_job j ON j.id = q.id
-        JOIN v2_job_status s ON s.id = q.id
+        LEFT JOIN v2_job_status s ON s.id = q.id
         FOR UPDATE OF q
         "#,
         job_id,
@@ -2520,7 +2537,7 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI
         email: Some(result.email),
     };
 
-    Ok((flow_info, result.is_flow_level))
+    Ok((flow_info, result.is_flow_level, result.is_wac))
 }
 
 async fn get_suspended_flow_info<'c>(
@@ -5416,12 +5433,12 @@ async fn add_batch_jobs(
         if dedicated_worker && path.is_some() {
             windmill_common::worker::dedicated_worker_tag(&w_id, &path.clone().unwrap())
         } else {
-            language.as_worker_tag(false).to_string()
+            format!("{}", language.as_str())
         }
     } else if let Some(tag) = batch_info.tag {
         tag
     } else {
-        language.as_worker_tag(false).to_string()
+        format!("{}", language.as_str())
     };
 
     let mut tx = user_db.begin(&authed).await?;

backend/windmill-api/src/lib.rs

@@ -64,6 +64,7 @@ use crate::scim_oss::has_scim_token;
 use windmill_common::error::AppError;
 
 mod ai;
+mod google;
 mod apps;
 pub mod args;
 mod audit;
@@ -493,16 +494,12 @@ pub async fn run_server(
     };
 
     #[cfg(feature = "agent_worker_server")]
-    let (
-        agent_workers_router,
-        agent_workers_bg_processor,
-        agent_workers_job_completed_tx,
-        batch_buffer,
-    ) = if server_mode {
-        windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
-    } else {
-        (Router::new(), vec![], None, None)
-    };
+    let (agent_workers_router, agent_workers_bg_processor, agent_workers_job_completed_tx) =
+        if server_mode {
+            windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
+        } else {
+            (Router::new(), vec![], None)
+        };
 
     #[cfg(feature = "agent_worker_server")]
     let agent_cache = Arc::new(AgentCache::new());
@@ -688,7 +685,6 @@ pub async fn run_server(
                         {
                             windmill_api_agent_workers::global_service(
                                 agent_workers_job_completed_tx,
-                                batch_buffer.clone(),
                             )
                             .layer(Extension(agent_cache.clone()))
                         } else {

backend/windmill-common/src/ai_google.rs

@@ -0,0 +1,726 @@
+//! Shared Google AI (Gemini API) types and conversion utilities.
+//!
+//! This module provides:
+//! - Gemini request/response types
+//! - OpenAI → Gemini message conversion
+//! - Gemini SSE event parsing
+//!
+//! Used by both windmill-api (chat proxy) and windmill-worker (AI agent).
+
+use serde::{Deserialize, Serialize};
+
+use crate::ai_types::{ContentPart, ExtraContent, GoogleExtraContent, OpenAIContent, OpenAIMessage, ToolDef, UrlCitation};
+use crate::error::Error;
+
+// ============================================================================
+// Request / Content Types
+// ============================================================================
+
+/// Inline data for binary content (images).
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct GeminiInlineData {
+    #[serde(rename = "mimeType")]
+    pub mime_type: String,
+    pub data: String,
+}
+
+/// A part of content — text, inline data, function call, or function response.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(untagged)]
+pub enum GeminiPart {
+    Text {
+        text: String,
+    },
+    InlineData {
+        #[serde(rename = "inlineData")]
+        inline_data: GeminiInlineData,
+    },
+    FunctionCall {
+        #[serde(rename = "functionCall")]
+        function_call: GeminiFunctionCall,
+        /// Thought signature for Gemini 3+ models — required when replaying function calls.
+        #[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")]
+        thought_signature: Option<String>,
+    },
+    FunctionResponse {
+        #[serde(rename = "functionResponse")]
+        function_response: GeminiFunctionResponse,
+    },
+}
+
+/// A function call from the model.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct GeminiFunctionCall {
+    pub name: String,
+    pub args: serde_json::Value,
+}
+
+/// A function response sent back to the model.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct GeminiFunctionResponse {
+    pub name: String,
+    pub response: serde_json::Value,
+}
+
+/// Content message with an optional role and a list of parts.
+#[derive(Serialize, Clone, Debug)]
+pub struct GeminiContentMessage {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub role: Option<String>,
+    pub parts: Vec<GeminiPart>,
+}
+
+/// Main request body for `generateContent` / `streamGenerateContent`.
+#[derive(Serialize)]
+pub struct GeminiTextRequest {
+    pub contents: Vec<GeminiContentMessage>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tools: Option<Vec<GeminiTool>>,
+    #[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")]
+    pub tool_config: Option<GeminiToolConfig>,
+    #[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")]
+    pub system_instruction: Option<GeminiContentMessage>,
+    #[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")]
+    pub generation_config: Option<GeminiGenerationConfig>,
+}
+
+/// Tool definition — function declarations and/or Google Search grounding.
+#[derive(Serialize)]
+pub struct GeminiTool {
+    #[serde(rename = "functionDeclarations", skip_serializing_if = "Option::is_none")]
+    pub function_declarations: Option<Vec<GeminiFunctionDeclaration>>,
+    #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")]
+    pub google_search: Option<serde_json::Value>,
+}
+
+/// A single function declaration.
+///
+/// `parameters` holds a pre-serialized (and, for the worker, pre-sanitized) JSON Schema.
+#[derive(Serialize)]
+pub struct GeminiFunctionDeclaration {
+    pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+    pub parameters: serde_json::Value,
+}
+
+/// Tool configuration controlling when and how functions are called.
+#[derive(Serialize)]
+pub struct GeminiToolConfig {
+    #[serde(rename = "functionCallingConfig")]
+    pub function_calling_config: GeminiFunctionCallingConfig,
+}
+
+/// Function calling mode and optional allow-list.
+#[derive(Serialize)]
+pub struct GeminiFunctionCallingConfig {
+    pub mode: String,
+    #[serde(rename = "allowedFunctionNames", skip_serializing_if = "Option::is_none")]
+    pub allowed_function_names: Option<Vec<String>>,
+}
+
+/// Generation parameters (temperature, token limits, structured output).
+#[derive(Serialize)]
+pub struct GeminiGenerationConfig {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub temperature: Option<f32>,
+    #[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")]
+    pub max_output_tokens: Option<u32>,
+    #[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")]
+    pub response_mime_type: Option<String>,
+    #[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")]
+    pub response_schema: Option<serde_json::Value>,
+}
+
+// ============================================================================
+// Image Generation Types
+// ============================================================================
+
+/// Request body for Imagen / Gemini image generation.
+#[derive(Serialize)]
+pub struct GeminiImageRequest {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub contents: Option<Vec<GeminiImageContent>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub instances: Option<Vec<GeminiPredictContent>>,
+}
+
+/// Content wrapper used in `generateContent` image requests.
+#[derive(Serialize)]
+pub struct GeminiImageContent {
+    pub parts: Vec<GeminiPart>,
+}
+
+/// Prompt wrapper for Imagen `predict` endpoint.
+#[derive(Serialize)]
+pub struct GeminiPredictContent {
+    pub prompt: String,
+}
+
+/// Top-level response from Gemini/Imagen image generation.
+#[derive(Deserialize)]
+pub struct GeminiImageResponse {
+    pub candidates: Option<Vec<GeminiImageCandidate>>,
+    pub predictions: Option<Vec<GeminiPredictCandidate>>,
+}
+
+#[derive(Deserialize)]
+pub struct GeminiImageCandidate {
+    pub content: GeminiImageCandidateContent,
+}
+
+#[derive(Deserialize)]
+pub struct GeminiImageCandidateContent {
+    pub parts: Vec<GeminiImageCandidatePart>,
+}
+
+#[derive(Deserialize)]
+pub struct GeminiImageCandidatePart {
+    #[serde(rename = "inlineData")]
+    pub inline_data: Option<GeminiInlineData>,
+}
+
+#[derive(Deserialize)]
+pub struct GeminiPredictCandidate {
+    #[serde(rename = "bytesBase64Encoded")]
+    pub bytes_base64_encoded: String,
+}
+
+// ============================================================================
+// SSE Response Types
+// ============================================================================
+
+/// One part inside a streaming candidate — text, function call, or thought signature.
+#[derive(Deserialize, Debug)]
+pub struct GeminiSSEPart {
+    #[serde(default)]
+    pub text: Option<String>,
+    #[serde(rename = "functionCall")]
+    pub function_call: Option<GeminiSSEFunctionCall>,
+    /// Thought signature for Gemini 3+ models.
+    #[serde(rename = "thoughtSignature")]
+    pub thought_signature: Option<String>,
+}
+
+/// Function call contained in a streaming part.
+#[derive(Deserialize, Debug)]
+pub struct GeminiSSEFunctionCall {
+    pub name: String,
+    pub args: serde_json::Value,
+}
+
+/// Content block inside a streaming candidate.
+#[derive(Deserialize, Debug)]
+pub struct GeminiSSEContent {
+    pub parts: Option<Vec<GeminiSSEPart>>,
+}
+
+/// Web source from a Gemini grounding chunk.
+#[derive(Deserialize, Debug)]
+pub struct GeminiGroundingChunkWeb {
+    pub uri: String,
+    #[serde(default)]
+    pub title: Option<String>,
+}
+
+/// One grounding chunk (search result) from Gemini web search.
+#[derive(Deserialize, Debug)]
+pub struct GeminiGroundingChunk {
+    pub web: Option<GeminiGroundingChunkWeb>,
+}
+
+/// Grounding metadata attached to a streaming candidate.
+#[derive(Deserialize, Debug)]
+pub struct GeminiGroundingMetadata {
+    #[serde(rename = "groundingChunks", default)]
+    pub grounding_chunks: Vec<GeminiGroundingChunk>,
+    #[serde(rename = "webSearchQueries", default)]
+    pub web_search_queries: Vec<String>,
+}
+
+/// One candidate inside a streaming Gemini response.
+#[derive(Deserialize, Debug)]
+pub struct GeminiSSECandidate {
+    pub content: Option<GeminiSSEContent>,
+    #[serde(rename = "finishReason")]
+    pub finish_reason: Option<String>,
+    #[serde(rename = "groundingMetadata")]
+    pub grounding_metadata: Option<GeminiGroundingMetadata>,
+}
+
+/// Token usage from the `usageMetadata` field of a Gemini SSE event.
+#[derive(Deserialize, Debug, Clone)]
+pub struct GeminiUsageMetadata {
+    #[serde(rename = "promptTokenCount", default)]
+    pub prompt_token_count: Option<i32>,
+    #[serde(rename = "candidatesTokenCount", default)]
+    pub candidates_token_count: Option<i32>,
+    #[serde(rename = "totalTokenCount", default)]
+    pub total_token_count: Option<i32>,
+}
+
+/// Top-level structure of one Gemini SSE event.
+#[derive(Deserialize, Debug)]
+pub struct GeminiSSEEvent {
+    pub candidates: Option<Vec<GeminiSSECandidate>>,
+    #[serde(rename = "usageMetadata")]
+    pub usage_metadata: Option<GeminiUsageMetadata>,
+}
+
+// ============================================================================
+// Parsed Event Result
+// ============================================================================
+
+/// A single function call extracted from a Gemini SSE event.
+#[derive(Debug)]
+pub struct GeminiToolCallEvent {
+    pub name: String,
+    pub args: serde_json::Value,
+    pub thought_signature: Option<String>,
+}
+
+impl GeminiToolCallEvent {
+    /// Convert the thought signature (if present) into an [`ExtraContent`].
+    pub fn to_extra_content(&self) -> Option<ExtraContent> {
+        self.thought_signature.as_ref().map(|sig| ExtraContent {
+            google: Some(GoogleExtraContent { thought_signature: Some(sig.clone()) }),
+        })
+    }
+}
+
+/// Structured result of parsing a Gemini response (streaming SSE event or non-streaming body).
+#[derive(Debug, Default)]
+pub struct GeminiParsedEvent {
+    pub text: Option<String>,
+    pub tool_calls: Vec<GeminiToolCallEvent>,
+    pub annotations: Vec<UrlCitation>,
+    pub used_websearch: bool,
+    pub usage: Option<GeminiUsageMetadata>,
+    pub finish_reason: Option<String>,
+}
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/// Parse a data URL into `(mime_type, base64_data)`.
+///
+/// Expected format: `data:<mime_type>;base64,<data>`.
+pub fn parse_data_url(url: &str) -> Option<(String, String)> {
+    let rest = url.strip_prefix("data:")?;
+    let (header, data) = rest.split_once(',')?;
+    let media_type = header.strip_suffix(";base64")?;
+    Some((media_type.to_string(), data.to_string()))
+}
+
+/// Find the function name associated with a `tool_call_id` by scanning prior messages.
+pub fn find_gemini_function_name(messages: &[OpenAIMessage], tool_call_id: &str) -> String {
+    messages
+        .iter()
+        .filter_map(|msg| msg.tool_calls.as_ref())
+        .flatten()
+        .find(|tc| tc.id == tool_call_id)
+        .map(|tc| tc.function.name.clone())
+        .unwrap_or_else(|| "unknown_function".to_string())
+}
+
+/// Convert an [`OpenAIContent`] value to a list of [`GeminiPart`]s.
+///
+/// Handles text and `image_url` (data URLs). `S3Object` variants are skipped here;
+/// the worker handles them by downloading and injecting inline data beforehand.
+pub fn convert_content_to_gemini_parts(content: &OpenAIContent) -> Vec<GeminiPart> {
+    match content {
+        OpenAIContent::Text(text) if !text.is_empty() => {
+            vec![GeminiPart::Text { text: text.clone() }]
+        }
+        OpenAIContent::Text(_) => vec![],
+        OpenAIContent::Parts(parts) => parts
+            .iter()
+            .filter_map(|part| match part {
+                ContentPart::Text { text } if !text.is_empty() => {
+                    Some(GeminiPart::Text { text: text.clone() })
+                }
+                ContentPart::ImageUrl { image_url } => {
+                    parse_data_url(&image_url.url).map(|(mime_type, data)| {
+                        GeminiPart::InlineData {
+                            inline_data: GeminiInlineData { mime_type, data },
+                        }
+                    })
+                }
+                // S3Objects are handled by the worker
+                _ => None,
+            })
+            .collect(),
+    }
+}
+
+/// Convert OpenAI-format messages to Gemini `contents` and an optional `systemInstruction`.
+///
+/// Returns `(contents, system_instruction)`.
+///
+/// `S3Object` images in content parts are skipped (the worker pre-converts them).
+/// Tool call history is preserved correctly for multi-turn agent conversations.
+pub fn openai_messages_to_gemini(
+    messages: &[OpenAIMessage],
+) -> (Vec<GeminiContentMessage>, Option<GeminiContentMessage>) {
+    let mut contents: Vec<GeminiContentMessage> = Vec::new();
+    let mut system_instruction: Option<GeminiContentMessage> = None;
+
+    for msg in messages {
+        match msg.role.as_str() {
+            "system" => {
+                if let Some(content) = &msg.content {
+                    let parts = convert_content_to_gemini_parts(content);
+                    if !parts.is_empty() {
+                        system_instruction =
+                            Some(GeminiContentMessage { role: None, parts });
+                    }
+                }
+            }
+            "tool" => {
+                if let (Some(tool_call_id), Some(content)) =
+                    (&msg.tool_call_id, &msg.content)
+                {
+                    let func_name = find_gemini_function_name(messages, tool_call_id);
+                    let response_text = match content {
+                        OpenAIContent::Text(text) => text.clone(),
+                        OpenAIContent::Parts(parts) => parts
+                            .iter()
+                            .filter_map(|p| {
+                                if let ContentPart::Text { text } = p {
+                                    Some(text.as_str())
+                                } else {
+                                    None
+                                }
+                            })
+                            .collect::<Vec<_>>()
+                            .join(" "),
+                    };
+                    contents.push(GeminiContentMessage {
+                        role: Some("user".to_string()),
+                        parts: vec![GeminiPart::FunctionResponse {
+                            function_response: GeminiFunctionResponse {
+                                name: func_name,
+                                response: serde_json::json!({ "result": response_text }),
+                            },
+                        }],
+                    });
+                }
+            }
+            role => {
+                let gemini_role = if role == "assistant" { "model" } else { "user" };
+                let mut parts: Vec<GeminiPart> = Vec::new();
+
+                if let Some(content) = &msg.content {
+                    parts.extend(convert_content_to_gemini_parts(content));
+                }
+
+                if let Some(tool_calls) = &msg.tool_calls {
+                    for tc in tool_calls {
+                        let args: serde_json::Value =
+                            serde_json::from_str(&tc.function.arguments).unwrap_or_default();
+                        let thought_signature = tc
+                            .extra_content
+                            .as_ref()
+                            .and_then(|ec| ec.google.as_ref())
+                            .and_then(|g| g.thought_signature.clone());
+                        parts.push(GeminiPart::FunctionCall {
+                            function_call: GeminiFunctionCall {
+                                name: tc.function.name.clone(),
+                                args,
+                            },
+                            thought_signature,
+                        });
+                    }
+                }
+
+                if !parts.is_empty() {
+                    contents.push(GeminiContentMessage {
+                        role: Some(gemini_role.to_string()),
+                        parts,
+                    });
+                }
+            }
+        }
+    }
+
+    (contents, system_instruction)
+}
+
+/// Convert OpenAI tool definitions to Gemini format.
+///
+/// `tool_params` must be pre-serialized (and, for the worker, pre-sanitized for Google)
+/// JSON schema values, one per entry in `tools` in the same order.
+pub fn openai_tools_to_gemini(
+    tools: &[ToolDef],
+    tool_params: &[serde_json::Value],
+    has_websearch: bool,
+) -> Option<Vec<GeminiTool>> {
+    let mut gemini_tools: Vec<GeminiTool> = Vec::new();
+
+    let declarations: Vec<GeminiFunctionDeclaration> = tools
+        .iter()
+        .zip(tool_params.iter())
+        .map(|(t, params)| GeminiFunctionDeclaration {
+            name: t.function.name.clone(),
+            description: t.function.description.clone(),
+            parameters: params.clone(),
+        })
+        .collect();
+
+    if !declarations.is_empty() {
+        gemini_tools.push(GeminiTool {
+            function_declarations: Some(declarations),
+            google_search: None,
+        });
+    }
+
+    if has_websearch {
+        gemini_tools.push(GeminiTool {
+            function_declarations: None,
+            google_search: Some(serde_json::json!({})),
+        });
+    }
+
+    if gemini_tools.is_empty() {
+        None
+    } else {
+        Some(gemini_tools)
+    }
+}
+
+/// Parse one Gemini SSE data line into a [`GeminiParsedEvent`].
+///
+/// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`).
+/// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating.
+pub fn parse_gemini_sse_event(data: &str) -> Result<Option<GeminiParsedEvent>, Error> {
+    if data.is_empty() || data == "[DONE]" {
+        return Ok(None);
+    }
+
+    let event: GeminiSSEEvent = match serde_json::from_str(data) {
+        Ok(e) => e,
+        Err(e) => {
+            tracing::error!("Failed to parse Gemini SSE event {}: {}", data, e);
+            return Ok(None);
+        }
+    };
+
+    let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() };
+
+    let Some(candidates) = event.candidates else {
+        return Ok(Some(parsed));
+    };
+
+    extract_candidates_into(&candidates, &mut parsed);
+
+    Ok(Some(parsed))
+}
+
+/// Parse a non-streaming Gemini `generateContent` response body.
+pub fn parse_gemini_response(data: &[u8]) -> Result<GeminiParsedEvent, Error> {
+    let event: GeminiSSEEvent = serde_json::from_slice(data)
+        .map_err(|e| Error::internal_err(format!("Failed to parse Gemini response: {}", e)))?;
+
+    let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() };
+
+    if let Some(candidates) = event.candidates {
+        extract_candidates_into(&candidates, &mut parsed);
+    }
+
+    Ok(parsed)
+}
+
+// ============================================================================
+// Gemini → OpenAI Format Conversion
+// ============================================================================
+
+/// Convert a `GeminiParsedEvent` from a non-streaming response to an OpenAI chat completion JSON.
+pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> serde_json::Value {
+    let content = parsed.text.as_deref().unwrap_or_default();
+
+    let tool_calls: Vec<serde_json::Value> = parsed
+        .tool_calls
+        .iter()
+        .enumerate()
+        .map(|(i, tc)| {
+            serde_json::json!({
+                "index": i,
+                "id": format!("call_{}", uuid::Uuid::new_v4().simple()),
+                "type": "function",
+                "function": {
+                    "name": tc.name,
+                    "arguments": serde_json::to_string(&tc.args).unwrap_or_default()
+                }
+            })
+        })
+        .collect();
+
+    let finish_reason = parsed
+        .finish_reason
+        .as_deref()
+        .map(|r| r.to_lowercase())
+        .unwrap_or_else(|| "stop".to_string());
+
+    let usage = parsed.usage.as_ref().map(|u| {
+        serde_json::json!({
+            "prompt_tokens": u.prompt_token_count.unwrap_or(0),
+            "completion_tokens": u.candidates_token_count.unwrap_or(0),
+            "total_tokens": u.total_token_count.unwrap_or(0),
+        })
+    });
+
+    let mut message = serde_json::json!({
+        "role": "assistant",
+        "content": content,
+    });
+    if !tool_calls.is_empty() {
+        message["tool_calls"] = serde_json::json!(tool_calls);
+    }
+
+    serde_json::json!({
+        "id": format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()),
+        "object": "chat.completion",
+        "model": model,
+        "choices": [{
+            "index": 0,
+            "message": message,
+            "finish_reason": finish_reason,
+        }],
+        "usage": usage,
+    })
+}
+
+/// Convert a `GeminiParsedEvent` from a streaming SSE event into OpenAI-format SSE lines.
+///
+/// Returns the serialized `"data: {...}\n\n"` lines ready to be written to the response stream.
+/// `tool_call_index` is mutated to track the running index across multiple SSE events.
+pub fn gemini_event_to_openai_sse_chunks(
+    parsed: &GeminiParsedEvent,
+    id: &str,
+    model: &str,
+    tool_call_index: &mut usize,
+) -> Vec<String> {
+    let mut chunks = Vec::new();
+
+    if let Some(text) = &parsed.text {
+        let chunk = serde_json::json!({
+            "id": id,
+            "object": "chat.completion.chunk",
+            "model": model,
+            "choices": [{
+                "index": 0,
+                "delta": { "content": text },
+                "finish_reason": null,
+            }]
+        });
+        chunks.push(format!("data: {}\n\n", chunk));
+    }
+
+    for tc in &parsed.tool_calls {
+        let args_str = serde_json::to_string(&tc.args).unwrap_or_default();
+        let call_id = format!("call_{}", uuid::Uuid::new_v4().simple());
+        let chunk = serde_json::json!({
+            "id": id,
+            "object": "chat.completion.chunk",
+            "model": model,
+            "choices": [{
+                "index": 0,
+                "delta": {
+                    "tool_calls": [{
+                        "index": *tool_call_index,
+                        "id": call_id,
+                        "type": "function",
+                        "function": {
+                            "name": tc.name,
+                            "arguments": args_str,
+                        }
+                    }]
+                },
+                "finish_reason": null,
+            }]
+        });
+        chunks.push(format!("data: {}\n\n", chunk));
+        *tool_call_index += 1;
+    }
+
+    chunks
+}
+
+/// Recursively remove JSON Schema fields unsupported by the Gemini API.
+pub fn sanitize_schema_for_google(value: &mut serde_json::Value) {
+    const UNSUPPORTED: &[&str] = &[
+        "additionalProperties",
+        "strict",
+        "$schema",
+        "default",
+        "exclusiveMinimum",
+        "exclusiveMaximum",
+        "const",
+        "multipleOf",
+    ];
+
+    if let Some(obj) = value.as_object_mut() {
+        for field in UNSUPPORTED {
+            obj.remove(*field);
+        }
+        for v in obj.values_mut() {
+            sanitize_schema_for_google(v);
+        }
+    } else if let Some(arr) = value.as_array_mut() {
+        for v in arr.iter_mut() {
+            sanitize_schema_for_google(v);
+        }
+    }
+}
+
+// ============================================================================
+// Internal Helpers
+// ============================================================================
+
+fn extract_candidates_into(candidates: &[GeminiSSECandidate], parsed: &mut GeminiParsedEvent) {
+    for candidate in candidates {
+        if let Some(content) = &candidate.content {
+            if let Some(parts) = &content.parts {
+                for part in parts {
+                    if let Some(text) = &part.text {
+                        if !text.is_empty() {
+                            match parsed.text.as_mut() {
+                                Some(existing) => existing.push_str(text),
+                                None => parsed.text = Some(text.clone()),
+                            }
+                        }
+                    }
+
+                    if let Some(function_call) = &part.function_call {
+                        parsed.tool_calls.push(GeminiToolCallEvent {
+                            name: function_call.name.clone(),
+                            args: function_call.args.clone(),
+                            thought_signature: part.thought_signature.clone(),
+                        });
+                    }
+                }
+            }
+        }
+
+        if candidate.finish_reason.is_some() {
+            parsed.finish_reason = candidate.finish_reason.clone();
+        }
+
+        if let Some(grounding) = &candidate.grounding_metadata {
+            if !grounding.web_search_queries.is_empty() || !grounding.grounding_chunks.is_empty() {
+                parsed.used_websearch = true;
+            }
+            for chunk in &grounding.grounding_chunks {
+                if let Some(web) = &chunk.web {
+                    parsed.annotations.push(UrlCitation {
+                        start_index: 0,
+                        end_index: 0,
+                        url: web.uri.clone(),
+                        title: web.title.clone(),
+                    });
+                }
+            }
+        }
+    }
+}

backend/windmill-common/src/error.rs

@@ -78,6 +78,8 @@ pub enum Error {
     AIError(String),
     #[error("{0}")]
     AlreadyCompleted(String),
+    #[error("WAC job suspended: {0}")]
+    WacSuspended(String),
     #[error("Find python error: {0}")]
     FindPythonError(String),
     #[error("Problem with arguments: {0}")]
@@ -108,6 +110,7 @@ impl Error {
             Self::JsonErr(_) => "JsonErr",
             Self::AIError(_) => "AIError",
             Self::AlreadyCompleted(_) => "AlreadyCompleted",
+            Self::WacSuspended(_) => "WacSuspended",
             Self::FindPythonError(_) => "FindPythonError",
             Self::ArgumentErr(_) => "ArgumentErr",
             Self::Generic(_, _) => "Generic",

backend/windmill-common/src/instance_config.rs

@@ -44,7 +44,7 @@ pub struct EnvRefWrapper {
 ///
 /// `Literal` serializes back to a plain JSON string, preserving backwards
 /// compatibility with existing consumers.
-#[derive(Deserialize, Serialize, Clone, Debug)]
+#[derive(Deserialize, Serialize, Clone)]
 #[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))]
 #[serde(untagged)]
 pub enum StringOrSecretRef {
@@ -53,6 +53,16 @@ pub enum StringOrSecretRef {
     EnvRef(EnvRefWrapper),
 }
 
+impl fmt::Debug for StringOrSecretRef {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Literal(_) => f.write_str("Literal(****)"),
+            Self::SecretRef(w) => f.debug_tuple("SecretRef").field(w).finish(),
+            Self::EnvRef(w) => f.debug_tuple("EnvRef").field(w).finish(),
+        }
+    }
+}
+
 impl StringOrSecretRef {
     /// Returns the literal string value, or `None` if this is an unresolved ref.
     pub fn as_literal(&self) -> Option<&str> {
@@ -255,25 +265,25 @@ pub struct GlobalSettings {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub instance_python_version: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub pip_index_url: Option<String>,
+    pub pip_index_url: Option<StringOrSecretRef>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub pip_extra_index_url: Option<String>,
+    pub pip_extra_index_url: Option<StringOrSecretRef>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub npm_config_registry: Option<String>,
+    pub npm_config_registry: Option<StringOrSecretRef>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub bunfig_install_scopes: Option<String>,
+    pub bunfig_install_scopes: Option<StringOrSecretRef>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub npmrc: Option<String>,
+    pub npmrc: Option<StringOrSecretRef>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub nuget_config: Option<String>,
+    pub nuget_config: Option<StringOrSecretRef>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub maven_repos: Option<String>,
+    pub maven_repos: Option<StringOrSecretRef>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub ruby_repos: Option<String>,
+    pub ruby_repos: Option<StringOrSecretRef>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub powershell_repo_url: Option<String>,
+    pub powershell_repo_url: Option<StringOrSecretRef>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub powershell_repo_pat: Option<String>,
+    pub powershell_repo_pat: Option<StringOrSecretRef>,
 
     // Array settings
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -924,7 +934,7 @@ fn redact_string(s: &str) -> String {
     }
 }
 
-fn format_setting_value(key: &str, value: &serde_json::Value) -> String {
+pub fn format_setting_value(key: &str, value: &serde_json::Value) -> String {
     if SENSITIVE_SETTINGS.contains(&key) {
         return match value {
             serde_json::Value::String(s) => format!("\"{}\"", redact_string(s)),
@@ -2209,6 +2219,25 @@ mod tests {
         assert_eq!(v, *"world");
     }
 
+    #[test]
+    fn string_or_secret_ref_debug_masks_literal() {
+        let v = StringOrSecretRef::Literal("super-secret-value".to_string());
+        let debug = format!("{v:?}");
+        assert_eq!(debug, "Literal(****)");
+        assert!(!debug.contains("super-secret-value"));
+    }
+
+    #[test]
+    fn format_setting_value_redacts_oauth_secrets() {
+        let val = serde_json::json!({
+            "google": {"id": "client-id", "secret": "my-super-secret-12345"}
+        });
+        let formatted = format_setting_value("oauths", &val);
+        assert!(!formatted.contains("my-super-secret-12345"));
+        assert!(formatted.contains("client-id"));
+        assert!(formatted.contains("****"));
+    }
+
     #[test]
     #[should_panic(expected = "literal_value() called on unresolved secret ref")]
     fn string_or_secret_ref_literal_value_panics_on_ref() {

backend/windmill-common/src/lib.rs

@@ -29,6 +29,7 @@ use sqlx::{Acquire, Postgres};
 pub mod agent_workers;
 #[cfg(feature = "bedrock")]
 pub mod ai_bedrock;
+pub mod ai_google;
 pub mod ai_providers;
 pub mod ai_types;
 pub mod apps;

backend/windmill-common/src/worker.rs

@@ -288,10 +288,6 @@ pub fn is_native_mode_from_env() -> bool {
 /// Use this for hot-path checks (e.g. per-job dispatch) to avoid read-locking WORKER_CONFIG.
 pub static NATIVE_MODE_RESOLVED: AtomicBool = AtomicBool::new(false);
 
-/// Whether this worker uses HTTP batch pull (set at startup in main.rs).
-/// Reported in worker_ping so the server knows which native workers to batch-pull for.
-pub static USES_BATCH_HTTP_PULL: AtomicBool = AtomicBool::new(false);
-
 pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false);
 #[derive(Clone)]
 pub struct HttpClient {
@@ -520,62 +516,6 @@ pub fn make_pull_query(tags: &[String]) -> String {
     query
 }
 
-pub fn make_batch_pull_query(tags: &[String], limit: u32) -> String {
-    format_batch_pull_query(format!(
-        "SELECT id
-        FROM v2_job_queue
-        WHERE running = false AND tag IN ({}) AND scheduled_for <= now()
-        ORDER BY priority DESC NULLS LAST, scheduled_for
-        FOR UPDATE SKIP LOCKED
-        LIMIT {limit}",
-        tags.iter().map(|x| format!("'{x}'")).join(", ")
-    ))
-}
-
-fn format_batch_pull_query(peek: String) -> String {
-    // Optimizations vs single-row format_pull_query:
-    // 1. ANY(ARRAY(SELECT ...)) instead of IN (SELECT ...) — forces PG to materialize IDs
-    //    into an array, enabling Bitmap Index Scan instead of Hash Semi Join / Nested Loop
-    // 2. r CTE chains off q (not peek) — only updates runtime for actually-locked rows,
-    //    avoids re-scanning peek
-    // 3. No separate j CTE — join v2_job directly in final SELECT off q's IDs
-    format!(
-        "WITH peek AS (
-            {}
-        ), q AS NOT MATERIALIZED (
-            UPDATE v2_job_queue SET
-                running = true,
-                started_at = coalesce(started_at, now()),
-                suspend_until = null,
-                worker = $1
-            WHERE id = ANY(ARRAY(SELECT id FROM peek))
-            RETURNING
-                id, started_at, scheduled_for,
-                canceled_by, canceled_reason, worker, cache_ignore_s3_path, runnable_settings_handle
-        ), r AS NOT MATERIALIZED (
-            UPDATE v2_job_runtime SET
-                ping = now()
-            WHERE id = ANY(ARRAY(SELECT id FROM q))
-        ) SELECT j.id, j.workspace_id, j.parent_job, j.created_by, q.started_at, q.scheduled_for,
-            j.runnable_id, j.runnable_path, j.args, q.canceled_by,
-            q.canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as,
-            f.flow_status, j.script_lang,
-            j.same_worker, j.pre_run_error, j.visible_to_owner,
-            j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job,
-            j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow,
-            j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path,
-            COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
-            p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email
-        FROM q
-            JOIN v2_job j ON q.id = j.id
-            LEFT JOIN v2_job_status f ON f.id = q.id
-            LEFT JOIN job_perms p ON p.job_id = q.id
-            LEFT JOIN v2_job pj ON j.parent_job = pj.id
-            ",
-        peek
-    )
-}
-
 pub async fn store_pull_query(wc: &WorkerConfig) {
     let mut queries = vec![];
     for tags in wc.priority_tags_sorted.iter() {
@@ -1254,8 +1194,6 @@ pub struct Ping {
     pub occupancy_rate_30m: Option<f32>,
     pub job_isolation: Option<String>,
     pub native_mode: Option<bool>,
-    #[serde(default)]
-    pub uses_batch_http_pull: Option<bool>,
     pub ping_type: PingType,
 }
 pub async fn update_ping_http(
@@ -1280,7 +1218,6 @@ pub async fn update_ping_http(
                 insert_ping.occupancy_rate_5m,
                 insert_ping.occupancy_rate_30m,
                 insert_ping.native_mode.unwrap_or(false),
-                insert_ping.uses_batch_http_pull.unwrap_or(false),
                 db,
             )
             .await?
@@ -1308,7 +1245,6 @@ pub async fn update_ping_http(
                 insert_ping.memory,
                 insert_ping.job_isolation,
                 insert_ping.native_mode.unwrap_or(false),
-                insert_ping.uses_batch_http_pull.unwrap_or(false),
                 db,
             )
             .await?;
@@ -1441,12 +1377,11 @@ pub async fn insert_ping_query(
     memory: Option<i64>,
     job_isolation: Option<String>,
     native_mode: bool,
-    uses_batch_http_pull: bool,
     db: &DB,
 ) -> anyhow::Result<()> {
     sqlx::query!(
-        "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)
-        DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
+        "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)
+        DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode",
         worker_instance,
         worker_name,
         ip,
@@ -1459,7 +1394,6 @@ pub async fn insert_ping_query(
         memory,
         job_isolation.as_deref(),
         native_mode,
-        uses_batch_http_pull,
         )
         .execute(db)
         .await?;
@@ -1551,13 +1485,12 @@ pub async fn update_worker_ping_main_loop_query(
     occupancy_rate_5m: Option<f32>,
     occupancy_rate_30m: Option<f32>,
     native_mode: bool,
-    uses_batch_http_pull: bool,
     db: &DB,
 ) -> anyhow::Result<()> {
     timeout(Duration::from_secs(10), sqlx::query!(
         "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,
          occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),
-         memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
+         memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6",
         jobs_executed,
         tags,
         occupancy_rate,
@@ -1570,7 +1503,6 @@ pub async fn update_worker_ping_main_loop_query(
         occupancy_rate_5m,
         occupancy_rate_30m,
         native_mode,
-        uses_batch_http_pull,
     )
         .execute(db))
     .await??;

backend/windmill-jseval/src/lib.rs

@@ -152,6 +152,21 @@ pub fn try_exact_property_access(
     None
 }
 
+/// JS runtime properties (not methods) that cannot be resolved by PostgreSQL's
+/// #> JSON path operator. Function calls like .map(...) already don't match the
+/// RE_FULL regex due to parentheses, so only property accesses need listing here.
+const JS_ONLY_PROPERTIES: &[&str] = &["length"];
+
+fn ends_with_js_only_property(rest: Option<&str>) -> bool {
+    match rest {
+        None => false,
+        Some(rest) => {
+            let last_segment = rest.rsplit('.').next().unwrap_or("");
+            JS_ONLY_PROPERTIES.contains(&last_segment)
+        }
+    }
+}
+
 pub async fn handle_full_regex(
     expr: &str,
     authed_client: &AuthedClient,
@@ -162,6 +177,13 @@ pub async fn handle_full_regex(
         let obj_key = captures.get(2).unwrap().as_str();
         let idx_o = captures.get(3).map(|y| y.as_str());
         let rest = captures.get(4).map(|y| y.as_str());
+
+        // Skip the SQL fast path when the expression accesses a JS runtime
+        // property (e.g. .length) that the PostgreSQL #> operator can't resolve.
+        if ends_with_js_only_property(rest) {
+            return None;
+        }
+
         let query = if let Some(idx) = idx_o {
             match rest {
                 Some(rest) => Some(format!("{}{}", idx, rest)),

backend/windmill-oauth/src/lib.rs

@@ -94,7 +94,7 @@ pub struct OAuthConfig {
 }
 
 /// OAuth client credentials
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Serialize, Deserialize)]
 pub struct OAuthClient {
     #[serde(default = "empty_string")]
     pub id: String,
@@ -110,6 +110,21 @@ pub struct OAuthClient {
     pub grant_types: Vec<String>,
 }
 
+impl std::fmt::Debug for OAuthClient {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("OAuthClient")
+            .field("id", &self.id)
+            .field("secret", &"***")
+            .field("display_name", &self.display_name)
+            .field("allowed_domains", &self.allowed_domains)
+            .field("connect_config", &self.connect_config)
+            .field("login_config", &self.login_config)
+            .field("tenant", &self.tenant)
+            .field("grant_types", &self.grant_types)
+            .finish()
+    }
+}
+
 fn empty_string() -> String {
     "".to_string()
 }
@@ -608,7 +623,18 @@ pub async fn refresh_token<'c>(
     .await?;
     let account = windmill_common::utils::not_found_if_none(account, "Account", &id.to_string())?;
 
-    refresh_token_for_account(tx, path, w_id, id, db, account, oauth_clients, http_client, connect_configs_json).await
+    refresh_token_for_account(
+        tx,
+        path,
+        w_id,
+        id,
+        db,
+        account,
+        oauth_clients,
+        http_client,
+        connect_configs_json,
+    )
+    .await
 }
 
 /// Refresh an OAuth token given pre-fetched account info (no additional SELECT).

backend/windmill-queue/src/jobs.rs

@@ -818,7 +818,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
     flow_is_done: bool,
     duration: Option<i64>,
     from_cache: bool,
-) -> Result<(Uuid, i64), Error> {
+) -> Result<(Uuid, i64, Option<serde_json::Value>), Error> {
     // tracing::error!("Start");
     // let start = tokio::time::Instant::now();
 
@@ -830,7 +830,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
     }
 
     let result_columns = result_columns.as_ref();
-    let (opt_uuid, duration, _skip_downstream_error_handlers) = (|| {
+    let (opt_uuid, duration, _skip_downstream_error_handlers, wac_job_ids) = (|| {
         commit_completed_job(
             db,
             completed_job,
@@ -866,7 +866,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
 
     // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout
     if let Some(job_id) = opt_uuid {
-        return Ok((job_id, duration));
+        return Ok((job_id, duration, None));
     }
 
     #[cfg(feature = "cloud")]
@@ -887,7 +887,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
 
     // tracing::error!("4 {:?}", start.elapsed());
 
-    Ok((completed_job.id, duration))
+    Ok((completed_job.id, duration, wac_job_ids))
 }
 
 async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
@@ -902,7 +902,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
     flow_is_done: bool,
     duration: Option<i64>,
     from_cache: bool,
-) -> windmill_common::error::Result<(Option<Uuid>, i64, bool)> {
+) -> windmill_common::error::Result<(Option<Uuid>, i64, bool, Option<serde_json::Value>)> {
     // let start = std::time::Instant::now();
 
     let mut tx = db.begin().warn_after_seconds(10).await?;
@@ -1003,25 +1003,31 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
         .map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?;
     }
 
+    let mut wac_job_ids: Option<serde_json::Value> = None;
     if !completed_job.is_flow_step() {
         if let Some(parent_job) = completed_job.parent_job {
-            let _ = sqlx::query_scalar!(
-                "UPDATE v2_job_status SET
+            // Only update WAC parents (v1 or v2). The WHERE condition skips
+            // non-WAC parents entirely (error handlers, run_script children, etc.).
+            // Also returns pending_steps.job_ids so WAC v2 child completion
+            // doesn't need a separate read.
+            let row = sqlx::query_scalar!(
+                r#"UPDATE v2_job_status SET
                         workflow_as_code_status = jsonb_set(
                             jsonb_set(
-                                COALESCE(workflow_as_code_status, '{}'::jsonb),
+                                workflow_as_code_status,
                                 array[$1],
                                 COALESCE(workflow_as_code_status->$1, '{}'::jsonb)
                             ),
                             array[$1, 'duration_ms'],
                             to_jsonb($2::bigint)
                         )
-                    WHERE id = $3",
+                    WHERE id = $3 AND workflow_as_code_status IS NOT NULL
+                    RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS "job_ids: serde_json::Value""#,
                 &completed_job.id.to_string(),
                 duration,
                 parent_job
             )
-            .execute(&mut *tx)
+            .fetch_optional(&mut *tx)
             .warn_after_seconds(10)
             .await
             .inspect_err(|e| {
@@ -1029,7 +1035,10 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
                     "Could not update parent job `duration_ms` in workflow as code status: {}",
                     e,
                 )
-            });
+            })
+            .ok()
+            .flatten();
+            wac_job_ids = row.flatten();
         }
     }
     // tracing::error!("Added completed job {:#?}", queued_job);
@@ -1250,14 +1259,14 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
         completed_job.id
     );
     // tracing::info!("completed job: {:?}", start.elapsed().as_micros());
-    Ok((None, duration, _skip_downstream_error_handlers))
+    Ok((None, duration, _skip_downstream_error_handlers, wac_job_ids))
 }
 
 async fn check_result_size<T: ValidableJson>(
     db: &Pool<Postgres>,
     queued_job: &MiniCompletedJob,
     result: Json<&T>,
-) -> Option<Result<(Option<Uuid>, i64, bool), Error>> {
+) -> Option<Result<(Option<Uuid>, i64, bool, Option<serde_json::Value>), Error>> {
     let result_size = result.size() / 1024 / 1024;
     if result_size > 2 {
         if result_size > *MAX_RESULT_SIZE_MB {
@@ -3434,38 +3443,6 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
     Ok(job_and_suspended)
 }
 
-/// Batch-pull up to `limit` jobs in a single query, marking them all as running.
-/// The caller controls which tags are queried, so flow/dependency jobs are never
-/// pulled (they use distinct tags like "flow" / "dependency").
-pub async fn batch_pull(
-    db: &Pool<Postgres>,
-    worker_name: &str,
-    tags: &[String],
-    limit: u32,
-) -> windmill_common::error::Result<Vec<PulledJob>> {
-    use windmill_common::worker::make_batch_pull_query;
-
-    if limit == 0 || tags.is_empty() {
-        return Ok(vec![]);
-    }
-
-    let query = make_batch_pull_query(tags, limit);
-    let jobs: Vec<PulledJob> = timeout(
-        Duration::from_secs(15),
-        sqlx::query_as::<_, PulledJob>(&query)
-            .bind(worker_name)
-            .fetch_all(db),
-    )
-    .await
-    .map_err(|_| {
-        windmill_common::error::Error::internal_err(
-            "batch_pull query timed out after 15s".to_string(),
-        )
-    })??;
-
-    Ok(jobs)
-}
-
 pub async fn custom_concurrency_key(
     db: &Pool<Postgres>,
     job_id: &Uuid,
@@ -5405,7 +5382,15 @@ async fn push_inner<'c, 'd>(
             language
                 .as_ref()
                 .map(|x| {
-                    let tag_lang = x.as_worker_tag(job_kind == JobKind::Dependencies);
+                    let tag_lang = if x == &ScriptLang::Bunnative {
+                        if job_kind == JobKind::Dependencies {
+                            ScriptLang::Bun.as_str()
+                        } else {
+                            ScriptLang::Nativets.as_str()
+                        }
+                    } else {
+                        x.as_str()
+                    };
                     if per_workspace {
                         format!("{}-{}", tag_lang, workspace_id)
                     } else {

backend/windmill-test-utils/src/lib.rs

@@ -421,7 +421,6 @@ pub fn spawn_test_worker(
             rx,
             tx2,
             &base_internal_url,
-            None,
         )
         .await
     };

backend/windmill-types/src/scripts.rs

@@ -88,20 +88,6 @@ impl ScriptLang {
         }
     }
 
-    /// Returns the worker tag for this language.
-    /// Bunnative scripts run on nativets workers (not bun), except dependency jobs which use bun.
-    pub fn as_worker_tag(&self, is_dependency_job: bool) -> &'static str {
-        if *self == ScriptLang::Bunnative {
-            if is_dependency_job {
-                ScriptLang::Bun.as_str()
-            } else {
-                ScriptLang::Nativets.as_str()
-            }
-        } else {
-            self.as_str()
-        }
-    }
-
     pub fn as_dependencies_filename(&self) -> Option<String> {
         use ScriptLang::*;
         Some(
@@ -473,6 +459,7 @@ pub struct NewScript {
     pub path: String,
     pub parent_hash: Option<ScriptHash>,
     pub summary: String,
+    #[serde(default)]
     pub description: String,
     pub content: String,
     pub schema: Option<Schema>,

backend/windmill-worker/nsjail/run.bun.config.proto

@@ -142,6 +142,13 @@ mount {
     rw: true
 }
 
+mount {
+    src: "{JOB_DIR}/checkpoint.json"
+    dst: "/tmp/{LANG}/checkpoint.json"
+    is_bind: true
+    mandatory: false
+}
+
 mount {
     src: "{JOB_DIR}/result.json"
     dst: "/tmp/{LANG}/result.json"

backend/windmill-worker/src/ai/image_handler.rs

@@ -2,8 +2,8 @@ use base64::Engine;
 use futures;
 use ulid;
 use windmill_common::{client::AuthedClient, error::Error};
-use windmill_types::s3::S3Object;
 use windmill_queue::MiniPulledJob;
+use windmill_types::s3::S3Object;
 
 use crate::ai::types::*;
 

backend/windmill-worker/src/ai/providers/anthropic.rs

@@ -1,14 +1,16 @@
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 use serde_json::value::RawValue;
-use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error};
+use windmill_common::{
+    ai_google::parse_data_url, ai_providers::AIProvider, client::AuthedClient, error::Error,
+};
 
 use crate::ai::{
     image_handler::prepare_messages_for_api,
     query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor},
     sse::{AnthropicSSEParser, SSEParser},
     types::*,
-    utils::{extract_text_content, parse_data_url, should_use_structured_output_tool},
+    utils::{extract_text_content, should_use_structured_output_tool},
 };
 
 /// Anthropic API version for standard API

backend/windmill-worker/src/ai/providers/google_ai.rs

@@ -1,215 +1,21 @@
 use async_trait::async_trait;
-use serde::{Deserialize, Serialize};
-use windmill_common::{client::AuthedClient, error::Error};
+use windmill_common::{
+    ai_google::{
+        openai_messages_to_gemini, openai_tools_to_gemini, GeminiGenerationConfig,
+        GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart,
+        GeminiPredictContent, GeminiTextRequest, GeminiTool,
+    },
+    client::AuthedClient,
+    error::Error,
+};
 
 use crate::ai::{
-    image_handler::download_and_encode_s3_image,
+    image_handler::{download_and_encode_s3_image, prepare_messages_for_api},
     query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor},
     sse::{GeminiSSEParser, SSEParser},
     types::*,
-    utils::parse_data_url,
 };
 
-// ============================================================================
-// Gemini API Types - Shared between text and image
-// ============================================================================
-
-/// Inline data for binary content (images)
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct GeminiInlineData {
-    #[serde(rename = "mimeType")]
-    pub mime_type: String,
-    pub data: String,
-}
-
-/// A part of content - can be text, inline data, function call, or function response
-#[derive(Serialize, Deserialize, Clone, Debug)]
-#[serde(untagged)]
-pub enum GeminiPart {
-    Text {
-        text: String,
-    },
-    InlineData {
-        #[serde(rename = "inlineData")]
-        inline_data: GeminiInlineData,
-    },
-    FunctionCall {
-        #[serde(rename = "functionCall")]
-        function_call: GeminiFunctionCall,
-        /// Thought signature for Gemini 3+ models - required for function calling
-        #[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")]
-        thought_signature: Option<String>,
-    },
-    FunctionResponse {
-        #[serde(rename = "functionResponse")]
-        function_response: GeminiFunctionResponse,
-    },
-}
-
-/// A function call from the model
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct GeminiFunctionCall {
-    pub name: String,
-    pub args: serde_json::Value,
-}
-
-/// A function response to send back to the model
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct GeminiFunctionResponse {
-    pub name: String,
-    pub response: serde_json::Value,
-}
-
-// ============================================================================
-// Gemini Text API Request Types
-// ============================================================================
-
-/// Main request structure for Gemini generateContent
-#[derive(Serialize)]
-pub struct GeminiTextRequest {
-    pub contents: Vec<GeminiContentMessage>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub tools: Option<Vec<GeminiTool>>,
-    #[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")]
-    pub tool_config: Option<GeminiToolConfig>,
-    #[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")]
-    pub system_instruction: Option<GeminiContentMessage>,
-    #[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")]
-    pub generation_config: Option<GeminiGenerationConfig>,
-}
-
-/// Content message with role and parts
-#[derive(Serialize)]
-pub struct GeminiContentMessage {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub role: Option<String>,
-    pub parts: Vec<GeminiPart>,
-}
-
-/// Tool definition - either function declarations or Google Search
-#[derive(Serialize)]
-pub struct GeminiTool {
-    #[serde(
-        rename = "functionDeclarations",
-        skip_serializing_if = "Option::is_none"
-    )]
-    pub function_declarations: Option<Vec<GeminiFunctionDeclaration>>,
-    #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")]
-    pub google_search: Option<serde_json::Value>,
-}
-
-/// Function declaration for tool use
-#[derive(Serialize)]
-pub struct GeminiFunctionDeclaration {
-    pub name: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub description: Option<String>,
-    pub parameters: OpenAPISchema,
-}
-
-/// Tool configuration for controlling function calling behavior
-#[derive(Serialize)]
-pub struct GeminiToolConfig {
-    #[serde(rename = "functionCallingConfig")]
-    pub function_calling_config: GeminiFunctionCallingConfig,
-}
-
-/// Function calling configuration
-#[derive(Serialize)]
-pub struct GeminiFunctionCallingConfig {
-    pub mode: String,
-    #[serde(
-        rename = "allowedFunctionNames",
-        skip_serializing_if = "Option::is_none"
-    )]
-    pub allowed_function_names: Option<Vec<String>>,
-}
-
-/// Generation configuration for output format
-#[derive(Serialize)]
-pub struct GeminiGenerationConfig {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub temperature: Option<f32>,
-    #[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")]
-    pub max_output_tokens: Option<u32>,
-    #[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")]
-    pub response_mime_type: Option<String>,
-    #[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")]
-    pub response_schema: Option<serde_json::Value>,
-}
-
-// ============================================================================
-// Gemini API Response Types
-// ============================================================================
-
-/// Grounding metadata from Google Search
-#[derive(Deserialize)]
-#[allow(dead_code)]
-pub struct GeminiGroundingMetadata {
-    #[serde(rename = "webSearchQueries")]
-    pub web_search_queries: Option<Vec<String>>,
-    #[serde(rename = "groundingChunks")]
-    pub grounding_chunks: Option<Vec<serde_json::Value>>,
-}
-
-// ============================================================================
-// Gemini Image API Types (for Imagen models)
-// ============================================================================
-
-/// Request for image generation (Imagen models)
-#[derive(Serialize)]
-pub struct GeminiImageRequest {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub contents: Option<Vec<GeminiImageContent>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub instances: Option<Vec<GeminiPredictContent>>,
-}
-
-/// Content for image generation
-#[derive(Serialize)]
-pub struct GeminiImageContent {
-    pub parts: Vec<GeminiPart>,
-}
-
-/// Content for Imagen predict endpoint
-#[derive(Serialize)]
-pub struct GeminiPredictContent {
-    pub prompt: String,
-}
-
-/// Response for image generation
-#[derive(Deserialize)]
-pub struct GeminiImageResponse {
-    pub candidates: Option<Vec<GeminiImageCandidate>>,
-    pub predictions: Option<Vec<GeminiPredictCandidate>>,
-}
-
-/// Image candidate from generateContent
-#[derive(Deserialize)]
-pub struct GeminiImageCandidate {
-    pub content: GeminiImageCandidateContent,
-}
-
-/// Content in image candidate
-#[derive(Deserialize)]
-pub struct GeminiImageCandidateContent {
-    pub parts: Vec<GeminiImageCandidatePart>,
-}
-
-/// Part of image candidate
-#[derive(Deserialize)]
-pub struct GeminiImageCandidatePart {
-    #[serde(rename = "inlineData", skip_serializing_if = "Option::is_none")]
-    pub inline_data: Option<GeminiInlineData>,
-}
-
-/// Prediction candidate from Imagen
-#[derive(Deserialize)]
-pub struct GeminiPredictCandidate {
-    #[serde(rename = "bytesBase64Encoded")]
-    pub bytes_base64_encoded: String,
-}
-
 // ============================================================================
 // Query Builder Implementation
 // ============================================================================
@@ -221,34 +27,24 @@ impl GoogleAIQueryBuilder {
         Self
     }
 
-    /// Build a text request using the native Gemini API format
     async fn build_text_request(
         &self,
         args: &BuildRequestArgs<'_>,
         client: &AuthedClient,
         workspace_id: &str,
     ) -> Result<String, Error> {
-        // Convert messages to Gemini format
-        let contents = self
-            .convert_messages_to_gemini(args.messages, client, workspace_id)
-            .await?;
+        let prepared_messages =
+            prepare_messages_for_api(args.messages, client, workspace_id).await?;
+        let (contents, system_instruction) = openai_messages_to_gemini(&prepared_messages);
 
-        // Build tools array
         let tools = self.convert_tools_to_gemini(args.tools, args.has_websearch);
 
-        // Build generation config
         let generation_config = self.build_generation_config(args);
 
-        // Build system instruction from system_prompt
-        let system_instruction = args.system_prompt.map(|s| GeminiContentMessage {
-            role: None,
-            parts: vec![GeminiPart::Text { text: s.to_string() }],
-        });
-
         let request = GeminiTextRequest {
             contents,
             tools,
-            tool_config: None, // Use AUTO mode by default
+            tool_config: None,
             system_instruction,
             generation_config,
         };
@@ -257,7 +53,6 @@ impl GoogleAIQueryBuilder {
             .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e)))
     }
 
-    /// Build an image generation request
     async fn build_image_request(
         &self,
         args: &BuildRequestArgs<'_>,
@@ -267,7 +62,6 @@ impl GoogleAIQueryBuilder {
         let is_imagen = args.model.contains("imagen");
 
         let request = if is_imagen {
-            // For Imagen models, use simple prompt format
             GeminiImageRequest {
                 instances: Some(vec![GeminiPredictContent {
                     prompt: args.user_message.trim().to_string(),
@@ -275,7 +69,6 @@ impl GoogleAIQueryBuilder {
                 contents: None,
             }
         } else {
-            // For Gemini models with image generation, build parts
             let mut parts = vec![GeminiPart::Text { text: args.user_message.trim().to_string() }];
 
             if let Some(system_prompt) = args.system_prompt {
@@ -285,7 +78,6 @@ impl GoogleAIQueryBuilder {
                 );
             }
 
-            // Add input images if provided
             if let Some(images) = args.images {
                 for image in images.iter() {
                     if !image.s3.is_empty() {
@@ -308,218 +100,39 @@ impl GoogleAIQueryBuilder {
             .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e)))
     }
 
-    /// Convert OpenAI-format messages to Gemini format
-    async fn convert_messages_to_gemini(
-        &self,
-        messages: &[OpenAIMessage],
-        client: &AuthedClient,
-        workspace_id: &str,
-    ) -> Result<Vec<GeminiContentMessage>, Error> {
-        let mut gemini_messages = Vec::new();
-
-        for msg in messages {
-            match msg.role.as_str() {
-                "system" => {
-                    // Skip - handled via args.system_prompt in build_text_request
-                }
-                "tool" => {
-                    // Handle tool responses
-                    if let (Some(tool_call_id), Some(content)) = (&msg.tool_call_id, &msg.content) {
-                        let func_name = self.find_function_name_by_id(messages, tool_call_id);
-                        let response_text = match content {
-                            OpenAIContent::Text(text) => text.clone(),
-                            OpenAIContent::Parts(parts) => parts
-                                .iter()
-                                .filter_map(|p| match p {
-                                    ContentPart::Text { text } => Some(text.clone()),
-                                    _ => None,
-                                })
-                                .collect::<Vec<_>>()
-                                .join(" "),
-                        };
-
-                        gemini_messages.push(GeminiContentMessage {
-                            role: Some("user".to_string()),
-                            parts: vec![GeminiPart::FunctionResponse {
-                                function_response: GeminiFunctionResponse {
-                                    name: func_name,
-                                    response: serde_json::json!({ "result": response_text }),
-                                },
-                            }],
-                        });
-                    }
-                }
-                _ => {
-                    // Handle user/assistant messages
-                    let role = match msg.role.as_str() {
-                        "assistant" => "model",
-                        _ => "user",
-                    };
-
-                    let mut parts = Vec::new();
-
-                    // Handle regular content
-                    if let Some(content) = &msg.content {
-                        let content_parts = self
-                            .convert_content_to_parts(&Some(content.clone()), client, workspace_id)
-                            .await?;
-                        parts.extend(content_parts);
-                    }
-
-                    // Handle tool calls from assistant
-                    if let Some(tool_calls) = &msg.tool_calls {
-                        for tc in tool_calls {
-                            let args: serde_json::Value =
-                                serde_json::from_str(&tc.function.arguments).unwrap_or_default();
-                            // Extract thought_signature from extra_content if present
-                            let thought_signature = tc
-                                .extra_content
-                                .as_ref()
-                                .and_then(|ec| ec.google.as_ref())
-                                .and_then(|g| g.thought_signature.clone());
-                            parts.push(GeminiPart::FunctionCall {
-                                function_call: GeminiFunctionCall {
-                                    name: tc.function.name.clone(),
-                                    args,
-                                },
-                                thought_signature,
-                            });
-                        }
-                    }
-
-                    if !parts.is_empty() {
-                        gemini_messages
-                            .push(GeminiContentMessage { role: Some(role.to_string()), parts });
-                    }
-                }
-            }
-        }
-
-        Ok(gemini_messages)
-    }
-
-    /// Convert OpenAI content to Gemini parts
-    async fn convert_content_to_parts(
-        &self,
-        content: &Option<OpenAIContent>,
-        client: &AuthedClient,
-        workspace_id: &str,
-    ) -> Result<Vec<GeminiPart>, Error> {
-        let mut parts = Vec::new();
-
-        if let Some(content) = content {
-            match content {
-                OpenAIContent::Text(text) => {
-                    if !text.is_empty() {
-                        parts.push(GeminiPart::Text { text: text.clone() });
-                    }
-                }
-                OpenAIContent::Parts(content_parts) => {
-                    for part in content_parts {
-                        match part {
-                            ContentPart::Text { text } => {
-                                if !text.is_empty() {
-                                    parts.push(GeminiPart::Text { text: text.clone() });
-                                }
-                            }
-                            ContentPart::ImageUrl { image_url } => {
-                                // Parse data URL format: data:mime_type;base64,data
-                                if let Some((mime_type, data)) = parse_data_url(&image_url.url) {
-                                    parts.push(GeminiPart::InlineData {
-                                        inline_data: GeminiInlineData { mime_type, data },
-                                    });
-                                }
-                            }
-                            ContentPart::S3Object { s3_object } => {
-                                if !s3_object.s3.is_empty() {
-                                    let (mime_type, data) = download_and_encode_s3_image(
-                                        s3_object,
-                                        client,
-                                        workspace_id,
-                                    )
-                                    .await?;
-                                    parts.push(GeminiPart::InlineData {
-                                        inline_data: GeminiInlineData { mime_type, data },
-                                    });
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        Ok(parts)
-    }
-
-    /// Find function name by tool call ID from previous messages
-    fn find_function_name_by_id(&self, messages: &[OpenAIMessage], tool_call_id: &str) -> String {
-        for msg in messages {
-            if let Some(tool_calls) = &msg.tool_calls {
-                for tc in tool_calls {
-                    if tc.id == tool_call_id {
-                        return tc.function.name.clone();
-                    }
-                }
-            }
-        }
-        "unknown_function".to_string()
-    }
-
-    /// Convert OpenAI tools to Gemini format
+    /// Convert OpenAI tool definitions to Gemini format.
+    ///
+    /// Sanitizes each tool's JSON schema for Google compatibility before delegating
+    /// to the shared [`openai_tools_to_gemini`] function.
     fn convert_tools_to_gemini(
         &self,
         tools: Option<&[ToolDef]>,
         has_websearch: bool,
     ) -> Option<Vec<GeminiTool>> {
-        let mut gemini_tools = Vec::new();
-
-        // Add function declarations
-        if let Some(tool_defs) = tools {
-            let declarations: Vec<GeminiFunctionDeclaration> = tool_defs
-                .iter()
-                .filter_map(|t| {
-                    // Deserialize RawValue into OpenAPISchema, sanitize, then use
-                    let mut schema: OpenAPISchema =
-                        serde_json::from_str(t.function.parameters.get()).ok()?;
-                    schema.sanitize_for_google();
-
-                    Some(GeminiFunctionDeclaration {
-                        name: t.function.name.clone(),
-                        description: t.function.description.clone(),
-                        parameters: schema,
-                    })
-                })
-                .collect();
-
-            if !declarations.is_empty() {
-                gemini_tools.push(GeminiTool {
-                    function_declarations: Some(declarations),
-                    google_search: None,
-                });
+        let Some(tool_defs) = tools else {
+            if has_websearch {
+                return Some(vec![GeminiTool {
+                    function_declarations: None,
+                    google_search: Some(serde_json::json!({})),
+                }]);
             }
-        }
+            return None;
+        };
 
-        // Add Google Search tool if enabled
-        if has_websearch {
-            gemini_tools.push(GeminiTool {
-                function_declarations: None,
-                google_search: Some(serde_json::json!({})),
-            });
-        }
+        let tool_params: Vec<serde_json::Value> = tool_defs
+            .iter()
+            .map(|t| {
+                let mut schema: OpenAPISchema =
+                    serde_json::from_str(t.function.parameters.get()).unwrap_or_default();
+                schema.sanitize_for_google();
+                serde_json::to_value(&schema).unwrap_or_default()
+            })
+            .collect();
 
-        if gemini_tools.is_empty() {
-            None
-        } else {
-            Some(gemini_tools)
-        }
+        openai_tools_to_gemini(tool_defs, &tool_params, has_websearch)
     }
 
-    /// Build generation config for structured output and other settings
-    fn build_generation_config(
-        &self,
-        args: &BuildRequestArgs<'_>,
-    ) -> Option<GeminiGenerationConfig> {
+    fn build_generation_config(&self, args: &BuildRequestArgs<'_>) -> Option<GeminiGenerationConfig> {
         let has_output_schema = args
             .output_schema
             .and_then(|s| s.properties.as_ref())
@@ -529,15 +142,11 @@ impl GoogleAIQueryBuilder {
         let (response_mime_type, response_schema) = if has_output_schema {
             let mut schema = args.output_schema.unwrap().clone();
             schema.sanitize_for_google();
-            (
-                Some("application/json".to_string()),
-                serde_json::to_value(&schema).ok(),
-            )
+            (Some("application/json".to_string()), serde_json::to_value(&schema).ok())
         } else {
             (None, None)
         };
 
-        // Only create config if there's something to configure
         if args.temperature.is_some() || args.max_tokens.is_some() || response_mime_type.is_some() {
             Some(GeminiGenerationConfig {
                 temperature: args.temperature,
@@ -554,7 +163,6 @@ impl GoogleAIQueryBuilder {
 #[async_trait]
 impl QueryBuilder for GoogleAIQueryBuilder {
     fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool {
-        // Google AI supports tools only for text output
         matches!(output_type, OutputType::Text)
     }
 
@@ -578,7 +186,6 @@ impl QueryBuilder for GoogleAIQueryBuilder {
             Error::internal_err(format!("Failed to parse Gemini image response: {}", e))
         })?;
 
-        // First, check Gemini models (candidates -> content -> parts -> inline_data)
         let image_data_from_gemini = gemini_response.candidates.as_ref().and_then(|candidates| {
             candidates.iter().find_map(|candidate| {
                 candidate
@@ -589,13 +196,11 @@ impl QueryBuilder for GoogleAIQueryBuilder {
             })
         });
 
-        // Then, check Imagen models (predictions -> bytes_base64_encoded)
         let image_data_from_imagen = gemini_response
             .predictions
             .as_ref()
             .and_then(|predictions| predictions.first().map(|p| &p.bytes_base64_encoded));
 
-        // Image data, preferring Gemini first then Imagen models
         let image_data = image_data_from_gemini.or(image_data_from_imagen);
 
         match image_data {
@@ -627,7 +232,6 @@ impl QueryBuilder for GoogleAIQueryBuilder {
             ..
         } = gemini_sse_parser;
 
-        // Send tool call arguments events for accumulated tool calls
         for tool_call in accumulated_tool_calls.values() {
             let event = StreamingEvent::ToolCallArguments {
                 call_id: tool_call.id.clone(),
@@ -637,7 +241,6 @@ impl QueryBuilder for GoogleAIQueryBuilder {
             stream_event_processor.send(event, &mut events_str).await?;
         }
 
-        // Convert Gemini usage metadata to TokenUsage
         let usage = gemini_usage.map(|u| {
             TokenUsage::new(
                 u.prompt_token_count,
@@ -647,11 +250,7 @@ impl QueryBuilder for GoogleAIQueryBuilder {
         });
 
         Ok(ParsedResponse::Text {
-            content: if accumulated_content.is_empty() {
-                None
-            } else {
-                Some(accumulated_content)
-            },
+            content: if accumulated_content.is_empty() { None } else { Some(accumulated_content) },
             tool_calls: accumulated_tool_calls.into_values().collect(),
             events_str: Some(events_str),
             annotations,
@@ -663,17 +262,11 @@ impl QueryBuilder for GoogleAIQueryBuilder {
     fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String {
         match output_type {
             OutputType::Text => {
-                format!(
-                    "{}/models/{}:streamGenerateContent?alt=sse",
-                    base_url, model
-                )
+                format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model)
             }
             OutputType::Image => {
-                let url_suffix = if model.contains("imagen") {
-                    "predict"
-                } else {
-                    "generateContent"
-                };
+                let url_suffix =
+                    if model.contains("imagen") { "predict" } else { "generateContent" };
                 format!("{}/models/{}:{}", base_url, model, url_suffix)
             }
         }
@@ -685,7 +278,6 @@ impl QueryBuilder for GoogleAIQueryBuilder {
         _base_url: &str,
         _output_type: &OutputType,
     ) -> Vec<(&'static str, String)> {
-        // Native Gemini API always uses x-goog-api-key
         vec![("x-goog-api-key", api_key.to_string())]
     }
 }

backend/windmill-worker/src/ai/query_builder.rs

@@ -1,7 +1,5 @@
 use async_trait::async_trait;
-use windmill_common::{
-    client::AuthedClient, error::Error, worker::Connection,
-};
+use windmill_common::{client::AuthedClient, error::Error, worker::Connection};
 use windmill_queue::MiniPulledJob;
 use windmill_types::s3::S3Object;
 

backend/windmill-worker/src/ai/sse.rs

@@ -5,15 +5,18 @@ use reqwest::Response;
 use serde::Deserialize;
 use serde_json;
 use tokio_stream::StreamExt;
-use windmill_common::{error::Error, utils::rd_string};
+use windmill_common::{
+    ai_google::{parse_gemini_sse_event, GeminiUsageMetadata},
+    ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall},
+    error::Error,
+    utils::rd_string,
+};
 
 use crate::ai::{
     query_builder::StreamEventProcessor,
     types::{StreamingEvent, UrlCitation},
 };
 
-use windmill_common::ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall};
-
 #[derive(Deserialize)]
 pub struct OpenAIChoiceDeltaToolCallFunction {
     pub name: Option<String>,
@@ -457,96 +460,19 @@ impl SSEParser for AnthropicSSEParser {
 // Gemini SSE Parser
 // ============================================================================
 
-/// Gemini streaming response part - can be text or function call
-#[derive(Deserialize, Debug)]
-pub struct GeminiSSEPart {
-    #[serde(default)]
-    pub text: Option<String>,
-    #[serde(rename = "functionCall")]
-    pub function_call: Option<GeminiSSEFunctionCall>,
-    /// Thought signature for Gemini 3+ models - required for function calling
-    #[serde(rename = "thoughtSignature")]
-    pub thought_signature: Option<String>,
-}
-
-/// Function call in Gemini streaming response
-#[derive(Deserialize, Debug)]
-pub struct GeminiSSEFunctionCall {
-    pub name: String,
-    pub args: serde_json::Value,
-}
-
-/// Content in Gemini streaming candidate
-#[derive(Deserialize, Debug)]
-pub struct GeminiSSEContent {
-    pub parts: Option<Vec<GeminiSSEPart>>,
-}
-
-/// Web reference in Gemini grounding chunk
-#[derive(Deserialize, Debug)]
-pub struct GeminiGroundingChunkWeb {
-    pub uri: String,
-    #[serde(default)]
-    pub title: Option<String>,
-}
-
-/// Grounding chunk from Gemini web search
-#[derive(Deserialize, Debug)]
-pub struct GeminiGroundingChunk {
-    pub web: Option<GeminiGroundingChunkWeb>,
-}
-
-/// Grounding metadata from Gemini web search
-#[derive(Deserialize, Debug)]
-pub struct GeminiGroundingMetadata {
-    #[serde(rename = "groundingChunks", default)]
-    pub grounding_chunks: Vec<GeminiGroundingChunk>,
-    #[serde(rename = "webSearchQueries", default)]
-    pub web_search_queries: Vec<String>,
-}
-
-/// Candidate in Gemini streaming response
-#[derive(Deserialize, Debug)]
-pub struct GeminiSSECandidate {
-    pub content: Option<GeminiSSEContent>,
-    #[serde(rename = "finishReason")]
-    #[allow(dead_code)]
-    pub finish_reason: Option<String>,
-    #[serde(rename = "groundingMetadata")]
-    pub grounding_metadata: Option<GeminiGroundingMetadata>,
-}
-
-/// Gemini usage metadata from SSE response
-#[derive(Deserialize, Debug, Clone)]
-pub struct GeminiUsageMetadata {
-    #[serde(rename = "promptTokenCount", default)]
-    pub prompt_token_count: Option<i32>,
-    #[serde(rename = "candidatesTokenCount", default)]
-    pub candidates_token_count: Option<i32>,
-    #[serde(rename = "totalTokenCount", default)]
-    pub total_token_count: Option<i32>,
-}
-
-/// Gemini SSE event structure
-#[derive(Deserialize, Debug)]
-pub struct GeminiSSEEvent {
-    pub candidates: Option<Vec<GeminiSSECandidate>>,
-    #[serde(rename = "usageMetadata")]
-    pub usage_metadata: Option<GeminiUsageMetadata>,
-}
-
-/// Gemini SSE Parser for streaming responses
+/// Accumulates Gemini streaming events and converts them into the worker's
+/// internal [`OpenAIToolCall`] / [`StreamingEvent`] representation.
+///
+/// The actual SSE parsing is delegated to [`parse_gemini_sse_event`] from
+/// `windmill_common::ai_google` so the logic can be shared with the API proxy.
 pub struct GeminiSSEParser {
     pub accumulated_content: String,
     pub accumulated_tool_calls: HashMap<i64, OpenAIToolCall>,
     pub events_str: String,
     pub stream_event_processor: StreamEventProcessor,
     tool_call_index: i64,
-    /// Collected URL citation annotations from web search
     pub annotations: Vec<UrlCitation>,
-    /// Whether web search was used in this response
     pub used_websearch: bool,
-    /// Token usage from usageMetadata
     pub usage: Option<GeminiUsageMetadata>,
 }
 
@@ -567,101 +493,57 @@ impl GeminiSSEParser {
 
 impl SSEParser for GeminiSSEParser {
     async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
-        let event: Option<GeminiSSEEvent> = serde_json::from_str(data)
-            .inspect_err(|e| {
-                tracing::error!("Failed to parse SSE as a Gemini event {}: {}", data, e);
-            })
-            .ok();
+        let Some(parsed) = parse_gemini_sse_event(data)? else {
+            return Ok(());
+        };
 
-        if let Some(event) = event {
-            if let Some(candidates) = event.candidates {
-                for candidate in candidates {
-                    if let Some(content) = candidate.content {
-                        if let Some(parts) = content.parts {
-                            for part in parts {
-                                // Handle text content
-                                if let Some(text) = part.text {
-                                    if !text.is_empty() {
-                                        self.accumulated_content.push_str(&text);
-                                        let event = StreamingEvent::TokenDelta { content: text };
-                                        self.stream_event_processor
-                                            .send(event, &mut self.events_str)
-                                            .await?;
-                                    }
-                                }
+        if let Some(text) = parsed.text {
+            self.accumulated_content.push_str(&text);
+            self.stream_event_processor
+                .send(StreamingEvent::TokenDelta { content: text }, &mut self.events_str)
+                .await?;
+        }
 
-                                // Handle function calls
-                                if let Some(function_call) = part.function_call {
-                                    let call_id = format!("call_{}", rd_string(24));
-                                    let idx = self.tool_call_index;
-                                    self.tool_call_index += 1;
+        for tool_call in parsed.tool_calls {
+            let call_id = format!("call_{}", rd_string(24));
+            let idx = self.tool_call_index;
+            self.tool_call_index += 1;
 
-                                    // Send tool call start event
-                                    let event = StreamingEvent::ToolCall {
-                                        call_id: call_id.clone(),
-                                        function_name: function_call.name.clone(),
-                                    };
-                                    self.stream_event_processor
-                                        .send(event, &mut self.events_str)
-                                        .await?;
+            self.stream_event_processor
+                .send(
+                    StreamingEvent::ToolCall {
+                        call_id: call_id.clone(),
+                        function_name: tool_call.name.clone(),
+                    },
+                    &mut self.events_str,
+                )
+                .await?;
 
-                                    // Build extra_content with thought_signature if present
-                                    let extra_content =
-                                        part.thought_signature.map(|sig| ExtraContent {
-                                            google: Some(GoogleExtraContent {
-                                                thought_signature: Some(sig),
-                                            }),
-                                        });
+            let extra_content = tool_call.thought_signature.map(|sig| ExtraContent {
+                google: Some(GoogleExtraContent { thought_signature: Some(sig) }),
+            });
 
-                                    // Store accumulated tool call
-                                    self.accumulated_tool_calls.insert(
-                                        idx,
-                                        OpenAIToolCall {
-                                            id: call_id,
-                                            function: OpenAIFunction {
-                                                name: function_call.name,
-                                                arguments: serde_json::to_string(
-                                                    &function_call.args,
-                                                )
-                                                .unwrap_or_else(|_| "{}".to_string()),
-                                            },
-                                            r#type: "function".to_string(),
-                                            extra_content,
-                                        },
-                                    );
-                                }
-                            }
-                        }
-                    }
+            self.accumulated_tool_calls.insert(
+                idx,
+                OpenAIToolCall {
+                    id: call_id,
+                    function: OpenAIFunction {
+                        name: tool_call.name,
+                        arguments: serde_json::to_string(&tool_call.args)
+                            .unwrap_or_else(|_| "{}".to_string()),
+                    },
+                    r#type: "function".to_string(),
+                    extra_content,
+                },
+            );
+        }
 
-                    // Handle grounding metadata (web search results)
-                    if let Some(ref grounding_metadata) = candidate.grounding_metadata {
-                        // Set used_websearch if there are search queries or grounding chunks
-                        if !grounding_metadata.web_search_queries.is_empty()
-                            || !grounding_metadata.grounding_chunks.is_empty()
-                        {
-                            self.used_websearch = true;
-                        }
-
-                        // Extract citations from grounding chunks
-                        for chunk in &grounding_metadata.grounding_chunks {
-                            if let Some(ref web) = chunk.web {
-                                self.annotations.push(UrlCitation {
-                                    start_index: 0, // Gemini doesn't provide character indices
-                                    end_index: 0,
-                                    url: web.uri.clone(),
-                                    title: web.title.clone(),
-                                });
-                            }
-                        }
-                    }
-                }
-            }
-
-            // Extract usage metadata
-            if let Some(usage_metadata) = event.usage_metadata {
-                self.usage = Some(usage_metadata);
-            }
+        self.annotations.extend(parsed.annotations);
+        if parsed.used_websearch {
+            self.used_websearch = true;
+        }
+        if let Some(usage) = parsed.usage {
+            self.usage = Some(usage);
         }
 
         Ok(())

backend/windmill-worker/src/ai/types.rs

@@ -20,8 +20,8 @@ use windmill_common::{
     flow_status::AgentAction,
     flows::FlowModule,
 };
-use windmill_types::s3::S3Object;
 use windmill_parser::Typ;
+use windmill_types::s3::S3Object;
 
 // Re-export shared types from windmill_common::ai_types
 pub use windmill_common::ai_types::{
@@ -1603,10 +1603,7 @@ mod tests {
 
         schema.sanitize_for_google();
 
-        assert!(
-            schema.multiple_of.is_none(),
-            "multipleOf should be removed"
-        );
+        assert!(schema.multiple_of.is_none(), "multipleOf should be removed");
     }
 
     #[test]
@@ -1639,7 +1636,10 @@ mod tests {
         assert!(schema.default.is_none());
 
         let value_prop = schema.properties.as_ref().unwrap().get("value").unwrap();
-        assert!(value_prop.default.is_none(), "nested default should be removed");
+        assert!(
+            value_prop.default.is_none(),
+            "nested default should be removed"
+        );
         assert!(
             value_prop.exclusive_minimum.is_none(),
             "nested exclusiveMinimum should be removed"
@@ -1652,7 +1652,10 @@ mod tests {
             value_prop.multiple_of.is_none(),
             "nested multipleOf should be removed"
         );
-        assert!(value_prop.r#const.is_none(), "nested const should be removed");
+        assert!(
+            value_prop.r#const.is_none(),
+            "nested const should be removed"
+        );
 
         assert!(schema.properties.is_some());
         assert!(matches!(&schema.r#type, Some(SchemaType::Single(t)) if t == "object"));

backend/windmill-worker/src/ai/utils.rs

@@ -731,16 +731,3 @@ pub fn extract_text_content(content: &OpenAIContent) -> String {
             .join(""),
     }
 }
-
-/// Parse a data URL to extract media type and base64 data
-/// Format: data:mime_type;base64,data
-/// Returns (media_type, data) tuple if successful
-pub fn parse_data_url(url: &str) -> Option<(String, String)> {
-    if !url.starts_with("data:") {
-        return None;
-    }
-    let rest = url.strip_prefix("data:")?;
-    let (header, data) = rest.split_once(",")?;
-    let media_type = header.strip_suffix(";base64")?;
-    Some((media_type.to_string(), data.to_string()))
-}

backend/windmill-worker/src/bash_executor.rs

@@ -215,7 +215,10 @@ exit $exit_status
             .current_dir(job_dir)
             .env_clear()
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?)
+            .envs(
+                get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn)
+                    .await?,
+            )
             .env("PATH", PATH_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .args(cmd_args)
@@ -241,7 +244,10 @@ exit $exit_status
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?)
+            .envs(
+                get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn)
+                    .await?,
+            )
             .env("PATH", PATH_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .env("HOME", HOME_ENV.as_str())

backend/windmill-worker/src/bigquery_executor.rs

@@ -6,9 +6,9 @@ use reqwest::Client;
 use serde_json::{json, value::RawValue, Value};
 use windmill_common::client::AuthedClient;
 use windmill_common::error::to_anyhow;
-use windmill_object_store::convert_json_line_stream;
 use windmill_common::worker::{Connection, SqlResultCollectionStrategy};
 use windmill_common::{error::Error, worker::to_raw_value};
+use windmill_object_store::convert_json_line_stream;
 use windmill_parser_sql::{
     parse_bigquery_sig, parse_db_resource, parse_s3_mode, parse_sql_blocks,
     parse_sql_statement_named_params,

backend/windmill-worker/src/common.rs

@@ -563,7 +563,6 @@ pub async fn update_worker_ping_for_failed_init_script(
                         wm_memory_usage: None,
                         job_isolation: None,
                         native_mode: None,
-                        uses_batch_http_pull: None,
                         ping_type: PingType::InitScript,
                     },
                 )

backend/windmill-worker/src/csharp_executor.rs

@@ -600,7 +600,10 @@ pub async fn handle_csharp_job(
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?)
+            .envs(
+                get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn)
+                    .await?,
+            )
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
@@ -633,7 +636,10 @@ pub async fn handle_csharp_job(
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?)
+            .envs(
+                get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn)
+                    .await?,
+            )
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
             .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR)

backend/windmill-worker/src/deno_executor.rs

@@ -121,11 +121,13 @@ async fn get_common_deno_proc_envs(
     }
 
     // Add proxy envs (including OTEL tracing proxy if enabled for deno)
-    for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno)
-        .await
-        .unwrap_or_default()
-    {
-        deno_envs.insert(k.to_string(), v);
+    if let Some(conn) = conn {
+        for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno, job_id, w_id, conn)
+            .await
+            .unwrap_or_default()
+        {
+            deno_envs.insert(k.to_string(), v);
+        }
     }
 
     return deno_envs;

backend/windmill-worker/src/go_executor.rs

@@ -354,7 +354,7 @@ func Run(req Req) (interface{{}}, error){{
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?)
+            .envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?)
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
@@ -375,7 +375,7 @@ func Run(req Req) (interface{{}}, error){{
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?)
+            .envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?)
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)

backend/windmill-worker/src/java_executor.rs

@@ -618,6 +618,7 @@ async fn run<'a>(
             .env("BASE_INTERNAL_URL", base_internal_url)
             .envs(envs)
             .envs(reserved_variables)
+            .envs(crate::get_otel_context_envs(&job.id))
             .args(vec![
                 "--config",
                 "run.config.proto",
@@ -675,7 +676,8 @@ async fn run<'a>(
             .env("HOME", &*JAVA_HOME_DIR)
             .env("BASE_INTERNAL_URL", base_internal_url)
             .envs(envs)
-            .envs(reserved_variables);
+            .envs(reserved_variables)
+            .envs(crate::get_otel_context_envs(&job.id));
         if metadata(TRUST_STORE_PATH.clone()).await.is_ok() {
             cmd.args(&[
                 &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH),

backend/windmill-worker/src/lib.rs

@@ -73,6 +73,7 @@ mod universal_pkg_installer;
 #[cfg(feature = "private")]
 mod volume_ee;
 mod volume_oss;
+pub mod wac_executor;
 mod worker;
 mod worker_flow;
 mod worker_lockfiles;

backend/windmill-worker/src/nu_executor.rs

@@ -264,7 +264,7 @@ async fn run<'a>(
             .env("BASE_INTERNAL_URL", base_internal_url)
             .envs(envs)
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?)
+            .envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?)
             .args(vec![
                 "--config",
                 "run.config.proto",
@@ -303,7 +303,7 @@ async fn run<'a>(
             .env("BASE_INTERNAL_URL", base_internal_url)
             .envs(envs)
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?)
+            .envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?)
             // TODO(v1):
             // "--plugins",
             // &format!(

backend/windmill-worker/src/php_executor.rs

@@ -22,7 +22,7 @@ use crate::{
         get_reserved_variables, read_result, start_child_process, MaybeLock, OccupancyMetrics,
     },
     handle_child::handle_child,
-    COMPOSER_CACHE_DIR, COMPOSER_PATH, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH,
+    is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH,
 };
 use windmill_common::client::AuthedClient;
 
@@ -316,6 +316,7 @@ try {{
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
+            .envs(crate::get_otel_context_envs(&job.id))
             .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR)
             .env("BASE_INTERNAL_URL", base_internal_url)
             .args(args)
@@ -332,6 +333,7 @@ try {{
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
+            .envs(crate::get_otel_context_envs(&job.id))
             .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR)
             .env("BASE_INTERNAL_URL", base_internal_url)
             .stdin(Stdio::null())

backend/windmill-worker/src/python_executor.rs

@@ -567,6 +567,9 @@ pub async fn handle_python_job(
 
     let annotations = PythonAnnotations::parse(inner_content);
 
+    let is_wac_v2 = job.script_entrypoint_override.is_none()
+        && crate::wac_executor::is_wac_v2_py(inner_content);
+
     if annotations.sandbox && NSJAIL_AVAILABLE.is_none() {
         return Err(Error::ExecutionErr(
             "Script has #sandbox annotation but nsjail is not available on this worker. \
@@ -677,8 +680,68 @@ pub async fn handle_python_job(
         String::new()
     };
     let main_override = main_name.unwrap_or_else(|| "main".to_string());
-    let wrapper_content: String = format!(
-        r#"
+    let wrapper_content: String = if is_wac_v2 {
+        format!(
+            r#"
+import os
+import json
+{import_loader}
+{import_base64}
+{import_datetime}
+import traceback
+import sys
+from {module_dir_dot} import {last} as inner_script
+from wmill.client import _run_workflow
+
+with open("args.json") as f:
+    kwargs = json.load(f, strict=False)
+args = {{}}
+{transforms}
+
+with open("checkpoint.json") as f:
+    checkpoint = json.load(f, strict=False)
+
+result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json")
+
+# Find the @workflow-decorated function
+workflow_fn = None
+for name in dir(inner_script):
+    obj = getattr(inner_script, name)
+    if callable(obj) and getattr(obj, '_is_workflow', False):
+        workflow_fn = obj
+        break
+
+if workflow_fn is None:
+    raise ValueError("No @workflow function found in script")
+
+for k, v in list(args.items()):
+    if v == '<function call>':
+        del args[k]
+
+try:
+    output = _run_workflow(workflow_fn, checkpoint, args)
+    output_json = json.dumps(output, separators=(',', ':'), default=str)
+    with open(result_json, 'w') as f:
+        f.write(output_json)
+except BaseException as e:
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    tb = traceback.format_tb(exc_traceback)
+    with open(result_json, 'w') as f:
+        err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}
+        extra = e.__dict__
+        if extra and len(extra) > 0:
+            err['extra'] = extra
+        flow_node_id = os.environ.get('WM_FLOW_STEP_ID')
+        if flow_node_id:
+            err['step_id'] = flow_node_id
+        err_json = json.dumps(err, separators=(',', ':'), default=str).replace('\n', '')
+        f.write(err_json)
+        sys.exit(1)
+"#,
+        )
+    } else {
+        format!(
+            r#"
 import os
 import json
 {import_loader}
@@ -751,9 +814,26 @@ except BaseException as e:
         f.write(err_json)
         sys.exit(1)
 "#,
-    );
+        )
+    };
     write_file(job_dir, "wrapper.py", &wrapper_content)?;
 
+    // For WAC v2, write checkpoint.json before python runs.
+    if is_wac_v2 {
+        if let Connection::Sql(db) = conn {
+            let checkpoint = crate::wac_executor::load_checkpoint(db, &job.id).await?;
+            let checkpoint =
+                crate::wac_executor::prepare_checkpoint_for_resume(db, &job.id, checkpoint).await?;
+
+            let checkpoint_json = serde_json::to_string(&checkpoint).map_err(|e| {
+                error::Error::internal_err(format!("Failed to serialize checkpoint: {e}"))
+            })?;
+            write_file(job_dir, "checkpoint.json", &checkpoint_json)?;
+        } else {
+            write_file(job_dir, "checkpoint.json", r#"{"completed_steps":{}}"#)?;
+        }
+    }
+
     tracing::debug!("Finished writing wrapper");
 
     let mut reserved_variables =
@@ -841,7 +921,10 @@ mount {{
             .env_clear()
             // inject PYTHONPATH here - for some reason I had to do it in nsjail conf
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?)
+            .envs(
+                get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn)
+                    .await?,
+            )
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
@@ -867,7 +950,10 @@ mount {{
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?)
+            .envs(
+                get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn)
+                    .await?,
+            )
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
@@ -930,7 +1016,15 @@ mount {{
         *new_args = Some(args.clone());
     }
 
-    read_result(job_dir, handle_result.result_stream).await
+    let result = read_result(job_dir, handle_result.result_stream).await?;
+
+    // WAC v2 post-execution: parse output and handle dispatch/suspend.
+    // Box::pin to avoid bloating handle_python_job's async state machine (stack overflow).
+    if is_wac_v2 {
+        return Box::pin(crate::bun_executor::handle_wac_v2_output(result, job, conn)).await;
+    }
+
+    Ok(result)
 }
 
 async fn prepare_wrapper(

backend/windmill-worker/src/result_processor.rs

@@ -665,7 +665,7 @@ pub async fn process_completed_job(
 
         add_time!(bench, "pre add_completed_job");
 
-        let (_, duration) = add_completed_job(
+        let (_, duration, wac_job_ids) = add_completed_job(
             db,
             &job,
             true,
@@ -717,6 +717,29 @@ pub async fn process_completed_job(
                 }
                 return Ok(r);
             }
+        } else if let Some(parent_job) = parent_job {
+            // wac_job_ids is piggybacked from the duration write in
+            // add_completed_job — no extra query needed.
+            if let Some(job_ids) = wac_job_ids {
+                if let Ok(Some(_)) = handle_wac_child_completion(
+                    db,
+                    &job_id,
+                    parent_job,
+                    &workspace_id,
+                    result,
+                    true,
+                    job_ids,
+                )
+                .await
+                {
+                    if let Some(done_tx) = done_tx {
+                        done_tx
+                            .send(())
+                            .expect("done receiver should still be alive");
+                    }
+                    return Ok(None);
+                }
+            }
         }
     } else {
         let result = add_completed_job_error(
@@ -770,11 +793,229 @@ pub async fn process_completed_job(
                 }
                 return Ok(r);
             }
+        } else if let Some(parent_job) = job.parent_job {
+            // WAC child failed — query job_ids from parent (errors are rare,
+            // so the extra read is acceptable here).
+            let job_ids_json: Option<Option<Value>> = sqlx::query_scalar(
+                "SELECT workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \
+                 FROM v2_job_status WHERE id = $1",
+            )
+            .bind(&parent_job)
+            .fetch_optional(db)
+            .await?;
+            if let Some(Some(job_ids)) = job_ids_json {
+                let err_result = Arc::new(serde_json::value::to_raw_value(&result).unwrap());
+                if let Ok(Some(_)) = handle_wac_child_completion(
+                    db,
+                    &job.id,
+                    parent_job,
+                    &job.workspace_id,
+                    err_result,
+                    false,
+                    job_ids,
+                )
+                .await
+                {
+                    if let Some(done_tx) = done_tx {
+                        done_tx
+                            .send(())
+                            .expect("done receiver should still be alive");
+                    }
+                    return Ok(None);
+                }
+            }
         }
     }
     return Ok(None);
 }
 
+/// Handle a WAC v2 child job completion.
+/// Returns Ok(Some(())) if the parent was a WAC job and was handled,
+/// Ok(None) if the parent is not a WAC job (caller should fall through).
+///
+/// CONCURRENCY: Multiple parallel children may complete simultaneously on
+/// different workers.  We use atomic SQL operations throughout:
+///   - `completed_steps` is merged via `jsonb_set(... || jsonb_build_object(...))`
+///     — PostgreSQL serialises concurrent UPDATEs on the same row, so each
+///     worker sees the previous worker's writes.
+///   - The suspend counter (set to N at dispatch time) is decremented atomically
+///     with `RETURNING` to determine the "all done" condition.
+pub(crate) async fn handle_wac_child_completion(
+    db: &DB,
+    child_job_id: &Uuid,
+    parent_job_id: Uuid,
+    workspace_id: &str,
+    result: Arc<Box<RawValue>>,
+    success: bool,
+    job_ids_value: Value,
+) -> error::Result<Option<()>> {
+    let job_ids = match job_ids_value {
+        Value::Object(m) => m,
+        _ => return Ok(None), // Not a WAC parent or no pending steps
+    };
+
+    let child_id_str = child_job_id.to_string();
+    let step_key = job_ids.iter().find_map(|(key, val)| {
+        if val.as_str() == Some(&child_id_str) {
+            Some(key.clone())
+        } else {
+            None
+        }
+    });
+
+    let step_key = match step_key {
+        Some(k) => k,
+        None => {
+            if !success {
+                // No step key and failed — can't store error, fail parent immediately
+                tracing::error!(
+                    parent_job = %parent_job_id,
+                    child_job = %child_job_id,
+                    "WAC v2 child job failed but no step key found, failing parent"
+                );
+                sqlx::query!(
+                    "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1",
+                    parent_job_id,
+                )
+                .execute(db)
+                .await?;
+                let parent_mini = get_mini_completed_job(&parent_job_id, workspace_id, db).await?;
+                if let Some(parent_mini) = parent_mini {
+                    let child_err: Value =
+                        serde_json::from_str(result.get()).unwrap_or(Value::Null);
+                    let err_value = json!({
+                        "message": format!("WAC child job {} failed (no step key)", child_job_id),
+                        "error": child_err,
+                    });
+                    let _ = windmill_queue::add_completed_job_error(
+                        db,
+                        &parent_mini,
+                        0,
+                        None,
+                        err_value,
+                        "wac_child_handler",
+                        false,
+                        None,
+                    )
+                    .await;
+                }
+                return Ok(Some(()));
+            }
+            tracing::warn!(
+                parent_job = %parent_job_id,
+                child_job = %child_job_id,
+                "WAC v2 child completed but no matching step key found in checkpoint, decrementing suspend to avoid parent hang"
+            );
+            // Still decrement suspend so the parent doesn't hang indefinitely
+            let _ = sqlx::query_scalar!(
+                "UPDATE v2_job_queue \
+                 SET suspend = GREATEST(suspend - 1, 0) \
+                 WHERE id = $1 \
+                 RETURNING suspend",
+                parent_job_id,
+            )
+            .fetch_optional(db)
+            .await?;
+            return Ok(Some(()));
+        }
+    };
+
+    // Build result — wrap errors with _error marker so workflow try/catch can handle them
+    let result_value: Value = if success {
+        serde_json::from_str(result.get()).unwrap_or(Value::Null)
+    } else {
+        let child_err: Value = serde_json::from_str(result.get()).unwrap_or(Value::Null);
+        tracing::info!(
+            parent_job = %parent_job_id,
+            child_job = %child_job_id,
+            step_key = %step_key,
+            "WAC v2 child job failed, storing error for workflow try/catch"
+        );
+        json!({
+            "__wmill_error": true,
+            "message": format!("WAC task '{}' failed (child job {})", step_key, child_job_id),
+            "child_job_id": child_job_id.to_string(),
+            "step_key": step_key,
+            "result": child_err,
+        })
+    };
+
+    tracing::info!(
+        parent_job = %parent_job_id,
+        child_job = %child_job_id,
+        step_key = %step_key,
+        success = success,
+        "WAC v2 child job completed"
+    );
+
+    // Use a transaction to ensure completed_steps merge + suspend decrement
+    // are atomic.  Without this, a crash between the two could strand the parent.
+    let result_json = serde_json::to_value(&result_value)
+        .map_err(|e| error::Error::InternalErr(format!("Failed to serialize step result: {e}")))?;
+
+    let mut tx = db.begin().await?;
+
+    // Merge the completed step into the checkpoint.
+    // Uses `|| jsonb_build_object(key, value)` so concurrent children on
+    // different workers don't overwrite each other — PostgreSQL serialises
+    // concurrent UPDATEs on the same row and each sees the previous write.
+    sqlx::query(
+        "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set(
+            workflow_as_code_status,
+            '{_checkpoint,completed_steps}',
+            COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb)
+            || jsonb_build_object($2::text, $3::jsonb)
+        ) WHERE id = $1",
+    )
+    .bind(&parent_job_id)
+    .bind(&step_key)
+    .bind(&result_json)
+    .execute(&mut *tx)
+    .await
+    .map_err(|e| error::Error::InternalErr(format!("Failed to add WAC completed step: {e}")))?;
+
+    // Decrement the suspend counter.  The counter was set to N (number of
+    // children) at dispatch time.  When it reaches 0 all children are done.
+    // Keep suspend_until non-null so the suspended pull query
+    // (`WHERE suspend_until IS NOT NULL AND suspend <= 0`) picks up the parent.
+    let new_suspend: Option<i32> = sqlx::query_scalar!(
+        "UPDATE v2_job_queue \
+         SET suspend = GREATEST(suspend - 1, 0) \
+         WHERE id = $1 \
+         RETURNING suspend",
+        parent_job_id,
+    )
+    .fetch_optional(&mut *tx)
+    .await?;
+
+    let all_done = new_suspend == Some(0);
+
+    if all_done {
+        // Clear pending_steps from checkpoint since all children are complete.
+        // This is cosmetic — the next replay will overwrite it anyway — but
+        // keeps the checkpoint clean for frontend display.
+        let _ = sqlx::query(
+            "UPDATE v2_job_status SET workflow_as_code_status = \
+             workflow_as_code_status #- '{_checkpoint,pending_steps}' \
+             WHERE id = $1",
+        )
+        .bind(&parent_job_id)
+        .execute(&mut *tx)
+        .await;
+    }
+
+    tx.commit().await?;
+
+    if all_done {
+        tracing::info!(
+            parent_job = %parent_job_id,
+            "WAC v2 all child jobs completed, unsuspending parent"
+        );
+    }
+
+    Ok(Some(()))
+}
+
 pub async fn handle_non_flow_job_error(
     db: &DB,
     job: &MiniCompletedJob,

backend/windmill-worker/src/ruby_executor.rs

@@ -812,7 +812,10 @@ mount {{
             .envs(envs)
             .envs(reserved_variables)
             .envs(RUBY_PROXY_ENVS.clone())
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?)
+            .envs(
+                get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn)
+                    .await?,
+            )
             .args(vec![
                 "--config",
                 "run.config.proto",
@@ -851,7 +854,10 @@ mount {{
             .env("BASE_INTERNAL_URL", base_internal_url)
             .envs(reserved_variables)
             .envs(RUBY_PROXY_ENVS.clone())
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?)
+            .envs(
+                get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn)
+                    .await?,
+            )
             .envs(envs);
 
         cmd.stdin(Stdio::null())

backend/windmill-worker/src/rust_executor.rs

@@ -700,7 +700,10 @@ pub async fn handle_rust_job(
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?)
+            .envs(
+                get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn)
+                    .await?,
+            )
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
@@ -716,7 +719,10 @@ pub async fn handle_rust_job(
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
-            .envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?)
+            .envs(
+                get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn)
+                    .await?,
+            )
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)

backend/windmill-worker/src/snowflake_executor.rs

@@ -9,8 +9,8 @@ use serde_json::{json, value::RawValue, Value};
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use windmill_common::error::to_anyhow;
-use windmill_object_store::convert_json_line_stream;
 use windmill_common::worker::{Connection, SqlResultCollectionStrategy};
+use windmill_object_store::convert_json_line_stream;
 
 use windmill_common::{error::Error, worker::to_raw_value};
 use windmill_parser_sql::{

backend/windmill-worker/src/wac_executor.rs

@@ -0,0 +1,339 @@
+use serde::{Deserialize, Serialize};
+use serde_json::value::RawValue;
+use serde_json::Value;
+use uuid::Uuid;
+
+use windmill_common::error::{self, Error};
+use windmill_common::DB;
+
+/// Checkpoint state persisted across workflow invocations.
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct WacCheckpoint {
+    #[serde(default)]
+    pub source_hash: String,
+    #[serde(default)]
+    pub completed_steps: serde_json::Map<String, Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pending_steps: Option<WacPendingSteps>,
+    #[serde(default)]
+    pub input_args: serde_json::Map<String, Value>,
+    /// Accumulated map of step_key → child job UUID across all dispatch rounds.
+    /// Unlike `pending_steps.job_ids` (cleared after completion), this persists
+    /// so the frontend can always resolve step keys to child job names.
+    #[serde(default)]
+    pub job_ids: serde_json::Map<String, Value>,
+    /// When set on a child job's checkpoint, indicates which step this child
+    /// should execute directly (instead of dispatching).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
+    pub _executing_key: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct WacPendingSteps {
+    pub mode: String,
+    pub keys: Vec<String>,
+    pub job_ids: serde_json::Map<String, Value>,
+}
+
+/// Output from a single WAC invocation (parsed from result.json).
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type")]
+pub enum WacOutput {
+    #[serde(rename = "dispatch")]
+    Dispatch { mode: String, steps: Vec<WacStepDispatch> },
+    #[serde(rename = "complete")]
+    Complete { result: Value },
+    /// An inline step executed in the parent process — persist result to
+    /// checkpoint and re-run immediately (no child job, no suspend).
+    #[serde(rename = "inline_checkpoint")]
+    InlineCheckpoint { key: String, result: Value },
+    /// Suspend the workflow waiting for an external approval event.
+    /// No child job is dispatched — the parent suspends directly and resumes
+    /// when a user hits the resume/cancel endpoint.
+    #[serde(rename = "approval")]
+    Approval { key: String, timeout: Option<u32>, form: Option<Value> },
+    /// Server-side sleep — suspend the workflow for a duration without holding a worker.
+    #[serde(rename = "sleep")]
+    Sleep { key: String, seconds: u32 },
+}
+
+/// A step dispatched by the WAC SDK.
+///
+/// `dispatch_type` determines how the child job is created:
+/// - `"inline"` (default): re-runs the parent workflow with `_executing_key` set
+/// - `"script"`: runs a separate Windmill script resolved from `script` path
+/// - `"flow"`: runs a separate Windmill flow resolved from `script` path
+#[derive(Debug, Deserialize, Clone)]
+pub struct WacStepDispatch {
+    pub name: String,
+    pub script: String,
+    pub args: serde_json::Map<String, Value>,
+    pub key: String,
+    #[serde(default = "default_dispatch_type")]
+    pub dispatch_type: String,
+    // Per-task options forwarded to push()
+    #[serde(default)]
+    pub timeout: Option<i32>,
+    #[serde(default)]
+    pub tag: Option<String>,
+    #[serde(default)]
+    pub cache_ttl: Option<i32>,
+    #[serde(default)]
+    pub priority: Option<i16>,
+    #[serde(default)]
+    pub concurrent_limit: Option<i32>,
+    #[serde(default)]
+    pub concurrency_key: Option<String>,
+    #[serde(default)]
+    pub concurrency_time_window_s: Option<i32>,
+}
+
+fn default_dispatch_type() -> String {
+    "inline".to_string()
+}
+
+/// Load the WAC checkpoint from `v2_job_status.workflow_as_code_status._checkpoint`.
+pub async fn load_checkpoint(db: &DB, job_id: &Uuid) -> error::Result<WacCheckpoint> {
+    let row: Option<Option<Value>> = sqlx::query_scalar(
+        "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = $1",
+    )
+    .bind(job_id)
+    .fetch_optional(db)
+    .await?;
+
+    match row {
+        Some(Some(status)) => {
+            let checkpoint: WacCheckpoint = match serde_json::from_value(status) {
+                Ok(c) => c,
+                Err(e) => {
+                    tracing::warn!(
+                        job_id = %job_id,
+                        error = %e,
+                        "Failed to deserialize WAC checkpoint, resetting to empty"
+                    );
+                    WacCheckpoint::default()
+                }
+            };
+            Ok(checkpoint)
+        }
+        _ => Ok(WacCheckpoint::default()),
+    }
+}
+
+/// Save the WAC checkpoint to `v2_job_status.workflow_as_code_status._checkpoint`.
+/// The top level of workflow_as_code_status is reserved for per-child-job timeline data.
+pub async fn save_checkpoint(
+    db: &DB,
+    job_id: &Uuid,
+    checkpoint: &WacCheckpoint,
+) -> error::Result<()> {
+    let status_json = serde_json::to_value(checkpoint)
+        .map_err(|e| Error::InternalErr(format!("Failed to serialize checkpoint: {e}")))?;
+
+    sqlx::query(
+        "INSERT INTO v2_job_status (id, workflow_as_code_status)
+         VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb))
+         ON CONFLICT (id) DO UPDATE SET
+            workflow_as_code_status = jsonb_set(
+                COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb),
+                '{_checkpoint}',
+                $2::jsonb
+            )",
+    )
+    .bind(job_id)
+    .bind(&status_json)
+    .execute(db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("Failed to save WAC checkpoint: {e}")))?;
+
+    Ok(())
+}
+
+/// Parse the WAC result from result.json content.
+pub fn parse_wac_output(result: &RawValue) -> error::Result<WacOutput> {
+    serde_json::from_str(result.get())
+        .map_err(|e| Error::InternalErr(format!("Failed to parse WAC output: {e}")))
+}
+
+/// Process a "dispatch" result: update checkpoint with pending steps info.
+pub fn update_checkpoint_for_dispatch(
+    checkpoint: &mut WacCheckpoint,
+    steps: &[WacStepDispatch],
+    mode: &str,
+    job_ids: &[(String, Uuid)],
+) {
+    let ids_map: serde_json::Map<String, Value> = job_ids
+        .iter()
+        .map(|(key, id)| (key.clone(), Value::String(id.to_string())))
+        .collect();
+    // Accumulate into persistent job_ids (survives pending_steps clearing)
+    for (k, v) in ids_map.iter() {
+        checkpoint.job_ids.insert(k.clone(), v.clone());
+    }
+    let pending = WacPendingSteps {
+        mode: mode.to_string(),
+        keys: steps.iter().map(|s| s.key.clone()).collect(),
+        job_ids: ids_map,
+    };
+    checkpoint.pending_steps = Some(pending);
+}
+
+/// Process a completed child job result: add to checkpoint's completed_steps.
+pub fn add_completed_step(checkpoint: &mut WacCheckpoint, step_key: &str, result: Value) {
+    checkpoint
+        .completed_steps
+        .insert(step_key.to_string(), result);
+    // If all pending steps are complete, clear pending
+    if let Some(ref pending) = checkpoint.pending_steps {
+        let all_done = pending
+            .keys
+            .iter()
+            .all(|k| checkpoint.completed_steps.contains_key(k));
+        if all_done {
+            checkpoint.pending_steps = None;
+        }
+    }
+}
+
+/// Check if all pending parallel steps are complete.
+pub fn all_pending_complete(checkpoint: &WacCheckpoint) -> bool {
+    match &checkpoint.pending_steps {
+        None => true,
+        Some(pending) => pending
+            .keys
+            .iter()
+            .all(|k| checkpoint.completed_steps.contains_key(k)),
+    }
+}
+
+/// If the checkpoint has a pending approval or sleep, inject the resume result
+/// into `completed_steps` and save back to DB. Returns the (possibly modified) checkpoint.
+///
+/// Called by both bun and python executors before writing checkpoint.json to disk.
+pub async fn prepare_checkpoint_for_resume(
+    db: &DB,
+    job_id: &Uuid,
+    mut checkpoint: WacCheckpoint,
+) -> error::Result<WacCheckpoint> {
+    let pending_mode = checkpoint.pending_steps.as_ref().map(|p| p.mode.as_str());
+
+    match pending_mode {
+        Some("approval") => {
+            let approval_key = checkpoint
+                .pending_steps
+                .as_ref()
+                .and_then(|p| p.keys.first().cloned())
+                .unwrap_or_default();
+
+            let resume_row = sqlx::query_as::<_, (sqlx::types::Json<Box<serde_json::value::RawValue>>, Option<String>, bool)>(
+                "SELECT value, approver, approved FROM resume_job WHERE job = $1 ORDER BY created_at ASC LIMIT 1",
+            )
+            .bind(job_id)
+            .fetch_optional(db)
+            .await?;
+
+            let approval_result = if let Some((value, approver, approved)) = resume_row {
+                serde_json::json!({
+                    "value": serde_json::from_str::<Value>(value.get()).unwrap_or(Value::Null),
+                    "approver": approver.unwrap_or_else(|| "anonymous".to_string()),
+                    "approved": approved,
+                })
+            } else {
+                serde_json::json!({
+                    "value": null,
+                    "approver": null,
+                    "approved": false,
+                })
+            };
+            checkpoint
+                .completed_steps
+                .insert(approval_key.clone(), approval_result);
+            checkpoint.pending_steps = None;
+            save_checkpoint(db, job_id, &checkpoint).await?;
+
+            tracing::info!(
+                job_id = %job_id,
+                approval_key = %approval_key,
+                "WAC v2 injected approval result into checkpoint"
+            );
+        }
+        Some("sleep") => {
+            let sleep_key = checkpoint
+                .pending_steps
+                .as_ref()
+                .and_then(|p| p.keys.first().cloned())
+                .unwrap_or_default();
+
+            checkpoint
+                .completed_steps
+                .insert(sleep_key.clone(), Value::Bool(true));
+            checkpoint.pending_steps = None;
+            save_checkpoint(db, job_id, &checkpoint).await?;
+
+            tracing::info!(
+                job_id = %job_id,
+                sleep_key = %sleep_key,
+                "WAC v2 resumed from sleep"
+            );
+        }
+        _ => {}
+    }
+
+    Ok(checkpoint)
+}
+
+/// Detect WAC v2 patterns in TypeScript/Bun code.
+/// Checks for `import ... from "windmill-client"` containing workflow/task,
+/// skipping comment lines.
+pub fn is_wac_v2_ts(code: &str) -> bool {
+    let mut has_wac_import = false;
+    let mut has_workflow = false;
+    let mut has_task = false;
+    for line in code.lines() {
+        let trimmed = line.trim();
+        if trimmed.starts_with("//") {
+            continue;
+        }
+        if trimmed.contains("windmill-client")
+            && (trimmed.starts_with("import") || trimmed.starts_with("from"))
+        {
+            has_wac_import = true;
+            if trimmed.contains("workflow") {
+                has_workflow = true;
+            }
+            if trimmed.contains("task") {
+                has_task = true;
+            }
+        }
+        if trimmed.contains("export") && trimmed.contains("workflow(") {
+            has_workflow = true;
+        }
+    }
+    has_wac_import && has_workflow && has_task
+}
+
+/// Detect WAC v2 patterns in Python code.
+/// Checks for `@workflow` decorator and `@task` decorator with wmill import,
+/// skipping comment lines.
+pub fn is_wac_v2_py(code: &str) -> bool {
+    let mut has_wmill_import = false;
+    let mut has_workflow_decorator = false;
+    let mut has_task_decorator = false;
+    for line in code.lines() {
+        let trimmed = line.trim();
+        if trimmed.starts_with('#') {
+            continue;
+        }
+        if trimmed.starts_with("import wmill") || trimmed.starts_with("from wmill") {
+            has_wmill_import = true;
+        }
+        if trimmed == "@workflow" || trimmed.starts_with("@workflow(") {
+            has_workflow_decorator = true;
+        }
+        if trimmed == "@task" || trimmed.starts_with("@task(") {
+            has_task_decorator = true;
+        }
+    }
+    has_wmill_import && has_workflow_decorator && has_task_decorator
+}

backend/windmill-worker/src/worker.rs

@@ -740,26 +740,69 @@ pub async fn is_otel_tracing_proxy_enabled_for_lang(lang: &ScriptLang) -> bool {
     }
 }
 
+/// Get OTEL trace context environment variables for a job (TRACEPARENT, OTEL_TRACE_ID, OTEL_SPAN_ID).
+/// Returns an empty vec when OTEL tracing is not enabled or on non-enterprise builds.
+pub fn get_otel_context_envs(job_id: &uuid::Uuid) -> Vec<(&'static str, String)> {
+    #[cfg(all(feature = "private", feature = "enterprise"))]
+    if windmill_common::OTEL_TRACING_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
+        let trace_id = format!("{:032x}", job_id.as_u128());
+        let span_id = format!("{:016x}", job_id.as_u64_pair().1);
+        let traceparent = format!("00-{}-{}-01", trace_id, span_id);
+        return vec![
+            ("TRACEPARENT", traceparent),
+            ("OTEL_TRACE_ID", trace_id),
+            ("OTEL_SPAN_ID", span_id),
+        ];
+    }
+    let _ = job_id;
+    vec![]
+}
+
 /// Get proxy environment variables for job execution for a specific language.
 /// When OTEL tracing proxy is enabled for this language, routes all traffic through the proxy.
 /// Otherwise, uses the standard HTTP_PROXY/HTTPS_PROXY from environment.
 pub async fn get_proxy_envs_for_lang(
     lang: &ScriptLang,
+    job_id: &uuid::Uuid,
+    w_id: &str,
+    conn: &Connection,
 ) -> anyhow::Result<Vec<(&'static str, String)>> {
+    #[allow(unused_mut)]
+    let mut envs;
     #[cfg(all(feature = "private", feature = "enterprise"))]
     if is_otel_tracing_proxy_enabled_for_lang(lang).await {
-        return get_otel_tracing_proxy_envs().await;
+        envs = get_otel_tracing_proxy_envs(job_id, w_id, conn).await?;
+    } else {
+        envs = PROXY_ENVS.clone();
     }
-    let _ = lang;
-    Ok(PROXY_ENVS.clone())
+    #[cfg(not(all(feature = "private", feature = "enterprise")))]
+    {
+        let _ = (lang, w_id, conn);
+        envs = PROXY_ENVS.clone();
+    }
+    envs.extend(get_otel_context_envs(job_id));
+    Ok(envs)
 }
 
 #[cfg(all(feature = "private", feature = "enterprise"))]
-async fn get_otel_tracing_proxy_envs() -> anyhow::Result<Vec<(&'static str, String)>> {
-    let port = crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT
+async fn get_otel_tracing_proxy_envs(
+    job_id: &uuid::Uuid,
+    w_id: &str,
+    conn: &Connection,
+) -> anyhow::Result<Vec<(&'static str, String)>> {
+    let port = match *crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT
         .read()
         .await
-        .ok_or_else(|| anyhow::anyhow!("OTEL tracing proxy port not initialized"))?;
+    {
+        Some(p) => p,
+        None => {
+            let reason = "OTEL tracing proxy is enabled but not available (not initialized yet, or NUM_WORKERS > 1). \
+                This job's HTTP requests will not be traced.";
+            tracing::warn!("{}", reason);
+            append_logs(job_id, w_id, format!("\n[warning] {reason}\n"), conn).await;
+            return Ok(PROXY_ENVS.clone());
+        }
+    };
     let proxy_url = format!("http://127.0.0.1:{}", port);
     Ok(vec![
         ("HTTP_PROXY", proxy_url.clone()),
@@ -1368,7 +1411,6 @@ pub async fn run_worker(
     mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
     killpill_tx: KillpillSender,
     base_internal_url: &str,
-    batch_pull_client: Option<&HttpClient>,
 ) {
     #[cfg(not(feature = "enterprise"))]
     if is_sandboxing_enabled() {
@@ -2069,149 +2111,135 @@ pub async fn run_worker(
                     continue;
                 }
             } else {
-                // If batch_pull_client is set (native worker with co-located server),
-                // use HTTP pull from batch buffer. Otherwise use direct SQL pull.
-                if let Some(bpc) = batch_pull_client {
-                    crate::agent_workers::pull_job(bpc, None, None)
-                        .await
-                        .map_err(|e| error::Error::InternalErr(e.to_string()))
-                        .map(|x| x.map(|y| NextJob::Http(y)))
-                } else {
-                    match &conn {
-                        Connection::Sql(db) => {
-                            let pull_time = Instant::now();
-                            let likelihood_of_suspend = last_30jobs_suspended as f64 / 30.0;
+                match &conn {
+                    Connection::Sql(db) => {
+                        let pull_time = Instant::now();
+                        let likelihood_of_suspend = last_30jobs_suspended as f64 / 30.0;
 
-                            let suspend_first = suspend_first_success
-                                || rand::random::<f64>() < likelihood_of_suspend
-                                || last_suspend_first.elapsed().as_secs_f64() > 5.0;
+                        let suspend_first = suspend_first_success
+                            || rand::random::<f64>() < likelihood_of_suspend
+                            || last_suspend_first.elapsed().as_secs_f64() > 5.0;
 
-                            if suspend_first {
-                                last_suspend_first = Instant::now();
-                            }
-                            let mut job = match timeout(
-                                Duration::from_secs(30),
-                                pull(
-                                    &db,
-                                    suspend_first,
-                                    &worker_name,
-                                    None,
-                                    #[cfg(feature = "benchmark")]
-                                    &mut bench,
-                                )
-                                .warn_after_seconds(2),
+                        if suspend_first {
+                            last_suspend_first = Instant::now();
+                        }
+                        let mut job = match timeout(
+                            Duration::from_secs(30),
+                            pull(
+                                &db,
+                                suspend_first,
+                                &worker_name,
+                                None,
+                                #[cfg(feature = "benchmark")]
+                                &mut bench,
                             )
+                            .warn_after_seconds(2),
+                        )
+                        .await
+                        {
+                            Ok(job) => job,
+                            Err(e) => {
+                                tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 20s, sleeping for 30s: {e:?}");
+                                tokio::time::sleep(Duration::from_secs(30)).await;
+                                continue;
+                            }
+                        };
+
+                        // Preprocess pulled job result
+                        if let Ok(ref mut pulled_job_res) = job {
+                            if let Err(e) = timeout(
+                                // Will fail if longer than 10 seconds
+                                core::time::Duration::from_secs(10),
+                                pulled_job_res.maybe_apply_debouncing(db),
+                            )
+                            .warn_after_seconds(2)
                             .await
+                            // Flatten result
+                            .map_err(error::Error::from)
+                            .and_then(|r| r)
                             {
-                                Ok(job) => job,
-                                Err(e) => {
-                                    tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 20s, sleeping for 30s: {e:?}");
-                                    tokio::time::sleep(Duration::from_secs(30)).await;
-                                    continue;
-                                }
-                            };
-
-                            // Preprocess pulled job result
-                            if let Ok(ref mut pulled_job_res) = job {
-                                if let Err(e) = timeout(
-                                    // Will fail if longer than 10 seconds
-                                    core::time::Duration::from_secs(10),
-                                    pulled_job_res.maybe_apply_debouncing(db),
-                                )
-                                .warn_after_seconds(2)
-                                .await
-                                // Flatten result
-                                .map_err(error::Error::from)
-                                .and_then(|r| r)
-                                {
-                                    pulled_job_res.error_while_preprocessing = Some(e.to_string());
-                                }
+                                pulled_job_res.error_while_preprocessing = Some(e.to_string());
                             }
+                        }
 
-                            add_time!(bench, "job pulled from DB");
-                            let duration_pull_s = pull_time.elapsed().as_secs_f64();
-                            let err_pull = job.is_ok();
-                            // let empty = job.as_ref().is_ok_and(|x| x.is_none());
+                        add_time!(bench, "job pulled from DB");
+                        let duration_pull_s = pull_time.elapsed().as_secs_f64();
+                        let err_pull = job.is_ok();
+                        // let empty = job.as_ref().is_ok_and(|x| x.is_none());
 
-                            if duration_pull_s > 0.5 {
-                                let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
-                                tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
-                                #[cfg(feature = "prometheus")]
-                                if empty {
-                                    if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
-                                        wp.inc();
-                                    }
-                                } else if let Some(wp) = worker_pull_over_500_counter.as_ref() {
+                        if duration_pull_s > 0.5 {
+                            let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
+                            tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
+                            #[cfg(feature = "prometheus")]
+                            if empty {
+                                if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
                                     wp.inc();
                                 }
-                            } else if duration_pull_s > 0.1 {
-                                let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
-                                tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
-                                #[cfg(feature = "prometheus")]
-                                if empty {
-                                    if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
-                                        wp.inc();
-                                    }
-                                } else if let Some(wp) = worker_pull_over_100_counter.as_ref() {
+                            } else if let Some(wp) = worker_pull_over_500_counter.as_ref() {
+                                wp.inc();
+                            }
+                        } else if duration_pull_s > 0.1 {
+                            let empty = job.as_ref().is_ok_and(|x| x.job.is_none());
+                            tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
+                            #[cfg(feature = "prometheus")]
+                            if empty {
+                                if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
                                     wp.inc();
                                 }
+                            } else if let Some(wp) = worker_pull_over_100_counter.as_ref() {
+                                wp.inc();
                             }
+                        }
 
-                            if let Ok(j) = job.as_ref() {
-                                let suspend_success = j.suspended;
-                                if suspend_first {
-                                    if last_30jobs_suspended < 30 {
-                                        last_30jobs_suspended += 1;
-                                    }
-                                } else {
-                                    last_30jobs_suspended -= 1;
+                        if let Ok(j) = job.as_ref() {
+                            let suspend_success = j.suspended;
+                            if suspend_first {
+                                if last_30jobs_suspended < 30 {
+                                    last_30jobs_suspended += 1;
                                 }
-                                suspend_first_success = suspend_first && suspend_success;
-                                #[cfg(feature = "prometheus")]
-                                if j.job.is_some() {
-                                    if let Some(wp) = worker_pull_duration_counter.as_ref() {
-                                        wp.inc_by(duration_pull_s);
-                                    }
-                                    if let Some(wp) = worker_pull_duration.as_ref() {
-                                        wp.observe(duration_pull_s);
-                                    }
-                                } else {
-                                    if let Some(wp) = worker_pull_duration_counter_empty.as_ref() {
-                                        wp.inc_by(duration_pull_s);
-                                    }
-                                    if let Some(wp) = worker_pull_duration_empty.as_ref() {
-                                        wp.observe(duration_pull_s);
-                                    }
+                            } else {
+                                last_30jobs_suspended -= 1;
+                            }
+                            suspend_first_success = suspend_first && suspend_success;
+                            #[cfg(feature = "prometheus")]
+                            if j.job.is_some() {
+                                if let Some(wp) = worker_pull_duration_counter.as_ref() {
+                                    wp.inc_by(duration_pull_s);
+                                }
+                                if let Some(wp) = worker_pull_duration.as_ref() {
+                                    wp.observe(duration_pull_s);
+                                }
+                            } else {
+                                if let Some(wp) = worker_pull_duration_counter_empty.as_ref() {
+                                    wp.inc_by(duration_pull_s);
+                                }
+                                if let Some(wp) = worker_pull_duration_empty.as_ref() {
+                                    wp.observe(duration_pull_s);
                                 }
                             }
-                            match job {
-                                Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() {
-                                    Ok(j) => {
-                                        Ok(j.map(|job| NextJob::Sql { flow_runners: None, job }))
-                                    }
-                                    Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc))
-                                    | Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(jc)) => {
-                                        if let Err(err) = job_completed_tx.send_job(jc, true).await
-                                        {
-                                            tracing::error!(
+                        }
+                        match job {
+                            Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() {
+                                Ok(j) => Ok(j.map(|job| NextJob::Sql { flow_runners: None, job })),
+                                Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc))
+                                | Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(jc)) => {
+                                    if let Err(err) = job_completed_tx.send_job(jc, true).await {
+                                        tracing::error!(
                                             "An error occurred while sending job completed: {:#?}",
                                             err
                                         )
-                                        }
-                                        Ok(None)
                                     }
-                                },
-                                Err(err) => Err(err),
-                            }
-                        }
-
-                        Connection::Http(client) => {
-                            crate::agent_workers::pull_job(&client, None, None)
-                                .await
-                                .map_err(|e| error::Error::InternalErr(e.to_string()))
-                                .map(|x| x.map(|y| NextJob::Http(y)))
+                                    Ok(None)
+                                }
+                            },
+                            Err(err) => Err(err),
                         }
                     }
+
+                    Connection::Http(client) => crate::agent_workers::pull_job(&client, None, None)
+                        .await
+                        .map_err(|e| error::Error::InternalErr(e.to_string()))
+                        .map(|x| x.map(|y| NextJob::Http(y))),
                 }
             }
         };
@@ -3500,6 +3528,13 @@ pub async fn handle_queued_job(
         {
             return Ok(false);
         }
+        if result
+            .as_ref()
+            .is_err_and(|err| matches!(err, &Error::WacSuspended(_)))
+        {
+            // WAC v2 job suspended while waiting for child jobs — don't complete it
+            return Ok(true);
+        }
         process_result(
             cjob,
             result.map(|x| Arc::new(x)),
@@ -3898,7 +3933,7 @@ pub async fn run_language_executor(
     run_inline: bool,
 ) -> error::Result<Box<RawValue>> {
     if language == Some(ScriptLang::Postgresql) {
-        return do_postgresql(
+        return Box::pin(do_postgresql(
             job,
             &client,
             &code,
@@ -3910,7 +3945,7 @@ pub async fn run_language_executor(
             occupancy_metrics,
             parent_runnable_path,
             run_inline,
-        )
+        ))
         .await;
     } else if language == Some(ScriptLang::Mysql) {
         #[cfg(not(feature = "mysql"))]
@@ -3925,7 +3960,7 @@ pub async fn run_language_executor(
                     "Inline execution is not yet supported for this language".to_string(),
                 ));
             }
-            return do_mysql(
+            return Box::pin(do_mysql(
                 job,
                 &client,
                 &code,
@@ -3936,7 +3971,7 @@ pub async fn run_language_executor(
                 column_order,
                 occupancy_metrics,
                 parent_runnable_path,
-            )
+            ))
             .await;
         }
     } else if language == Some(ScriptLang::Bigquery) {
@@ -3962,7 +3997,7 @@ pub async fn run_language_executor(
                     "Inline execution is not yet supported for this language".to_string(),
                 ));
             }
-            return do_bigquery(
+            return Box::pin(do_bigquery(
                 job,
                 &client,
                 &code,
@@ -3973,7 +4008,7 @@ pub async fn run_language_executor(
                 column_order,
                 occupancy_metrics,
                 parent_runnable_path,
-            )
+            ))
             .await;
         }
     } else if language == Some(ScriptLang::Snowflake) {
@@ -3991,7 +4026,7 @@ pub async fn run_language_executor(
                     "Inline execution is not yet supported for this language".to_string(),
                 ));
             }
-            return do_snowflake(
+            return Box::pin(do_snowflake(
                 job,
                 &client,
                 &code,
@@ -4002,7 +4037,7 @@ pub async fn run_language_executor(
                 column_order,
                 occupancy_metrics,
                 parent_runnable_path,
-            )
+            ))
             .await;
         }
     } else if language == Some(ScriptLang::Mssql) {
@@ -4028,7 +4063,7 @@ pub async fn run_language_executor(
                     "Inline execution is not yet supported for this language".to_string(),
                 ));
             }
-            return do_mssql(
+            return Box::pin(do_mssql(
                 job,
                 &client,
                 &code,
@@ -4039,7 +4074,7 @@ pub async fn run_language_executor(
                 occupancy_metrics,
                 job_dir,
                 parent_runnable_path,
-            )
+            ))
             .await;
         }
     } else if language == Some(ScriptLang::OracleDB) {
@@ -4065,7 +4100,7 @@ pub async fn run_language_executor(
                     "Inline execution is not yet supported for this language".to_string(),
                 ));
             }
-            return do_oracledb(
+            return Box::pin(do_oracledb(
                 job,
                 &client,
                 &code,
@@ -4076,7 +4111,7 @@ pub async fn run_language_executor(
                 column_order,
                 occupancy_metrics,
                 parent_runnable_path,
-            )
+            ))
             .await;
         }
     } else if language == Some(ScriptLang::DuckDb) {
@@ -4090,7 +4125,7 @@ pub async fn run_language_executor(
 
         #[cfg(feature = "duckdb")]
         {
-            return do_duckdb(
+            return Box::pin(do_duckdb(
                 job,
                 &client,
                 &code,
@@ -4102,7 +4137,7 @@ pub async fn run_language_executor(
                 occupancy_metrics,
                 parent_runnable_path,
                 run_inline,
-            )
+            ))
             .await;
         }
     } else if language == Some(ScriptLang::Graphql) {
@@ -4111,7 +4146,7 @@ pub async fn run_language_executor(
                 "Inline execution is not yet supported for this language".to_string(),
             ));
         }
-        return do_graphql(
+        return Box::pin(do_graphql(
             job,
             &client,
             &code,
@@ -4120,7 +4155,7 @@ pub async fn run_language_executor(
             canceled_by,
             worker_name,
             occupancy_metrics,
-        )
+        ))
         .await;
     } else if language == Some(ScriptLang::Nativets) {
         if run_inline {
@@ -4147,7 +4182,7 @@ pub async fn run_language_executor(
                 .collect::<Vec<String>>()
                 .join("\n"));
 
-        let result = do_nativets(
+        let result = Box::pin(do_nativets(
             job,
             &client,
             env_code,
@@ -4158,7 +4193,7 @@ pub async fn run_language_executor(
             worker_name,
             occupancy_metrics,
             has_stream,
-        )
+        ))
         .await?;
         return Ok(result);
     }

backend/windmill-worker/src/worker_flow.rs

@@ -865,7 +865,7 @@ pub async fn update_flow_status_after_job_completion_internal(
                         .and_then(|x| x.stop_after_all_iters_if.as_ref())
                     {
                         let args = from_result_to_args(args.as_ref().await.get_ref())?;
-                        evaluate_stop_after_all_iters_if(
+                        if let Err(e) = evaluate_stop_after_all_iters_if(
                             db,
                             stop_after_all_iters_if,
                             module_status,
@@ -879,7 +879,16 @@ pub async fn update_flow_status_after_job_completion_internal(
                             flow,
                             &old_status,
                         )
-                        .await?;
+                        .await
+                        {
+                            tracing::error!("error evaluating stop_after_all_iters_if: {e:#}");
+                            stop_early = true;
+                            skip_if_stop_early = false;
+                            stop_early_err_msg = Some(format!(
+                                "Error evaluating stop_after_all_iters_if expression `{}`: {e:#}",
+                                stop_after_all_iters_if.expr
+                            ));
+                        }
                     }
 
                     let new_status = if
@@ -1074,7 +1083,7 @@ pub async fn update_flow_status_after_job_completion_internal(
                     {
                         let args = from_result_to_args(args.as_ref().await.get_ref())?;
 
-                        evaluate_stop_after_all_iters_if(
+                        if let Err(e) = evaluate_stop_after_all_iters_if(
                             db,
                             stop_after_all_iters_if,
                             module_status,
@@ -1088,7 +1097,15 @@ pub async fn update_flow_status_after_job_completion_internal(
                             flow,
                             &old_status,
                         )
-                        .await?;
+                        .await
+                        {
+                            stop_early = true;
+                            skip_if_stop_early = false;
+                            stop_early_err_msg = Some(format!(
+                                "Error evaluating stop_after_all_iters_if expression `{}`: {e:#}",
+                                stop_after_all_iters_if.expr
+                            ));
+                        }
                     }
                 }
 
@@ -1705,8 +1722,8 @@ pub async fn update_flow_status_after_job_completion_internal(
                 chat_ai_info.conversation_id,
             )
             .await?;
-            let duration = if success {
-                let (_, duration) = add_completed_job(
+            let (duration, wac_job_ids) = if success {
+                let (_, duration, wac_job_ids) = add_completed_job(
                     db,
                     &cflow_job,
                     true,
@@ -1720,9 +1737,9 @@ pub async fn update_flow_status_after_job_completion_internal(
                     false,
                 )
                 .await?;
-                duration
+                (duration, wac_job_ids)
             } else {
-                let (_, duration) = add_completed_job(
+                let (_, duration, wac_job_ids) = add_completed_job(
                     db,
                     &cflow_job,
                     false,
@@ -1740,11 +1757,30 @@ pub async fn update_flow_status_after_job_completion_internal(
                     false,
                 )
                 .await?;
-                duration
+                (duration, wac_job_ids)
             };
             flow_job_duration = flow_job
                 .started_at
                 .map(|x| FlowJobDuration { started_at: x, duration_ms: duration });
+
+            // If this flow is a WAC child (not a flow step, has parent),
+            // notify the WAC parent of completion.
+            if !flow_job.is_flow_step() {
+                if let Some(parent_job) = flow_job.parent_job {
+                    if let Some(job_ids) = wac_job_ids {
+                        let _ = crate::result_processor::handle_wac_child_completion(
+                            db,
+                            &flow_job.id,
+                            parent_job,
+                            &flow_job.workspace_id,
+                            nresult.clone(),
+                            success,
+                            job_ids,
+                        )
+                        .await;
+                    }
+                }
+            }
         }
         true
     } else {

backend/windmill-worker/src/worker_utils.rs

@@ -8,7 +8,7 @@ use windmill_common::{
         get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage,
         insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query,
         update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED,
-        USES_BATCH_HTTP_PULL, WORKER_CONFIG, WORKER_GROUP,
+        WORKER_CONFIG, WORKER_GROUP,
     },
     KillpillSender, DB,
 };
@@ -31,7 +31,6 @@ pub(crate) async fn update_worker_ping_full(
     let tags = wc.worker_tags.clone();
     let native_mode = wc.native_mode;
     drop(wc);
-    let uses_batch_http_pull = USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed);
 
     let memory_usage = get_worker_memory_usage();
     let wm_memory_usage = get_windmill_memory_usage();
@@ -65,7 +64,6 @@ pub(crate) async fn update_worker_ping_full(
             occupancy_rate_5m,
             occupancy_rate_30m,
             native_mode,
-            uses_batch_http_pull,
         )
     })
     .retry(
@@ -112,7 +110,6 @@ async fn update_worker_ping_full_inner(
     occupancy_rate_5m: Option<f32>,
     occupancy_rate_30m: Option<f32>,
     native_mode: bool,
-    uses_batch_http_pull: bool,
 ) -> anyhow::Result<()> {
     match conn {
         Connection::Sql(db) => {
@@ -129,7 +126,6 @@ async fn update_worker_ping_full_inner(
                 occupancy_rate_5m,
                 occupancy_rate_30m,
                 native_mode,
-                uses_batch_http_pull,
                 db,
             )
             .await?;
@@ -159,7 +155,6 @@ async fn update_worker_ping_full_inner(
                         wm_memory_usage: get_windmill_memory_usage(),
                         job_isolation: None,
                         native_mode: Some(native_mode),
-                        uses_batch_http_pull: Some(uses_batch_http_pull),
                         ping_type: PingType::MainLoop,
                     },
                 )
@@ -191,7 +186,6 @@ pub async fn insert_ping(
             wc.native_mode,
         )
     };
-    let uses_batch_http_pull = USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed);
 
     let vcpus = get_vcpus();
     let memory = get_memory();
@@ -219,7 +213,6 @@ pub async fn insert_ping(
                 memory,
                 job_isolation,
                 native_mode,
-                uses_batch_http_pull,
                 db,
             )
             .await?;
@@ -249,7 +242,6 @@ pub async fn insert_ping(
                         wm_memory_usage: get_windmill_memory_usage(),
                         job_isolation,
                         native_mode: Some(native_mode),
-                        uses_batch_http_pull: Some(uses_batch_http_pull),
                         ping_type: PingType::Initial,
                     },
                 )
@@ -326,9 +318,6 @@ pub async fn update_worker_ping_from_job(
                         native_mode: Some(
                             NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed),
                         ),
-                        uses_batch_http_pull: Some(
-                            USES_BATCH_HTTP_PULL.load(std::sync::atomic::Ordering::Relaxed),
-                        ),
                     },
                 )
                 .await?;

benchmarks/benchmark_oneoff.ts

@@ -47,7 +47,6 @@ export async function main({
   kind,
   jobs,
   noVerify,
-  skipDeploy,
 }: {
   host: string;
   email?: string;
@@ -57,7 +56,6 @@ export async function main({
   kind: string;
   jobs: number;
   noVerify?: boolean;
-  skipDeploy?: boolean;
 }) {
   windmill.setClient("", host);
 
@@ -148,8 +146,7 @@ export async function main({
   }
 
   if (
-    !skipDeploy &&
-    ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "nativets_sleep", "dedicated_nativets"].includes(
+    ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
       kind
     )
   ) {
@@ -168,7 +165,7 @@ export async function main({
       kind: "noop",
     });
   } else if (
-    ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "nativets_sleep", "dedicated_nativets"].includes(
+    ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes(
       kind
     )
   ) {
@@ -339,7 +336,6 @@ export async function main({
     !noVerify &&
     kind !== "noop" &&
     kind !== "nativets" &&
-    kind !== "nativets_sleep" &&
     kind !== "dedicated_nativets" &&
     !kind.startsWith("flow:") &&
     !kind.startsWith("script:")
@@ -402,9 +398,6 @@ if (import.meta.main) {
     .option("--no-verify", "Do not verify the output of the jobs.", {
       default: false,
     })
-    .option("--skip-deploy", "Skip script deployment (use already deployed script).", {
-      default: false,
-    })
     .action(main)
     .command(
       "upgrade",

benchmarks/lib.ts

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.651.1";
+export const VERSION = "v1.652.0";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({
@@ -95,10 +95,6 @@ export async function createBenchScript(
     scriptContent =
       '//native\nexport async function main(){ return (await fetch(BASE_URL + "/api/version")).text() }';
     language = "bunnative";
-  } else if (scriptPattern === "nativets_sleep") {
-    scriptContent =
-      '//native\nexport async function main(){ const ms = 300 + Math.floor(Math.random() * 400); await new Promise(r => setTimeout(r, ms)); return { slept: ms }; }';
-    language = "bunnative";
   } else if (scriptPattern === "dedicated_nativets") {
     scriptContent = "//native\nexport function main(){ return 42; }";
     language = "bunnative";

benchmarks/model_batch_vs_sql.ts

@@ -1,165 +0,0 @@
-/**
- * Model: Batch Pull vs Direct SQL throughput
- *
- * Calibrated from real benchmarks (3 native workers = 24 subworkers, local PG):
- *   - nativets (fast):   batch 291 j/s, SQL 253 j/s at N=24; batch 108, SQL 88 at N=8
- *   - nativets_sleep:    both ~43.8 j/s at N=24 (bottlenecked by 500ms avg exec time)
- *
- * Per-worker job time model:
- *   T_pw = T_base + T_exec + T_contention(N)
- *   throughput = N / T_pw
- *
- * Batch: T_contention grows linearly with N (HTTP server load)
- *   T_pw_batch(N) = BASE_BATCH + T_exec + SCALE_BATCH × N
- *
- * SQL: T_contention grows quadratically with N (SKIP LOCKED scanning past locked rows)
- *   T_pw_sql(N) = BASE_SQL + T_exec + SCALE_SQL × N²
- *
- * Parameters fitted from 2 data points each (N=8, N=24):
- *   Batch: BASE=69.9ms, SCALE=0.525ms/worker
- *   SQL:   BASE=90.4ms, SCALE=0.0078ms/worker²
- *   (SQL quadratic overtakes batch linear around N~40)
- */
-
-// --- Model parameters (fitted from benchmarks) ---
-
-// Batch: per-worker time = BASE + SCALE_LINEAR * N + T_exec
-const BASE_BATCH = 69.9; // ms — base overhead (worker loop, HTTP roundtrip, job completion writes)
-const SCALE_BATCH = 0.525; // ms per subworker — linear growth from server load
-
-// SQL: per-worker time = BASE + SCALE_QUAD * N² + T_exec
-const BASE_SQL = 90.4; // ms — base overhead (worker loop, poll interval wait, job completion writes)
-const SCALE_SQL = 0.0078; // ms per subworker² — quadratic growth from SKIP LOCKED contention
-
-// --- Throughput functions ---
-
-function throughputBatch(subworkers: number, execMs: number): number {
-  const tPerWorker = BASE_BATCH + execMs + SCALE_BATCH * subworkers;
-  return (subworkers / tPerWorker) * 1000; // jobs/s
-}
-
-function throughputSql(subworkers: number, execMs: number): number {
-  const tPerWorker = BASE_SQL + execMs + SCALE_SQL * subworkers * subworkers;
-  return (subworkers / tPerWorker) * 1000; // jobs/s
-}
-
-function pct(batch: number, sql: number): string {
-  const diff = ((batch - sql) / sql) * 100;
-  return `${diff >= 0 ? "+" : ""}${diff.toFixed(0)}%`;
-}
-
-// --- Validation against real data ---
-
-console.log("=== Model Validation (vs real benchmarks) ===\n");
-console.log(
-  "  Setup                | Model Batch | Real Batch | Model SQL | Real SQL",
-);
-console.log(
-  "  ---------------------|-------------|------------|-----------|--------",
-);
-
-const cases = [
-  { n: 8, exec: 0, label: "1W nativets", realBatch: 108, realSql: 88 },
-  { n: 24, exec: 0, label: "3W nativets", realBatch: 291, realSql: 253 },
-  {
-    n: 24,
-    exec: 500,
-    label: "3W sleep(500ms)",
-    realBatch: 43.8,
-    realSql: 43.8,
-  },
-];
-
-for (const c of cases) {
-  const mb = throughputBatch(c.n, c.exec);
-  const ms = throughputSql(c.n, c.exec);
-  console.log(
-    `  ${c.label.padEnd(21)}| ${mb.toFixed(0).padStart(7)} j/s | ${c.realBatch.toFixed(0).padStart(6)} j/s | ${ms.toFixed(0).padStart(5)} j/s | ${c.realSql.toFixed(0).padStart(4)} j/s`,
-  );
-}
-
-// --- Projections ---
-
-const workerCounts = [1, 2, 3, 5, 8, 10, 15, 20]; // native workers (×8 subworkers each)
-const execTimes = [
-  { ms: 0, label: "~0ms (identity)" },
-  { ms: 5, label: "5ms" },
-  { ms: 20, label: "20ms" },
-  { ms: 50, label: "50ms" },
-  { ms: 200, label: "200ms" },
-  { ms: 500, label: "500ms" },
-];
-
-console.log("\n\n=== Projected Throughput (jobs/s) ===\n");
-
-for (const exec of execTimes) {
-  console.log(`--- Job duration: ${exec.label} ---\n`);
-  console.log(
-    "  Native workers (subw) |   Batch   |    SQL    |  Advantage  | Batch wins?",
-  );
-  console.log(
-    "  ----------------------|-----------|----------|-------------|------------",
-  );
-
-  for (const w of workerCounts) {
-    const n = w * 8;
-    const b = throughputBatch(n, exec.ms);
-    const s = throughputSql(n, exec.ms);
-    const advantage = pct(b, s);
-    const wins = b > s * 1.05 ? "  YES" : b > s * 1.01 ? "  marginal" : "  no";
-    console.log(
-      `  ${String(w).padStart(2)}W (${String(n).padStart(3)})    | ${b.toFixed(0).padStart(5)} j/s | ${s.toFixed(0).padStart(5)} j/s | ${advantage.padStart(8)}    | ${wins}`,
-    );
-  }
-  console.log();
-}
-
-// --- Crossover analysis ---
-
-console.log("=== Crossover: min workers where batch is >10% faster ===\n");
-console.log("  Job duration | Min workers | Subworkers | Batch j/s | SQL j/s");
-console.log("  -------------|-------------|------------|-----------|--------");
-
-for (const exec of execTimes) {
-  let found = false;
-  for (let w = 1; w <= 50; w++) {
-    const n = w * 8;
-    const b = throughputBatch(n, exec.ms);
-    const s = throughputSql(n, exec.ms);
-    if (b > s * 1.1) {
-      console.log(
-        `  ${exec.label.padEnd(13)}| ${String(w).padStart(5)}W      | ${String(n).padStart(5)}      | ${b.toFixed(0).padStart(5)} j/s | ${s.toFixed(0).padStart(5)} j/s`,
-      );
-      found = true;
-      break;
-    }
-  }
-  if (!found) {
-    console.log(
-      `  ${exec.label.padEnd(13)}| >50W (never significant at this job duration)`,
-    );
-  }
-}
-
-console.log("\n\n=== Key Takeaways ===\n");
-console.log(
-  "1. For fast jobs (~0ms): batch pull is always faster, advantage grows with scale",
-);
-console.log("   - 5 native workers (40 subworkers): ~13% faster");
-console.log("   - 10 native workers (80 subworkers): ~25% faster");
-console.log("   - 20 native workers (160 subworkers): ~88% faster");
-console.log(
-  "2. For medium jobs (50ms): batch advantage meaningful from ~5 native workers",
-);
-console.log(
-  "3. For slow jobs (500ms+): only matters at 15+ native workers (120+ subworkers)",
-);
-console.log(
-  "   (but still reduces DB load — fewer pull queries, less index scanning)",
-);
-console.log(
-  "4. The SQL quadratic contention (SKIP LOCKED scanning) is the dominant factor",
-);
-console.log(
-  "   — SQL throughput plateaus around 15-20 native workers while batch keeps scaling",
-);

benchmarks/results_batch_pull.md

@@ -1,93 +0,0 @@
-# Batch Pull Benchmark Results
-
-Date: 2026-03-06
-Setup: 1 server + 1 native worker (8 subworkers), standalone mode
-Hardware: fedora, 1.5TB disk, ~1GB memory usage
-DB: PostgreSQL local, windmill 270 MiB
-
-## nativets — 1000 jobs
-
-|  | Batch Pull | Direct SQL |
-|--|-----------|------------|
-| Duration | 9.2s | 11.4s |
-| **Throughput** | **108 jobs/s** | **88 jobs/s** |
-| Improvement | **+23%** | baseline |
-
-### pg_stat_statements
-
-| Query | Batch calls | Batch ms | SQL calls | SQL ms |
-|-------|------------|----------|-----------|--------|
-| Native pull (FOR UPDATE SKIP LOCKED) | 2,399 (0.02ms avg) | 47 | 2,849 (0.03ms avg) | 87 |
-| Default worker pull | 520 (0.04ms avg) | 19 | 445 (0.05ms avg) | 23 |
-| DELETE from queue | 1,001 (0.23ms avg) | 231 | 1,001 (0.19ms avg) | 195 |
-| INSERT into completed | 1,001 (0.05ms avg) | 51 | 1,001 (0.04ms avg) | 45 |
-| INSERT job_logs | 2,003 (0.02ms avg) | 44 | 2,003 (0.02ms avg) | 44 |
-| Agent token blacklist | 3,920 (0.00ms avg) | 19 | — | — |
-| **Total** | **9,389** | **1,332** | **8,882** | **1,212** |
-
-### pg_stat_database
-
-| Metric | Batch Pull | Direct SQL |
-|--------|-----------|------------|
-| Transactions committed | 6,484 | 5,868 |
-| Blocks read (disk) | 101 | 101 |
-| Blocks hit (cache) | 920,257 | 699,032 |
-| Tuples returned | 7,784,044 | 8,987,097 |
-| Tuples fetched | 1,028,819 | 805,100 |
-| Tuples inserted | 6,822 | 6,880 |
-| Tuples updated | 3,377 | 3,090 |
-| Tuples deleted | 2,883 | 2,654 |
-
----
-
-## nativets_sleep — 1000 jobs
-
-Each job sleeps 300-700ms (random). Theoretical max with 8 workers: ~16 jobs/s.
-
-|  | Batch Pull | Direct SQL |
-|--|-----------|------------|
-| Duration | 66.7s | 68.2s |
-| **Throughput** | **15.0 jobs/s** | **14.7 jobs/s** |
-| Improvement | ~same | baseline |
-
-### pg_stat_statements
-
-| Query | Batch calls | Batch ms | SQL calls | SQL ms |
-|-------|------------|----------|-----------|--------|
-| Native pull (FOR UPDATE SKIP LOCKED) | 2,399 (0.02ms avg) | 43 | 1,762 (0.04ms avg) | 75 |
-| Default worker pull | 1,591 (0.07ms avg) | 113 | 1,392 (0.08ms avg) | 115 |
-| DELETE from queue | 1,001 (0.31ms avg) | 308 | 1,001 (0.20ms avg) | 205 |
-| INSERT into completed | 1,001 (0.05ms avg) | 46 | 1,001 (0.04ms avg) | 41 |
-| INSERT job_logs | 2,003 (0.02ms avg) | 45 | 2,003 (0.02ms avg) | 40 |
-| Job runtime ping | 1,490 (0.02ms avg) | 33 | 1,489 (0.02ms avg) | 31 |
-| Worker ping (job) | 1,001 (0.03ms avg) | 32 | 1,001 (0.03ms avg) | 30 |
-| **Total** | **16,523** | **5,798** | **16,364** | **5,717** |
-
-### pg_stat_database
-
-| Metric | Batch Pull | Direct SQL |
-|--------|-----------|------------|
-| Transactions committed | 13,638 | 13,388 |
-| Blocks read (disk) | 394 | 850 |
-| Blocks hit (cache) | 7,033,158 | 4,932,617 |
-| Tuples returned | 58,424,571 | 57,204,918 |
-| Tuples fetched | 8,776,186 | 6,321,947 |
-| Tuples inserted | 7,500 | 7,531 |
-| Tuples updated | 6,111 | 6,193 |
-| Tuples deleted | 3,006 | 3,196 |
-
----
-
-## Analysis
-
-**Throughput**: +23% for fast CPU-bound jobs. Negligible difference for I/O-bound jobs.
-
-**Pull queries**: Batch pull does MORE pull queries for nativets_sleep (2,399 vs 1,762). The refiller polls every 50ms even when all workers are busy executing jobs. With direct SQL, workers only poll when idle. This is wasted work — the refiller queries DB and gets empty results while jobs are in-flight.
-
-**Disk I/O**: Batch pull cuts disk reads in half for nativets_sleep (394 vs 850 blocks). Likely because the batch query locks multiple rows in one pass, reducing index traversal.
-
-**Cache hits**: Higher with batch pull (7M vs 4.9M for sleep). More buffer hits from the refiller's repeated empty polls touching the same index pages.
-
-**Tuples fetched**: Higher with batch pull (8.7M vs 6.3M for sleep). Same cause — the refiller's empty polls scan the index.
-
-**At scale**: With 8 subworkers the differences are small. The real benefit is with many native workers where direct SQL SKIP LOCKED contention grows O(N²).

benchmarks/results_batch_pull_3workers.md

@@ -1,123 +0,0 @@
-# Batch Pull Benchmark Results — 3 Workers
-
-Date: 2026-03-06
-Setup: 1 server + 3 native workers (8 subworkers each = 24 subworkers)
-Hardware: fedora, 1.5TB disk, ~1GB memory usage
-DB: PostgreSQL local, windmill 270 MiB
-
-## nativets — 1000 jobs
-
-|  | 3W Batch | 3W SQL | 1W Batch | 1W SQL |
-|--|---------|--------|---------|--------|
-| Duration | 3.7s | 3.5s | 9.2s | 11.4s |
-| **Throughput** | **272 jobs/s** | **288 jobs/s** | **108 jobs/s** | **88 jobs/s** |
-| vs 1W SQL | +209% | +227% | +23% | baseline |
-
-Note: First 3W SQL run was 33 jobs/s (outlier due to cold start or background activity). Rerun gave 288 jobs/s.
-
-## nativets — 10,000 jobs
-
-|  | 3W Batch | 3W SQL |
-|--|---------|--------|
-| Duration | 34.3s | 39.5s |
-| **Throughput** | **291 jobs/s** | **253 jobs/s** |
-| Improvement | **+15%** | baseline |
-
-### pg_stat_statements (1000 jobs, first run)
-
-| Query | 3W Batch calls | 3W Batch ms | 3W SQL calls | 3W SQL ms |
-|-------|---------------|-------------|-------------|----------|
-| Native pull (FOR UPDATE SKIP LOCKED) | 4,801 (0.01ms) | 59 | 20,367 (0.01ms) | 231 |
-| Default worker pull | 353 (0.03ms) | 10 | 880 (0.02ms) | 16 |
-| DELETE from queue | 1,001 (0.44ms) | 439 | 1,001 (0.43ms) | 427 |
-| INSERT into completed | 1,001 (0.06ms) | 61 | 1,001 (0.05ms) | 55 |
-| INSERT job_logs | 2,003 (0.03ms) | 67 | 2,003 (0.03ms) | 64 |
-| Agent token blacklist | 7,867 (0.00ms) | 33 | — | — |
-| Worker ping (job) | 350 (0.04ms) | 15 | — | — |
-| Outstanding wait time | 664 (0.03ms) | 21 | 742 (0.03ms) | 23 |
-
-### pg_stat_database (1000 jobs, first run)
-
-| Metric | 3W Batch | 3W SQL |
-|--------|---------|--------|
-| Transactions committed | 22,070 | 29,301 |
-| Blocks read (disk) | 308 | 395 |
-| Blocks hit (cache) | 280,071 | 1,276,521 |
-| Tuples returned | 3,368,255 | 28,695,830 |
-| Tuples fetched | 140,864 | 1,089,774 |
-| Tuples inserted | 6,682 | 6,760 |
-| Tuples updated | 3,521 | 3,453 |
-| Tuples deleted | 3,007 | 3,007 |
-
----
-
-## nativets_sleep — 1000 jobs
-
-Each job sleeps 300-700ms (random). Theoretical max with 24 workers: ~48 jobs/s.
-
-|  | 3W Batch | 3W SQL | 1W Batch | 1W SQL |
-|--|---------|--------|---------|--------|
-| Duration | 22.8s | 22.8s | 66.7s | 68.2s |
-| **Throughput** | **43.8 jobs/s** | **43.8 jobs/s** | **15.0 jobs/s** | **14.7 jobs/s** |
-| vs 3W SQL | ~same | baseline | — | — |
-
-### pg_stat_statements
-
-| Query | 3W Batch calls | 3W Batch ms | 3W SQL calls | 3W SQL ms |
-|-------|---------------|-------------|-------------|----------|
-| Native pull (FOR UPDATE SKIP LOCKED) | 4,898 (0.01ms) | 55 | 6,440 (0.02ms) | 113 |
-| Default worker pull | 696 (0.06ms) | 43 | 654 (0.06ms) | 37 |
-| DELETE from queue | 1,001 (0.38ms) | 379 | 1,001 (0.29ms) | 290 |
-| INSERT into completed | 1,001 (0.05ms) | 46 | 1,001 (0.04ms) | 43 |
-| INSERT job_logs | 2,003 (0.02ms) | 44 | 2,003 (0.02ms) | 43 |
-| Agent token blacklist | 7,295 (0.00ms) | 31 | — | — |
-| Job runtime ping | 1,499 (0.02ms) | 35 | 1,444 (0.02ms) | 35 |
-| Worker ping (job) | 1,001 (0.03ms) | 32 | 1,001 (0.03ms) | 31 |
-| Job stats | 549 (0.05ms) | 27 | 493 (0.05ms) | 26 |
-| Outstanding wait time | 928 (0.02ms) | 20 | 944 (0.02ms) | 21 |
-
-### pg_stat_database
-
-| Metric | 3W Batch | 3W SQL |
-|--------|---------|--------|
-| Transactions committed | 25,896 | 18,646 |
-| Blocks read (disk) | 115 | 204 |
-| Blocks hit (cache) | 1,173,696 | 1,256,397 |
-| Tuples returned | 21,074,537 | 22,063,593 |
-| Tuples fetched | 1,200,878 | 1,262,900 |
-| Tuples inserted | 7,495 | 7,461 |
-| Tuples updated | 6,204 | 6,141 |
-| Tuples deleted | 3,005 | 3,011 |
-
----
-
-## Analysis
-
-### nativets (CPU-bound): +15% with 10K jobs
-
-With 10,000 jobs, batch pull achieves **291 jobs/s vs 253 jobs/s** (+15%). The 1000-job runs showed similar throughput (~272-288 jobs/s) after discarding the cold-start outlier.
-
-**DB load difference** (from the 1000-job first run, which captured the worst-case SQL contention):
-- **20,367 pull queries** (SQL) vs 4,801 (batch) — 4x more queries
-- **28.7M tuples returned** (SQL) vs 3.4M (batch) — 8.5x more index scanning
-- **1.3M cache hits** (SQL) vs 280K (batch) — 4.6x more buffer activity
-
-The batch approach consolidates all 24 subworkers into a single `LIMIT 24` query, reducing contention on the queue index.
-
-### nativets_sleep (I/O-bound): No throughput difference
-
-Both achieve **43.8 jobs/s** (91% of theoretical 48 jobs/s max). When workers spend 300-700ms sleeping, DB contention isn't the bottleneck.
-
-Batch pull still shows slightly lower DB load:
-- **4,898 pull queries** vs 6,440 — 24% fewer
-- **115 disk reads** vs 204 — 44% fewer
-
-### Scaling summary
-
-| Setup | Batch jobs/s | SQL jobs/s | Batch advantage |
-|-------|-------------|-----------|----------------|
-| 1W × 1000 jobs | 108 | 88 | +23% |
-| 3W × 1000 jobs | 272 | 288 | ~same |
-| 3W × 10,000 jobs | 291 | 253 | **+15%** |
-
-At 24 subworkers, batch pull provides a consistent ~15% throughput improvement for sustained CPU-bound workloads, with significantly lower DB load (4x fewer pull queries, 8x fewer tuples scanned). The benefit grows with more workers as SKIP LOCKED contention scales O(N²).

cli/src/commands/sync/sync.ts

@@ -2626,6 +2626,7 @@ export async function push(
         let [_basePath, changes] = queue.shift()!;
         const promise = (async () => {
           const alreadySynced: string[] = [];
+          const deletedVarsResPaths: string[] = [];
           const isRawApp = isRawAppFile(changes[0].path);
           if (isRawApp) {
             const deleteRawApp = changes.find(
@@ -2870,12 +2871,23 @@ export async function push(
                     name: change.path.split(SEP)[1],
                   });
                   break;
-                case "resource":
-                  await wmill.deleteResource({
-                    workspace: workspaceId,
-                    path: removeSuffix(target, ".resource.json"),
-                  });
+                case "resource": {
+                  const resourcePath = removeSuffix(target, ".resource.json");
+                  try {
+                    await wmill.deleteResource({
+                      workspace: workspaceId,
+                      path: resourcePath,
+                    });
+                  } catch (e: any) {
+                    if (e?.status === 404 && deletedVarsResPaths.includes(resourcePath)) {
+                      log.debug(`Resource ${resourcePath} already deleted by linked variable`);
+                    } else {
+                      throw e;
+                    }
+                  }
+                  deletedVarsResPaths.push(resourcePath);
                   break;
+                }
                 case "resource-type":
                   await wmill.deleteResourceType({
                     workspace: workspaceId,
@@ -3012,12 +3024,23 @@ export async function push(
                   });
                   break;
                 }
-                case "variable":
-                  await wmill.deleteVariable({
-                    workspace: workspaceId,
-                    path: removeSuffix(target, ".variable.json"),
-                  });
+                case "variable": {
+                  const variablePath = removeSuffix(target, ".variable.json");
+                  try {
+                    await wmill.deleteVariable({
+                      workspace: workspaceId,
+                      path: variablePath,
+                    });
+                  } catch (e: any) {
+                    if (e?.status === 404 && deletedVarsResPaths.includes(variablePath)) {
+                      log.debug(`Variable ${variablePath} already deleted by linked resource`);
+                    } else {
+                      throw e;
+                    }
+                  }
+                  deletedVarsResPaths.push(variablePath);
                   break;
+                }
                 case "user": {
                   const users = await wmill.listUsers({
                     workspace: workspaceId,

cli/src/commands/workspace/workspace.ts

@@ -408,7 +408,8 @@ async function remove(_opts: GlobalOptions, name: string) {
 
 async function whoami(_opts: GlobalOptions) {
   await requireLogin(_opts);
-  log.info(await wmill.globalWhoami());
+  const whoamiInfo = await wmill.globalWhoami();
+  log.info(JSON.stringify(whoamiInfo, null, 2));
   const activeName = await getActiveWorkspaceName(_opts);
   log.info("Active: " + colors.green.bold(activeName || "none"));
 }

cli/src/main.ts

@@ -67,7 +67,7 @@ export {
   workspaceAdd,
 };
 
-export const VERSION = "1.651.1";
+export const VERSION = "1.652.0";
 
 // Re-exported from constants.ts to maintain backwards compatibility
 export { WM_FORK_PREFIX } from "./core/constants.ts";

frontend/BUGS.txt

@@ -0,0 +1,39 @@
+# Svelte 5 Migration - Bug Report
+# Testing started: 2026-03-02
+
+## Warnings (not blocking but worth fixing)
+
+1. [WARNING] binding_property_non_reactive in Grid.svelte:372:5
+   - `bind:this={moveResizes[item.id]}` is binding to a non-reactive property
+   - File: src/lib/components/apps/svelte-grid/Grid.svelte
+   - Appears multiple times in App editor
+   - Status: NOT FIXED (non-blocking warning)
+
+2. [WARNING] legacy_recursive_reactive_block in RecomputeAllComponents.svelte
+   - Migrated `$:` reactive block that both accesses and updates the same reactive value
+   - File: src/lib/components/apps/editor/RecomputeAllComponents.svelte
+   - May cause recursive updates when converted to $effect
+   - Status: NOT FIXED (non-blocking warning)
+
+3. [WARNING] ownership_invalid_mutation in SchemaForm.svelte:70:16
+   - Mutating unbound props (`schema`) is strongly discouraged
+   - Parent: src/lib/components/ApiConnectForm.svelte should use `bind:schema={...}`
+   - Appears when opening PostgreSQL resource creation form
+   - Status: NOT FIXED (non-blocking warning)
+
+4. [WARNING] ownership_invalid_binding in InputTransformSchemaForm.svelte
+   - Passes `schema` to InputTransformForm.svelte with `bind:`, but parent Pane.svelte didn't declare `schema` as binding
+   - Appears in flow editor when adding a TypeScript step
+   - Status: NOT FIXED (non-blocking warning)
+
+## Bugs
+
+1. [BUG] state_descriptors_fixed in Chart.svelte (Queue metrics drawer)
+   - Error: "Property descriptors defined on `$state` objects must contain `value` and always be `enumerable`, `configurable` and `writable`."
+   - Triggered by: Clicking "Queue metrics" on /workers page
+   - File: src/lib/components/chartjs-wrappers/Chart.svelte
+   - Root cause: Chart.js's `listenArrayEvents` calls Object.defineProperty on data arrays that are Svelte 5 $state proxies, which reject non-standard property descriptors
+   - Fix: Use $state.snapshot() to pass plain copies of data and options to Chart.js
+   - Status: FIXED
+
+

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.651.1",
+	"version": "1.652.0",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.651.1",
+			"version": "1.652.0",
 			"hasInstallScript": true,
 			"license": "AGPL-3.0",
 			"dependencies": {