Merge branch 'main' into di/data-tables

.github/workflows/backend-test.yml

@@ -86,4 +86,4 @@ jobs:
         run: |
           deno --version && bun -v && go version && python3 --version
           cd windmill-duckdb-ffi-internal && ./build_dev.sh && cd ..
-          DENO_PATH=$(which deno) BUN_PATH=$(which bun) GO_PATH=$(which go) UV_PATH=$(which uv) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,private --all -- --nocapture
+          DENO_PATH=$(which deno) BUN_PATH=$(which bun) GO_PATH=$(which go) UV_PATH=$(which uv) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private --all -- --nocapture

.github/workflows/claude.yml

@@ -104,7 +104,7 @@ jobs:
               }
             }
           claude_args: |
-            --allowedTools "Bash"
+            --allowedTools "Bash,WebFetch,WebSearch"
             --system-prompt "## IMPORTANT INSTRUCTIONS
             - Your branch name should be a short description of the requested changes.
             - Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a draft PR from that branch to main.

.github/workflows/git-commands.yaml

@@ -82,6 +82,10 @@ jobs:
         run: |
           set -e  # Exit on any command failure
           PR_NUMBER=${{ github.event.issue.number }}
+
+          # Set up error trap to comment on PR for any failure
+          trap 'gh pr comment $PR_NUMBER --body "❌ SQLx update failed. Please check the workflow logs for details."' ERR
+
           BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)
           echo "Checking out PR branch: $BRANCH_NAME"
           git checkout $BRANCH_NAME
@@ -93,10 +97,8 @@ jobs:
           cd backend
           cargo install sqlx-cli --version 0.8.5
           sqlx migrate run
-          if ! ./update_sqlx.sh --dir ./windmill-ee-private; then
-            gh pr comment $PR_NUMBER --body "❌ SQLx update failed. Please check the workflow logs for details."
-            exit 1
-          fi
+          ./substitute_ee_code.sh --dir ./windmill-ee-private
+          ./update_sqlx.sh
           # Pass the branch name to the next step
           echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
 
@@ -216,11 +218,12 @@ jobs:
         with:
           github-token: ${{ steps.app.outputs.token }}
           script: |
+            const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
             github.rest.issues.createComment({
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,
-              body: 'Starting ee ref update...'
+              body: `Starting ee ref update...\n\n[View workflow run](${runUrl})`
             })
 
       - name: Checkout repository
@@ -249,7 +252,12 @@ jobs:
         env:
           GH_TOKEN: ${{ steps.app.outputs.token }}
         run: |
+          set -e  # Exit on any command failure
           PR_NUMBER=${{ github.event.issue.number }}
+
+          # Set up error trap to comment on PR for any failure
+          trap 'gh pr comment $PR_NUMBER --body "❌ EE ref update failed. Please check the workflow logs for details."' ERR
+
           BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)
           echo "Checking out PR branch: $BRANCH_NAME"
           git checkout $BRANCH_NAME

CHANGELOG.md

@@ -1,5 +1,79 @@
 # Changelog
 
+## [1.585.1](https://github.com/windmill-labs/windmill/compare/v1.585.0...v1.585.1) (2025-11-26)
+
+
+### Bug Fixes
+
+* clone group members on workspace forks ([#7232](https://github.com/windmill-labs/windmill/issues/7232)) ([faad000](https://github.com/windmill-labs/windmill/commit/faad0006b430262f1ab6b749ccf60a6860a54405))
+
+## [1.585.0](https://github.com/windmill-labs/windmill/compare/v1.584.0...v1.585.0) (2025-11-26)
+
+
+### Features
+
+* flow for loop squashing ([#7107](https://github.com/windmill-labs/windmill/issues/7107)) ([93c34c6](https://github.com/windmill-labs/windmill/commit/93c34c6d769dd56da02064c0abdaa060efe0259d))
+* **frontend:** move app reports script to hub ([#7227](https://github.com/windmill-labs/windmill/issues/7227)) ([f8a0bfc](https://github.com/windmill-labs/windmill/commit/f8a0bfc20d1ccb6ce36e3adf4188533470270bdb))
+
+
+### Bug Fixes
+
+* **cli:** improve back-compatibility with app.yaml with policy still present ([c0a92f8](https://github.com/windmill-labs/windmill/commit/c0a92f83b00341b0eee36e2dc88d301a75b6fdfb))
+* **frontend:** workspace color race condition for superadmins ([#7229](https://github.com/windmill-labs/windmill/issues/7229)) ([8922be1](https://github.com/windmill-labs/windmill/commit/8922be11a6935d200e1909f14aa410c6eff67a13))
+* workspace forks shouldn't inherit promotion mode repo ([#7223](https://github.com/windmill-labs/windmill/issues/7223)) ([f56f3b0](https://github.com/windmill-labs/windmill/commit/f56f3b0fe6185167e9c4c8541ddefba01c509b8d))
+
+## [1.584.0](https://github.com/windmill-labs/windmill/compare/v1.583.3...v1.584.0) (2025-11-25)
+
+
+### Features
+
+* **app:** add chat component ([#7199](https://github.com/windmill-labs/windmill/issues/7199)) ([c47669b](https://github.com/windmill-labs/windmill/commit/c47669be0bffb0821afe9d4182cc071cb3b675f0))
+
+
+### Bug Fixes
+
+* **cli:** handle better public apps ([de7739a](https://github.com/windmill-labs/windmill/commit/de7739a3ee7ef76a8eb4fe6570ef0bdfbdb92293))
+* **frontend:** forking UI issues ([#7215](https://github.com/windmill-labs/windmill/issues/7215)) ([dccee1d](https://github.com/windmill-labs/windmill/commit/dccee1dba08b4c4b170336b3f3e959fbe7bca8ce))
+* **frontend:** improve preprocessor discoverability ([#7214](https://github.com/windmill-labs/windmill/issues/7214)) ([a2d3297](https://github.com/windmill-labs/windmill/commit/a2d3297343338ba0b559481723e64db5f4acf3a7))
+* **frontend:** loading saved/history inputs when json view is on ([#7217](https://github.com/windmill-labs/windmill/issues/7217)) ([e87f814](https://github.com/windmill-labs/windmill/commit/e87f81470904578467f42598cfce5fad2249f131))
+* **frontend:** show trigger table when no trigger selected ([#7219](https://github.com/windmill-labs/windmill/issues/7219)) ([c37dde0](https://github.com/windmill-labs/windmill/commit/c37dde096c301a502c6ff2c570b91153a5717fcd))
+* list scripts without 1000 limits ([897faf0](https://github.com/windmill-labs/windmill/commit/897faf085b49ada0648594797cfe627decb50952))
+
+## [1.583.3](https://github.com/windmill-labs/windmill/compare/v1.583.2...v1.583.3) (2025-11-24)
+
+
+### Bug Fixes
+
+* **cli:** fix pushing non existing apps with cli ([29d8ad2](https://github.com/windmill-labs/windmill/commit/29d8ad2e5de1447f39252edd5ceea22034801547))
+* fix deadlock on oidc ([0e717c9](https://github.com/windmill-labs/windmill/commit/0e717c9588c1a3811cc4d438b53a93b4b13cf239))
+
+## [1.583.2](https://github.com/windmill-labs/windmill/compare/v1.583.1...v1.583.2) (2025-11-24)
+
+
+### Bug Fixes
+
+* listScripts include description with opt-out query arg ([#7210](https://github.com/windmill-labs/windmill/issues/7210)) ([80c7911](https://github.com/windmill-labs/windmill/commit/80c79116416db8629135a8853df645d9fff286dd))
+* **scim:** use value instead of display for group updates ([#7203](https://github.com/windmill-labs/windmill/issues/7203)) ([193efe3](https://github.com/windmill-labs/windmill/commit/193efe31c8aeab3a5e3e28278855d518c000bda8))
+
+## [1.583.1](https://github.com/windmill-labs/windmill/compare/v1.583.0...v1.583.1) (2025-11-21)
+
+
+### Bug Fixes
+
+* **cli:** improve cli local policy generation ([b857d15](https://github.com/windmill-labs/windmill/commit/b857d15352ab422c1d477cd9172856956c2c4b7c))
+
+## [1.583.0](https://github.com/windmill-labs/windmill/compare/v1.582.2...v1.583.0) (2025-11-21)
+
+
+### Features
+
+* **cli:** app policies are generated locally ([#7200](https://github.com/windmill-labs/windmill/issues/7200)) ([f85ab0c](https://github.com/windmill-labs/windmill/commit/f85ab0c5dd64f01fbd0c0c9c2a4cea029a04cd50))
+
+
+### Bug Fixes
+
+* **backend:** handle malformed errors in flow error handler ([#7196](https://github.com/windmill-labs/windmill/issues/7196)) ([f531866](https://github.com/windmill-labs/windmill/commit/f53186653220a5213e0bffdf77187faaa9190a7e))
+
 ## [1.582.2](https://github.com/windmill-labs/windmill/compare/v1.582.1...v1.582.2) (2025-11-21)
 
 

backend/.sqlx/query-00c1dd0cfaf15aafdcfcabc1f123cebdf8d777f48e148bcb171fa15e8bf6f098.json

@@ -0,0 +1,58 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            workspace.id AS \"id!\",\n            workspace.name AS \"name!\",\n            workspace.owner AS \"owner!\",\n            workspace.deleted AS \"deleted!\",\n            workspace.premium AS \"premium!\",\n            workspace_settings.color AS \"color\",\n            workspace.parent_workspace_id AS \"parent_workspace_id\"\n        FROM workspace\n        LEFT JOIN workspace_settings ON workspace.id = workspace_settings.workspace_id\n        WHERE workspace.id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "name!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "owner!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "deleted!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 4,
+        "name": "premium!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
+        "name": "color",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "parent_workspace_id",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "00c1dd0cfaf15aafdcfcabc1f123cebdf8d777f48e148bcb171fa15e8bf6f098"
+}

backend/.sqlx/query-05f4663a0f58736e92fe7cbbef3c99a03bc74ab3be1bacdbbf3910a76a1beacc.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT name FROM group_ WHERE workspace_id = $1 UNION SELECT name FROM instance_group ORDER BY name asc",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "05f4663a0f58736e92fe7cbbef3c99a03bc74ab3be1bacdbbf3910a76a1beacc"
+}

backend/.sqlx/query-086fdf726b88e9f4fd9750bf9dd7f49c589465194548d88e5ae30872846b70a9.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM flow_conversation WHERE id = $1) as \"exists!\"",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists!",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "086fdf726b88e9f4fd9750bf9dd7f49c589465194548d88e5ae30872846b70a9"
+}

backend/.sqlx/query-348d64dc9f81f04538c5fec98df51312794c22d6337be897bc618585fa5b27f3.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE v2_job_status\n         SET flow_status = jsonb_set(\n             flow_status,\n             '{memory_id}',\n             to_jsonb($2::uuid)\n         )\n         WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "348d64dc9f81f04538c5fec98df51312794c22d6337be897bc618585fa5b27f3"
+}

backend/.sqlx/query-3f9004b5542afaaef40942a3ca8889ea164d9cfb4962997650d922d82f000df9.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usr_to_group (workspace_id, group_, usr)\n         SELECT $2, group_, usr\n         FROM usr_to_group\n         WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "3f9004b5542afaaef40942a3ca8889ea164d9cfb4962997650d922d82f000df9"
+}

backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json

@@ -15,7 +15,7 @@
       ]
     },
     "nullable": [
-      null
+      true
     ]
   },
   "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

backend/.sqlx/query-8830fc3736ec6dbbaee20f0ebeba5a87c2c94227798f4545fe0931af2818a6fa.json

@@ -0,0 +1,66 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "display_name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "owners",
+        "type_info": "VarcharArray"
+      },
+      {
+        "ordinal": 4,
+        "name": "extra_perms",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 5,
+        "name": "summary",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 6,
+        "name": "created_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 7,
+        "name": "edited_at",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "8830fc3736ec6dbbaee20f0ebeba5a87c2c94227798f4545fe0931af2818a6fa"
+}

backend/.sqlx/query-ba9ab074f466bc2c581f018e2592f5a453e8a766c35dbf919d29c96966d63c75.json

@@ -0,0 +1,42 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "summary",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "extra_perms",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      false
+    ]
+  },
+  "hash": "ba9ab074f466bc2c581f018e2592f5a453e8a766c35dbf919d29c96966d63c75"
+}

backend/.sqlx/query-c77fc86e0fa5d483889b0c4201f6d89b454523b31847798bedcc6c460f9ff7ea.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "c77fc86e0fa5d483889b0c4201f6d89b454523b31847798bedcc6c460f9ff7ea"
+}

backend/.sqlx/query-f5568a691ec5931634cf986f806f5eae7bb8ed0f5c6e54ca3f49a991c53ed50d.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE v2_job_status \n         SET flow_status = jsonb_set(\n             flow_status,\n             '{memory_id}',\n             to_jsonb($2::uuid)\n         )\n         WHERE id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Uuid",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "f5568a691ec5931634cf986f806f5eae7bb8ed0f5c6e54ca3f49a991c53ed50d"
-}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.582.2"
+version = "1.585.1"
 authors.workspace = true
 edition.workspace = true
 
@@ -34,7 +34,7 @@ members = [
 exclude = ["./windmill-duckdb-ffi-internal"]
 
 [workspace.package]
-version = "1.582.2"
+version = "1.585.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 

backend/ee-repo-ref.txt

@@ -1 +1 @@
-c90bfc11c7c643b2c8c5111f092e7ae142318e9d
+99828e4a3a27af428a45d1e6e65092ca15082b1e

backend/parsers/windmill-parser-wasm/build.nu

@@ -65,15 +65,19 @@ const targets = [
 # 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
 
 # Build all separately
-def 'main all' [ --no-opt(-n), --cli] {
+def 'main all' [ --no-opt(-n), --cli, --node ] {
   if ($no_opt) {
-    if ($cli) {
+    if ($node) {
+      $targets | each { main $in.ident -n --node }
+    } else if ($cli) {
       $targets | each { main $in.ident -n --cli }
     } else {
       $targets | each { main $in.ident -n }
     }
   } else {
-    if ($cli) {
+    if ($node) {
+      $targets | each { main $in.ident --node }
+    } else if ($cli) {
       $targets | each { main $in.ident --cli }
     } else {
       $targets | each { main $in.ident }
@@ -86,6 +90,7 @@ def main [
   target: string, # Language, e.g.: py, ts, rust
   --no-opt(-n), # Compile in debug mode for dev
   --cli # Compile and place binaries in cli.
+  --node
 ] {
   let t = $targets | where ident == $target;
 
@@ -97,7 +102,10 @@ def main [
     mut profile = "";
     mut tar = "";
 
-    if ($cli) {
+    if ($node) {
+      $env.OUT_DIR = $"node-pkg-($t.ident)"
+      $tar = "nodejs"
+    } else if ($cli) {
       $env.OUT_DIR = $"../../../cli/wasm/($t.ident)"
       $tar = "deno"
     } else {

backend/src/monitor.rs

@@ -2316,13 +2316,14 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, node_n
             );
 
             let error_message = format!(
-            "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?}).\nThis likely means that the job died on worker {}, OOM are a common reason for worker crashes.\nCheck the workers around the time of the last ping and the exit code if any.",
-            job.last_ping.unwrap_or_default(),
-            *ZOMBIE_JOB_TIMEOUT,
-            error_kind.to_string(),
-            job.worker.clone().unwrap_or_default(),
-        );
+                "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?}).\nThis likely means that the job died on worker {}, OOM are a common reason for worker crashes.\nCheck the workers around the time of the last ping and the exit code if any.",
+                job.last_ping.unwrap_or_default(),
+                *ZOMBIE_JOB_TIMEOUT,
+                error_kind.to_string(),
+                job.worker.clone().unwrap_or_default(),
+            );
             let memory_peak = job.memory_peak.unwrap_or(0);
+            let (_, killpill_rx_never_used) = KillpillSender::new(1);
             let _ = handle_job_error(
                 db,
                 &client,
@@ -2335,6 +2336,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, node_n
                 "",
                 node_name,
                 send_result_never_used,
+                &killpill_rx_never_used,
                 #[cfg(feature = "benchmark")]
                 &mut windmill_common::bench::BenchmarkIter::new(),
             )

backend/tests/worker.rs

@@ -217,6 +217,7 @@ async fn test_deno_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
                         iterator: InputTransform::Javascript { expr: "result".to_string() },
                         skip_failures: false,
                         parallel: false,
+                        squash: None,
                         parallelism: None,
                         modules: vec![FlowModule {
                             id: "c".to_string(),
@@ -396,6 +397,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
                         iterator: InputTransform::Static { value: windmill_common::worker::to_raw_value(&[1, 2, 3]) },
                         skip_failures: false,
                         parallel: false,
+                        squash: None,
                         parallelism: None,
                         modules: vec![
                             FlowModule {

backend/update_sqlx.sh

@@ -2,15 +2,6 @@
 
 set -e
 
-# Parse arguments
-while [[ "$#" -gt 0 ]]; do
-    case $1 in
-        --dir) EE_DIR="$2"; shift ;;
-        *) echo "Unknown parameter: $1"; exit 1 ;;
-    esac
-    shift
-done
-
 # Check if running on macOS
 if [[ "$(uname)" == "Darwin" ]]; then
     echo "Running on macOS - substituting samael..."

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.582.2
+  version: 1.585.1
   title: Windmill API
 
   contact:
@@ -688,6 +688,22 @@ paths:
               schema:
                 $ref: "#/components/schemas/UserWorkspaceList"
 
+  /w/{workspace}/workspaces/get_as_superadmin:
+    get:
+      summary: get workspace as super admin (require to be super admin)
+      operationId: getWorkspaceAsSuperAdmin
+      tags:
+        - workspace
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: workspace
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Workspace"
+
   /workspaces/list_as_superadmin:
     get:
       summary: list all workspaces as super admin (require to be super admin)
@@ -5217,6 +5233,14 @@ paths:
             Accepts multiple values as a comma-separated list.
           schema:
             type: string
+        - name: without_description
+          in: query
+          description: |
+            (default false)
+            If true, the description field will be omitted from the response.
+          schema:
+            type: boolean
+
       responses:
         "200":
           description: All scripts
@@ -6500,6 +6524,13 @@ paths:
           in: query
           schema:
             type: boolean
+        - name: without_description
+          in: query
+          description: |
+            (default false)
+            If true, the description field will be omitted from the response.
+          schema:
+            type: boolean
       responses:
         "200":
           description: All flow
@@ -7729,6 +7760,9 @@ paths:
                   type: array
                   items:
                     type: string
+                run_query_params:
+                  type: object
+                  description: Runnable query parameters
               required:
                 - args
                 - component
@@ -12059,32 +12093,32 @@ paths:
               schema:
                 type: boolean
   /w/{workspace}/email_triggers/setenabled/{path}:
-      post:
-        summary: enable/disable email trigger
-        operationId: setEmailTriggerEnabled
-        tags:
-          - email_trigger
-        parameters:
-          - $ref: "#/components/parameters/WorkspaceId"
-          - $ref: "#/components/parameters/Path"
-        requestBody:
-          required: true
+    post:
+      summary: enable/disable email trigger
+      operationId: setEmailTriggerEnabled
+      tags:
+        - email_trigger
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                enabled:
+                  type: boolean
+              required:
+                - enabled
+      responses:
+        "200":
+          description: email trigger enable/disable
           content:
-            application/json:
+            text/plain:
               schema:
-                type: object
-                properties:
-                  enabled:
-                    type: boolean
-                required:
-                  - enabled
-        responses:
-          "200":
-            description: email trigger enable/disable
-            content:
-              text/plain:
-                schema:
-                  type: string
+                type: string
 
   /groups/list:
     get:

backend/windmill-api/src/apps.rs

@@ -10,6 +10,7 @@ use std::{collections::HashMap, sync::Arc};
 use crate::{
     auth::OptTokened,
     db::{ApiAuthed, DB},
+    jobs::RunJobQuery,
     resources::get_resource_value_interpolated_internal,
     users::{require_owner_of_path, OptAuthed},
     utils::{check_scopes, WithStarredInfoQuery},
@@ -1654,6 +1655,8 @@ pub struct ExecuteApp {
     pub force_viewer_static_fields: Option<StaticFields>,
     pub force_viewer_one_of_fields: Option<OneOfFields>,
     pub force_viewer_allow_user_resources: Option<AllowUserResources>,
+    /// Runnable query parameters (e.g., memory_id for chat-enabled flows)
+    pub run_query_params: Option<RunJobQuery>,
 }
 
 fn digest(code: &str) -> String {
@@ -1894,6 +1897,12 @@ async fn execute_component(
     )
     .await?;
 
+    let is_flow = payload
+        .path
+        .as_ref()
+        .map(|p| p.starts_with("flow/"))
+        .unwrap_or(false);
+
     let (job_payload, tag, on_behalf_of) = match (payload.path, payload.raw_code, payload.id) {
         // flow or script:
         (Some(path), None, None) => get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?,
@@ -1920,7 +1929,7 @@ async fn execute_component(
 
     let end_user_email = opt_authed.as_ref().map(|a| a.email.clone());
 
-    let (uuid, tx) = push(
+    let (uuid, mut tx) = push(
         &db,
         tx,
         &w_id,
@@ -1954,6 +1963,14 @@ async fn execute_component(
         None,
     )
     .await?;
+
+    // Apply runnable query parameters if provided
+    if let Some(ref run_query) = payload.run_query_params {
+        if is_flow {
+            crate::jobs::process_flow_run_query_params(&mut tx, uuid, run_query).await?;
+        }
+    }
+
     tx.commit().await?;
 
     Ok(uuid.to_string())

backend/windmill-api/src/flow_conversations.rs

@@ -134,6 +134,7 @@ pub async fn get_or_create_conversation_with_id(
     } else {
         title.to_string()
     };
+
     // Create new conversation with provided ID
     let conversation = sqlx::query_as!(
         FlowConversation,
@@ -148,6 +149,7 @@ pub async fn get_or_create_conversation_with_id(
     )
     .fetch_one(&mut **tx)
     .await?;
+
     Ok(conversation)
 }
 

backend/windmill-api/src/flows.rs

@@ -133,7 +133,11 @@ async fn list_flows(
             "o.workspace_id",
             "o.path",
             "summary",
-            "description",
+            if !lq.without_description.unwrap_or(false) {
+                "description"
+            } else {
+                "NULL as description"
+            },
             "fv.created_by as edited_by",
             "fv.created_at as edited_at",
             "archived",
@@ -1579,6 +1583,7 @@ mod tests {
                         skip_failures: true,
                         parallel: false,
                         parallelism: None,
+                        squash: None,
                     }),
                     stop_after_if: Some(StopAfterIf {
                         expr: "previous.isEmpty()".to_string(),

backend/windmill-api/src/folders.rs

@@ -96,7 +96,7 @@ async fn list_folders(
 
     let rows = sqlx::query_as!(
         Folder,
-        "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
+        "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3",
         w_id,
         per_page as i64,
         offset as i64
@@ -117,7 +117,7 @@ async fn list_foldernames(
     let mut tx = user_db.begin(&authed).await?;
 
     let rows = sqlx::query_scalar!(
-        "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
+        "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3",
         w_id,
         per_page as i64,
         offset as i64

backend/windmill-api/src/groups.rs

@@ -103,7 +103,7 @@ async fn list_groups(
 
     let rows = sqlx::query_as!(
         Group,
-        "SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
+        "SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3",
         w_id,
         per_page as i64,
         offset as i64
@@ -126,7 +126,7 @@ async fn list_group_names(
 ) -> JsonResult<Vec<String>> {
     let rows = if !only_member_of.unwrap_or(false) {
         sqlx::query_scalar!(
-            "SELECT name FROM group_ WHERE workspace_id = $1 UNION SELECT name FROM instance_group ORDER BY name desc",
+            "SELECT name FROM group_ WHERE workspace_id = $1 UNION SELECT name FROM instance_group ORDER BY name asc",
             w_id
         )
         .fetch_all(&db)

backend/windmill-api/src/jobs.rs

@@ -1743,7 +1743,7 @@ pub struct ListableCompletedJob {
     pub labels: Option<serde_json::Value>,
 }
 
-#[derive(Deserialize, Clone, Default)]
+#[derive(Debug, Deserialize, Clone, Default)]
 pub struct RunJobQuery {
     pub scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
     pub scheduled_in_secs: Option<i64>,
@@ -3957,7 +3957,7 @@ async fn set_flow_memory_id(
     memory_id: Uuid,
 ) -> error::Result<()> {
     sqlx::query!(
-        "UPDATE v2_job_status 
+        "UPDATE v2_job_status
          SET flow_status = jsonb_set(
              flow_status,
              '{memory_id}',
@@ -3972,6 +3972,19 @@ async fn set_flow_memory_id(
     Ok(())
 }
 
+/// Apply flow-specific query parameters after job creation
+pub async fn process_flow_run_query_params(
+    tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+    job_id: Uuid,
+    run_query: &RunJobQuery,
+) -> error::Result<()> {
+    // Set memory_id if provided (for agent memory)
+    if let Some(memory_id) = run_query.memory_id {
+        set_flow_memory_id(tx, job_id, memory_id).await?;
+    }
+    Ok(())
+}
+
 async fn handle_chat_conversation_messages(
     tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
     authed: &ApiAuthed,

backend/windmill-api/src/scripts.rs

@@ -41,11 +41,11 @@ use windmill_audit::ActionKind;
 use windmill_worker::{process_relative_imports, scoped_dependency_map::ScopedDependencyMap};
 
 use windmill_common::{
-    assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType},
+    assets::{AssetUsageKind, AssetWithAltAccessType, clear_asset_usage, insert_asset_usage},
     error::to_anyhow,
     s3_helpers::upload_artifact_to_store,
     scripts::hash_script,
-    utils::WarnAfterExt,
+    utils::{WarnAfterExt, paginate_without_limits},
     worker::{CLOUD_HOSTED, MIN_VERSION_SUPPORTS_DEBOUNCING},
 };
 
@@ -61,7 +61,7 @@ use windmill_common::{
     },
     users::username_to_permissioned_as,
     utils::{
-        not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
+        not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath,
     },
     worker::to_raw_value,
     HUB_BASE_URL,
@@ -193,7 +193,7 @@ async fn list_search_scripts(
 ) -> JsonResult<Vec<SearchScript>> {
     let mut tx = user_db.begin(&authed).await?;
     #[cfg(feature = "enterprise")]
-    let n = 1000;
+    let n = 10000;
 
     #[cfg(not(feature = "enterprise"))]
     let n = 10;
@@ -219,7 +219,7 @@ async fn list_scripts(
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListScriptQuery>,
 ) -> JsonResult<Vec<ListableScript>> {
-    let (per_page, offset) = paginate(pagination);
+    let (per_page, offset) = paginate_without_limits(pagination);
     let mut sqlb = SqlBuilder::select_from("script as o")
         .fields(&[
             "hash",
@@ -228,6 +228,11 @@ async fn list_scripts(
             "COALESCE(draft.created_at, o.created_at) as created_at",
             "archived",
             "extra_perms",
+            if !lq.without_description.unwrap_or(false) {
+                "description"
+            } else {
+                "NULL as description"
+            },
             "CASE WHEN lock_error_logs IS NOT NULL THEN true ELSE false END as has_deploy_errors",
             "language",
             "favorite.path IS NOT NULL as starred",

backend/windmill-api/src/workspaces.rs

@@ -75,6 +75,7 @@ lazy_static::lazy_static! {
 
 pub fn workspaced_service() -> Router {
     let router = Router::new()
+        .route("/get_as_superadmin", get(get_workspace_as_superadmin))
         .route("/list_pending_invites", get(list_pending_invites))
         .route("/update", post(edit_workspace))
         .route("/archive", post(archive_workspace))
@@ -2342,6 +2343,35 @@ async fn get_used_triggers(
     Ok(Json(websocket_used))
 }
 
+async fn get_workspace_as_superadmin(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+) -> JsonResult<Workspace> {
+    require_super_admin(&db, &authed.email).await?;
+    let workspace = sqlx::query_as!(
+        Workspace,
+        "SELECT
+            workspace.id AS \"id!\",
+            workspace.name AS \"name!\",
+            workspace.owner AS \"owner!\",
+            workspace.deleted AS \"deleted!\",
+            workspace.premium AS \"premium!\",
+            workspace_settings.color AS \"color\",
+            workspace.parent_workspace_id AS \"parent_workspace_id\"
+        FROM workspace
+        LEFT JOIN workspace_settings ON workspace.id = workspace_settings.workspace_id
+        WHERE workspace.id = $1",
+        w_id
+    )
+    .fetch_optional(&db)
+    .await?;
+
+    let workspace = not_found_if_none(workspace, "workspace", w_id)?;
+
+    Ok(Json(workspace))
+}
+
 async fn list_workspaces_as_super_admin(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
@@ -2425,6 +2455,13 @@ lazy_static::lazy_static! {
         }
     };
 
+    pub static ref DISABLE_WORKSPACE_FORK: bool = {
+        match std::env::var("DISABLE_WORKSPACE_FORK") {
+            Ok(val) => val == "true",
+            Err(_) => false,
+        }
+    };
+
 }
 
 async fn create_workspace_require_superadmin() -> String {
@@ -2681,9 +2718,14 @@ async fn update_workspace_settings(
         WorkspaceGitSyncSettings::default()
     };
 
-    // We only keep the first git sync repo, since it is considered the main one
+    // We only keep the first git sync repo that is sync mode (use_individual_branch = false), since it is considered the main one
     // Context: see WIN-1559
-    git_sync_settings.repositories.truncate(1);
+    git_sync_settings.repositories = git_sync_settings
+        .repositories
+        .into_iter()
+        .filter(|r| !r.use_individual_branch.unwrap_or(false))
+        .take(1)
+        .collect();
 
     let serialized_config = serde_json::to_value::<WorkspaceGitSyncSettings>(git_sync_settings)
         .map_err(|err| Error::internal_err(err.to_string()))?;
@@ -2753,6 +2795,17 @@ async fn clone_groups(
     .execute(&mut **tx)
     .await?;
 
+    sqlx::query!(
+        "INSERT INTO usr_to_group (workspace_id, group_, usr)
+         SELECT $2, group_, usr
+         FROM usr_to_group
+         WHERE workspace_id = $1",
+        source_workspace_id,
+        target_workspace_id,
+    )
+    .execute(&mut **tx)
+    .await?;
+
     Ok(())
 }
 
@@ -3123,6 +3176,10 @@ async fn create_workspace_fork_branch(
         )));
     }
 
+    if *DISABLE_WORKSPACE_FORK {
+        require_super_admin(&db, &authed.email).await?;
+    }
+
     Ok(Json(
         handle_fork_branch_creation(&authed.email, &authed.username, &db, &w_id, &nw.id).await?,
     ))
@@ -3140,6 +3197,10 @@ async fn create_workspace_fork(
         )));
     }
 
+    if *DISABLE_WORKSPACE_FORK {
+        require_super_admin(&db, &authed.email).await?;
+    }
+
     let mut tx: Transaction<'_, Postgres> = db.begin().await?;
 
     // Generate unique forked workspace ID with wm-fork prefix

backend/windmill-common/src/flow_conversations.rs

@@ -15,6 +15,8 @@ pub enum MessageType {
 }
 
 /// Add a message to a conversation using an existing transaction
+/// If the conversation doesn't exist, logs a warning and returns Ok (no error thrown)
+/// This allows memory_id to be used for agent memory without requiring a conversation
 pub async fn add_message_to_conversation_tx(
     tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
     conversation_id: Uuid,
@@ -24,6 +26,23 @@ pub async fn add_message_to_conversation_tx(
     step_name: Option<&str>,
     success: bool,
 ) -> Result<()> {
+    // Check if conversation exists first
+    let conversation_exists = sqlx::query!(
+        "SELECT EXISTS(SELECT 1 FROM flow_conversation WHERE id = $1) as \"exists!\"",
+        conversation_id
+    )
+    .fetch_one(&mut **tx)
+    .await?
+    .exists;
+
+    if !conversation_exists {
+        tracing::warn!(
+            "Conversation {} does not exist. Skipping message insertion. This is expected when flows are called from apps (memory_id is used for agent memory only).",
+            conversation_id
+        );
+        return Ok(());
+    }
+
     // Insert the message
     sqlx::query!(
         "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success)

backend/windmill-common/src/flows.rs

@@ -78,7 +78,8 @@ pub struct ListableFlow {
     pub workspace_id: String,
     pub path: String,
     pub summary: String,
-    pub description: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
     pub edited_by: Option<String>,
     pub edited_at: Option<chrono::DateTime<chrono::Utc>>,
     pub archived: bool,
@@ -868,6 +869,8 @@ pub enum FlowModuleValue {
         parallel: bool,
         #[serde(skip_serializing_if = "Option::is_none")]
         parallelism: Option<InputTransform>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        squash: Option<bool>,
     },
 
     /// While loop node
@@ -877,6 +880,8 @@ pub enum FlowModuleValue {
         modules_node: Option<FlowNodeId>,
         #[serde(default = "default_false")]
         skip_failures: bool,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        squash: Option<bool>,
     },
 
     /// Branch-one node
@@ -988,6 +993,7 @@ struct UntaggedFlowModuleValue {
     assets: Option<Vec<AssetWithAltAccessType>>,
     tools: Option<Vec<AgentTool>>,
     pass_flow_input_directly: Option<bool>,
+    squash: Option<bool>,
 }
 
 impl<'de> Deserialize<'de> for FlowModuleValue {
@@ -1026,6 +1032,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
                 skip_failures: untagged.skip_failures.unwrap_or(true),
                 parallel: untagged.parallel.unwrap_or(false),
                 parallelism: untagged.parallelism,
+                squash: untagged.squash,
             }),
             "whileloopflow" => Ok(FlowModuleValue::WhileloopFlow {
                 modules: untagged
@@ -1033,6 +1040,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
                     .ok_or_else(|| serde::de::Error::missing_field("modules"))?,
                 modules_node: untagged.modules_node,
                 skip_failures: untagged.skip_failures.unwrap_or(false),
+                squash: untagged.squash,
             }),
             "branchone" => Ok(FlowModuleValue::BranchOne {
                 branches: untagged
@@ -1122,6 +1130,7 @@ where
 
 #[derive(Deserialize)]
 pub struct ListFlowQuery {
+    pub without_description: Option<bool>,
     pub path_start: Option<String>,
     pub path_exact: Option<String>,
     pub edited_by: Option<String>,

backend/windmill-common/src/scripts.rs

@@ -353,6 +353,8 @@ pub struct ListableScript {
     pub starred: bool,
     pub tag: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub has_draft: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub draft_only: Option<bool>,
@@ -492,6 +494,7 @@ where
 
 #[derive(Debug, Deserialize)]
 pub struct ListScriptQuery {
+    pub without_description: Option<bool>,
     pub path_start: Option<String>,
     pub path_exact: Option<String>,
     pub created_by: Option<String>,

backend/windmill-queue/src/jobs.rs

@@ -24,6 +24,9 @@ use serde::{ser::SerializeMap, Serialize};
 use serde_json::{json, value::RawValue};
 use sqlx::{types::Json, Pool, Postgres, Transaction};
 use sqlx::{Encode, PgExecutor};
+use tokio::sync::mpsc::Sender;
+use tokio::sync::oneshot;
+use tokio::task::JoinHandle;
 use tokio::{sync::RwLock, time::sleep};
 use ulid::Ulid;
 use uuid::Uuid;
@@ -137,7 +140,7 @@ pub struct CanceledBy {
     pub reason: Option<String>,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct JobCompleted {
     pub job: MiniCompletedJob,
     pub preprocessed_args: Option<HashMap<String, Box<RawValue>>>,
@@ -151,6 +154,10 @@ pub struct JobCompleted {
     pub duration: Option<i64>,
     pub has_stream: Option<bool>,
     pub from_cache: Option<bool>,
+    #[serde(skip)]
+    pub flow_runners: Option<Arc<FlowRunners>>,
+    #[serde(skip)]
+    pub done_tx: Option<oneshot::Sender<()>>,
 }
 
 pub async fn cancel_single_job<'c>(
@@ -2271,6 +2278,8 @@ pub struct JobAndPerms {
     pub parent_runnable_path: Option<String>,
     pub token: String,
     pub precomputed_agent_info: Option<PrecomputedAgentInfo>,
+    #[serde(skip)]
+    pub flow_runners: Option<Arc<FlowRunners>>,
 }
 impl PulledJob {
     pub async fn get_job_and_perms(self, db: &DB) -> JobAndPerms {
@@ -2302,6 +2311,7 @@ impl PulledJob {
             parent_runnable_path: self.parent_runnable_path,
             token,
             precomputed_agent_info: None,
+            flow_runners: None,
         }
     }
 }
@@ -2492,6 +2502,8 @@ impl PulledJobResult {
                     duration: None,
                     has_stream: Some(false),
                     from_cache: None,
+                    flow_runners: None,
+                    done_tx: None,
                 }),
             ),
             PulledJobResult { job: Some(job), error_while_preprocessing: Some(e), .. } => Err(
@@ -2511,6 +2523,8 @@ impl PulledJobResult {
                     duration: None,
                     has_stream: Some(false),
                     from_cache: None,
+                    flow_runners: None,
+                    done_tx: None,
                 }),
             ),
             PulledJobResult { job, .. } => Ok(job),
@@ -5715,10 +5729,74 @@ async fn restarted_flows_resolution(
     ))
 }
 
+
+
+// Wrapper struct to send both job and optional flow_runners to dedicated workers
+pub struct DedicatedWorkerJob {
+    pub job: Arc<MiniPulledJob>,
+    pub flow_runners: Option<Arc<FlowRunners>>,
+    pub done_tx: Option<oneshot::Sender<()>>,
+}
+
+#[derive(Debug)]
+pub struct FlowRunners {
+    pub runners: HashMap<String, Sender<DedicatedWorkerJob>>,
+    pub handles: Vec<JoinHandle<()>>,
+    pub job_id: Uuid,
+}
+
+impl Drop for FlowRunners {
+    fn drop(&mut self) {
+        let total_runners = self.handles.len();
+        tracing::info!("dropping {} flow runners for job {}", total_runners, self.job_id);
+
+        // First, drop all senders to signal workers to stop gracefully
+        self.runners.clear();
+
+        // Spawn a background task to wait with timeout and abort if needed
+        let handles = std::mem::take(&mut self.handles);
+        let job_id = self.job_id;
+
+        tokio::spawn(async move {
+            // Extract abort handles before consuming the join handles
+            let abort_handles: Vec<_> = handles.iter().map(|h| h.abort_handle()).collect();
+
+            // Wait up to 5 seconds for natural termination
+            let timeout_result = tokio::time::timeout(
+                tokio::time::Duration::from_secs(5),
+                futures::future::join_all(handles)
+            ).await;
+
+            match timeout_result {
+                Ok(_) => {
+                    tracing::info!("all {} flow runners for job {} terminated gracefully", total_runners, job_id);
+                }
+                Err(_) => {
+                    // Timeout reached, abort only the handles that haven't finished
+                    let mut aborted = 0;
+                    for abort_handle in abort_handles {
+                        if !abort_handle.is_finished() {
+                            abort_handle.abort();
+                            aborted += 1;
+                        }
+                    }
+                    let graceful = total_runners - aborted;
+                    tracing::warn!(
+                        "flow runners for job {}: {} terminated gracefully, {} aborted after 5s timeout",
+                        job_id, graceful, aborted
+                    );
+                }
+            }
+        });
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct SameWorkerPayload {
     pub job_id: Uuid,
     pub recoverable: bool,
+    #[serde(skip)]
+    pub flow_runners: Option<Arc<FlowRunners>>,
 }
 
 pub async fn get_same_worker_job(

backend/windmill-queue/src/lib.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-mod jobs;
+pub mod jobs;
 #[cfg(feature = "private")]
 pub mod jobs_ee;
 pub mod jobs_oss;

backend/windmill-worker/loader_builder.bun.js

@@ -17,7 +17,7 @@ const captureVersion =
   /^((?:\@[^\/\@]+\/[^\/\@]+)|(?:[^\/\@]+))(?:\@([^\/]+))?.*$/;
 
 import { semver } from "bun";
-
+import { isBuiltin } from "module";
 let content = await fs.readFile("./out/main.js", { encoding: "utf8" });
 const imports = new Bun.Transpiler().scanImports(
   content.replaceAll("__require", "require")
@@ -29,7 +29,7 @@ for (const i of imports) {
   if (name == undefined) {
     throw Error("Unrecognized import: " + i.path);
   }
-  if (name.startsWith("node:")) {
+  if (isBuiltin(name)) {
     continue;
   }
   let splitted = name.split("/");

backend/windmill-worker/src/ai/tools.rs

@@ -529,6 +529,7 @@ async fn execute_windmill_tool(
             &mut occupancy_metrics_spawn,
             &mut killpill_rx_spawn,
             None,
+            None,
             #[cfg(feature = "benchmark")]
             &mut bench_spawn,
         )
@@ -656,8 +657,9 @@ async fn handle_tool_execution_success(
         ..
     }) = send_result.as_ref()
     {
+        let result = result.clone();
         ctx.job_completed_tx
-            .send(send_result.as_ref().unwrap().result.clone(), true)
+            .send(send_result.unwrap().result, true)
             .await
             .map_err(to_anyhow)?;
         result

backend/windmill-worker/src/bun_executor.rs

@@ -9,10 +9,8 @@ use serde_json::value::RawValue;
 
 use uuid::Uuid;
 use windmill_parser_ts::remove_pinned_imports;
-use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PrecomputedAgentInfo};
 
-#[cfg(feature = "enterprise")]
-use crate::common::build_envs_map;
+use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PrecomputedAgentInfo};
 
 use crate::{
     common::{
@@ -38,12 +36,6 @@ use tokio::{fs::File, process::Command};
 
 use tokio::io::AsyncReadExt;
 
-#[cfg(feature = "enterprise")]
-use tokio::sync::mpsc::Receiver;
-
-#[cfg(feature = "enterprise")]
-use windmill_common::variables;
-
 use windmill_common::{
     error::{self, Result},
     get_latest_hash_for_path,
@@ -1567,10 +1559,18 @@ pub async fn get_common_bun_proc_envs(base_internal_url: Option<&str>) -> HashMa
     return bun_envs;
 }
 
-#[cfg(feature = "enterprise")]
-use crate::{dedicated_worker::handle_dedicated_process, JobCompletedSender};
+#[cfg(feature = "private")]
+use crate::{
+    common::build_envs_map, dedicated_worker_oss::handle_dedicated_process, JobCompletedSender,
+};
+#[cfg(feature = "private")]
+use tokio::sync::mpsc::Receiver;
+#[cfg(feature = "private")]
+use windmill_common::variables;
+#[cfg(feature = "private")]
+use windmill_queue::DedicatedWorkerJob;
 
-#[cfg(feature = "enterprise")]
+#[cfg(feature = "private")]
 pub async fn start_worker(
     requirements_o: Option<String>,
     codebase: Option<String>,
@@ -1584,8 +1584,9 @@ pub async fn start_worker(
     script_path: &str,
     token: &str,
     job_completed_tx: JobCompletedSender,
-    jobs_rx: Receiver<std::sync::Arc<MiniPulledJob>>,
+    jobs_rx: Receiver<DedicatedWorkerJob>,
     killpill_rx: tokio::sync::broadcast::Receiver<()>,
+    client: windmill_common::client::AuthedClient,
 ) -> Result<()> {
     let mut logs = "".to_string();
     let mut mem_peak: i32 = 0;
@@ -1737,6 +1738,12 @@ BigInt.prototype.toJSON = function () {{
 
 console.log('start');
 
+function getArgs(line) {{
+    let {{ {spread} }} = JSON.parse(line)
+    {dates}
+    return [ {spread} ];
+}}
+
 for await (const line of Readline.createInterface({{ input: process.stdin }})) {{
     {print_lines}
 
@@ -1744,9 +1751,8 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
         process.exit(0);
     }}
     try {{
-        let {{ {spread} }} = JSON.parse(line)
-        {dates}
-        let res = await Main.main(...[ {spread} ]);
+        const args = getArgs(line);
+        const res = await Main.main(...args);
         console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value));
     }} catch (e) {{
         console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }}));
@@ -1812,6 +1818,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
             db,
             &script_path,
             "nodejs",
+            client,
         )
         .await
     } else {
@@ -1838,6 +1845,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
             db,
             script_path,
             "bun",
+            client,
         )
         .await
     }

backend/windmill-worker/src/dedicated_worker.rs

@@ -1,759 +0,0 @@
-// use tokio::sync::mpsc;
-
-// pub fn create_dedicated_worker() {
-//     let (job_completed_tx, mut new_job) = mpsc::channel::<JobCompleted>(100);
-// }
-
-use async_recursion::async_recursion;
-use std::{collections::VecDeque, process::Stdio, sync::Arc};
-use tokio::sync::mpsc::Sender;
-use tokio::{
-    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
-    process::Command,
-    task::JoinHandle,
-};
-use windmill_common::error::Error;
-use windmill_common::flows::FlowValue;
-use windmill_common::worker::WORKER_CONFIG;
-use windmill_common::KillpillSender;
-use windmill_common::{
-    cache, error,
-    flows::{FlowModule, FlowModuleValue},
-    scripts::{ScriptHash, ScriptLang},
-    variables,
-    worker::to_raw_value,
-    DB,
-};
-use windmill_queue::append_logs;
-use windmill_queue::MiniPulledJob;
-
-use anyhow::Context;
-
-use crate::{common::start_child_process, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS};
-
-use futures::{future, Future};
-use std::{collections::HashMap, task::Poll};
-
-use tokio::sync::mpsc::Receiver;
-
-fn conditional_polling<T>(
-    fut: impl Future<Output = T>,
-    predicate: bool,
-) -> impl Future<Output = T> {
-    let mut fut = Box::pin(fut);
-    future::poll_fn(move |cx| {
-        if predicate {
-            fut.as_mut().poll(cx)
-        } else {
-            Poll::Pending
-        }
-    })
-}
-
-async fn write_stdin(stdin: &mut tokio::process::ChildStdin, s: &str) -> error::Result<()> {
-    let _ = &stdin.write_all(format!("{s}\n").as_bytes()).await?;
-    stdin.flush().await.context("stdin flush")?;
-    Ok(())
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn handle_dedicated_process(
-    command_path: &String,
-    job_dir: &str,
-    context_envs: HashMap<String, String>,
-    envs: HashMap<String, String>,
-    reserved_variables: Vec<variables::ContextualVariable>,
-    common_bun_proc_envs: HashMap<String, String>,
-    args: Vec<&str>,
-    mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
-    job_completed_tx: JobCompletedSender,
-    token: &str,
-    mut jobs_rx: Receiver<std::sync::Arc<MiniPulledJob>>,
-    worker_name: &str,
-    db: &DB,
-    script_path: &str,
-    mode: &str,
-) -> std::result::Result<(), error::Error> {
-    //do not cache local dependencies
-
-    use windmill_queue::{JobCompleted, MiniCompletedJob};
-
-    use crate::{handle_child::process_status, PROXY_ENVS};
-    let cmd_name = format!("dedicated {command_path}");
-    let mut child = {
-        let mut cmd = Command::new(command_path);
-        cmd.current_dir(job_dir)
-            .env_clear()
-            .envs(context_envs)
-            .envs(envs)
-            .envs(PROXY_ENVS.clone())
-            .envs(
-                reserved_variables
-                    .iter()
-                    .map(|x| (x.name.clone(), x.value.clone()))
-                    .collect::<Vec<_>>(),
-            )
-            .envs(common_bun_proc_envs)
-            .args(args)
-            .stdin(Stdio::piped())
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped());
-        start_child_process(cmd, command_path, false).await?
-    };
-
-    let stdout = child
-        .stdout()
-        .take()
-        .expect("child did not have a handle to stdout");
-
-    let stderr = child
-        .stderr()
-        .take()
-        .expect("child did not have a handle to stderr");
-
-    let mut reader = BufReader::new(stdout).lines();
-
-    let mut err_reader = BufReader::new(stderr).lines();
-
-    let mut stdin = child
-        .stdin()
-        .take()
-        .expect("child did not have a handle to stdin");
-
-    // Ensure the child process is spawned in the runtime so it can
-    // make progress on its own while we await for any output.
-    let child = tokio::spawn(async move {
-        let status = Box::into_pin(child.wait())
-            .await
-            .expect("child process encountered an error");
-        if let Err(e) = process_status(&cmd_name, status, vec![]) {
-            tracing::error!("child exit status was not success: {e:#}");
-        } else {
-            tracing::info!("child exit status was success");
-        }
-    });
-
-    let mut jobs: VecDeque<MiniCompletedJob> = VecDeque::with_capacity(MAX_BUFFERED_DEDICATED_JOBS);
-    // let mut i = 0;
-    // let mut j = 0;
-    let mut alive = true;
-
-    let init_log = format!("dedicated worker {mode}: {worker_name}\n\n");
-    let mut logs = init_log.clone();
-    loop {
-        tokio::select! {
-            biased;
-            _ = killpill_rx.recv(), if alive => {
-                println!("received killpill for dedicated worker");
-                alive = false;
-                if let Err(e) = write_stdin(&mut stdin, "end").await {
-                    tracing::info!("Could not write end message to stdin: {e:?}")
-                }
-                stdin.flush().await.context("stdin flush")?;
-            },
-            line = err_reader.next_line() => {
-                if let Some(line) = line.expect("line is ok") {
-                    tracing::error!("stderr dedicated worker: {line}");
-                    logs.push_str("[stderr] ");
-                    logs.push_str(&line);
-                    logs.push_str("\n");
-                } else {
-                    tracing::info!("dedicated worker process exited {script_path}");
-                    let mut last_stdout = "".to_string();
-                    while let Some(line) = reader.next_line().await.ok().flatten() {
-                        last_stdout = line;
-                        last_stdout.push_str("\n");
-                    }
-                    tracing::info!("Last stdout for {script_path}: {last_stdout}");
-                    break;
-                }
-            },
-            line = reader.next_line() => {
-                // j += 1;
-
-                if let Some(line) = line.expect("line is ok") {
-                    if line == "start" {
-                        tracing::info!("dedicated worker process started {script_path}");
-                        continue;
-                    }
-                    tracing::debug!("processed job: |{line}|");
-                    if line.starts_with("wm_res[") {
-                        let job = jobs.pop_front().expect("pop");
-                        tracing::info!("job completed on dedicated worker {script_path}: {}", job.id);
-                        match serde_json::from_str::<Box<serde_json::value::RawValue>>(&line.replace("wm_res[success]:", "").replace("wm_res[error]:", "")) {
-                            Ok(result) => {
-                                let result = Arc::new(result);
-                                append_logs(&job.id, &job.workspace_id,  logs.clone(), &db.into()).await;
-                                if line.starts_with("wm_res[success]:") {
-                                    job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false), from_cache: None }, true).await.unwrap()
-                                } else {
-                                    job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false), from_cache: None }, true).await.unwrap()
-                                }
-                            },
-                            Err(e) => {
-                                tracing::error!("Could not deserialize job result `{line}`: {e:?}");
-                                job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false), from_cache: None }, true).await.unwrap();
-                            },
-                        };
-                        logs = init_log.clone();
-                    } else {
-                        logs.push_str(&line);
-                        logs.push_str("\n");
-                    }
-                } else {
-                    tracing::info!("dedicated worker {script_path} process exited");
-                    let mut last_stderr = "".to_string();
-                    while let Some(line) = err_reader.next_line().await.ok().flatten() {
-                        last_stderr = line;
-                        last_stderr.push_str("\n");
-                    }
-                    tracing::info!("Last stderr for {script_path}: {last_stderr}");
-                    break;
-                }
-            },
-            job = conditional_polling(jobs_rx.recv(), alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS) => {
-                // i += 1;
-                if let Some(job) = job {
-                    let id = job.id;
-                    let args = serde_json::to_string(&job.args).expect("serialize");
-                    jobs.push_back(MiniCompletedJob::from(job));
-                    tracing::info!("received job and adding to queue on dedicated worker for {script_path}: {} (queue_size: {})", id, jobs.len());
-
-                    // write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({"x": job.id}))).expect("serialize")).await?;
-                    write_stdin(&mut stdin, &args).await?;
-                    stdin.flush().await.context("stdin flush")?;
-                    // tracing::info!("wrote job to stdin for {script_path}: {} (queue_size: {})", id, jobs.len());
-                } else {
-                    tracing::debug!("job channel closed");
-                    alive = false;
-                    if let Err(e) = write_stdin(&mut stdin, "end").await {
-                        tracing::error!("Could not write end message to stdin: {e:?}")
-                    }
-                }
-            }
-        }
-    }
-
-    child
-        .await
-        .map_err(|e| anyhow::anyhow!("child process {script_path} encountered an error: {e:#}"))?;
-
-    tracing::info!("dedicated worker {script_path} child process exited successfully");
-    Ok(())
-}
-
-type DedicatedWorker = (
-    String,
-    Sender<std::sync::Arc<MiniPulledJob>>,
-    Option<JoinHandle<()>>,
-);
-
-// spawn one dedicated worker per compatible steps of the flow, associating the node id to the dedicated worker channel send
-#[async_recursion]
-#[cfg(feature = "enterprise")]
-async fn spawn_dedicated_workers_for_flow(
-    modules: &Vec<FlowModule>,
-    w_id: &str,
-    path: &str,
-    killpill_tx: KillpillSender,
-    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
-    db: &DB,
-    worker_dir: &str,
-    base_internal_url: &str,
-    worker_name: &str,
-    job_completed_tx: &JobCompletedSender,
-) -> Vec<DedicatedWorker> {
-    let mut workers = vec![];
-    let mut script_path_to_worker: HashMap<String, Sender<std::sync::Arc<MiniPulledJob>>> =
-        HashMap::new();
-    for module in modules.iter() {
-        let value = module.get_value();
-        if let Ok(value) = value {
-            match &value {
-                FlowModuleValue::Script { path, hash, .. } => {
-                    let key = format!(
-                        "{}:{}",
-                        path,
-                        hash.clone()
-                            .map(|x| x.to_string())
-                            .unwrap_or_else(|| "".to_string())
-                    );
-                    if let Some(sender) = script_path_to_worker.get(&key) {
-                        workers.push((module.id.clone(), sender.clone(), None));
-                    } else {
-                        if let Some(dedi_w) = spawn_dedicated_worker(
-                            SpawnWorker::Script { path: path.to_string(), hash: hash.clone() },
-                            w_id,
-                            killpill_tx.clone(),
-                            killpill_rx,
-                            db,
-                            worker_dir,
-                            base_internal_url,
-                            worker_name,
-                            job_completed_tx,
-                            Some(module.id.clone()),
-                        )
-                        .await
-                        {
-                            script_path_to_worker.insert(key, dedi_w.1.clone());
-                            workers.push(dedi_w);
-                        }
-                    }
-                }
-                FlowModuleValue::ForloopFlow { modules, .. } => {
-                    let w = spawn_dedicated_workers_for_flow(
-                        &modules,
-                        w_id,
-                        path,
-                        killpill_tx.clone(),
-                        killpill_rx,
-                        db,
-                        worker_dir,
-                        base_internal_url,
-                        worker_name,
-                        job_completed_tx,
-                    )
-                    .await;
-                    workers.extend(w);
-                }
-                FlowModuleValue::WhileloopFlow { modules, .. } => {
-                    let w = spawn_dedicated_workers_for_flow(
-                        &modules,
-                        w_id,
-                        path,
-                        killpill_tx.clone(),
-                        killpill_rx,
-                        db,
-                        worker_dir,
-                        base_internal_url,
-                        worker_name,
-                        job_completed_tx,
-                    )
-                    .await;
-                    workers.extend(w);
-                }
-                FlowModuleValue::BranchOne { branches, default, .. } => {
-                    for modules in branches
-                        .iter()
-                        .map(|x| &x.modules)
-                        .chain(std::iter::once(default))
-                    {
-                        let w = spawn_dedicated_workers_for_flow(
-                            &modules,
-                            w_id,
-                            path,
-                            killpill_tx.clone(),
-                            killpill_rx,
-                            db,
-                            worker_dir,
-                            base_internal_url,
-                            worker_name,
-                            job_completed_tx,
-                        )
-                        .await;
-                        workers.extend(w);
-                    }
-                }
-                FlowModuleValue::BranchAll { branches, .. } => {
-                    for branch in branches {
-                        let w = spawn_dedicated_workers_for_flow(
-                            &branch.modules,
-                            w_id,
-                            path,
-                            killpill_tx.clone(),
-                            killpill_rx,
-                            db,
-                            worker_dir,
-                            base_internal_url,
-                            worker_name,
-                            job_completed_tx,
-                        )
-                        .await;
-                        workers.extend(w);
-                    }
-                }
-                FlowModuleValue::RawScript { content, lock, path: spath, language, .. } => {
-                    if let Some(dedi_w) = spawn_dedicated_worker(
-                        SpawnWorker::RawScript {
-                            path: spath.clone().unwrap_or(path.to_string()),
-                            content: content.to_string(),
-                            lock: lock.clone(),
-                            lang: language.clone(),
-                        },
-                        w_id,
-                        killpill_tx.clone(),
-                        killpill_rx,
-                        db,
-                        worker_dir,
-                        base_internal_url,
-                        worker_name,
-                        job_completed_tx,
-                        Some(module.id.clone()),
-                    )
-                    .await
-                    {
-                        workers.push(dedi_w);
-                    }
-                }
-                FlowModuleValue::FlowScript { id, language, .. } => {
-                    let spawn = cache::flow::fetch_script(
-                        &windmill_common::worker::Connection::Sql(db.clone()),
-                        *id,
-                    )
-                    .await
-                    .map(|data| SpawnWorker::RawScript {
-                        path: "".to_string(),
-                        content: data.code.clone(),
-                        lock: data.lock.clone(),
-                        lang: *language,
-                    });
-                    match spawn {
-                        Ok(spawn) => {
-                            if let Some(dedi_w) = spawn_dedicated_worker(
-                                spawn,
-                                w_id,
-                                killpill_tx.clone(),
-                                killpill_rx,
-                                db,
-                                worker_dir,
-                                base_internal_url,
-                                worker_name,
-                                job_completed_tx,
-                                Some(module.id.clone()),
-                            )
-                            .await
-                            {
-                                workers.push(dedi_w);
-                            }
-                        }
-                        Err(err) => tracing::error!(
-                            "failed to get script for module: {:?}, err: {:?}",
-                            module,
-                            err
-                        ),
-                    }
-                }
-                FlowModuleValue::Flow { .. } => (),
-                FlowModuleValue::Identity => (),
-                FlowModuleValue::AIAgent { .. } => (),
-            }
-        } else {
-            tracing::error!("failed to get value for module: {:?}", module);
-        }
-    }
-    workers
-}
-
-pub async fn create_dedicated_worker_map(
-    killpill_tx: &KillpillSender,
-    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
-    db: &DB,
-    worker_dir: &str,
-    base_internal_url: &str,
-    worker_name: &str,
-    job_completed_tx: &JobCompletedSender,
-) -> (
-    HashMap<String, Sender<std::sync::Arc<MiniPulledJob>>>,
-    bool,
-    Vec<JoinHandle<()>>,
-) {
-    let mut dedicated_handles = vec![];
-    if let Some(_wp) = WORKER_CONFIG.read().await.dedicated_worker.clone() {
-        let mut hm = HashMap::new();
-        let is_flow_worker;
-        if let Some(flow_path) = _wp.path.strip_prefix("flow/") {
-            is_flow_worker = true;
-            let value = sqlx::query_scalar!(
-                "SELECT flow_version.value AS \"value!: sqlx::types::Json<Box<sqlx::types::JsonRawValue>>\" 
-                FROM flow 
-                LEFT JOIN flow_version 
-                    ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
-                WHERE flow.path = $1 AND flow.workspace_id = $2",
-                flow_path,
-                _wp.workspace_id
-            )
-            .fetch_optional(db)
-            .await;
-            if let Ok(v) = value {
-                if let Some(v) = v {
-                    let value = serde_json::from_str::<FlowValue>(v.get()).map_err(|err| {
-                        Error::internal_err(format!(
-                            "could not convert json to flow for {flow_path}: {err:?}"
-                        ))
-                    });
-                    if let Ok(flow) = value {
-                        let workers = spawn_dedicated_workers_for_flow(
-                            &flow.modules,
-                            &_wp.workspace_id,
-                            &_wp.path,
-                            killpill_tx.clone(),
-                            &killpill_rx,
-                            db,
-                            &worker_dir,
-                            base_internal_url,
-                            &worker_name,
-                            &job_completed_tx,
-                        )
-                        .await;
-                        workers.into_iter().for_each(|(path, sender, handle)| {
-                            tracing::info!("spawned dedicated worker for flow: {}", path.as_str());
-                            if let Some(h) = handle {
-                                dedicated_handles.push(h);
-                            }
-                            hm.insert(path, sender);
-                        });
-                    }
-                } else {
-                    tracing::error!(
-                        "flow present but value not found for dedicated worker. {}",
-                        flow_path
-                    );
-                }
-            } else {
-                tracing::error!("flow not found for dedicated worker: {}. Waiting for dependency job and expected to restart.", flow_path);
-            }
-        } else {
-            is_flow_worker = false;
-            if let Some((path, sender, handle)) = spawn_dedicated_worker(
-                SpawnWorker::Script { path: _wp.path.clone(), hash: None },
-                &_wp.workspace_id,
-                killpill_tx.clone(),
-                &killpill_rx,
-                db,
-                &worker_dir,
-                base_internal_url,
-                &worker_name,
-                &job_completed_tx,
-                None,
-            )
-            .await
-            {
-                if let Some(h) = handle {
-                    dedicated_handles.push(h);
-                }
-                hm.insert(path, sender);
-            } else {
-                tracing::error!(
-                    "failed to spawn dedicated worker for {}, script not found",
-                    _wp.path
-                );
-            }
-        }
-        (hm, is_flow_worker, dedicated_handles)
-    } else {
-        (HashMap::new(), false, dedicated_handles)
-    }
-}
-
-#[derive(Debug, Clone)]
-pub enum SpawnWorker {
-    Script { path: String, hash: Option<ScriptHash> },
-    RawScript { path: String, content: String, lock: Option<String>, lang: ScriptLang },
-}
-
-// spawn one dedicated worker and return the key, the channel sender and the join handle
-// note that for it will return none for language that do not support dedicated workers
-// note that go using cache binary does not need dedicated workers so all languages are supported
-#[cfg(feature = "enterprise")]
-async fn spawn_dedicated_worker(
-    sw: SpawnWorker,
-    w_id: &str,
-    killpill_tx: KillpillSender,
-    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
-    db: &DB,
-    worker_dir: &str,
-    base_internal_url: &str,
-    worker_name: &str,
-    job_completed_tx: &JobCompletedSender,
-    node_id: Option<String>,
-) -> Option<DedicatedWorker> {
-    use windmill_common::{
-        error::Error,
-        scripts::{ScriptHash, ScriptLang},
-        utils::rd_string,
-    };
-    use windmill_queue::MiniPulledJob;
-
-    use crate::{build_envs, get_script_content_by_hash, ContentReqLangEnvs};
-
-    #[cfg(not(feature = "enterprise"))]
-    {
-        tracing::error!("Dedicated worker is an enterprise feature");
-        killpill_tx.send(()).expect("send");
-        return None;
-    }
-
-    #[cfg(feature = "enterprise")]
-    {
-        let (dedicated_worker_tx, dedicated_worker_rx) = tokio::sync::mpsc::channel::<
-            std::sync::Arc<MiniPulledJob>,
-        >(MAX_BUFFERED_DEDICATED_JOBS);
-        let killpill_rx = killpill_rx.resubscribe();
-        let db2 = db.clone();
-        let base_internal_url = base_internal_url.to_string();
-        let worker_name = worker_name.to_string();
-        let job_completed_tx = job_completed_tx.clone();
-        let job_dir = format!(
-            "{}/dedicated{}",
-            worker_dir,
-            node_id
-                .as_ref()
-                .map(|x| format!("-{x}"))
-                .unwrap_or_else(|| "".to_string())
-        );
-        tokio::fs::create_dir_all(&job_dir)
-            .await
-            .expect("create dir");
-
-        let path = match &sw {
-            SpawnWorker::RawScript { path, .. } => path.to_string(),
-            SpawnWorker::Script { path, .. } => path.to_string(),
-        };
-
-        let path2 = path.clone();
-        let w_id = w_id.to_string();
-
-        let (content, lock, language, envs, codebase) = match sw.clone() {
-            SpawnWorker::Script { path, hash } => {
-                let q = if let Some(hash) = hash {
-                    get_script_content_by_hash(&hash, &w_id, &db2.into())
-                        .await
-                        .map(|r: ContentReqLangEnvs| {
-                            Some((r.content, r.lockfile, r.language, r.envs, r.codebase))
-                        })
-                } else {
-                    sqlx::query_as::<_, (String, Option<String>, Option<ScriptLang>, Option<Vec<String>>, bool, Option<ScriptHash>)>(
-                        "SELECT content, lock, language, envs, codebase IS NOT NULL, hash FROM script WHERE path = $1 AND workspace_id = $2 AND
-                            archived = false AND lock IS not NULL AND lock_error_logs IS NULL ORDER BY created_at DESC LIMIT 1",
-                    )
-                    .bind(&path)
-                    .bind(&w_id)
-                    .fetch_optional(&db2)
-                    .await
-                    .map_err(|e| Error::internal_err(format!("expected content and lock: {e:#}")))
-                    .map(|x| x.map(|y| (y.0, y.1, y.2, y.3, if y.4 { y.5.map(|z| z.to_string()) } else { None })))
-                };
-                if let Ok(q) = q {
-                    if let Some(wp) = q {
-                        wp
-                    } else {
-                        tracing::error!(
-                            "Failed to fetch script `{}` in workspace {} for dedicated worker.",
-                            path,
-                            w_id
-                        );
-                        return None;
-                    }
-                } else {
-                    tracing::error!("Failed to fetch script for dedicated worker");
-                    killpill_tx.send();
-                    return None;
-                }
-            }
-            SpawnWorker::RawScript { content, lock, lang, .. } => {
-                (content, lock, Some(lang), None, None)
-            }
-        };
-
-        match language {
-            Some(ScriptLang::Python3) | Some(ScriptLang::Bun) | Some(ScriptLang::Deno) => {}
-            _ => return None,
-        }
-
-        let db = db.clone();
-        let handle = tokio::spawn(async move {
-            let token = {
-                let token = rd_string(32);
-                if let Err(e) = sqlx::query_scalar!(
-                    "INSERT INTO token
-                    (token, label, super_admin, email)
-                    VALUES ($1, $2, $3, $4)",
-                    token,
-                    "dedicated_worker",
-                    true,
-                    "dedicated_worker@windmill.dev"
-                )
-                .execute(&db)
-                .await
-                {
-                    tracing::error!("failed to create token for dedicated worker: {:?}", e);
-                    killpill_tx.clone().send();
-                };
-                token
-            };
-
-            let worker_envs = build_envs(envs.as_ref()).expect("failed to build envs");
-
-            if let Err(e) = match language {
-                Some(ScriptLang::Python3) => {
-                    #[cfg(not(feature = "python"))]
-                    {
-                        tracing::error!("Python requires the python feature to be enabled");
-                        killpill_tx.send();
-                        return;
-                    }
-
-                    #[cfg(feature = "python")]
-                    crate::python_executor::start_worker(
-                        lock.as_ref(),
-                        &db,
-                        &content,
-                        &base_internal_url,
-                        &job_dir,
-                        &worker_name,
-                        worker_envs,
-                        &w_id,
-                        &path,
-                        &token,
-                        job_completed_tx,
-                        dedicated_worker_rx,
-                        killpill_rx,
-                    )
-                    .await
-                }
-                Some(ScriptLang::Bun) => {
-                    crate::bun_executor::start_worker(
-                        lock,
-                        codebase,
-                        &db,
-                        &content,
-                        &base_internal_url,
-                        &job_dir,
-                        &worker_name,
-                        worker_envs,
-                        &w_id,
-                        &path,
-                        &token,
-                        job_completed_tx,
-                        dedicated_worker_rx,
-                        killpill_rx,
-                    )
-                    .await
-                }
-                Some(ScriptLang::Deno) => {
-                    crate::deno_executor::start_worker(
-                        &content,
-                        &base_internal_url,
-                        &job_dir,
-                        &worker_name,
-                        worker_envs,
-                        &w_id,
-                        &path,
-                        &token,
-                        job_completed_tx,
-                        dedicated_worker_rx,
-                        killpill_rx,
-                        &db,
-                    )
-                    .await
-                }
-                _ => unreachable!("Non supported language for dedicated worker"),
-            } {
-                tracing::error!("error in dedicated worker for {sw:#?}: {:?}", e);
-            };
-            killpill_tx.clone().send();
-        });
-        return Some((node_id.unwrap_or(path2), dedicated_worker_tx, Some(handle)));
-        // (Some(dedi_path), Some(dedicated_worker_tx), Some(handle))
-    }
-}

backend/windmill-worker/src/dedicated_worker_oss.rs

@@ -0,0 +1,26 @@
+#[cfg(feature = "private")]
+// #[allow(unused)]
+pub use crate::dedicated_worker_ee::*;
+
+#[cfg(not(feature = "private"))]
+pub async fn spawn_flow_module_runners(
+    _job: &windmill_queue::MiniPulledJob,
+    _module: &windmill_common::flows::FlowModule,
+    _failure_module: Option<&Box<windmill_common::flows::FlowModule>>,
+    _killpill_rx: &tokio::sync::broadcast::Receiver<()>,
+    _db: &windmill_common::db::DB,
+    _worker_dir: &str,
+    _base_internal_url: &str,
+    _worker_name: &str,
+    _job_completed_tx: &crate::JobCompletedSender,
+) -> windmill_common::error::Result<(
+    std::collections::HashMap<
+        String,
+        tokio::sync::mpsc::Sender<windmill_queue::DedicatedWorkerJob>,
+    >,
+    Vec<tokio::task::JoinHandle<()>>,
+)> {
+    Err(windmill_common::error::Error::internal_err(
+        "Squashing flow loops is not available in OSS".to_string(),
+    ))
+}

backend/windmill-worker/src/deno_executor.rs

@@ -514,13 +514,14 @@ async fn build_import_map(
     Ok(()) as error::Result<()>
 }
 
-#[cfg(feature = "enterprise")]
-use crate::{dedicated_worker::handle_dedicated_process, JobCompletedSender};
-
-#[cfg(feature = "enterprise")]
+#[cfg(feature = "private")]
+use crate::{dedicated_worker_oss::handle_dedicated_process, JobCompletedSender};
+#[cfg(feature = "private")]
 use tokio::sync::mpsc::Receiver;
+#[cfg(feature = "private")]
+use windmill_queue::DedicatedWorkerJob;
 
-#[cfg(feature = "enterprise")]
+#[cfg(feature = "private")]
 pub async fn start_worker(
     inner_content: &str,
     base_internal_url: &str,
@@ -531,9 +532,10 @@ pub async fn start_worker(
     script_path: &str,
     token: &str,
     job_completed_tx: JobCompletedSender,
-    jobs_rx: Receiver<std::sync::Arc<MiniPulledJob>>,
+    jobs_rx: Receiver<DedicatedWorkerJob>,
     killpill_rx: tokio::sync::broadcast::Receiver<()>,
     db: &sqlx::Pool<sqlx::Postgres>,
+    client: windmill_common::client::AuthedClient,
 ) -> Result<()> {
     use windmill_common::variables;
 
@@ -654,6 +656,7 @@ for await (const chunk of Deno.stdin.readable) {{
         db,
         script_path,
         "deno",
+        client,
     )
     .await
 }

backend/windmill-worker/src/duckdb_executor.rs

@@ -603,7 +603,10 @@ async fn transform_s3_uris(query: &str) -> Result<String> {
 // BigQuery extension requires a json file as credentials
 // The file path is set as an env var by do_duckdb
 // It is created by transform_attach_db_resource_query (when bigquery is detected)
-// and deleted by do_duckdb after the query is executed
+// and deleted by do_duckdb after the query is executed.
+//
+// This relies on the fact that DuckDB does not run in native worker, so
+// a worker will only run a single job at a time.
 pub struct UseBigQueryCredentialsFile {
     path: String,
 }
@@ -665,6 +668,8 @@ fn remove_comments(stmt: &str) -> &str {
 #[cfg(test)]
 mod tests {
     use super::*;
+
+    // Tests for remove_comments function
     #[test]
     fn test_remove_comments_single_line() {
         let sql = "-- This is a comment\nSELECT * FROM table;";
@@ -695,4 +700,377 @@ mod tests {
         let sql = "   -- Comment\n  -- Comment2\n  -- Comment3\n   SELECT\n\n * FROM\n table\n;\n\n -- end comment   ";
         assert_eq!(remove_comments(sql), "SELECT\n\n * FROM\n table\n;");
     }
+
+    // Tests for parse_attach_db_resource function
+    #[test]
+    fn test_parse_attach_db_resource_postgres_res_prefix() {
+        let query = "ATTACH '$res:u/user/my_postgres' AS mydb (TYPE POSTGRES)";
+        let result = parse_attach_db_resource(query);
+        assert!(result.is_some());
+        let parsed = result.unwrap();
+        assert_eq!(parsed.resource_path, "u/user/my_postgres");
+        assert_eq!(parsed.name, "mydb");
+        assert_eq!(parsed.db_type, "POSTGRES");
+        assert!(parsed.extra_args.is_none() || parsed.extra_args.unwrap().is_empty());
+    }
+
+    #[test]
+    fn test_parse_attach_db_resource_res_protocol() {
+        let query = "ATTACH 'res://f/folder/database' AS db (TYPE postgresql)";
+        let result = parse_attach_db_resource(query);
+        assert!(result.is_some());
+        let parsed = result.unwrap();
+        assert_eq!(parsed.resource_path, "f/folder/database");
+        assert_eq!(parsed.name, "db");
+        assert_eq!(parsed.db_type, "postgresql");
+    }
+
+    #[test]
+    fn test_parse_attach_db_resource_mysql() {
+        let query = "ATTACH '$res:u/admin/mysql_prod' AS mysql_db (TYPE mysql)";
+        let result = parse_attach_db_resource(query);
+        assert!(result.is_some());
+        let parsed = result.unwrap();
+        assert_eq!(parsed.resource_path, "u/admin/mysql_prod");
+        assert_eq!(parsed.name, "mysql_db");
+        assert_eq!(parsed.db_type, "mysql");
+    }
+
+    #[test]
+    fn test_parse_attach_db_resource_bigquery() {
+        let query = "ATTACH '$res:u/user/bq_resource' AS bq (TYPE bigquery)";
+        let result = parse_attach_db_resource(query);
+        assert!(result.is_some());
+        let parsed = result.unwrap();
+        assert_eq!(parsed.resource_path, "u/user/bq_resource");
+        assert_eq!(parsed.name, "bq");
+        assert_eq!(parsed.db_type, "bigquery");
+    }
+
+    #[test]
+    fn test_parse_attach_db_resource_with_extra_args() {
+        let query = "ATTACH '$res:u/user/db' AS mydb (TYPE POSTGRES, READ_ONLY)";
+        let result = parse_attach_db_resource(query);
+        assert!(result.is_some());
+        let parsed = result.unwrap();
+        assert_eq!(parsed.resource_path, "u/user/db");
+        assert_eq!(parsed.name, "mydb");
+        assert_eq!(parsed.db_type, "POSTGRES");
+        assert_eq!(parsed.extra_args.unwrap(), ", READ_ONLY");
+    }
+
+    #[test]
+    fn test_parse_attach_db_resource_case_insensitive() {
+        let query = "attach '$res:u/user/db' as mydb (type postgres)";
+        let result = parse_attach_db_resource(query);
+        assert!(result.is_some());
+        let parsed = result.unwrap();
+        assert_eq!(parsed.resource_path, "u/user/db");
+        assert_eq!(parsed.name, "mydb");
+        assert_eq!(parsed.db_type, "postgres");
+    }
+
+    #[test]
+    fn test_parse_attach_db_resource_no_match() {
+        let query = "SELECT * FROM table";
+        let result = parse_attach_db_resource(query);
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_parse_attach_db_resource_regular_attach() {
+        // Regular ATTACH without $res: or res:// should not match
+        let query = "ATTACH 'mydb.duckdb' AS mydb (TYPE duckdb)";
+        let result = parse_attach_db_resource(query);
+        assert!(result.is_none());
+    }
+
+    // Tests for format_attach_db_conn_str function
+    #[test]
+    fn test_format_attach_db_conn_str_postgres_full() {
+        let db_resource = json!({
+            "host": "localhost",
+            "port": 5432,
+            "user": "admin",
+            "password": "secret123",
+            "dbname": "mydb",
+            "sslmode": "require"
+        });
+        let result = format_attach_db_conn_str(db_resource, "postgres").unwrap();
+        assert!(result.contains("dbname=mydb"));
+        assert!(result.contains("user=admin"));
+        assert!(result.contains("host=localhost"));
+        assert!(result.contains("password=secret123"));
+        assert!(result.contains("port=5432"));
+        assert!(result.contains("sslmode=require"));
+    }
+
+    #[test]
+    fn test_format_attach_db_conn_str_postgres_minimal() {
+        let db_resource = json!({
+            "host": "db.example.com",
+            "dbname": "production"
+        });
+        let result = format_attach_db_conn_str(db_resource, "postgres").unwrap();
+        assert!(result.contains("dbname=production"));
+        assert!(result.contains("host=db.example.com"));
+        // Optional fields should result in empty strings
+        assert!(!result.contains("user="));
+        assert!(!result.contains("password="));
+    }
+
+    #[test]
+    fn test_format_attach_db_conn_str_postgresql_alias() {
+        let db_resource = json!({
+            "host": "localhost",
+            "dbname": "test"
+        });
+        let result = format_attach_db_conn_str(db_resource, "postgresql").unwrap();
+        assert!(result.contains("dbname=test"));
+        assert!(result.contains("host=localhost"));
+    }
+
+    #[test]
+    fn test_format_attach_db_conn_str_bigquery() {
+        let db_resource = json!({
+            "project_id": "my-gcp-project"
+        });
+        let result = format_attach_db_conn_str(db_resource, "bigquery").unwrap();
+        assert_eq!(result, "project=my-gcp-project");
+    }
+
+    #[test]
+    fn test_format_attach_db_conn_str_bigquery_missing_project_id() {
+        let db_resource = json!({
+            "other_field": "value"
+        });
+        let result = format_attach_db_conn_str(db_resource, "bigquery");
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("project_id"));
+    }
+
+    #[test]
+    fn test_format_attach_db_conn_str_unsupported_type() {
+        let db_resource = json!({});
+        let result = format_attach_db_conn_str(db_resource, "oracle");
+        assert!(result.is_err());
+        assert!(result
+            .unwrap_err()
+            .to_string()
+            .contains("Unsupported db type"));
+    }
+
+    #[test]
+    fn test_format_attach_db_conn_str_case_insensitive() {
+        let db_resource = json!({
+            "host": "localhost",
+            "dbname": "test"
+        });
+        let result = format_attach_db_conn_str(db_resource, "POSTGRES").unwrap();
+        assert!(result.contains("dbname=test"));
+    }
+
+    #[cfg(feature = "mysql")]
+    #[test]
+    fn test_format_attach_db_conn_str_mysql_full() {
+        let db_resource = json!({
+            "host": "mysql.example.com",
+            "port": 3306,
+            "user": "root",
+            "password": "mysecret",
+            "database": "app_db",
+            "ssl": true
+        });
+        let result = format_attach_db_conn_str(db_resource, "mysql").unwrap();
+        assert!(result.contains("database=app_db"));
+        assert!(result.contains("host=mysql.example.com"));
+        assert!(result.contains("ssl_mode=required"));
+        assert!(result.contains("password=mysecret"));
+        assert!(result.contains("port=3306"));
+        assert!(result.contains("user=root"));
+    }
+
+    #[cfg(feature = "mysql")]
+    #[test]
+    fn test_format_attach_db_conn_str_mysql_ssl_disabled() {
+        let db_resource = json!({
+            "host": "localhost",
+            "database": "test",
+            "ssl": false
+        });
+        let result = format_attach_db_conn_str(db_resource, "mysql").unwrap();
+        assert!(result.contains("ssl_mode=disabled"));
+    }
+
+    // Tests for get_attach_db_install_str function
+    #[test]
+    fn test_get_attach_db_install_str_postgres() {
+        let result = get_attach_db_install_str("postgres").unwrap();
+        assert_eq!(result, "INSTALL postgres;");
+    }
+
+    #[test]
+    fn test_get_attach_db_install_str_bigquery() {
+        let result = get_attach_db_install_str("bigquery").unwrap();
+        assert_eq!(result, "INSTALL bigquery FROM community;");
+    }
+
+    #[test]
+    fn test_get_attach_db_install_str_unsupported() {
+        let result = get_attach_db_install_str("sqlite");
+        assert!(result.is_err());
+        assert!(result
+            .unwrap_err()
+            .to_string()
+            .contains("Unsupported db type"));
+    }
+
+    #[test]
+    fn test_get_attach_db_install_str_case_insensitive() {
+        let result = get_attach_db_install_str("POSTGRES").unwrap();
+        assert_eq!(result, "INSTALL postgres;");
+    }
+
+    #[cfg(feature = "mysql")]
+    #[test]
+    fn test_get_attach_db_install_str_mysql() {
+        let result = get_attach_db_install_str("mysql").unwrap();
+        assert_eq!(result, "INSTALL mysql;");
+    }
+
+    #[cfg(not(feature = "mysql"))]
+    #[test]
+    fn test_get_attach_db_install_str_mysql_disabled() {
+        let result = get_attach_db_install_str("mysql");
+        assert!(result.is_err());
+        assert!(result
+            .unwrap_err()
+            .to_string()
+            .contains("MySQL feature is not enabled"));
+    }
+
+    // Tests for transform_s3_uris function
+    #[tokio::test]
+    async fn test_transform_s3_uris_empty_storage() {
+        let query = "SELECT * FROM read_parquet('s3:///path/to/file.parquet')";
+        let result = transform_s3_uris(query).await.unwrap();
+        assert_eq!(
+            result,
+            "SELECT * FROM read_parquet('s3://_default_/path/to/file.parquet')"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_transform_s3_uris_with_storage() {
+        // URIs with explicit storage should not be transformed
+        let query = "SELECT * FROM read_parquet('s3://mybucket/path/to/file.parquet')";
+        let result = transform_s3_uris(query).await.unwrap();
+        assert_eq!(result, query);
+    }
+
+    #[tokio::test]
+    async fn test_transform_s3_uris_multiple_empty() {
+        let query = "SELECT * FROM read_parquet('s3:///file1.parquet') UNION SELECT * FROM read_parquet('s3:///file2.parquet')";
+        let result = transform_s3_uris(query).await.unwrap();
+        assert!(result.contains("s3://_default_/file1.parquet"));
+        assert!(result.contains("s3://_default_/file2.parquet"));
+    }
+
+    #[tokio::test]
+    async fn test_transform_s3_uris_no_s3() {
+        let query = "SELECT * FROM my_table";
+        let result = transform_s3_uris(query).await.unwrap();
+        assert_eq!(result, query);
+    }
+
+    #[tokio::test]
+    async fn test_transform_s3_uris_mixed() {
+        let query = "SELECT * FROM read_parquet('s3:///default.parquet'), read_csv('s3://explicit/file.csv')";
+        let result = transform_s3_uris(query).await.unwrap();
+        assert!(result.contains("s3://_default_/default.parquet"));
+        assert!(result.contains("s3://explicit/file.csv"));
+    }
+
+    #[tokio::test]
+    async fn test_transform_s3_uris_nested_path() {
+        let query = "SELECT * FROM read_parquet('s3:///deep/nested/path/file.parquet')";
+        let result = transform_s3_uris(query).await.unwrap();
+        assert_eq!(
+            result,
+            "SELECT * FROM read_parquet('s3://_default_/deep/nested/path/file.parquet')"
+        );
+    }
+
+    // Tests for Arg struct
+    #[test]
+    fn test_arg_serialization() {
+        let arg = Arg {
+            name: "test_arg".to_string(),
+            arg_type: "string".to_string(),
+            json_value: json!("hello"),
+        };
+        let serialized = serde_json::to_string(&arg).unwrap();
+        assert!(serialized.contains("\"name\":\"test_arg\""));
+        assert!(serialized.contains("\"arg_type\":\"string\""));
+        assert!(serialized.contains("\"json_value\":\"hello\""));
+    }
+
+    #[test]
+    fn test_arg_serialization_number() {
+        let arg = Arg {
+            name: "count".to_string(),
+            arg_type: "integer".to_string(),
+            json_value: json!(42),
+        };
+        let serialized = serde_json::to_string(&arg).unwrap();
+        assert!(serialized.contains("\"json_value\":42"));
+    }
+
+    #[test]
+    fn test_arg_serialization_null() {
+        let arg = Arg {
+            name: "optional".to_string(),
+            arg_type: "text".to_string(),
+            json_value: json!(null),
+        };
+        let serialized = serde_json::to_string(&arg).unwrap();
+        assert!(serialized.contains("\"json_value\":null"));
+    }
+
+    #[test]
+    fn test_arg_serialization_array() {
+        let arg = Arg {
+            name: "items".to_string(),
+            arg_type: "array".to_string(),
+            json_value: json!([1, 2, 3]),
+        };
+        let serialized = serde_json::to_string(&arg).unwrap();
+        assert!(serialized.contains("\"json_value\":[1,2,3]"));
+    }
+
+    #[test]
+    fn test_arg_serialization_object() {
+        let arg = Arg {
+            name: "config".to_string(),
+            arg_type: "object".to_string(),
+            json_value: json!({"key": "value"}),
+        };
+        let serialized = serde_json::to_string(&arg).unwrap();
+        assert!(serialized.contains("\"json_value\":{\"key\":\"value\"}"));
+    }
+
+    #[test]
+    fn test_remove_comments_comment_in_string() {
+        let sql = "SELECT '-- not a comment' FROM table;";
+        let result = remove_comments(sql);
+        assert_eq!(result, "SELECT '-- not a comment' FROM table;");
+    }
+
+    #[test]
+    fn test_remove_comments_multiple_dashes() {
+        let sql = "SELECT 5 - - 3;";
+        let result = remove_comments(sql);
+        // This correctly handles the subtraction of negative number
+        assert_eq!(result, sql);
+    }
 }

backend/windmill-worker/src/js_eval.rs

@@ -190,9 +190,24 @@ fn should_add_return(expr: &str) -> bool {
 
     // Check for common statement patterns that shouldn't have return prepended
     let statement_prefixes = [
-        "const ", "let ", "var ", "if ", "if(", "for ", "for(",
-        "while ", "while(", "switch ", "switch(", "try ", "try{",
-        "throw ", "function ", "class ", "async ", "await "
+        "const ",
+        "let ",
+        "var ",
+        "if ",
+        "if(",
+        "for ",
+        "for(",
+        "while ",
+        "while(",
+        "switch ",
+        "switch(",
+        "try ",
+        "try{",
+        "throw ",
+        "function ",
+        "class ",
+        "async ",
+        "await ",
     ];
 
     for prefix in &statement_prefixes {
@@ -677,7 +692,33 @@ function get_from_env(name) {{
             .map(|a| { format!("let {a} = get_from_env(\"{a}\");\n",) })
             .join(""),
         if expr.contains("error") && transform_context.contains(&"previous_result".to_string()) {
-            "let error = previous_result?.error;"
+            r#"let error = previous_result?.error;
+if (!error) {
+    if (Array.isArray(previous_result)) {
+        const errors = previous_result.filter(item => item && typeof item === 'object' && 'error' in item);
+        if (errors.length === 1) {
+            error = errors[0].error;
+        } else if (errors.length > 1) {
+            error = {
+                name: 'MultipleErrors',
+                message: errors.map(({ error: e }, i) => `[${e.step_id || i}] ${e.message || e.name}`).join('; '),
+                errors: previous_result
+            };
+        } else {
+            error = {
+                name: 'MultipleErrors',
+                message: "Could not parse errors",
+                errors: previous_result
+            };
+        }
+    } else {
+        if (previous_result) {
+            error = { name: 'UnknownError', message: 'Could not parse the error', error: previous_result };
+        } else {
+            error = { name: 'UnknownError', message: 'No error found' };
+        }
+    }
+}"#
         } else {
             ""
         },
@@ -1614,9 +1655,15 @@ multiline template`";
         assert_eq!(contains_semicolon_outside_strings("x; y"), true);
 
         // Semicolons inside strings (should NOT be detected)
-        assert_eq!(contains_semicolon_outside_strings("\"hello; world\""), false);
+        assert_eq!(
+            contains_semicolon_outside_strings("\"hello; world\""),
+            false
+        );
         assert_eq!(contains_semicolon_outside_strings("'test; string'"), false);
-        assert_eq!(contains_semicolon_outside_strings("`template; literal`"), false);
+        assert_eq!(
+            contains_semicolon_outside_strings("`template; literal`"),
+            false
+        );
 
         // Mixed cases
         assert_eq!(

backend/windmill-worker/src/lib.rs

@@ -23,8 +23,10 @@ mod bun_executor;
 pub mod common;
 mod config;
 mod csharp_executor;
-#[cfg(feature = "enterprise")]
-mod dedicated_worker;
+
+#[cfg(feature = "private")]
+mod dedicated_worker_ee;
+mod dedicated_worker_oss;
 mod deno_executor;
 #[cfg(feature = "duckdb")]
 mod duckdb_executor;

backend/windmill-worker/src/python_executor.rs

@@ -18,6 +18,8 @@ use tokio::{
     sync::Semaphore,
     task,
 };
+use windmill_queue::MiniPulledJob;
+
 use uuid::Uuid;
 #[cfg(all(feature = "enterprise", feature = "parquet", unix))]
 use windmill_common::ee_oss::{get_license_plan, LicensePlan};
@@ -546,7 +548,6 @@ pub async fn handle_python_job(
     precomputed_agent_info: Option<PrecomputedAgentInfo>,
     has_stream: &mut bool,
 ) -> windmill_common::error::Result<Box<RawValue>> {
-
     let script_path = crate::common::use_flow_root_path(job.runnable_path());
 
     let annotations = PythonAnnotations::parse(inner_content);
@@ -2130,16 +2131,16 @@ fn get_result_postprocessor<'a>(skip: bool) -> &'a str {
     }
 }
 
-#[cfg(feature = "enterprise")]
+#[cfg(feature = "private")]
 use crate::JobCompletedSender;
-#[cfg(feature = "enterprise")]
-use crate::{common::build_envs_map, dedicated_worker::handle_dedicated_process};
-#[cfg(feature = "enterprise")]
+#[cfg(feature = "private")]
+use crate::{common::build_envs_map, dedicated_worker_oss::handle_dedicated_process};
+#[cfg(feature = "private")]
 use windmill_common::variables;
+#[cfg(feature = "private")]
+use windmill_queue::DedicatedWorkerJob;
 
-use windmill_queue::MiniPulledJob;
-
-#[cfg(feature = "enterprise")]
+#[cfg(feature = "private")]
 pub async fn start_worker(
     requirements_o: Option<&String>,
     db: &sqlx::Pool<sqlx::Postgres>,
@@ -2152,10 +2153,12 @@ pub async fn start_worker(
     script_path: &str,
     token: &str,
     job_completed_tx: JobCompletedSender,
-    jobs_rx: tokio::sync::mpsc::Receiver<std::sync::Arc<MiniPulledJob>>,
+    jobs_rx: tokio::sync::mpsc::Receiver<DedicatedWorkerJob>,
     killpill_rx: tokio::sync::broadcast::Receiver<()>,
+    client: windmill_common::client::AuthedClient,
 ) -> error::Result<()> {
     use crate::{PyV, PyVAlias};
+    tracing::info!("script path: {}", script_path);
 
     let mut mem_peak: i32 = 0;
     let mut canceled_by: Option<CanceledBy> = None;
@@ -2346,6 +2349,7 @@ for line in sys.stdin:
         db,
         script_path,
         "python",
+        client,
     )
     .await
 }

backend/windmill-worker/src/result_processor.rs

@@ -31,8 +31,8 @@ use windmill_common::{
 use windmill_common::bench::{BenchmarkInfo, BenchmarkIter};
 
 use windmill_queue::{
-    append_logs, get_mini_completed_job, CanceledBy, JobCompleted, MiniCompletedJob, MiniPulledJob,
-    ValidableJson, WrappedError, INIT_SCRIPT_TAG,
+    append_logs, get_mini_completed_job, CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob,
+    MiniPulledJob, ValidableJson, WrappedError, INIT_SCRIPT_TAG,
 };
 
 use serde_json::{json, value::RawValue, Value};
@@ -66,6 +66,7 @@ async fn process_jc(
     same_worker_tx: Option<&SameWorkerSender>,
     job_completed_sender: &JobCompletedSender,
     stats_map: &JobStatsMap,
+    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
     #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) {
     let success: bool = jc.success;
@@ -155,6 +156,7 @@ async fn process_jc(
         same_worker_tx,
         &worker_name,
         job_completed_sender.clone(),
+        killpill_rx,
         #[cfg(feature = "benchmark")]
         bench,
     )
@@ -285,6 +287,7 @@ pub fn start_background_processor(
                         Some(&same_worker_tx),
                         &job_completed_sender,
                         &stats_map,
+                        &killpill_rx,
                         #[cfg(feature = "benchmark")]
                         &mut bench,
                     )
@@ -350,6 +353,8 @@ pub fn start_background_processor(
                         stop_early_override,
                         &worker_name,
                         job_completed_sender.clone(),
+                        None,
+                        &killpill_rx,
                         #[cfg(feature = "benchmark")]
                         &mut bench,
                     )
@@ -413,6 +418,7 @@ pub async fn process_result(
     conn: &Connection,
     duration: Option<i64>,
     has_stream: bool,
+    flow_runners: Option<Arc<FlowRunners>>,
 ) -> error::Result<bool> {
     match result {
         Ok(result) => {
@@ -431,6 +437,8 @@ pub async fn process_result(
                     duration,
                     has_stream: Some(has_stream),
                     from_cache: None,
+                    flow_runners,
+                    done_tx: None,
                 },
             )
             .with_context(windmill_common::otel_oss::otel_ctx())
@@ -495,6 +503,8 @@ pub async fn process_result(
                     duration,
                     has_stream: Some(has_stream),
                     from_cache: None,
+                    flow_runners,
+                    done_tx: None,
                 },
             )
             .with_context(windmill_common::otel_oss::otel_ctx())
@@ -512,6 +522,7 @@ pub async fn handle_receive_completed_job(
     same_worker_tx: Option<&SameWorkerSender>,
     worker_name: &str,
     job_completed_tx: JobCompletedSender,
+    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
     #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) -> Option<Arc<MiniPulledJob>> {
     let token = jc.token.clone();
@@ -529,6 +540,7 @@ pub async fn handle_receive_completed_job(
         same_worker_tx.clone(),
         worker_name,
         job_completed_tx.clone(),
+        killpill_rx,
         #[cfg(feature = "benchmark")]
         bench,
     )
@@ -548,6 +560,7 @@ pub async fn handle_receive_completed_job(
                 &worker_dir,
                 worker_name,
                 job_completed_tx,
+                killpill_rx,
                 #[cfg(feature = "benchmark")]
                 bench,
             )
@@ -571,6 +584,8 @@ pub async fn process_completed_job(
         preprocessed_args,
         has_stream,
         from_cache,
+        flow_runners,
+        done_tx,
         ..
     }: JobCompleted,
     client: &AuthedClient,
@@ -579,6 +594,7 @@ pub async fn process_completed_job(
     same_worker_tx: Option<&SameWorkerSender>,
     worker_name: &str,
     job_completed_tx: JobCompletedSender,
+    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
     #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) -> error::Result<Option<Arc<MiniPulledJob>>> {
     if success {
@@ -661,12 +677,19 @@ pub async fn process_completed_job(
                     None,
                     worker_name,
                     job_completed_tx,
+                    flow_runners,
+                    killpill_rx,
                     #[cfg(feature = "benchmark")]
                     bench,
                 )
                 .warn_after_seconds(10)
                 .await?;
                 add_time!(bench, "updated flow status END");
+                if let Some(done_tx) = done_tx {
+                    done_tx
+                        .send(())
+                        .expect("done receiver should still be alive");
+                }
                 return Ok(r);
             }
         }
@@ -707,11 +730,18 @@ pub async fn process_completed_job(
                     None,
                     worker_name,
                     job_completed_tx,
+                    flow_runners,
+                    killpill_rx,
                     #[cfg(feature = "benchmark")]
                     bench,
                 )
                 .warn_after_seconds(10)
                 .await?;
+                if let Some(done_tx) = done_tx {
+                    done_tx
+                        .send(())
+                        .expect("done receiver should still be alive");
+                }
                 return Ok(r);
             }
         }
@@ -761,6 +791,7 @@ pub async fn handle_job_error(
     worker_dir: &str,
     worker_name: &str,
     job_completed_tx: JobCompletedSender,
+    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
     #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) {
     let err_string = format!("{}: {}", err.name(), err.to_string());
@@ -809,6 +840,8 @@ pub async fn handle_job_error(
             None,
             worker_name,
             job_completed_tx.clone(),
+            None,
+            killpill_rx,
             #[cfg(feature = "benchmark")]
             bench,
         )

backend/windmill-worker/src/worker.rs

@@ -11,6 +11,7 @@
 
 use anyhow::anyhow;
 use futures::TryFutureExt;
+use tokio::sync::Mutex;
 use tokio::time::sleep;
 use tokio::time::timeout;
 use windmill_common::client::AuthedClient;
@@ -61,6 +62,8 @@ use std::{
     time::Duration,
 };
 use windmill_parser::MainArgSignature;
+use windmill_queue::DedicatedWorkerJob;
+use windmill_queue::FlowRunners;
 use windmill_queue::MiniCompletedJob;
 use windmill_queue::PulledJobResultToJobErr;
 
@@ -169,8 +172,8 @@ use crate::duckdb_executor::do_duckdb;
 #[cfg(all(feature = "enterprise", feature = "oracledb"))]
 use crate::oracledb_executor::do_oracledb;
 
-#[cfg(feature = "enterprise")]
-use crate::dedicated_worker::create_dedicated_worker_map;
+#[cfg(all(feature = "private", feature = "enterprise"))]
+use crate::dedicated_worker_oss::create_dedicated_worker_map;
 
 #[cfg(feature = "enterprise")]
 use crate::snowflake_executor::do_snowflake;
@@ -534,6 +537,8 @@ lazy_static::lazy_static! {
         .ok()
         .and_then(|x| x.parse::<i64>().ok())
         .unwrap_or(1000);
+
+    pub static ref FLOW_RUNNER_RUNNING: Mutex<bool> = Mutex::new(false);
 }
 
 type Envs = Vec<(String, String)>;
@@ -560,14 +565,14 @@ lazy_static::lazy_static! {
 
 #[derive(Debug)]
 pub enum NextJob {
-    Sql(PulledJob),
+    Sql { job: PulledJob, flow_runners: Option<Arc<FlowRunners>> },
     Http(JobAndPerms),
 }
 
 impl NextJob {
     pub fn job(self) -> MiniPulledJob {
         match self {
-            NextJob::Sql(job) => job.job,
+            NextJob::Sql { job, .. } => job.job,
             NextJob::Http(job) => job.job,
         }
     }
@@ -577,7 +582,7 @@ impl std::ops::Deref for NextJob {
     type Target = MiniPulledJob;
     fn deref(&self) -> &Self::Target {
         match self {
-            NextJob::Sql(job) => &job.job,
+            NextJob::Sql { job, .. } => &job.job,
             NextJob::Http(job) => &job.job,
         }
     }
@@ -735,10 +740,10 @@ impl JobCompletedSender {
 impl SameWorkerSender {
     pub async fn send(
         &self,
-        payload: SameWorkerPayload,
+        message: SameWorkerPayload,
     ) -> Result<(), tokio::sync::mpsc::error::SendError<SameWorkerPayload>> {
         self.1.fetch_add(1, Ordering::Relaxed);
-        self.0.send(payload).await
+        self.0.send(message).await
     }
 }
 
@@ -831,18 +836,36 @@ fn add_outstanding_wait_time(
 
 async fn extract_job_and_perms(job: NextJob, conn: &Connection) -> JobAndPerms {
     match (job, conn) {
-        (NextJob::Sql(job), Connection::Sql(db)) => job.get_job_and_perms(db).await,
-        (NextJob::Sql(_), Connection::Http(_)) => panic!("sql job on http connection"),
+        (NextJob::Sql { job, flow_runners, .. }, Connection::Sql(db)) => {
+            JobAndPerms { flow_runners, ..job.get_job_and_perms(db).await }
+        }
+        (NextJob::Sql { .. }, Connection::Http(_)) => panic!("sql job on http connection"),
         (NextJob::Http(job), _) => job,
     }
 }
 
-fn create_span(arc_job: &Arc<MiniPulledJob>, worker_name: &str, hostname: &str) -> Span {
-    let span = tracing::span!(tracing::Level::INFO, "job",
-        job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id,  worker = %worker_name, hostname = %hostname, tag = %arc_job.tag,
+pub fn create_span_with_name(
+    arc_job: &MiniPulledJob,
+    worker_name: &str,
+    hostname: Option<&str>,
+    span_name: &str,
+) -> Span {
+    // The span macro requires a literal, so we use a fixed name and set otel.name dynamically
+    let span = tracing::span!(
+        tracing::Level::INFO,
+        "job",
+        job_id = %arc_job.id,
+        root_job = field::Empty,
+        workspace_id = %arc_job.workspace_id,
+        worker = %worker_name,
+        hostname = field::Empty,
+        tag = %arc_job.tag,
         language = field::Empty,
-        script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty,
-        otel.name = field::Empty);
+        script_path = field::Empty,
+        flow_step_id = field::Empty,
+        parent_job = field::Empty,
+        otel.name = field::Empty
+    );
 
     let rj = arc_job.flow_innermost_root_job.unwrap_or(arc_job.id);
 
@@ -850,10 +873,10 @@ fn create_span(arc_job: &Arc<MiniPulledJob>, worker_name: &str, hostname: &str)
         span.record("language", lg.as_str());
     }
     if let Some(step_id) = arc_job.flow_step_id.as_ref() {
-        span.record("otel.name", format!("job {}", step_id).as_str());
+        span.record("otel.name", format!("{} {}", span_name, step_id).as_str());
         span.record("flow_step_id", step_id.as_str());
     } else {
-        span.record("otel.name", "job");
+        span.record("otel.name", span_name);
     }
     if let Some(parent_job) = arc_job.parent_job.as_ref() {
         span.record("parent_job", parent_job.to_string().as_str());
@@ -864,6 +887,9 @@ fn create_span(arc_job: &Arc<MiniPulledJob>, worker_name: &str, hostname: &str)
     if let Some(root_job) = arc_job.flow_innermost_root_job.as_ref() {
         span.record("root_job", root_job.to_string().as_str());
     }
+    if let Some(hostname) = hostname {
+        span.record("hostname", hostname);
+    }
 
     windmill_common::otel_oss::set_span_parent(&span, &rj);
     span
@@ -878,6 +904,7 @@ pub async fn handle_all_job_kind_error(
     worker_dir: &str,
     worker_name: &str,
     job_completed_tx: JobCompletedSender,
+    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
     #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) {
     match conn {
@@ -894,6 +921,7 @@ pub async fn handle_all_job_kind_error(
                 &worker_dir,
                 &worker_name,
                 job_completed_tx.clone(),
+                &killpill_rx,
                 #[cfg(feature = "benchmark")]
                 bench,
             )
@@ -917,6 +945,8 @@ pub async fn handle_all_job_kind_error(
                         duration: None,
                         has_stream: Some(false),
                         from_cache: None,
+                        flow_runners: None,
+                        done_tx: None,
                     },
                     false,
                 )
@@ -964,27 +994,21 @@ pub fn start_interactive_worker_shell(
                         .await;
 
                         match job {
-                            Ok(j) => match j.to_pulled_job() {
-                                Ok(j) => Ok(j.clone().map(NextJob::Sql)),
-                                ref e @ (Err(PulledJobResultToJobErr::MissingConcurrencyKey(
-                                    ref jc,
-                                ))
-                                | Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(
-                                    ref jc,
-                                ))) => {
-                                    if let Err(err) =
-                                        job_completed_tx.send_job(jc.clone(), true).await
-                                    {
-                                        let e_fmt = match e {
-                                            Ok(_) => "unknown error".to_owned(),
-                                            Err(e) => e.to_string(),
-                                        };
-
-                                        tracing::error!("An error occurred while sending job completed ({e_fmt}): {:#?}", err)
+                            Ok(j) => {
+                                match j.to_pulled_job() {
+                                    Ok(j) => Ok(j
+                                        .clone()
+                                        .map(|job| NextJob::Sql { flow_runners: None, job })),
+                                    Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc))
+                                    | Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(jc)) => {
+                                        if let Err(err) = job_completed_tx.send_job(jc, true).await
+                                        {
+                                            tracing::error!("An error occurred while sending job completed: {:#?}", err)
+                                        }
+                                        Ok(None)
                                     }
-                                    Ok(None)
                                 }
-                            },
+                            }
                             Err(err) => Err(err),
                         }
                     }
@@ -1012,6 +1036,7 @@ pub fn start_interactive_worker_shell(
                             parent_runnable_path,
                             token,
                             precomputed_agent_info: precomputed_bundle,
+                            flow_runners,
                         } = extract_job_and_perms(job, &conn).await;
 
                         let authed_client = AuthedClient::new(
@@ -1041,6 +1066,7 @@ pub fn start_interactive_worker_shell(
                             &mut occupancy_metrics,
                             &mut killpill_rx,
                             precomputed_bundle,
+                            flow_runners,
                             #[cfg(feature = "benchmark")]
                             &mut bench,
                         )
@@ -1478,9 +1504,9 @@ pub async fn run_worker(
     // Option<Sender<Arc<QueuedJob>>>,
     // Option<JoinHandle<()>>,
 
-    #[cfg(feature = "enterprise")]
+    #[cfg(all(feature = "private", feature = "enterprise"))]
     let (dedicated_workers, is_flow_worker, dedicated_handles): (
-        HashMap<String, Sender<Arc<MiniPulledJob>>>,
+        HashMap<String, Sender<DedicatedWorkerJob>>,
         bool,
         Vec<JoinHandle<()>>,
     ) = match conn {
@@ -1499,9 +1525,9 @@ pub async fn run_worker(
         Connection::Http(_) => (HashMap::new(), false, vec![]),
     };
 
-    #[cfg(not(feature = "enterprise"))]
+    #[cfg(any(not(feature = "private"), not(feature = "enterprise")))]
     let (dedicated_workers, is_flow_worker, dedicated_handles): (
-        HashMap<String, Sender<Arc<MiniPulledJob>>>,
+        HashMap<String, Sender<DedicatedWorkerJob>>,
         bool,
         Vec<JoinHandle<()>>,
     ) = (HashMap::new(), false, vec![]);
@@ -1658,7 +1684,7 @@ pub async fn run_worker(
 
             if let Ok(same_worker_job) = same_worker_rx.try_recv() {
                 same_worker_queue_size.fetch_sub(1, Ordering::SeqCst);
-                tracing::debug!(
+                tracing::info!(
                     worker = %worker_name, hostname = %hostname,
                     "received {} from same worker channel",
                     same_worker_job.job_id
@@ -1678,7 +1704,12 @@ pub async fn run_worker(
                                 .expect("send kill to job completed tx");
                             break;
                         } else {
-                            job.map(|x| x.map(NextJob::Sql))
+                            job.map(|x| {
+                                x.map(|job| NextJob::Sql {
+                                    flow_runners: same_worker_job.flow_runners,
+                                    job,
+                                })
+                            })
                         }
                     }
                     Connection::Http(client) => client
@@ -1829,22 +1860,14 @@ pub async fn run_worker(
                         }
                         match job {
                             Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() {
-                                Ok(j) => Ok(j.map(NextJob::Sql)),
-                                ref e @ (Err(PulledJobResultToJobErr::MissingConcurrencyKey(
-                                    ref jc,
-                                ))
-                                | Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(
-                                    ref jc,
-                                ))) => {
-                                    if let Err(err) =
-                                        job_completed_tx.send_job(jc.clone(), true).await
-                                    {
-                                        let e_fmt = match e {
-                                            Ok(_) => "unknown error".to_owned(),
-                                            Err(e) => e.to_string(),
-                                        };
-
-                                        tracing::error!("An error occurred while sending job completed ({e_fmt}): {:#?}", err)
+                                Ok(j) => Ok(j.map(|job| NextJob::Sql { flow_runners: None, job })),
+                                Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc))
+                                | Err(PulledJobResultToJobErr::ErrorWhilePreprocessing(jc)) => {
+                                    if let Err(err) = job_completed_tx.send_job(jc, true).await {
+                                        tracing::error!(
+                                            "An error occurred while sending job completed: {:#?}",
+                                            err
+                                        )
                                     }
                                     Ok(None)
                                 }
@@ -1876,7 +1899,10 @@ pub async fn run_worker(
 
                 tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id);
 
-                if matches!(job.kind, JobKind::Script | JobKind::Preview) {
+                if matches!(
+                    job.kind,
+                    JobKind::Script | JobKind::Preview | JobKind::FlowScript
+                ) {
                     if !dedicated_workers.is_empty() {
                         let key_o = if is_flow_worker {
                             job.flow_step_id.as_ref().map(|x| x.to_string())
@@ -1885,8 +1911,12 @@ pub async fn run_worker(
                         };
                         if let Some(key) = key_o {
                             if let Some(dedicated_worker_tx) = dedicated_workers.get(&key) {
-                                if let Err(e) = dedicated_worker_tx.send(Arc::new(job.job())).await
-                                {
+                                let dedicated_job = DedicatedWorkerJob {
+                                    job: Arc::new(job.job()),
+                                    flow_runners: None,
+                                    done_tx: None,
+                                };
+                                if let Err(e) = dedicated_worker_tx.send(dedicated_job).await {
                                     tracing::info!("failed to send jobs to dedicated workers. Likely dedicated worker has been shut down. This is normal: {e:?}");
                                 }
 
@@ -1900,6 +1930,77 @@ pub async fn run_worker(
                             }
                         }
                     }
+
+                    // Extract flow_runners early to use in both dedicated workers and flow runners
+                    let flow_runners = match &job {
+                        NextJob::Sql { flow_runners, .. } => flow_runners.clone(),
+                        NextJob::Http(_) => None,
+                    };
+
+                    if let Some(flow_runners) = flow_runners {
+                        let key_o = job.flow_step_id.as_ref().map(|x| x.to_string());
+                        if let Some(key) = key_o {
+                            if let Some(flow_runner_tx) = flow_runners.runners.get(&key) {
+                                tracing::info!(
+                                    "sending job {} to flow runner step {}",
+                                    job.id,
+                                    key
+                                );
+                                let (done_tx, done_rx) = tokio::sync::oneshot::channel::<()>();
+                                let flow_runners = flow_runners.clone();
+
+                                let job = job.job();
+
+                                let dedicated_job = DedicatedWorkerJob {
+                                    job: Arc::new(job.clone()),
+                                    flow_runners: Some(flow_runners),
+                                    done_tx: Some(done_tx),
+                                };
+
+                                if let Err(e) = flow_runner_tx.send(dedicated_job).await {
+                                    let token = match &conn {
+                                        Connection::Sql(db) => {
+                                            windmill_queue::jobs::create_token(db, &job, None).await
+                                        }
+                                        _ => "".to_string(),
+                                    };
+                                    handle_all_job_kind_error(
+                                        &conn,
+                                        &AuthedClient::new(
+                                            base_internal_url.to_owned(),
+                                            job.workspace_id.clone(),
+                                            token,
+                                            None,
+                                        ),
+                                        MiniCompletedJob::from(job),
+                                        error::Error::InternalErr(format!(
+                                            "failed to send jobs to flow runners: {e:?}"
+                                        )),
+                                        Some(&same_worker_tx),
+                                        &worker_dir,
+                                        &worker_name,
+                                        job_completed_tx.clone(),
+                                        &killpill_rx,
+                                        #[cfg(feature = "benchmark")]
+                                        &mut bench,
+                                    )
+                                    .await;
+                                } else {
+                                    if let Err(err) = done_rx.await {
+                                        tracing::error!("Flow runner done channel has been dropped without being received: {err:?}");
+                                    }
+                                }
+
+                                #[cfg(feature = "benchmark")]
+                                {
+                                    add_time!(bench, "sent to flow runner");
+                                    infos.add_iter(bench, true);
+                                }
+
+                                continue;
+                            }
+                        }
+                    }
                 }
 
                 if matches!(job.kind, JobKind::Noop) {
@@ -1919,6 +2020,8 @@ pub async fn run_worker(
                                 duration: None,
                                 has_stream: Some(false),
                                 from_cache: None,
+                                flow_runners: None,
+                                done_tx: None,
                             },
                             true,
                         )
@@ -2050,6 +2153,7 @@ pub async fn run_worker(
                         parent_runnable_path,
                         token,
                         precomputed_agent_info: precomputed_bundle,
+                        flow_runners,
                     } = extract_job_and_perms(job, &conn).await;
 
                     let authed_client = AuthedClient::new(
@@ -2061,7 +2165,7 @@ pub async fn run_worker(
 
                     let arc_job = Arc::new(job);
 
-                    let span = create_span(&arc_job, &worker_name, hostname);
+                    let span = create_span_with_name(&arc_job, &worker_name, Some(hostname), "job");
 
                     let job_result = handle_queued_job(
                         arc_job.clone(),
@@ -2081,6 +2185,7 @@ pub async fn run_worker(
                         &mut occupancy_metrics,
                         &mut killpill_rx2,
                         precomputed_bundle,
+                        flow_runners,
                         #[cfg(feature = "benchmark")]
                         &mut bench,
                     )
@@ -2140,6 +2245,7 @@ pub async fn run_worker(
                                 &worker_dir,
                                 &worker_name,
                                 job_completed_tx.clone(),
+                                &killpill_rx,
                                 #[cfg(feature = "benchmark")]
                                 &mut bench,
                             )
@@ -2304,7 +2410,7 @@ async fn queue_init_bash_maybe<'c>(
     };
     if let Some((uuid, content)) = uuid_content {
         same_worker_tx
-            .send(SameWorkerPayload { job_id: uuid, recoverable: false })
+            .send(SameWorkerPayload { job_id: uuid, recoverable: false, flow_runners: None })
             .await
             .map_err(to_anyhow)?;
         tracing::info!("Creating initial job {uuid} from initial script script: {content}");
@@ -2422,7 +2528,7 @@ async fn queue_periodic_script_bash_maybe<'c>(
     };
 
     same_worker_tx
-        .send(SameWorkerPayload { job_id: uuid, recoverable: false })
+        .send(SameWorkerPayload { job_id: uuid, recoverable: false, flow_runners: None })
         .await
         .map_err(to_anyhow)?;
     tracing::info!("Creating periodic script job {uuid} from periodic script: {content}");
@@ -2434,7 +2540,6 @@ pub struct SendResult {
     pub time: Instant,
 }
 
-#[derive(Clone)]
 pub enum SendResultPayload {
     JobCompleted(JobCompleted),
     UpdateFlow(UpdateFlow),
@@ -2521,6 +2626,7 @@ pub async fn handle_queued_job(
     occupancy_metrics: &mut OccupancyMetrics,
     killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
     precomputed_agent_info: Option<PrecomputedAgentInfo>,
+    flow_runners: Option<Arc<FlowRunners>>,
     #[cfg(feature = "benchmark")] _bench: &mut BenchmarkIter,
 ) -> windmill_common::error::Result<bool> {
     // Extract the active span from the context
@@ -2648,6 +2754,8 @@ pub async fn handle_queued_job(
                             duration: None,
                             has_stream: Some(false),
                             from_cache: Some(true),
+                            flow_runners: None,
+                            done_tx: None,
                         },
                         true,
                     )
@@ -2683,6 +2791,8 @@ pub async fn handle_queued_job(
                 worker_dir,
                 job_completed_tx.clone(),
                 worker_name,
+                flow_runners,
+                &killpill_rx,
             ))
             .warn_after_seconds(10)
             .await?;
@@ -2913,6 +3023,7 @@ pub async fn handle_queued_job(
             conn,
             Some(started.elapsed().as_millis() as i64),
             has_stream,
+            flow_runners,
         )
         .await
     }

backend/windmill-worker/src/worker_flow.rs

@@ -64,7 +64,7 @@ use windmill_common::{
 use windmill_queue::schedule::get_schedule_opt;
 use windmill_queue::{
     add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job,
-    handle_maybe_scheduled_job, insert_concurrency_key, interpolate_args, CanceledBy,
+    handle_maybe_scheduled_job, insert_concurrency_key, interpolate_args, CanceledBy, FlowRunners,
     MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError,
 };
 
@@ -88,6 +88,8 @@ pub async fn update_flow_status_after_job_completion(
     stop_early_override: Option<bool>,
     worker_name: &str,
     job_completed_tx: JobCompletedSender,
+    flow_runners: Option<Arc<FlowRunners>>,
+    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
     #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) -> error::Result<Option<Arc<MiniPulledJob>>> {
     // this is manual tailrecursion because async_recursion blows up the stack
@@ -121,6 +123,8 @@ pub async fn update_flow_status_after_job_completion(
             rec.has_triggered_error_handler,
             worker_name,
             job_completed_tx.clone(),
+            flow_runners.clone(),
+            killpill_rx,
             #[cfg(feature = "benchmark")]
             bench,
         ))
@@ -147,6 +151,8 @@ pub async fn update_flow_status_after_job_completion(
                     rec.has_triggered_error_handler,
                     worker_name,
                     job_completed_tx.clone(),
+                    flow_runners.clone(),
+                    killpill_rx,
                     #[cfg(feature = "benchmark")]
                     bench,
                 ))
@@ -285,6 +291,8 @@ pub async fn update_flow_status_after_job_completion_internal(
     has_triggered_error_handler: bool,
     worker_name: &str,
     job_completed_tx: JobCompletedSender,
+    flow_runners: Option<Arc<FlowRunners>>,
+    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
     #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
 ) -> error::Result<UpdateFlowStatusAfterJobCompletion> {
     let mut has_triggered_error_handler = has_triggered_error_handler;
@@ -434,7 +442,6 @@ pub async fn update_flow_status_after_job_completion_internal(
             args
         }));
 
-
         let from_result_to_args = |args: &Result<HashMap<String, Box<RawValue>>, sqlx::Error>| {
             let args = args
                 .as_ref()
@@ -473,7 +480,7 @@ pub async fn update_flow_status_after_job_completion_internal(
                                 _ => None,
                             };
                         let args = from_result_to_args(args.as_ref().await.get_ref())?;
-                       
+
                         compute_bool_from_expr(
                             &expr,
                             Marc::new(args),
@@ -742,7 +749,6 @@ pub async fn update_flow_status_after_job_completion_internal(
                             &mut stop_early_err_msg,
                             &mut nresult,
                             args,
-
                         )
                         .await?;
                     }
@@ -1565,6 +1571,8 @@ pub async fn update_flow_status_after_job_completion_internal(
             worker_dir,
             job_completed_tx,
             worker_name,
+            flow_runners,
+            &killpill_rx,
         ))
         .warn_after_seconds(10)
         .await
@@ -2044,6 +2052,8 @@ pub async fn handle_flow(
     worker_dir: &str,
     job_completed_tx: JobCompletedSender,
     worker_name: &str,
+    flow_runners: Option<Arc<FlowRunners>>,
+    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
 ) -> anyhow::Result<()> {
     let flow = flow_data.value();
     let status = flow_job
@@ -2101,6 +2111,9 @@ pub async fn handle_flow(
             same_worker_tx,
             worker_dir,
             worker_name,
+            flow_runners.clone(),
+            job_completed_tx.clone(),
+            &killpill_rx,
         ))
         .warn_after_seconds(10)
         .await?;
@@ -2185,6 +2198,7 @@ struct PushNextFlowJobRec {
     flow_job: Arc<MiniPulledJob>,
     status: FlowStatus,
 }
+
 // #[async_recursion]
 // #[instrument(level = "trace", skip_all)]
 async fn push_next_flow_job(
@@ -2197,6 +2211,9 @@ async fn push_next_flow_job(
     same_worker_tx: &SameWorkerSender,
     worker_dir: &str,
     worker_name: &str,
+    flow_runners: Option<Arc<FlowRunners>>,
+    job_completed_tx: JobCompletedSender,
+    killpill_rx: &tokio::sync::broadcast::Receiver<()>,
 ) -> error::Result<PushNextFlowJob> {
     let job_root = flow_job
         .flow_innermost_root_job
@@ -3028,6 +3045,22 @@ async fn push_next_flow_job(
         }
     };
 
+    // only start runners if we're not already in a squash for loop
+    let start_runners = flow_runners.is_none()
+        && matches!(
+            next_status,
+            NextStatus::NextLoopIteration { start_runners: true, .. }
+        );
+
+    let do_not_pass_runners = matches!(next_status, NextStatus::NextStep { .. })
+        && flow_runners
+            .as_ref()
+            .is_some_and(|fr| fr.job_id == flow_job.id);
+
+    let continue_with_runners = (start_runners || (flow_runners.is_some() && !do_not_pass_runners))
+        && module.suspend.is_none()
+        && module.sleep.is_none();
+
     // Also check `flow_job.same_worker` for [`JobKind::Flow`] jobs as it's no
     // more reflected to the flow value on push.
     let job_same_worker = flow_job.same_worker
@@ -3083,6 +3116,7 @@ async fn push_next_flow_job(
             NextStatus::NextLoopIteration {
                 next: ForloopNextIteration { new_args, .. },
                 simple_input_transforms,
+                ..
             } => {
                 let mut args = if let Ok(args) = nargs {
                     args.as_ref().clone()
@@ -3300,7 +3334,7 @@ async fn push_next_flow_job(
             Some(module.id.clone()),
             new_job_priority_override,
             job_perms.as_ref(),
-            false,
+            continue_with_runners,
             None,
             None,
             None,
@@ -3308,7 +3342,7 @@ async fn push_next_flow_job(
         .warn_after_seconds(2)
         .await?;
 
-        if continue_on_same_worker {
+        if continue_on_same_worker || continue_with_runners {
             let _ = sqlx::query!(
                 "UPDATE v2_job_queue SET worker = $2 WHERE id = $1",
                 uuid,
@@ -3583,7 +3617,7 @@ async fn push_next_flow_job(
     .warn_after_seconds(3)
     .await?;
 
-    if continue_on_same_worker {
+    if continue_on_same_worker || continue_with_runners {
         if !is_one_uuid {
             return Err(Error::BadRequest(
                  "Cannot continue on same worker with multiple jobs, parallel cannot be used in conjunction with same_worker".to_string(),
@@ -3593,9 +3627,43 @@ async fn push_next_flow_job(
     tx.commit().warn_after_seconds(3).await?;
     tracing::info!(id = %flow_job.id, root_id = %job_root, "all next flow jobs pushed: {uuids:?}");
 
-    if continue_on_same_worker {
+    if continue_on_same_worker || continue_with_runners {
+        let flow_runners = if start_runners {
+            tracing::info!(id = %flow_job.id, "starting flow runners for module {}", module.id);
+            let (new_flow_runners, new_flow_runner_handles) =
+                crate::dedicated_worker_oss::spawn_flow_module_runners(
+                    &flow_job,
+                    module,
+                    flow.failure_module.as_ref(),
+                    &killpill_rx,
+                    db,
+                    worker_dir,
+                    &client.base_internal_url,
+                    worker_name,
+                    &job_completed_tx,
+                )
+                .await
+                .with_context(|| {
+                    format!(
+                        "failed to spawn flow module runners for job: {}",
+                        flow_job.id
+                    )
+                })?;
+
+            let flow_runners = FlowRunners {
+                runners: new_flow_runners,
+                handles: new_flow_runner_handles,
+                job_id: flow_job.id,
+            };
+            Some(Arc::new(flow_runners))
+        } else if !do_not_pass_runners {
+            flow_runners
+        } else {
+            None
+        };
+
         same_worker_tx
-            .send(SameWorkerPayload { job_id: first_uuid, recoverable: true })
+            .send(SameWorkerPayload { job_id: first_uuid, recoverable: true, flow_runners })
             .warn_after_seconds(3)
             .await
             .map_err(to_anyhow)?;
@@ -3702,6 +3770,7 @@ enum NextStatus {
     NextLoopIteration {
         next: ForloopNextIteration,
         simple_input_transforms: Option<HashMap<String, InputTransform>>,
+        start_runners: bool,
     },
     AllFlowJobs {
         branchall: Option<BranchAllStatus>,
@@ -3943,7 +4012,7 @@ async fn compute_next_flow_transform(
                 NextStatus::NextStep,
             ))
         }
-        FlowModuleValue::WhileloopFlow { modules, modules_node, .. } => {
+        FlowModuleValue::WhileloopFlow { modules, modules_node, squash, .. } => {
             // if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg
             let is_simple = is_simple_modules(&modules, flow.failure_module.as_ref());
             let (flow_jobs, flow_jobs_success, flow_jobs_duration) = match status_module {
@@ -3960,6 +4029,8 @@ async fn compute_next_flow_transform(
                 _ => (vec![], Some(vec![]), Some(FlowJobsDuration::new(0))),
             };
             let next_loop_idx = flow_jobs.len();
+
+            let start_runners = next_loop_idx == 0 && squash.unwrap_or(false);
             next_loop_iteration(
                 flow,
                 status,
@@ -3982,11 +4053,14 @@ async fn compute_next_flow_transform(
                 db,
                 module,
                 delete_after_use,
+                start_runners,
             )
             .await
         }
         /* forloop modules are expected set `iter: { value: Value, index: usize }` as job arguments */
-        FlowModuleValue::ForloopFlow { modules, modules_node, iterator, parallel, .. } => {
+        FlowModuleValue::ForloopFlow {
+            modules, modules_node, iterator, parallel, squash, ..
+        } => {
             // if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg
             let is_simple = !matches!(flow_job.kind, JobKind::FlowPreview)
                 && !parallel
@@ -4025,6 +4099,7 @@ async fn compute_next_flow_transform(
                     Ok(NextFlowTransform::EmptyInnerFlows { branch_chosen: None })
                 }
                 ForLoopStatus::NextIteration(ns) => {
+                    let start_runners = ns.index == 0 && squash.unwrap_or(false);
                     next_loop_iteration(
                         flow,
                         status,
@@ -4036,6 +4111,7 @@ async fn compute_next_flow_transform(
                         db,
                         module,
                         delete_after_use,
+                        start_runners,
                     )
                     .await
                 }
@@ -4300,6 +4376,7 @@ async fn next_loop_iteration(
     db: &sqlx::Pool<sqlx::Postgres>,
     module: &FlowModule,
     delete_after_use: bool,
+    start_runners: bool,
 ) -> Result<NextFlowTransform, Error> {
     let inner_path = || format!("{}/loop-{}", flow_job.runnable_path(), ns.index);
     if is_simple {
@@ -4317,7 +4394,7 @@ async fn next_loop_iteration(
             ContinuePayload::SingleJob(
                 payload_from_simple_module(value, db, flow_job, module, inner_path()).await?,
             ),
-            NextStatus::NextLoopIteration { next: ns, simple_input_transforms },
+            NextStatus::NextLoopIteration { next: ns, simple_input_transforms, start_runners },
         ));
     }
 
@@ -4341,7 +4418,7 @@ async fn next_loop_iteration(
             timeout: None,
             on_behalf_of: None,
         }),
-        NextStatus::NextLoopIteration { next: ns, simple_input_transforms: None },
+        NextStatus::NextLoopIteration { next: ns, simple_input_transforms: None, start_runners },
     ))
 }
 

backend/windmill-worker/src/worker_lockfiles.rs

@@ -1193,6 +1193,7 @@ async fn lock_modules<'c>(
                     skip_failures,
                     parallel,
                     parallelism,
+                    squash,
                 } => {
                     let nmodules;
                     (nmodules, tx, nmodified_ids, nerrors) = Box::pin(lock_modules(
@@ -1222,6 +1223,7 @@ async fn lock_modules<'c>(
                         skip_failures,
                         parallel,
                         parallelism,
+                        squash,
                     }
                     .into()
                 }
@@ -1258,7 +1260,7 @@ async fn lock_modules<'c>(
                     }
                     e.value = FlowModuleValue::BranchAll { branches: nbranches, parallel }.into()
                 }
-                FlowModuleValue::WhileloopFlow { modules, modules_node, skip_failures } => {
+                FlowModuleValue::WhileloopFlow { modules, modules_node, skip_failures, squash } => {
                     let nmodules;
                     (nmodules, tx, nmodified_ids, nerrors) = Box::pin(lock_modules(
                         modules,
@@ -1284,6 +1286,7 @@ async fn lock_modules<'c>(
                         modules: nmodules,
                         modules_node,
                         skip_failures,
+                        squash,
                     }
                     .into()
                 }

benchmarks/lib.ts

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.582.2";
+export const VERSION = "v1.585.1";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({

cli/deno.lock

@@ -1,5 +1,5 @@
 {
-  "version": "4",
+  "version": "5",
   "specifiers": {
     "jsr:@david/code-block-writer@^13.0.2": "13.0.2",
     "jsr:@deno/cache-dir@~0.10.3": "0.10.3",
@@ -60,6 +60,8 @@
     "jsr:@windmill-labs/cliffy-table@^1.0.0-rc.5": "1.0.0-rc.5",
     "npm:@ayonli/jsext@*": "1.8.0",
     "npm:@types/node@*": "22.12.0",
+    "npm:@windmill-labs/shared-utils@1.0.1": "1.0.1",
+    "npm:@windmill-labs/shared-utils@1.0.2": "1.0.2",
     "npm:centdix-utils@*": "1.0.15",
     "npm:diff@*": "8.0.2",
     "npm:es-main@*": "1.3.0",
@@ -67,6 +69,7 @@
     "npm:express@*": "5.1.0",
     "npm:get-port@7.1.0": "7.1.0",
     "npm:jszip@3.7.1": "3.7.1",
+    "npm:jszip@3.8.0": "3.8.0",
     "npm:minimatch@*": "10.0.3",
     "npm:open@*": "10.2.0",
     "npm:ws@*": "8.18.3"
@@ -293,82 +296,134 @@
       ]
     },
     "@esbuild/aix-ppc64@0.25.8": {
-      "integrity": "sha512-urAvrUedIqEiFR3FYSLTWQgLu5tb+m0qZw0NBEasUeo6wuqatkMDaRT+1uABiGXEu5vqgPd7FGE1BhsAIy9QVA=="
+      "integrity": "sha512-urAvrUedIqEiFR3FYSLTWQgLu5tb+m0qZw0NBEasUeo6wuqatkMDaRT+1uABiGXEu5vqgPd7FGE1BhsAIy9QVA==",
+      "os": ["aix"],
+      "cpu": ["ppc64"]
     },
     "@esbuild/android-arm64@0.25.8": {
-      "integrity": "sha512-OD3p7LYzWpLhZEyATcTSJ67qB5D+20vbtr6vHlHWSQYhKtzUYrETuWThmzFpZtFsBIxRvhO07+UgVA9m0i/O1w=="
+      "integrity": "sha512-OD3p7LYzWpLhZEyATcTSJ67qB5D+20vbtr6vHlHWSQYhKtzUYrETuWThmzFpZtFsBIxRvhO07+UgVA9m0i/O1w==",
+      "os": ["android"],
+      "cpu": ["arm64"]
     },
     "@esbuild/android-arm@0.25.8": {
-      "integrity": "sha512-RONsAvGCz5oWyePVnLdZY/HHwA++nxYWIX1atInlaW6SEkwq6XkP3+cb825EUcRs5Vss/lGh/2YxAb5xqc07Uw=="
+      "integrity": "sha512-RONsAvGCz5oWyePVnLdZY/HHwA++nxYWIX1atInlaW6SEkwq6XkP3+cb825EUcRs5Vss/lGh/2YxAb5xqc07Uw==",
+      "os": ["android"],
+      "cpu": ["arm"]
     },
     "@esbuild/android-x64@0.25.8": {
-      "integrity": "sha512-yJAVPklM5+4+9dTeKwHOaA+LQkmrKFX96BM0A/2zQrbS6ENCmxc4OVoBs5dPkCCak2roAD+jKCdnmOqKszPkjA=="
+      "integrity": "sha512-yJAVPklM5+4+9dTeKwHOaA+LQkmrKFX96BM0A/2zQrbS6ENCmxc4OVoBs5dPkCCak2roAD+jKCdnmOqKszPkjA==",
+      "os": ["android"],
+      "cpu": ["x64"]
     },
     "@esbuild/darwin-arm64@0.25.8": {
-      "integrity": "sha512-Jw0mxgIaYX6R8ODrdkLLPwBqHTtYHJSmzzd+QeytSugzQ0Vg4c5rDky5VgkoowbZQahCbsv1rT1KW72MPIkevw=="
+      "integrity": "sha512-Jw0mxgIaYX6R8ODrdkLLPwBqHTtYHJSmzzd+QeytSugzQ0Vg4c5rDky5VgkoowbZQahCbsv1rT1KW72MPIkevw==",
+      "os": ["darwin"],
+      "cpu": ["arm64"]
     },
     "@esbuild/darwin-x64@0.25.8": {
-      "integrity": "sha512-Vh2gLxxHnuoQ+GjPNvDSDRpoBCUzY4Pu0kBqMBDlK4fuWbKgGtmDIeEC081xi26PPjn+1tct+Bh8FjyLlw1Zlg=="
+      "integrity": "sha512-Vh2gLxxHnuoQ+GjPNvDSDRpoBCUzY4Pu0kBqMBDlK4fuWbKgGtmDIeEC081xi26PPjn+1tct+Bh8FjyLlw1Zlg==",
+      "os": ["darwin"],
+      "cpu": ["x64"]
     },
     "@esbuild/freebsd-arm64@0.25.8": {
-      "integrity": "sha512-YPJ7hDQ9DnNe5vxOm6jaie9QsTwcKedPvizTVlqWG9GBSq+BuyWEDazlGaDTC5NGU4QJd666V0yqCBL2oWKPfA=="
+      "integrity": "sha512-YPJ7hDQ9DnNe5vxOm6jaie9QsTwcKedPvizTVlqWG9GBSq+BuyWEDazlGaDTC5NGU4QJd666V0yqCBL2oWKPfA==",
+      "os": ["freebsd"],
+      "cpu": ["arm64"]
     },
     "@esbuild/freebsd-x64@0.25.8": {
-      "integrity": "sha512-MmaEXxQRdXNFsRN/KcIimLnSJrk2r5H8v+WVafRWz5xdSVmWLoITZQXcgehI2ZE6gioE6HirAEToM/RvFBeuhw=="
+      "integrity": "sha512-MmaEXxQRdXNFsRN/KcIimLnSJrk2r5H8v+WVafRWz5xdSVmWLoITZQXcgehI2ZE6gioE6HirAEToM/RvFBeuhw==",
+      "os": ["freebsd"],
+      "cpu": ["x64"]
     },
     "@esbuild/linux-arm64@0.25.8": {
-      "integrity": "sha512-WIgg00ARWv/uYLU7lsuDK00d/hHSfES5BzdWAdAig1ioV5kaFNrtK8EqGcUBJhYqotlUByUKz5Qo6u8tt7iD/w=="
+      "integrity": "sha512-WIgg00ARWv/uYLU7lsuDK00d/hHSfES5BzdWAdAig1ioV5kaFNrtK8EqGcUBJhYqotlUByUKz5Qo6u8tt7iD/w==",
+      "os": ["linux"],
+      "cpu": ["arm64"]
     },
     "@esbuild/linux-arm@0.25.8": {
-      "integrity": "sha512-FuzEP9BixzZohl1kLf76KEVOsxtIBFwCaLupVuk4eFVnOZfU+Wsn+x5Ryam7nILV2pkq2TqQM9EZPsOBuMC+kg=="
+      "integrity": "sha512-FuzEP9BixzZohl1kLf76KEVOsxtIBFwCaLupVuk4eFVnOZfU+Wsn+x5Ryam7nILV2pkq2TqQM9EZPsOBuMC+kg==",
+      "os": ["linux"],
+      "cpu": ["arm"]
     },
     "@esbuild/linux-ia32@0.25.8": {
-      "integrity": "sha512-A1D9YzRX1i+1AJZuFFUMP1E9fMaYY+GnSQil9Tlw05utlE86EKTUA7RjwHDkEitmLYiFsRd9HwKBPEftNdBfjg=="
+      "integrity": "sha512-A1D9YzRX1i+1AJZuFFUMP1E9fMaYY+GnSQil9Tlw05utlE86EKTUA7RjwHDkEitmLYiFsRd9HwKBPEftNdBfjg==",
+      "os": ["linux"],
+      "cpu": ["ia32"]
     },
     "@esbuild/linux-loong64@0.25.8": {
-      "integrity": "sha512-O7k1J/dwHkY1RMVvglFHl1HzutGEFFZ3kNiDMSOyUrB7WcoHGf96Sh+64nTRT26l3GMbCW01Ekh/ThKM5iI7hQ=="
+      "integrity": "sha512-O7k1J/dwHkY1RMVvglFHl1HzutGEFFZ3kNiDMSOyUrB7WcoHGf96Sh+64nTRT26l3GMbCW01Ekh/ThKM5iI7hQ==",
+      "os": ["linux"],
+      "cpu": ["loong64"]
     },
     "@esbuild/linux-mips64el@0.25.8": {
-      "integrity": "sha512-uv+dqfRazte3BzfMp8PAQXmdGHQt2oC/y2ovwpTteqrMx2lwaksiFZ/bdkXJC19ttTvNXBuWH53zy/aTj1FgGw=="
+      "integrity": "sha512-uv+dqfRazte3BzfMp8PAQXmdGHQt2oC/y2ovwpTteqrMx2lwaksiFZ/bdkXJC19ttTvNXBuWH53zy/aTj1FgGw==",
+      "os": ["linux"],
+      "cpu": ["mips64el"]
     },
     "@esbuild/linux-ppc64@0.25.8": {
-      "integrity": "sha512-GyG0KcMi1GBavP5JgAkkstMGyMholMDybAf8wF5A70CALlDM2p/f7YFE7H92eDeH/VBtFJA5MT4nRPDGg4JuzQ=="
+      "integrity": "sha512-GyG0KcMi1GBavP5JgAkkstMGyMholMDybAf8wF5A70CALlDM2p/f7YFE7H92eDeH/VBtFJA5MT4nRPDGg4JuzQ==",
+      "os": ["linux"],
+      "cpu": ["ppc64"]
     },
     "@esbuild/linux-riscv64@0.25.8": {
-      "integrity": "sha512-rAqDYFv3yzMrq7GIcen3XP7TUEG/4LK86LUPMIz6RT8A6pRIDn0sDcvjudVZBiiTcZCY9y2SgYX2lgK3AF+1eg=="
+      "integrity": "sha512-rAqDYFv3yzMrq7GIcen3XP7TUEG/4LK86LUPMIz6RT8A6pRIDn0sDcvjudVZBiiTcZCY9y2SgYX2lgK3AF+1eg==",
+      "os": ["linux"],
+      "cpu": ["riscv64"]
     },
     "@esbuild/linux-s390x@0.25.8": {
-      "integrity": "sha512-Xutvh6VjlbcHpsIIbwY8GVRbwoviWT19tFhgdA7DlenLGC/mbc3lBoVb7jxj9Z+eyGqvcnSyIltYUrkKzWqSvg=="
+      "integrity": "sha512-Xutvh6VjlbcHpsIIbwY8GVRbwoviWT19tFhgdA7DlenLGC/mbc3lBoVb7jxj9Z+eyGqvcnSyIltYUrkKzWqSvg==",
+      "os": ["linux"],
+      "cpu": ["s390x"]
     },
     "@esbuild/linux-x64@0.25.8": {
-      "integrity": "sha512-ASFQhgY4ElXh3nDcOMTkQero4b1lgubskNlhIfJrsH5OKZXDpUAKBlNS0Kx81jwOBp+HCeZqmoJuihTv57/jvQ=="
+      "integrity": "sha512-ASFQhgY4ElXh3nDcOMTkQero4b1lgubskNlhIfJrsH5OKZXDpUAKBlNS0Kx81jwOBp+HCeZqmoJuihTv57/jvQ==",
+      "os": ["linux"],
+      "cpu": ["x64"]
     },
     "@esbuild/netbsd-arm64@0.25.8": {
-      "integrity": "sha512-d1KfruIeohqAi6SA+gENMuObDbEjn22olAR7egqnkCD9DGBG0wsEARotkLgXDu6c4ncgWTZJtN5vcgxzWRMzcw=="
+      "integrity": "sha512-d1KfruIeohqAi6SA+gENMuObDbEjn22olAR7egqnkCD9DGBG0wsEARotkLgXDu6c4ncgWTZJtN5vcgxzWRMzcw==",
+      "os": ["netbsd"],
+      "cpu": ["arm64"]
     },
     "@esbuild/netbsd-x64@0.25.8": {
-      "integrity": "sha512-nVDCkrvx2ua+XQNyfrujIG38+YGyuy2Ru9kKVNyh5jAys6n+l44tTtToqHjino2My8VAY6Lw9H7RI73XFi66Cg=="
+      "integrity": "sha512-nVDCkrvx2ua+XQNyfrujIG38+YGyuy2Ru9kKVNyh5jAys6n+l44tTtToqHjino2My8VAY6Lw9H7RI73XFi66Cg==",
+      "os": ["netbsd"],
+      "cpu": ["x64"]
     },
     "@esbuild/openbsd-arm64@0.25.8": {
-      "integrity": "sha512-j8HgrDuSJFAujkivSMSfPQSAa5Fxbvk4rgNAS5i3K+r8s1X0p1uOO2Hl2xNsGFppOeHOLAVgYwDVlmxhq5h+SQ=="
+      "integrity": "sha512-j8HgrDuSJFAujkivSMSfPQSAa5Fxbvk4rgNAS5i3K+r8s1X0p1uOO2Hl2xNsGFppOeHOLAVgYwDVlmxhq5h+SQ==",
+      "os": ["openbsd"],
+      "cpu": ["arm64"]
     },
     "@esbuild/openbsd-x64@0.25.8": {
-      "integrity": "sha512-1h8MUAwa0VhNCDp6Af0HToI2TJFAn1uqT9Al6DJVzdIBAd21m/G0Yfc77KDM3uF3T/YaOgQq3qTJHPbTOInaIQ=="
+      "integrity": "sha512-1h8MUAwa0VhNCDp6Af0HToI2TJFAn1uqT9Al6DJVzdIBAd21m/G0Yfc77KDM3uF3T/YaOgQq3qTJHPbTOInaIQ==",
+      "os": ["openbsd"],
+      "cpu": ["x64"]
     },
     "@esbuild/openharmony-arm64@0.25.8": {
-      "integrity": "sha512-r2nVa5SIK9tSWd0kJd9HCffnDHKchTGikb//9c7HX+r+wHYCpQrSgxhlY6KWV1nFo1l4KFbsMlHk+L6fekLsUg=="
+      "integrity": "sha512-r2nVa5SIK9tSWd0kJd9HCffnDHKchTGikb//9c7HX+r+wHYCpQrSgxhlY6KWV1nFo1l4KFbsMlHk+L6fekLsUg==",
+      "os": ["openharmony"],
+      "cpu": ["arm64"]
     },
     "@esbuild/sunos-x64@0.25.8": {
-      "integrity": "sha512-zUlaP2S12YhQ2UzUfcCuMDHQFJyKABkAjvO5YSndMiIkMimPmxA+BYSBikWgsRpvyxuRnow4nS5NPnf9fpv41w=="
+      "integrity": "sha512-zUlaP2S12YhQ2UzUfcCuMDHQFJyKABkAjvO5YSndMiIkMimPmxA+BYSBikWgsRpvyxuRnow4nS5NPnf9fpv41w==",
+      "os": ["sunos"],
+      "cpu": ["x64"]
     },
     "@esbuild/win32-arm64@0.25.8": {
-      "integrity": "sha512-YEGFFWESlPva8hGL+zvj2z/SaK+pH0SwOM0Nc/d+rVnW7GSTFlLBGzZkuSU9kFIGIo8q9X3ucpZhu8PDN5A2sQ=="
+      "integrity": "sha512-YEGFFWESlPva8hGL+zvj2z/SaK+pH0SwOM0Nc/d+rVnW7GSTFlLBGzZkuSU9kFIGIo8q9X3ucpZhu8PDN5A2sQ==",
+      "os": ["win32"],
+      "cpu": ["arm64"]
     },
     "@esbuild/win32-ia32@0.25.8": {
-      "integrity": "sha512-hiGgGC6KZ5LZz58OL/+qVVoZiuZlUYlYHNAmczOm7bs2oE1XriPFi5ZHHrS8ACpV5EjySrnoCKmcbQMN+ojnHg=="
+      "integrity": "sha512-hiGgGC6KZ5LZz58OL/+qVVoZiuZlUYlYHNAmczOm7bs2oE1XriPFi5ZHHrS8ACpV5EjySrnoCKmcbQMN+ojnHg==",
+      "os": ["win32"],
+      "cpu": ["ia32"]
     },
     "@esbuild/win32-x64@0.25.8": {
-      "integrity": "sha512-cn3Yr7+OaaZq1c+2pe+8yxC8E144SReCQjN6/2ynubzYjvyqZjTXfQJpAcQpsdJq3My7XADANiYGHoFC69pLQw=="
+      "integrity": "sha512-cn3Yr7+OaaZq1c+2pe+8yxC8E144SReCQjN6/2ynubzYjvyqZjTXfQJpAcQpsdJq3My7XADANiYGHoFC69pLQw==",
+      "os": ["win32"],
+      "cpu": ["x64"]
     },
     "@isaacs/balanced-match@4.0.1": {
       "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ=="
@@ -385,6 +440,12 @@
         "undici-types"
       ]
     },
+    "@windmill-labs/shared-utils@1.0.1": {
+      "integrity": "sha512-DUMzPIFCKImuGpbuHXXmGGUT3VXYlgrv/jIIEOW+Iig+9tZvYqOUxfgn32lDhm73k82xBg8MdAf+0qABzfqFeQ=="
+    },
+    "@windmill-labs/shared-utils@1.0.2": {
+      "integrity": "sha512-3LwALmwMeO3MqglGlyTtBUF05/ogpdDM5GiZKGN7271AEctS+ZJi3pXMHZ+YZdLxdgi2qLNNnVHO8qG5vFud2Q=="
+    },
     "accepts@2.0.0": {
       "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==",
       "dependencies": [
@@ -509,7 +570,7 @@
     },
     "esbuild@0.25.8": {
       "integrity": "sha512-vVC0USHGtMi8+R4Kz8rt6JhEWLxsv9Rnu/lGYbPR8u47B+DCBksq9JarW0zOO7bs37hyOK1l2/oqtbciutL5+Q==",
-      "dependencies": [
+      "optionalDependencies": [
         "@esbuild/aix-ppc64",
         "@esbuild/android-arm",
         "@esbuild/android-arm64",
@@ -536,7 +597,9 @@
         "@esbuild/win32-arm64",
         "@esbuild/win32-ia32",
         "@esbuild/win32-x64"
-      ]
+      ],
+      "scripts": true,
+      "bin": true
     },
     "escape-html@1.0.3": {
       "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
@@ -659,13 +722,15 @@
       "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
     },
     "is-docker@3.0.0": {
-      "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="
+      "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==",
+      "bin": true
     },
     "is-inside-container@1.0.0": {
       "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==",
       "dependencies": [
         "is-docker"
-      ]
+      ],
+      "bin": true
     },
     "is-promise@4.0.0": {
       "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="
@@ -688,6 +753,15 @@
         "set-immediate-shim"
       ]
     },
+    "jszip@3.8.0": {
+      "integrity": "sha512-cnpQrXvFSLdsR9KR5/x7zdf6c3m8IhZfZzSblFEHSqBaVwD2nvJ4CuCKLyvKvwBgZm08CgfSoiTBQLm5WW9hGw==",
+      "dependencies": [
+        "lie",
+        "pako",
+        "readable-stream",
+        "set-immediate-shim"
+      ]
+    },
     "lie@3.3.0": {
       "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
       "dependencies": [
@@ -899,7 +973,8 @@
       ]
     },
     "sudo-prompt@9.2.1": {
-      "integrity": "sha512-Mu7R0g4ig9TUuGSxJavny5Rv0egCEtpZRNMrZaYS1vxkiIxGiGUwoezU3LazIQ+KE04hTrTfNPgxU5gzi7F5Pw=="
+      "integrity": "sha512-Mu7R0g4ig9TUuGSxJavny5Rv0egCEtpZRNMrZaYS1vxkiIxGiGUwoezU3LazIQ+KE04hTrTfNPgxU5gzi7F5Pw==",
+      "deprecated": true
     },
     "toidentifier@1.0.1": {
       "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="

cli/deps.ts

@@ -49,7 +49,7 @@ export function yamlParseContent(
 
 export * as Diff from "npm:diff";
 export { minimatch } from "npm:minimatch";
-export { default as JSZip } from "npm:jszip@3.7.1";
+export { default as JSZip } from "npm:jszip@3.8.0";
 
 export * as express from "npm:express";
 export * as http from "node:http";
@@ -57,6 +57,7 @@ export { WebSocketServer, WebSocket } from "npm:ws";
 export * as getPort from "npm:get-port@7.1.0";
 export * as open from "npm:open";
 export * as esMain from "npm:es-main";
+export * as windmillUtils from "npm:@windmill-labs/shared-utils@1.0.2";
 
 import { OpenAPI } from "./gen/index.ts";
 

cli/src/commands/app/apps.ts

@@ -1,7 +1,15 @@
 // deno-lint-ignore-file no-explicit-any
 import { requireLogin } from "../../core/auth.ts";
 import { resolveWorkspace, validatePath } from "../../core/context.ts";
-import { colors, Command, log, SEP, Table, yamlParseFile } from "../../../deps.ts";
+import {
+  colors,
+  Command,
+  log,
+  SEP,
+  Table,
+  windmillUtils,
+  yamlParseFile,
+} from "../../../deps.ts";
 import * as wmill from "../../../gen/services.gen.ts";
 import { ListableApp, Policy } from "../../../gen/types.gen.ts";
 
@@ -10,12 +18,16 @@ import { readInlinePathSync } from "../../utils/utils.ts";
 
 export interface AppFile {
   value: any;
+  public?: boolean;
   summary: string;
   policy: Policy;
 }
 
 const alreadySynced: string[] = [];
 
+export function isExecutionModeAnonymous(app: any) {
+  return app?.["policy"]?.["execution_mode"] == "anonymous";
+}
 export async function pushApp(
   workspace: string,
   remotePath: string,
@@ -37,6 +49,13 @@ export async function pushApp(
   } catch {
     //ignore
   }
+  if (isExecutionModeAnonymous(app)) {
+    app.public = true;
+  }
+  // console.log(app);
+  if (app) {
+    app.policy = undefined;
+  }
 
   if (!localPath.endsWith(SEP)) {
     localPath += SEP;
@@ -69,7 +88,9 @@ export async function pushApp(
   }
 
   replaceInlineScripts(localApp.value);
-
+  // console.log(localApp, localApp?.["policy"]);
+  await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? (localApp.policy ? isExecutionModeAnonymous(localApp) : false));
+  // console.log(localApp, localApp?.["policy"]);
   if (app) {
     if (isSuperset(localApp, app)) {
       log.info(colors.green(`App ${remotePath} is up to date`));
@@ -98,6 +119,17 @@ export async function pushApp(
   }
 }
 
+async function generatingPolicy(app: any, path: string, publicApp: boolean) {
+  log.info(colors.gray(`Generating fresh policy for app ${path}...`));
+  try {
+    app.policy = await windmillUtils.updatePolicy(app.value, undefined);
+    app.policy.execution_mode = publicApp ? "anonymous" : "publisher";
+  } catch (e) {
+    log.error(colors.red(`Error generating policy for app ${path}: ${e}`));
+    throw e;
+  }
+}
+
 async function list(opts: GlobalOptions & { includeDraftOnly?: boolean }) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);

cli/src/commands/sync/sync.ts

@@ -67,6 +67,7 @@ import {
   PathAssigner,
 } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts";
 import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
+import { isExecutionModeAnonymous } from "../app/apps.ts";
 
 // Merge CLI options with effective settings, preserving CLI flags as overrides
 function mergeCliWithEffectiveOptions<
@@ -415,6 +416,10 @@ function ZipFSElement(
               };
             }
 
+            if (isExecutionModeAnonymous(app)) {
+              app.public = true;
+            }
+            app.policy = undefined;
             yield {
               isDirectory: false,
               path: path.join(finalPath, "app.yaml"),
@@ -661,7 +666,6 @@ export async function elementsToMap(
   const map: { [key: string]: string } = {};
   const processedBasePaths = new Set<string>();
 
-
   for await (const entry of readDirRecursiveWithIgnore(ignore, els)) {
     if (entry.isDirectory || entry.ignored) continue;
     const path = entry.path;
@@ -853,25 +857,6 @@ async function compareDynFSElement(
         }
       }
       return o;
-    } else if (k.endsWith(".app.yaml")) {
-      let o: any;
-      try {
-        o = yamlParseContent(k, v);
-      } catch (error) {
-        log.error(`Failed to parse app YAML content at path: ${k}`);
-        throw error;
-      }
-      const o2 = o["policy"];
-
-      if (typeof o2 == "object") {
-        if (o2["on_behalf_of"] != undefined) {
-          delete o2["on_behalf_of"];
-        }
-        if (o2["on_behalf_of_email"] != undefined) {
-          delete o2["on_behalf_of_email"];
-        }
-      }
-      return o;
     } else {
       try {
         return yamlParseContent(k, v);

cli/src/main.ts

@@ -68,7 +68,7 @@ export {
 //   }
 // });
 
-export const VERSION = "1.582.2";
+export const VERSION = "1.585.1";
 
 export const WM_FORK_PREFIX = "wm-fork";
 

frontend/package.json

@@ -1,9 +1,10 @@
 {
 	"name": "windmill-components",
-	"version": "1.582.2",
+	"version": "1.585.1",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
+		"build:utils": "vite build --config vite.sharedUtils.config.js",
 		"preview": "vite preview",
 		"postinstall": "node -e \"if (require('fs').existsSync('./scripts/untar_ui_builder.js')) { require('child_process').execSync('node ./scripts/untar_ui_builder.js', {stdio: 'inherit'}) }\"",
 		"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --threshold warning",
@@ -64,7 +65,8 @@
 		"tar": "^7.4.3",
 		"tslib": "^2.6.1",
 		"typescript": "^5.5.0",
-		"vite": "npm:rolldown-vite@latest",
+		"vite": "npm:rolldown-vite@7.2.8",
+		"vite-plugin-dts": "^4.5.4",
 		"vite-plugin-mkcert": "^1.17.5",
 		"yootils": "^0.3.1"
 	},

frontend/package.sharedUtils.json

@@ -0,0 +1,12 @@
+{
+	"name": "@windmill-labs/shared-utils",
+	"version": "1.0.2",
+	"type": "module",
+	"private": false,
+	"main": "./lib.es.js",
+	"module": "./lib.es.js",
+	"exports": {
+		".": "./lib.es.js"
+	},
+	"types": "./lib.d.ts"
+}

frontend/src/lib/components/ArgInput.svelte

@@ -1049,6 +1049,8 @@
 					{#if oneOf && oneOf.length >= 2}
 						<ToggleButtonGroup
 							selected={oneOfSelected}
+							wrap
+							class="mb-4"
 							on:selected={({ detail }) => {
 								oneOfSelected = detail
 								const selectedObjProperties =

frontend/src/lib/components/DBManagerDrawer.svelte

@@ -7,10 +7,7 @@
 	import { ArrowLeft, Expand, Loader2, Minimize, RefreshCcw } from 'lucide-svelte'
 	import {
 		dbSupportsSchemas,
-		getDbSchemas,
 		getLanguageByResourceType,
-		loadAllTablesMetaData,
-		loadTableMetaData,
 		type TableMetadata
 	} from './apps/components/display/dbtable/utils'
 	import DbManager from './DBManager.svelte'
@@ -18,9 +15,7 @@
 		dbDeleteTableActionWithPreviewScript,
 		dbTableOpsWithPreviewScripts,
 		getDatabaseArg,
-		getDucklakeSchema,
-		wrapDucklakeQuery,
-		type DbInput
+		getDucklakeSchema
 	} from './dbOps'
 	import { makeCreateTableQuery } from './apps/components/display/dbtable/queries/createTable'
 	import { runScriptAndPollResult } from './jobs/utils'
@@ -28,6 +23,13 @@
 	import SqlRepl from './SqlRepl.svelte'
 	import SimpleAgTable from './SimpleAgTable.svelte'
 	import { untrack } from 'svelte'
+	import type { DbInput } from './dbTypes'
+	import { wrapDucklakeQuery } from './ducklake'
+	import {
+		getDbSchemas,
+		loadAllTablesMetaData,
+		loadTableMetaData
+	} from './apps/components/display/dbtable/metadata'
 
 	let input: DbInput | undefined = $state()
 	let open = $derived(!!input)

frontend/src/lib/components/DBTableEditor.svelte

@@ -53,11 +53,7 @@
 	import { Cell } from './table'
 	import DataTable from './table/DataTable.svelte'
 	import Head from './table/Head.svelte'
-	import {
-		datatypeHasLength,
-		dbSupportsSchemas,
-		type DbType
-	} from './apps/components/display/dbtable/utils'
+	import { datatypeHasLength, dbSupportsSchemas } from './apps/components/display/dbtable/utils'
 	import { DB_TYPES } from '$lib/consts'
 	import Popover from './meltComponents/Popover.svelte'
 	import Tooltip from './meltComponents/Tooltip.svelte'
@@ -74,6 +70,7 @@
 	import Select from './select/Select.svelte'
 	import { safeSelectItems } from './select/utils.svelte'
 	import TextInput from './text_input/TextInput.svelte'
+	import type { DbType } from './dbTypes'
 
 	const { onConfirm, dbType, previewSql, dbSchema, currentSchema }: DBTableEditorProps = $props()
 

frontend/src/lib/components/Editor.svelte

@@ -91,10 +91,10 @@
 	import { extToLang, langToExt } from '$lib/editorLangUtils'
 	import { aiChatManager } from './copilot/chat/AIChatManager.svelte'
 	import type { Selection } from 'monaco-editor'
-	import { getDbSchemas } from './apps/components/display/dbtable/utils'
 	import { canHavePreprocessor, getPreprocessorModuleCode } from '$lib/script_helpers'
 	import { setMonacoTypescriptOptions } from './monacoLanguagesOptions'
 	import { copilotInfo } from '$lib/aiStore'
+	import { getDbSchemas } from './apps/components/display/dbtable/metadata'
 	// import EditorTheme from './EditorTheme.svelte'
 
 	let divEl: HTMLDivElement | null = $state(null)

frontend/src/lib/components/ExploreAssetButton.svelte

@@ -13,7 +13,7 @@
 </script>
 
 <script lang="ts">
-	import { isDbType } from '$lib/components/apps/components/display/dbtable/utils'
+	import { isDbType } from '$lib/components/dbTypes'
 	import { formatAsset, type Asset } from '$lib/components/assets/lib'
 	import { Button, ButtonType } from '$lib/components/common'
 	import DbManagerDrawer from '$lib/components/DBManagerDrawer.svelte'

frontend/src/lib/components/FlowGraphViewerStep.svelte

@@ -121,6 +121,7 @@
 					{:else if stepDetail.value.type == 'forloopflow'}
 						For loop {#if stepDetail.value.parallel}(parallel){/if}
 						{#if stepDetail.value.skip_failures}(skip failures){/if}
+						{#if stepDetail.value.squash}(squash){/if}
 					{:else if stepDetail.value.type == 'branchall'}
 						Run all branches {#if stepDetail.value.parallel}(parallel){/if}
 					{:else if stepDetail.value.type == 'branchone'}
@@ -128,7 +129,8 @@
 					{:else if stepDetail.value.type == 'flow'}
 						Inner flow
 					{:else if stepDetail.value.type == 'whileloopflow'}
-						While loop
+						While loop {#if stepDetail.value.skip_failures}(skip failures){/if}
+						{#if stepDetail.value.squash}(squash){/if}
 					{:else if stepDetail.id === 'failure'}
 						Error handler
 					{:else if stepDetail.id === 'preprocessor'}

frontend/src/lib/components/MapResult.svelte

@@ -1,12 +1,12 @@
 <script lang="ts">
 	import { Map, View, Feature } from 'ol'
 	import { Fill, Stroke, Style, Text } from 'ol/style.js'
-	import { useGeographic } from 'ol/proj'
-	import { OSM, Vector as VectorSource } from 'ol/source'
-	import { Vector as VectorLayer, Tile as TileLayer } from 'ol/layer'
-	import { Point } from 'ol/geom'
-	import { defaults as defaultControls } from 'ol/control'
-	import CircleStyle from 'ol/style/Circle'
+	import { useGeographic } from 'ol/proj.js'
+	import { OSM, Vector as VectorSource } from 'ol/source.js'
+	import { Vector as VectorLayer, Tile as TileLayer } from 'ol/layer.js'
+	import { Point } from 'ol/geom.js'
+	import { defaults as defaultControls } from 'ol/control.js'
+	import CircleStyle from 'ol/style/Circle.js'
 
 	interface Marker {
 		lon: number

frontend/src/lib/components/Path.svelte

@@ -138,7 +138,7 @@
 	}
 
 	export async function reset() {
-		if (path == '' || path == 'u//') {
+		if (path == '' || path == 'u//' || path?.startsWith('tmp/')) {
 			if ($lastMetaUsed == undefined || $lastMetaUsed.owner != $userStore?.username) {
 				meta = {
 					ownerKind: hideUser ? 'folder' : 'user',
@@ -323,12 +323,12 @@
 
 	async function initPath() {
 		await tick()
-		if (path != undefined && path != '') {
+		if (path != undefined && path != '' && !path?.startsWith('tmp/')) {
 			meta = pathToMeta(path, hideUser)
 			onMetaChange()
 			return
 		}
-		if (initialPath == undefined || initialPath == '') {
+		if (initialPath == undefined || initialPath == '' || initialPath?.startsWith('tmp/')) {
 			reset()
 		} else {
 			meta = pathToMeta(initialPath, hideUser)

frontend/src/lib/components/QueuePosition.svelte

@@ -87,7 +87,7 @@
 </script>
 
 {#if queueState}
-	<div class="text-small ml-4">
+	<div class="text-xs ml-4">
 		<span class="text-orange-600">Queue position: <b>{queueState.position}</b></span>
 		{#if !minimal}
 			<span class="ml-2 text-primary">(Waiting for an available worker)</span>

frontend/src/lib/components/ScriptBuilder.svelte

@@ -379,7 +379,7 @@
 		if (templateScript) {
 			script.content += '\r\n' + templateScript
 		}
-		scriptEditor?.inferSchema(script.content, language, true)
+		scriptEditor?.inferSchema(script.content, { nlang: language, resetArgs: true })
 		if (script.content != editor?.getCode()) {
 			setCode(script.content)
 		}
@@ -567,7 +567,7 @@
 			if (!disableHistoryChange) {
 				history.replaceState(history.state, '', `/scripts/edit/${script.path}`)
 			}
-			if (stay || script.kind !== 'script' || script.no_main_func) {
+			if (stay || (script.no_main_func && script.kind !== 'preprocessor')) {
 				script.parent_hash = newHash
 				sendUserToast('Deployed')
 			} else {
@@ -849,7 +849,7 @@
 		captureOn.set(true)
 	}
 
-	function addPreprocessor() {
+	function addPreprocessor(e?: { detail?: { args: Record<string, any> } }) {
 		const code = editor?.getCode()
 		if (code) {
 			const preprocessorCode = getPreprocessorFullCode(script.language, false)
@@ -865,6 +865,11 @@
 			}
 		}
 		selectedInputTab = 'preprocessor'
+
+		// Apply provided args to the preprocessor
+		if (e?.detail?.args && Object.keys(e.detail.args).length > 0) {
+			args = { ...args, ...e.detail.args }
+		}
 	}
 
 	function handleDeployTrigger(trigger: Trigger) {

frontend/src/lib/components/ScriptEditor.svelte

@@ -123,6 +123,8 @@
 		enablePreprocessorSnippet = false
 	}: Props = $props()
 
+	let initialArgs = structuredClone($state.snapshot(args))
+
 	$effect.pre(() => {
 		if (schema == undefined) {
 			schema = emptySchema()
@@ -269,7 +271,18 @@
 		})
 	}
 
-	export async function inferSchema(code: string, nlang?: SupportedLanguage, resetArgs = false) {
+	export async function inferSchema(
+		code: string,
+		{
+			nlang,
+			resetArgs = false,
+			applyInitialArgs = false
+		}: {
+			nlang?: SupportedLanguage
+			resetArgs?: boolean
+			applyInitialArgs?: boolean
+		} = {}
+	) {
 		let nschema = schema ?? emptySchema()
 
 		try {
@@ -297,6 +310,10 @@
 			if (resetArgs) {
 				args = {}
 			}
+			if (applyInitialArgs) {
+				// we reapply initial args as the schema form might have cleared them between mount and the schema inference
+				args = initialArgs
+			}
 			schema = nschema
 		} catch (e) {
 			validCode = false
@@ -338,7 +355,7 @@
 	}
 
 	onMount(() => {
-		inferSchema(code)
+		inferSchema(code, { applyInitialArgs: true })
 		loadPastTests()
 		aiChatManager.saveAndClear()
 		aiChatManager.changeMode(AIMode.SCRIPT)

frontend/src/lib/components/ScriptEditorSkeleton.svelte

@@ -0,0 +1,55 @@
+<div class="flex flex-col h-screen bg-surface">
+	<!-- Top bar -->
+	<div class="h-12 px-4 border-b flex items-center gap-4">
+		<div class="w-8 h-8 rounded animate-skeleton dark:bg-frost-900/50"></div>
+		<div class="flex-1 h-6 max-w-md rounded animate-skeleton dark:bg-frost-900/50"></div>
+	</div>
+
+	<!-- Editor bar -->
+	<div class="border-b shadow-sm px-4 py-2 flex gap-2">
+		<div class="w-20 h-6 rounded animate-skeleton dark:bg-frost-900/50"></div>
+		<div class="w-20 h-6 rounded animate-skeleton dark:bg-frost-900/50"></div>
+		<div class="w-20 h-6 rounded animate-skeleton dark:bg-frost-900/50"></div>
+		<div class="flex-1"></div>
+		<div class="w-24 h-6 rounded animate-skeleton dark:bg-frost-900/50"></div>
+	</div>
+
+	<!-- Main content area -->
+	<div class="flex flex-1 overflow-hidden">
+		<!-- Left panel - Editor -->
+		<div class="flex-1 min-w-0 border-r bg-surface-secondary">
+			<div class="w-full h-full p-4">
+				<div class="w-32 h-4 mb-3 rounded animate-skeleton dark:bg-frost-900/50"></div>
+				<div class="w-48 h-4 mb-3 rounded animate-skeleton dark:bg-frost-900/50"></div>
+				<div class="w-40 h-4 mb-3 rounded animate-skeleton dark:bg-frost-900/50"></div>
+				<div class="w-56 h-4 mb-3 rounded animate-skeleton dark:bg-frost-900/50"></div>
+			</div>
+		</div>
+
+		<!-- Right panel - Test/Preview with horizontal split -->
+		<div class="w-2/5 min-w-0 flex flex-col bg-surface">
+			<!-- Top section - test inputs -->
+			<div class="flex-1 border-b flex flex-col">
+				<div class="h-10 border-b px-4 flex items-center">
+					<div class="w-24 h-4 rounded animate-skeleton dark:bg-frost-900/50"></div>
+				</div>
+				<div class="flex-1 p-4 space-y-3">
+					<div class="w-full h-8 rounded animate-skeleton dark:bg-frost-900/50"></div>
+					<div class="w-full h-8 rounded animate-skeleton dark:bg-frost-900/50"></div>
+					<div class="w-32 h-8 rounded animate-skeleton dark:bg-frost-900/50"></div>
+				</div>
+			</div>
+
+			<!-- Bottom section - results -->
+			<div class="flex-1 flex flex-col">
+				<div class="h-10 border-b px-4 flex items-center">
+					<div class="w-24 h-4 rounded animate-skeleton dark:bg-frost-900/50"></div>
+				</div>
+				<div class="flex-1 p-4">
+					<div class="w-full h-4 mb-2 rounded animate-skeleton dark:bg-frost-900/50"></div>
+					<div class="w-3/4 h-4 rounded animate-skeleton dark:bg-frost-900/50"></div>
+				</div>
+			</div>
+		</div>
+	</div>
+</div>

frontend/src/lib/components/ScriptPicker.svelte

@@ -57,15 +57,19 @@
 
 	async function loadItems(): Promise<void> {
 		if (itemKind == 'flow') {
-			items = (await FlowService.listFlows({ workspace: $workspaceStore! })).map((flow) => ({
+			items = (
+				await FlowService.listFlows({ workspace: $workspaceStore!, withoutDescription: true })
+			).map((flow) => ({
 				value: flow.path,
-				label: `${flow.path}${flow.summary ? ` | ${truncate(flow.summary, 20)}` : ''}`
+				label: `${flow.path}${flow.summary ? ` | ${truncate(flow.summary, 20)}` : ''}`,
+				withoutDescription: true
 			}))
 		} else if (itemKind == 'script') {
 			items = (
 				await ScriptService.listScripts({
 					workspace: $workspaceStore!,
-					kinds: kinds.join(',')
+					kinds: kinds.join(','),
+					withoutDescription: true
 				})
 			).map((script) => ({
 				value: script.path,

frontend/src/lib/components/SqlRepl.svelte

@@ -55,7 +55,9 @@
 	import { getLanguageByResourceType } from './apps/components/display/dbtable/utils'
 	import StepHistory, { type StepHistoryData } from './flows/propPicker/StepHistory.svelte'
 	import { Pane, Splitpanes } from 'svelte-splitpanes'
-	import { getDatabaseArg, getDbType, wrapDucklakeQuery, type DbInput } from './dbOps'
+	import { getDatabaseArg, getDbType } from './dbOps'
+	import type { DbInput } from './dbTypes'
+	import { wrapDucklakeQuery } from './ducklake'
 
 	type Props = {
 		input: DbInput

frontend/src/lib/components/TagsToListenTo.svelte

@@ -29,6 +29,9 @@
 	{disabled}
 	class={disabled ? 'border-0' : ''}
 	allowClear={!disabled}
-	onCreateItem={(c) => worker_tags.push(c)}
+	onCreateItem={(c) => {
+		worker_tags.push(c)
+		dispatch('dirty')
+	}}
 	createText="Press Enter to use this tag"
 />

frontend/src/lib/components/apps/components/componentManagedFields.ts

@@ -0,0 +1,97 @@
+/**
+ * Component-Managed Fields Registry
+ *
+ * This module provides a centralized registry of fields that are automatically
+ * managed by components at runtime, rather than being configured by users.
+ *
+ * These fields should never be treated as static fields in force_viewer_static_fields,
+ * as their values are determined dynamically by user interactions (pagination,
+ * sorting, search, chat input, etc.)
+ */
+
+import type { AppInput } from '$lib/components/apps/inputType'
+
+/**
+ * Registry of component types to their managed field names
+ */
+export const COMPONENT_MANAGED_FIELDS: Record<string, string[]> = {
+	aggridinfinitecomponent: ['offset', 'limit', 'orderBy', 'isDesc', 'search'],
+	aggridinfinitecomponentee: ['offset', 'limit', 'orderBy', 'isDesc', 'search'],
+	chatcomponent: ['user_message']
+}
+
+/**
+ * Get the list of managed fields for a given component type
+ */
+export function getManagedFields(componentType: string): string[] {
+	return COMPONENT_MANAGED_FIELDS[componentType] ?? []
+}
+
+/**
+ * Check if a field is managed by a specific component type
+ */
+export function isFieldManagedByComponent(componentType: string, fieldName: string): boolean {
+	const managedFields = getManagedFields(componentType)
+	return managedFields.includes(fieldName)
+}
+
+/**
+ * Convert component-managed fields from static to evalv2 type
+ * This ensures they are properly handled at runtime and not added to force_viewer_static_fields
+ */
+export function convertManagedFieldsToEvalv2(
+	componentType: string,
+	componentId: string,
+	fields: Record<string, AppInput>
+): Record<string, AppInput> {
+	const managedFieldNames = getManagedFields(componentType)
+
+	if (managedFieldNames.length === 0) {
+		return fields
+	}
+
+	const convertedFields = { ...fields }
+
+	for (const fieldName of managedFieldNames) {
+		if (convertedFields[fieldName]) {
+			// Determine the expression based on component type
+			let expr: string
+
+			if (
+				componentType === 'aggridinfinitecomponent' ||
+				componentType === 'aggridinfinitecomponentee'
+			) {
+				// AgGrid components use params.{fieldName}
+				expr = `${componentId}.params.${fieldName}`
+			} else if (componentType === 'chatcomponent') {
+				// Chat component uses userMessage output
+				expr = `${componentId}.userMessage`
+			} else {
+				// Default pattern for future components
+				expr = `${componentId}.${fieldName}`
+			}
+
+			// Convert to evalv2 type while preserving fieldType
+			convertedFields[fieldName] = {
+				type: 'evalv2',
+				expr,
+				fieldType: convertedFields[fieldName].fieldType ?? 'string',
+				connections: []
+			} as AppInput
+		}
+	}
+
+	return convertedFields
+}
+
+/**
+ * Get all managed field names across all component types
+ * Useful for generic checks
+ */
+export function getAllManagedFieldNames(): string[] {
+	const allFields = new Set<string>()
+	Object.values(COMPONENT_MANAGED_FIELDS).forEach((fields) => {
+		fields.forEach((field) => allFields.add(field))
+	})
+	return Array.from(allFields)
+}

frontend/src/lib/components/apps/components/display/AppChat.svelte

@@ -0,0 +1,338 @@
+<script lang="ts">
+	import { getContext } from 'svelte'
+	import type { AppInput } from '../../inputType'
+	import type { AppViewerContext, ComponentCustomCSS, RichConfigurations } from '../../types'
+	import RunnableWrapper from '../helpers/RunnableWrapper.svelte'
+	import type RunnableComponent from '../helpers/RunnableComponent.svelte'
+	import { initConfig, initOutput } from '../../editor/appUtils'
+	import { components } from '../../editor/component'
+	import ResolveConfig from '../helpers/ResolveConfig.svelte'
+	import ResolveStyle from '../helpers/ResolveStyle.svelte'
+	import InitializeComponent from '../helpers/InitializeComponent.svelte'
+	import { twMerge } from 'tailwind-merge'
+	import { Loader2 } from 'lucide-svelte'
+	import { initCss } from '../../utils'
+	import ChatMessage from '$lib/components/chat/ChatMessage.svelte'
+	import ChatInput from '$lib/components/chat/ChatInput.svelte'
+	import { parseStreamDeltas } from '$lib/components/chat/utils'
+	import { randomUUID } from '$lib/components/flows/conversations/FlowChatManager.svelte'
+
+	interface Message {
+		id: string
+		role: 'user' | 'assistant'
+		content: string
+	}
+
+	interface Props {
+		id: string
+		componentInput: AppInput | undefined
+		configuration: RichConfigurations
+		customCss?: ComponentCustomCSS<'chatcomponent'> | undefined
+		recomputeIds?: string[] | undefined
+		render: boolean
+		errorHandledByComponent?: boolean
+	}
+
+	let {
+		id,
+		componentInput,
+		configuration,
+		customCss = undefined,
+		recomputeIds = undefined,
+		render,
+		errorHandledByComponent = $bindable(false)
+	}: Props = $props()
+
+	const { worldStore, app, componentControl } = getContext<AppViewerContext>('AppViewerContext')
+
+	// Initialize outputs
+	let outputs = initOutput($worldStore, id, {
+		result: undefined as any,
+		loading: false,
+		jobId: undefined as string | undefined,
+		messages: [] as Message[],
+		userMessage: '' as string // Output for evalv2 field
+	})
+
+	// Resolve configuration
+	let resolvedConfig = $state(
+		initConfig(components['chatcomponent'].initialData.configuration, configuration)
+	)
+
+	// Initialize CSS
+	let css = $state(initCss($app.css?.chatcomponent, customCss))
+
+	// State
+	let runnableComponent: RunnableComponent | undefined = $state()
+	let runnableWrapper: RunnableWrapper | undefined = $state()
+	let loading = $state(false)
+	let result: any = $state(undefined)
+	let messages: Message[] = $state([])
+	let inputValue = $state('')
+	let messagesContainer: HTMLDivElement | undefined = $state()
+
+	// Streaming state management
+	let currentStreamingMessageIndex: number | undefined = $state(undefined)
+	let accumulatedContent = $state('')
+
+	// Generate stable memory_id for chat session (for agent memory persistence)
+	let chatMemoryId = $state(randomUUID())
+
+	// Register component control for programmatic access
+	$componentControl[id] = {
+		sendMessage: (message: string) => {
+			if (message && !loading) {
+				inputValue = message
+				handleSend()
+			}
+		}
+	}
+
+	// Auto-scroll to bottom when messages change
+	$effect(() => {
+		if (messages.length > 0 && messagesContainer) {
+			setTimeout(() => {
+				messagesContainer?.scrollTo({
+					top: messagesContainer.scrollHeight,
+					behavior: 'smooth'
+				})
+			}, 50)
+		}
+	})
+
+	// Handle streaming updates
+	function handleStreamUpdate(e: CustomEvent<{ id: string; result_stream: string }>) {
+		const streamContent = e.detail.result_stream
+
+		const parsed = parseStreamDeltas(streamContent)
+		if (parsed.content) {
+			accumulatedContent = parsed.content
+		} else {
+			accumulatedContent = streamContent
+		}
+
+		// Update or create streaming message
+		if (currentStreamingMessageIndex !== undefined) {
+			messages = messages.map((msg, idx) =>
+				idx === currentStreamingMessageIndex ? { ...msg, content: accumulatedContent } : msg
+			)
+		} else {
+			const assistantMessage: Message = {
+				id: randomUUID(),
+				role: 'assistant',
+				content: accumulatedContent
+			}
+			messages = [...messages, assistantMessage]
+			currentStreamingMessageIndex = messages.length - 1
+		}
+	}
+
+	// Handle job completion
+	function handleJobComplete(e: CustomEvent<{ id: string; result: any }>) {
+		const finalResult = e.detail.result
+
+		// Extract final content
+		let finalContent = ''
+		if (typeof finalResult === 'string') {
+			finalContent = finalResult
+		} else if (finalResult && typeof finalResult === 'object' && 'output' in finalResult) {
+			finalContent =
+				typeof finalResult.output === 'string'
+					? finalResult.output
+					: JSON.stringify(finalResult.output, null, 2)
+		} else {
+			finalContent = JSON.stringify(finalResult, null, 2)
+		}
+
+		// If we were streaming, update the message with final result to ensure completeness
+		if (currentStreamingMessageIndex !== undefined && finalContent) {
+			messages = messages.map((msg, idx) =>
+				idx === currentStreamingMessageIndex ? { ...msg, content: finalContent } : msg
+			)
+		}
+		// If not streaming, create new message
+		else if (finalContent.length > 0) {
+			messages = [
+				...messages,
+				{
+					id: randomUUID(),
+					role: 'assistant',
+					content: finalContent
+				}
+			]
+		}
+
+		// Finalize streaming
+		currentStreamingMessageIndex = undefined
+		accumulatedContent = ''
+	}
+
+	// Handle job error
+	function handleJobError(e: CustomEvent<{ id: string; error: any }>) {
+		const error = e.detail.error
+
+		// Add error message
+		messages = [
+			...messages,
+			{
+				id: randomUUID(),
+				role: 'assistant',
+				content: `Error: ${error.message || JSON.stringify(error)}`
+			}
+		]
+
+		// Reset streaming state
+		currentStreamingMessageIndex = undefined
+		accumulatedContent = ''
+	}
+
+	// Handle send message
+	async function handleSend() {
+		if (!inputValue.trim() || loading) return
+
+		const userMessage = inputValue.trim()
+		inputValue = ''
+
+		// Add user message to chat
+		const newUserMessage: Message = {
+			id: randomUUID(),
+			role: 'user',
+			content: userMessage
+		}
+		messages = [...messages, newUserMessage]
+
+		// Reset streaming state for new message
+		currentStreamingMessageIndex = undefined
+		accumulatedContent = ''
+
+		// Update output so evalv2 field can reference it
+		outputs.userMessage.set(userMessage)
+
+		// Trigger the runnable
+		if (!runnableComponent) {
+			runnableWrapper?.handleSideEffect(true)
+		} else {
+			await runnableComponent?.runComponent()
+		}
+	}
+
+	// Handle enter key
+	function handleKeydown(e: KeyboardEvent) {
+		if (e.key === 'Enter' && !e.shiftKey) {
+			e.preventDefault()
+			handleSend()
+		}
+	}
+
+	$effect(() => {
+		errorHandledByComponent = resolvedConfig?.onError?.selected !== 'errorOverlay'
+	})
+</script>
+
+<InitializeComponent {id} />
+
+{#each Object.keys(components['chatcomponent'].initialData.configuration) as key (key)}
+	<ResolveConfig
+		{id}
+		{key}
+		bind:resolvedConfig={resolvedConfig[key]}
+		configuration={configuration[key]}
+	/>
+{/each}
+
+{#each Object.keys(css ?? {}) as key (key)}
+	<ResolveStyle
+		{id}
+		{customCss}
+		{key}
+		bind:css={css[key]}
+		componentStyle={$app.css?.chatcomponent}
+	/>
+{/each}
+
+<RunnableWrapper
+	bind:this={runnableWrapper}
+	bind:runnableComponent
+	bind:loading
+	bind:result
+	{componentInput}
+	{id}
+	{recomputeIds}
+	{outputs}
+	doOnSuccess={resolvedConfig.onSuccess}
+	doOnError={resolvedConfig.onError}
+	on:streamupdate={handleStreamUpdate}
+	on:done={handleJobComplete}
+	on:doneError={handleJobError}
+	{errorHandledByComponent}
+	autoRefresh={false}
+	{render}
+	extraQueryParams={{ memory_id: chatMemoryId }}
+>
+	{#if render}
+		<div
+			class={twMerge(
+				'flex flex-col h-full border rounded-lg bg-surface',
+				css?.container?.class,
+				'wm-chat-container'
+			)}
+			style={css?.container?.style}
+		>
+			<!-- Messages Container -->
+			<div
+				bind:this={messagesContainer}
+				class={twMerge(
+					'flex-1 overflow-y-auto p-4 bg-background',
+					css?.messagesContainer?.class,
+					'wm-chat-messages'
+				)}
+				style={css?.messagesContainer?.style}
+			>
+				{#if messages.length === 0}
+					<div class="flex items-center justify-center h-full text-tertiary text-sm">
+						No messages yet. Start a conversation!
+					</div>
+				{:else}
+					<div class="w-full space-y-4 xl:max-w-7xl mx-auto">
+						{#each messages as message (message.id)}
+							<ChatMessage
+								role={message.role}
+								content={message.content}
+								enableMarkdown={true}
+								enableS3Display={true}
+								customCss={{
+									userMessage: css?.userMessage,
+									assistantMessage: css?.assistantMessage
+								}}
+							/>
+						{/each}
+						{#if loading}
+							<div class="flex items-center gap-2 text-tertiary">
+								<Loader2 size={16} class="animate-spin" />
+								<span class="text-sm">Processing...</span>
+							</div>
+						{/if}
+					</div>
+				{/if}
+			</div>
+
+			<!-- Input Container -->
+			<div
+				class={twMerge('border-t p-3', css?.inputContainer?.class, 'wm-chat-input')}
+				style={css?.inputContainer?.style}
+			>
+				<ChatInput
+					bind:value={inputValue}
+					placeholder={resolvedConfig.placeholder}
+					disabled={loading}
+					onSend={handleSend}
+					onKeydown={handleKeydown}
+					customCss={{
+						input: css?.input,
+						button: css?.button
+					}}
+				/>
+			</div>
+		</div>
+	{/if}
+</RunnableWrapper>

frontend/src/lib/components/apps/components/display/AppMap.svelte

@@ -6,17 +6,18 @@
 	import type { AppViewerContext, ComponentCustomCSS, RichConfigurations } from '../../types'
 	import { twMerge } from 'tailwind-merge'
 	import { Map, View, Feature } from 'ol'
-	import { useGeographic } from 'ol/proj'
-	import { OSM, Vector as VectorSource } from 'ol/source'
-	import { Vector as VectorLayer, Tile as TileLayer } from 'ol/layer'
-	import { Point } from 'ol/geom'
-	import { defaults as defaultControls } from 'ol/control'
-	import { findGridItem, initConfig, initOutput } from '../../editor/appUtils'
+	import { useGeographic } from 'ol/proj.js'
+	import { OSM, Vector as VectorSource } from 'ol/source.js'
+	import { Vector as VectorLayer, Tile as TileLayer } from 'ol/layer.js'
+	import { Point } from 'ol/geom.js'
+	import { defaults as defaultControls } from 'ol/control.js'
+	import { initConfig, initOutput } from '../../editor/appUtils'
 	import InitializeComponent from '../helpers/InitializeComponent.svelte'
 	import ResolveStyle from '../helpers/ResolveStyle.svelte'
 	import { components } from '../../editor/component'
 	import ResolveConfig from '../helpers/ResolveConfig.svelte'
-	import { Style, Circle, Fill, Stroke, Text } from 'ol/style'
+	import { Style, Circle, Fill, Stroke, Text } from 'ol/style.js'
+	import { findGridItem } from '../../editor/appUtilsCore'
 	interface Marker {
 		lon: number
 		lat: number

frontend/src/lib/components/apps/components/display/InsertRowDrawerButton.svelte

@@ -2,7 +2,8 @@
 	import { Drawer, DrawerContent, Button } from '$lib/components/common'
 	import { CornerDownLeft, Plus } from 'lucide-svelte'
 	import InsertRow from './dbtable/InsertRow.svelte'
-	import type { ColumnDef, DbType } from './dbtable/utils'
+	import type { ColumnDef } from './dbtable/utils'
+	import type { DbType } from '$lib/components/dbTypes'
 
 	type Props = {
 		columnDefs: ColumnDef[]

frontend/src/lib/components/apps/components/display/dbtable/AppDbExplorer.svelte

@@ -8,16 +8,8 @@
 	} from '../../../types'
 	import { components, type TableAction } from '$lib/components/apps/editor/component'
 	import ResolveConfig from '../../helpers/ResolveConfig.svelte'
-	import { findGridItem, initConfig, initOutput } from '$lib/components/apps/editor/appUtils'
-	import {
-		getDbSchemas,
-		loadTableMetaData,
-		type TableMetadata,
-		getPrimaryKeys,
-		type ColumnDef,
-		type DbType,
-		getTablesByResource
-	} from './utils'
+	import { initConfig, initOutput } from '$lib/components/apps/editor/appUtils'
+	import { type TableMetadata, getPrimaryKeys, type ColumnDef } from './utils'
 	import { getContext, tick, untrack } from 'svelte'
 	import UpdateCell from './UpdateCell.svelte'
 	import { workspaceStore, type DBSchemas } from '$lib/stores'
@@ -38,7 +30,10 @@
 	import RefreshButton from '$lib/components/apps/components/helpers/RefreshButton.svelte'
 	import RunnableWrapper from '../../helpers/RunnableWrapper.svelte'
 	import InsertRowDrawerButton from '../InsertRowDrawerButton.svelte'
-	import { getDucklakeSchema, type DbInput } from '$lib/components/dbOps'
+	import { getDucklakeSchema } from '$lib/components/dbOps'
+	import { findGridItem } from '$lib/components/apps/editor/appUtilsCore'
+	import type { DbInput, DbType } from '$lib/components/dbTypes'
+	import { getDbSchemas, getTablesByResource, loadTableMetaData } from './metadata'
 
 	interface Props {
 		id: string

frontend/src/lib/components/apps/components/display/dbtable/DbExplorerCount.svelte

@@ -7,7 +7,7 @@
 	import { initOutput } from '../../../editor/appUtils'
 	import { type ColumnDef } from './utils'
 	import { getCountInput } from './queries/count'
-	import type { DbInput } from '$lib/components/dbOps'
+	import type { DbInput } from '$lib/components/dbTypes'
 
 	interface Props {
 		id: string

frontend/src/lib/components/apps/components/display/dbtable/DeleteRow.svelte

@@ -8,7 +8,7 @@
 	import { getPrimaryKeys, type ColumnDef } from './utils'
 	import { sendUserToast } from '$lib/toast'
 	import { getDeleteInput } from './queries/delete'
-	import type { DbInput } from '$lib/components/dbOps'
+	import type { DbInput } from '$lib/components/dbTypes'
 
 	interface Props {
 		id: string

frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
 	import type { Schema, SchemaProperty } from '$lib/common'
-	import { ColumnIdentity, type ColumnMetadata, type DbType, type ColumnDef } from './utils'
+	import { ColumnIdentity, type ColumnMetadata, type ColumnDef } from './utils'
 
 	import init, {
 		parse_sql,
@@ -19,6 +19,7 @@
 	import { untrack } from 'svelte'
 	import Toggle from '$lib/components/Toggle.svelte'
 	import { usePromise } from '$lib/svelte5Utils.svelte'
+	import type { DbType } from '$lib/components/dbTypes'
 
 	type FieldMetadata = {
 		type: string

frontend/src/lib/components/apps/components/display/dbtable/InsertRowRunnable.svelte

@@ -8,7 +8,7 @@
 	import { type ColumnDef } from './utils'
 	import { sendUserToast } from '$lib/toast'
 	import { getInsertInput } from './queries/insert'
-	import type { DbInput } from '$lib/components/dbOps'
+	import type { DbInput } from '$lib/components/dbTypes'
 
 	interface Props {
 		id: string

frontend/src/lib/components/apps/components/display/dbtable/UpdateCell.svelte

@@ -8,7 +8,7 @@
 	import { getPrimaryKeys, type ColumnDef } from './utils'
 	import { sendUserToast } from '$lib/toast'
 	import { getUpdateInput } from './queries/update'
-	import type { DbInput } from '$lib/components/dbOps'
+	import type { DbInput } from '$lib/components/dbTypes'
 
 	interface Props {
 		id: string

frontend/src/lib/components/apps/components/display/dbtable/metadata.ts

@@ -0,0 +1,484 @@
+import { JobService, ResourceService } from '$lib/gen'
+
+import { runScriptAndPollResult } from '$lib/components/jobs/utils'
+import type { DbInput } from '$lib/components/dbTypes'
+import { getLanguageByResourceType, resourceTypeToLang, scripts, type TableMetadata } from './utils'
+
+import { type Preview } from '$lib/gen'
+import type { DBSchema, DBSchemas, GraphqlSchema, SQLSchema } from '$lib/stores'
+
+import { tryEvery } from '$lib/utils'
+import { stringifySchema } from '$lib/components/copilot/lib'
+import type { DbType } from '$lib/components/dbTypes'
+
+export async function loadTableMetaData(
+	input: DbInput,
+	workspace: string | undefined,
+	table: string | undefined
+): Promise<TableMetadata | undefined> {
+	if (!input || !table || !workspace) return undefined
+
+	let language = input.type == 'ducklake' ? 'duckdb' : getLanguageByResourceType(input.resourceType)
+	let content = await makeLoadTableMetaDataQuery(input, workspace, table)
+
+	const job = await JobService.runScriptPreview({
+		workspace: workspace,
+		requestBody: {
+			language,
+			content,
+			args: input.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
+		}
+	})
+
+	const maxRetries = 8
+	let attempts = 0
+	while (attempts < maxRetries) {
+		try {
+			await new Promise((resolve) => setTimeout(resolve, 1000 * (attempts || 0.6)))
+
+			const testResult = (await JobService.getCompletedJob({
+				workspace: workspace,
+				id: job
+			})) as any
+
+			if (testResult.success) {
+				attempts = maxRetries
+
+				if (input.type === 'database' && input.resourceType === 'ms_sql_server') {
+					return testResult.result[0].map(lowercaseKeys)
+				} else {
+					return testResult.result.map(lowercaseKeys)
+				}
+			} else {
+				attempts++
+			}
+		} catch (error) {
+			attempts++
+		}
+	}
+
+	console.error('Failed to load table metadata after maximum retries.')
+	return undefined
+}
+
+export async function loadAllTablesMetaData(
+	workspace: string | undefined,
+	input: DbInput
+): Promise<Record<string, TableMetadata> | undefined> {
+	if (!input || !workspace) return undefined
+
+	let language = input.type == 'ducklake' ? 'duckdb' : getLanguageByResourceType(input.resourceType)
+
+	try {
+		let result = (await runScriptAndPollResult({
+			workspace: workspace,
+			requestBody: {
+				language,
+				content: await makeLoadTableMetaDataQuery(input, workspace, undefined),
+				args: input.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
+			}
+		})) as ({ table_name: string; schema_name?: string } & object)[]
+		if (input.type === 'database' && input.resourceType === 'ms_sql_server') {
+			result = (result as any)[0]
+		}
+		const map: Record<string, TableMetadata> = {}
+
+		for (const _col of result) {
+			const col = lowercaseKeys(_col)
+			const tableKey = col.schema_name ? `${col.schema_name}.${col.table_name}` : col.table_name
+			if (!(tableKey in map)) {
+				map[tableKey] = []
+			}
+			map[tableKey].push(col)
+		}
+		return map
+	} catch (e) {
+		throw new Error('Error loading all tables metadata: ' + e)
+	}
+}
+
+async function makeLoadTableMetaDataQuery(
+	input: DbInput,
+	workspace: string,
+	table: string | undefined
+): Promise<string> {
+	if (input.type === 'ducklake') {
+		return `ATTACH 'ducklake://${input.ducklake}' AS __ducklake__;
+		SELECT
+			COLUMN_NAME as field,
+			DATA_TYPE as DataType,
+			COLUMN_DEFAULT as DefaultValue,
+			false as IsPrimaryKey,
+			false as IsIdentity,
+			IS_NULLABLE as IsNullable,
+			false as IsEnum,
+			TABLE_NAME as table_name
+		FROM information_schema.columns c
+		WHERE table_catalog = '__ducklake__' AND table_schema = current_schema()`
+	} else if (input.resourceType === 'mysql') {
+		const resourceObj = (await ResourceService.getResourceValue({
+			workspace,
+			path: input.resourcePath
+		})) as any
+		return `
+	SELECT 
+			COLUMN_NAME as field,
+			COLUMN_TYPE as DataType,
+			COLUMN_DEFAULT as DefaultValue,
+			CASE WHEN COLUMN_KEY = 'PRI' THEN 'YES' ELSE 'NO' END as IsPrimaryKey,
+			CASE WHEN EXTRA like '%auto_increment%' THEN 'YES' ELSE 'NO' END as IsIdentity,
+			CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
+			CASE WHEN DATA_TYPE = 'enum' THEN true ELSE false END as IsEnum${
+				table
+					? ''
+					: `,
+			TABLE_NAME as table_name`
+			}
+	FROM 
+			INFORMATION_SCHEMA.COLUMNS${
+				table
+					? `
+	WHERE
+			TABLE_NAME = '${table.split('.').reverse()[0]}' AND TABLE_SCHEMA = '${
+				table.split('.').reverse()[1] ?? resourceObj?.database ?? ''
+			}'`
+					: `
+	WHERE
+			TABLE_SCHEMA NOT IN ('mysql', 'performance_schema', 'information_schema', 'sys')`
+			}
+	ORDER BY
+			TABLE_NAME,
+			ORDINAL_POSITION;
+	`
+	} else if (input.resourceType === 'postgresql') {
+		return `
+	SELECT 
+		a.attname as field,
+		pg_catalog.format_type(a.atttypid, a.atttypmod) as DataType,
+		(SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128)
+		 FROM pg_catalog.pg_attrdef d
+		 WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) as DefaultValue,
+		(SELECT CASE WHEN i.indisprimary THEN true ELSE 'NO' END
+		 FROM pg_catalog.pg_class tbl, pg_catalog.pg_class idx, pg_catalog.pg_index i, pg_catalog.pg_attribute att
+		 WHERE tbl.oid = a.attrelid AND idx.oid = i.indexrelid AND att.attrelid = tbl.oid
+								 AND i.indrelid = tbl.oid AND att.attnum = any(i.indkey) AND att.attname = a.attname LIMIT 1) as IsPrimaryKey,
+		CASE a.attidentity
+				WHEN 'd' THEN 'By Default'
+				WHEN 'a' THEN 'Always'
+				ELSE 'No'
+		END as IsIdentity,
+		CASE a.attnotnull
+				WHEN false THEN 'YES'
+				ELSE 'NO'
+		END as IsNullable,
+		(SELECT true
+		 FROM pg_catalog.pg_enum e
+		 WHERE e.enumtypid = a.atttypid FETCH FIRST ROW ONLY) as IsEnum${
+				table
+					? ''
+					: `,
+    ns.nspname AS schema_name,
+    c.relname AS table_name`
+			}
+	FROM pg_catalog.pg_attribute a${
+		table
+			? `
+	WHERE a.attrelid = (SELECT c.oid FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid WHERE relname = '${
+		table.split('.').reverse()[0]
+	}' AND ns.nspname = '${table.split('.').reverse()[1] ?? 'public'}')
+		AND a.attnum > 0 AND NOT a.attisdropped
+		`
+			: `
+	JOIN pg_catalog.pg_class c ON a.attrelid = c.oid
+	JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid
+	WHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped
+		AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'`
+	}
+	ORDER BY ${table ? 'a.attnum' : 'ns.nspname, c.relname, a.attnum'};
+	
+	`
+	} else if (input.resourceType === 'ms_sql_server') {
+		return `
+		SELECT 
+    COLUMN_NAME as field,
+    DATA_TYPE as DataType,
+    COLUMN_DEFAULT as DefaultValue,
+    CASE WHEN COLUMNPROPERTY(OBJECT_ID(TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1 THEN 'By Default' ELSE 'No' END as IsIdentity,
+    CASE WHEN COLUMNPROPERTY(OBJECT_ID(TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1 THEN 1 ELSE 0 END as IsPrimaryKey, -- This line still needs correction for primary key identification
+    CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
+    CASE WHEN DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum${
+			table
+				? ''
+				: `,
+		TABLE_NAME as table_name`
+		}
+FROM    
+    INFORMATION_SCHEMA.COLUMNS${
+			table
+				? `
+WHERE   
+    TABLE_NAME = '${table}'`
+				: ''
+		}
+ORDER BY
+    ORDINAL_POSITION;
+	`
+	} else if (
+		input.resourceType === 'snowflake' ||
+		(input.resourceType as any) === 'snowflake_oauth'
+	) {
+		return `
+		select COLUMN_NAME as field,
+		DATA_TYPE as DataType,
+		COLUMN_DEFAULT as DefaultValue,
+		CASE WHEN COLUMN_DEFAULT like 'AUTOINCREMENT%' THEN 'By Default' ELSE 'No' END as IsIdentity,
+		CASE WHEN COLUMN_DEFAULT like 'AUTOINCREMENT%' THEN 1 ELSE 0 END as IsPrimaryKey,
+		CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
+		CASE WHEN DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum${
+			table
+				? ''
+				: `,
+		table_name as table_name,
+		table_schema as schema_name`
+		}
+	from information_schema.columns${
+		table
+			? `
+	where table_name = '${table.split('.').reverse()[0]}' and table_schema = '${
+		table.split('.').reverse()[1] ?? 'PUBLIC'
+	}'`
+			: "\nwhere table_schema <> 'INFORMATION_SCHEMA'\n"
+	}
+	order by ORDINAL_POSITION;
+	`
+	} else if (input.resourceType === 'bigquery') {
+		// TODO: find a solution for this (query uses hardcoded dataset name)
+		if (!table) throw new Error('Table name is required for BigQuery')
+		return `SELECT 
+    c.COLUMN_NAME as field,
+    DATA_TYPE as DataType,
+    CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue,
+    CASE WHEN constraint_name is not null THEN true ELSE false END as IsPrimaryKey,
+    'No' as IsIdentity,
+    IS_NULLABLE as IsNullable,
+    false as IsEnum
+FROM
+    ${table.split('.')[0]}.INFORMATION_SCHEMA.COLUMNS c
+    LEFT JOIN
+    ${table.split('.')[0]}.INFORMATION_SCHEMA.KEY_COLUMN_USAGE p
+    on c.table_name = p.table_name AND c.column_name = p.COLUMN_NAME
+WHERE   
+    c.TABLE_NAME = '${table.split('.')[1]}'
+order by c.ORDINAL_POSITION;`
+	} else {
+		throw new Error('Unsupported database type:' + input.resourceType)
+	}
+}
+
+function lowercaseKeys(obj: Record<string, any>): any {
+	const newObj = {}
+	Object.keys(obj).forEach((key) => {
+		newObj[key.toLowerCase()] = obj[key]
+	})
+	return newObj
+}
+
+export async function getDbSchemas(
+	resourceType: string,
+	resourcePath: string,
+	workspace: string | undefined,
+	dbSchemas: DBSchemas,
+	errorCallback: (message: string) => void
+): Promise<void> {
+	if (!scripts[resourceType]) return
+
+	return new Promise(async (resolve, reject) => {
+		if (!resourceType || !resourcePath || !workspace) {
+			resolve()
+			return
+		}
+
+		const job = await JobService.runScriptPreview({
+			workspace: workspace,
+			requestBody: {
+				language: scripts[resourceType].lang as Preview['language'],
+				content: scripts[resourceType].code,
+				args: {
+					[scripts[resourceType].argName]: resourcePath.startsWith('datatable://')
+						? resourcePath
+						: '$res:' + resourcePath
+				}
+			}
+		})
+
+		tryEvery({
+			tryCode: async () => {
+				if (resourcePath) {
+					const testResult = await JobService.getCompletedJob({
+						workspace,
+						id: job
+					})
+					if (!testResult.success) {
+						console.error(testResult.result?.['error']?.['message'])
+					} else {
+						if (testResult.result === 'WINDMILL_TOO_BIG') {
+							console.info('Result is too big, fetching result separately')
+							const data = await JobService.getCompletedJobResult({
+								workspace,
+								id: job
+							})
+							testResult.result = data
+						}
+						if (resourceType !== undefined) {
+							if (resourceType !== 'graphql') {
+								const { processingFn } = scripts[resourceType]
+								let schema: any
+								try {
+									schema =
+										processingFn !== undefined ? processingFn(testResult.result) : testResult.result
+								} catch (e) {
+									console.error(e)
+									errorCallback('Error processing schema')
+									resolve()
+									return
+								}
+								const dbSchema = {
+									lang: resourceTypeToLang(resourceType) as SQLSchema['lang'],
+									schema,
+									publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
+								}
+								dbSchemas[resourcePath] = {
+									...dbSchema,
+									stringified: stringifySchema(dbSchema)
+								}
+							} else {
+								if (
+									typeof testResult.result !== 'object' ||
+									!('__schema' in (testResult?.result ?? {}))
+								) {
+									console.error('Invalid GraphQL schema')
+
+									errorCallback('Invalid GraphQL schema')
+								} else {
+									const dbSchema = {
+										lang: 'graphql' as GraphqlSchema['lang'],
+										schema: testResult.result
+									}
+									dbSchemas[resourcePath] = {
+										...(dbSchema as any),
+										stringified: stringifySchema(dbSchema as any)
+									}
+								}
+							}
+						}
+					}
+					resolve()
+				}
+			},
+			timeoutCode: async () => {
+				console.error('Could not query schema within 5s')
+				errorCallback('Could not query schema within 5s')
+				try {
+					await JobService.cancelQueuedJob({
+						workspace,
+						id: job,
+						requestBody: {
+							reason: 'Could not query schema within 5s'
+						}
+					})
+				} catch (err) {
+					console.error(err)
+				}
+				reject()
+			},
+			interval: 500,
+			timeout: 5000
+		})
+	})
+}
+
+export async function getTablesByResource(
+	schema: Partial<Record<string, DBSchema>>,
+	dbType: DbType | undefined,
+	dbPath: string,
+	workspace: string
+): Promise<string[]> {
+	const s = Object.values(schema)?.[0]
+	switch (dbType) {
+		case 'ms_sql_server': {
+			const paths: string[] = []
+			for (const key in s?.schema) {
+				for (const subKey in s.schema[key]) {
+					if (key === 'dbo') {
+						paths.push(`${subKey}`)
+					}
+				}
+			}
+			return paths
+		}
+		case 'mysql': {
+			const resourceObj = (await ResourceService.getResourceValue({
+				workspace,
+				path: dbPath.split('$res:')[1]
+			})) as any
+			const paths: string[] = []
+			for (const key in s?.schema) {
+				for (const subKey in s.schema[key]) {
+					if (key === resourceObj?.database) {
+						paths.push(`${subKey}`)
+					} else {
+						paths.push(`${key}.${subKey}`)
+					}
+				}
+			}
+			return paths
+		}
+		case 'snowflake': {
+			const paths: string[] = []
+			for (const key in s?.schema) {
+				for (const subKey in s.schema[key]) {
+					if (key === 'PUBLIC') {
+						paths.push(`${subKey}`)
+					} else {
+						paths.push(`${key}.${subKey}`)
+					}
+				}
+			}
+			return paths
+		}
+		case 'postgresql': {
+			const paths: string[] = []
+			for (const key in s?.schema) {
+				for (const subKey in s.schema[key]) {
+					if (key === 'public') {
+						paths.push(`${subKey}`)
+					} else {
+						paths.push(`${key}.${subKey}`)
+					}
+				}
+			}
+			return paths
+		}
+		case 'bigquery': {
+			const paths: string[] = []
+			for (const key in s?.schema) {
+				for (const subKey in s.schema[key]) {
+					paths.push(`${key}.${subKey}`)
+				}
+			}
+			return paths
+		}
+		case 'duckdb': {
+			const paths: string[] = []
+			for (const key in s?.schema) {
+				for (const subKey in s.schema[key]) {
+					paths.push(`${subKey}`)
+				}
+			}
+
+			return paths
+		}
+		default:
+			return []
+	}
+}

frontend/src/lib/components/apps/components/display/dbtable/queries/count.ts

@@ -1,6 +1,7 @@
 import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
-import { wrapDucklakeQuery, type DbInput } from '$lib/components/dbOps'
-import { buildParameters, type DbType } from '../utils'
+import { wrapDucklakeQuery } from '../../../../../ducklake'
+import type { DbType, DbInput } from '$lib/components/dbTypes'
+import { buildParameters } from '../utils'
 import { getLanguageByResourceType, type ColumnDef, buildVisibleFieldList } from '../utils'
 
 export function makeCountQuery(

frontend/src/lib/components/apps/components/display/dbtable/queries/createTable.ts

@@ -1,4 +1,5 @@
-import { dbSupportsSchemas, type DbType } from '../utils'
+import { dbSupportsSchemas } from '../utils'
+import type { DbType } from '$lib/components/dbTypes'
 
 export type CreateTableValues = {
 	name: string

frontend/src/lib/components/apps/components/display/dbtable/queries/delete.ts

@@ -1,6 +1,7 @@
 import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
-import { wrapDucklakeQuery, type DbInput } from '$lib/components/dbOps'
-import { getLanguageByResourceType, type ColumnDef, buildParameters, type DbType } from '../utils'
+import type { DbType, DbInput } from '$lib/components/dbTypes'
+import { wrapDucklakeQuery } from '../../../../../ducklake'
+import { getLanguageByResourceType, type ColumnDef, buildParameters } from '../utils'
 
 export function makeDeleteQuery(table: string, columns: ColumnDef[], dbType: DbType) {
 	let query = buildParameters(

frontend/src/lib/components/apps/components/display/dbtable/queries/deleteTable.ts

@@ -1,4 +1,4 @@
-import type { DbType } from '../utils'
+import type { DbType } from '$lib/components/dbTypes'
 
 export function makeDeleteTableQuery(tableKey: string, resourceType: DbType): string {
 	// same for all sql dbs

frontend/src/lib/components/apps/components/display/dbtable/queries/insert.ts

@@ -1,6 +1,7 @@
 import type { AppInput } from '$lib/components/apps/inputType'
-import { wrapDucklakeQuery, type DbInput } from '$lib/components/dbOps'
-import { buildParameters, ColumnIdentity, type DbType } from '../utils'
+import { wrapDucklakeQuery } from '../../../../../ducklake'
+import type { DbType, DbInput } from '$lib/components/dbTypes'
+import { buildParameters, ColumnIdentity } from '../utils'
 import { getLanguageByResourceType, type ColumnDef } from '../utils'
 
 function formatInsertValues(columns: ColumnDef[], dbType: DbType, startIndex: number = 1): string {

frontend/src/lib/components/apps/components/display/dbtable/queries/select.ts

@@ -1,6 +1,7 @@
 import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
-import { wrapDucklakeQuery, type DbInput } from '$lib/components/dbOps'
-import { buildParameters, type DbType } from '../utils'
+import { wrapDucklakeQuery } from '../../../../../ducklake'
+import type { DbType, DbInput } from '$lib/components/dbTypes'
+import { buildParameters } from '../utils'
 import { getLanguageByResourceType, type ColumnDef, buildVisibleFieldList } from '../utils'
 
 function makeSnowflakeSelectQuery(

frontend/src/lib/components/apps/components/display/dbtable/queries/update.ts

@@ -1,6 +1,7 @@
 import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
-import { wrapDucklakeQuery, type DbInput } from '$lib/components/dbOps'
-import { getLanguageByResourceType, type ColumnDef, buildParameters, type DbType } from '../utils'
+import { wrapDucklakeQuery } from '../../../../../ducklake'
+import type { DbInput, DbType } from '$lib/components/dbTypes'
+import { getLanguageByResourceType, type ColumnDef, buildParameters } from '../utils'
 
 export function makeUpdateQuery(
 	table: string,

frontend/src/lib/components/apps/components/display/dbtable/utils.ts

@@ -1,15 +1,13 @@
-import { JobService, type Preview, ResourceService, type ScriptLang } from '$lib/gen'
-import type { DBSchema, DBSchemas, GraphqlSchema, SQLSchema } from '$lib/stores'
+import type { ScriptLang } from '$lib/gen'
+import type { SQLSchema } from '$lib/stores'
 import {
 	buildClientSchema,
 	getIntrospectionQuery,
 	printSchema,
 	type IntrospectionQuery
 } from 'graphql'
-import { tryEvery } from '$lib/utils'
-import { stringifySchema } from '$lib/components/copilot/lib'
-import { runScriptAndPollResult } from '$lib/components/jobs/utils'
-import { getDatabaseArg, type DbInput } from '$lib/components/dbOps'
+
+import type { DbType } from '$lib/components/dbTypes'
 
 export enum ColumnIdentity {
 	ByDefault = 'By Default',
@@ -54,270 +52,6 @@ export type ColumnDef = {
 	defaultValueNull?: boolean
 } & ColumnMetadata
 
-export async function loadTableMetaData(
-	input: DbInput,
-	workspace: string | undefined,
-	table: string | undefined
-): Promise<TableMetadata | undefined> {
-	if (!input || !table || !workspace) return undefined
-
-	let language = input.type == 'ducklake' ? 'duckdb' : getLanguageByResourceType(input.resourceType)
-	let content = await makeLoadTableMetaDataQuery(input, workspace, table)
-
-	const job = await JobService.runScriptPreview({
-		workspace: workspace,
-		requestBody: {
-			language,
-			content,
-			args: getDatabaseArg(input)
-		}
-	})
-
-	const maxRetries = 8
-	let attempts = 0
-	while (attempts < maxRetries) {
-		try {
-			await new Promise((resolve) => setTimeout(resolve, 1000 * (attempts || 0.6)))
-
-			const testResult = (await JobService.getCompletedJob({
-				workspace: workspace,
-				id: job
-			})) as any
-
-			if (testResult.success) {
-				attempts = maxRetries
-
-				if (input.type === 'database' && input.resourceType === 'ms_sql_server') {
-					return testResult.result[0].map(lowercaseKeys)
-				} else {
-					return testResult.result.map(lowercaseKeys)
-				}
-			} else {
-				attempts++
-			}
-		} catch (error) {
-			attempts++
-		}
-	}
-
-	console.error('Failed to load table metadata after maximum retries.')
-	return undefined
-}
-
-export async function loadAllTablesMetaData(
-	workspace: string | undefined,
-	input: DbInput
-): Promise<Record<string, TableMetadata> | undefined> {
-	if (!input || !workspace) return undefined
-
-	let language = input.type == 'ducklake' ? 'duckdb' : getLanguageByResourceType(input.resourceType)
-
-	try {
-		let result = (await runScriptAndPollResult({
-			workspace: workspace,
-			requestBody: {
-				language,
-				content: await makeLoadTableMetaDataQuery(input, workspace, undefined),
-				args: getDatabaseArg(input)
-			}
-		})) as ({ table_name: string; schema_name?: string } & object)[]
-		if (input.type === 'database' && input.resourceType === 'ms_sql_server') {
-			result = (result as any)[0]
-		}
-		const map: Record<string, TableMetadata> = {}
-
-		for (const _col of result) {
-			const col = lowercaseKeys(_col)
-			const tableKey = col.schema_name ? `${col.schema_name}.${col.table_name}` : col.table_name
-			if (!(tableKey in map)) {
-				map[tableKey] = []
-			}
-			map[tableKey].push(col)
-		}
-		return map
-	} catch (e) {
-		throw new Error('Error loading all tables metadata: ' + e)
-	}
-}
-
-async function makeLoadTableMetaDataQuery(
-	input: DbInput,
-	workspace: string,
-	table: string | undefined
-): Promise<string> {
-	if (input.type === 'ducklake') {
-		return `ATTACH 'ducklake://${input.ducklake}' AS __ducklake__;
-		SELECT
-			COLUMN_NAME as field,
-			DATA_TYPE as DataType,
-			COLUMN_DEFAULT as DefaultValue,
-			false as IsPrimaryKey,
-			false as IsIdentity,
-			IS_NULLABLE as IsNullable,
-			false as IsEnum,
-			TABLE_NAME as table_name
-		FROM information_schema.columns c
-		WHERE table_catalog = '__ducklake__' AND table_schema = current_schema()`
-	} else if (input.resourceType === 'mysql') {
-		const resourceObj = (await ResourceService.getResourceValue({
-			workspace,
-			path: input.resourcePath
-		})) as any
-		return `
-	SELECT 
-			COLUMN_NAME as field,
-			COLUMN_TYPE as DataType,
-			COLUMN_DEFAULT as DefaultValue,
-			CASE WHEN COLUMN_KEY = 'PRI' THEN 'YES' ELSE 'NO' END as IsPrimaryKey,
-			CASE WHEN EXTRA like '%auto_increment%' THEN 'YES' ELSE 'NO' END as IsIdentity,
-			CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
-			CASE WHEN DATA_TYPE = 'enum' THEN true ELSE false END as IsEnum${
-				table
-					? ''
-					: `,
-			TABLE_NAME as table_name`
-			}
-	FROM 
-			INFORMATION_SCHEMA.COLUMNS${
-				table
-					? `
-	WHERE
-			TABLE_NAME = '${table.split('.').reverse()[0]}' AND TABLE_SCHEMA = '${
-				table.split('.').reverse()[1] ?? resourceObj?.database ?? ''
-			}'`
-					: `
-	WHERE
-			TABLE_SCHEMA NOT IN ('mysql', 'performance_schema', 'information_schema', 'sys')`
-			}
-	ORDER BY
-			TABLE_NAME,
-			ORDINAL_POSITION;
-	`
-	} else if (input.resourceType === 'postgresql') {
-		return `
-	SELECT 
-		a.attname as field,
-		pg_catalog.format_type(a.atttypid, a.atttypmod) as DataType,
-		(SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128)
-		 FROM pg_catalog.pg_attrdef d
-		 WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) as DefaultValue,
-		(SELECT CASE WHEN i.indisprimary THEN true ELSE 'NO' END
-		 FROM pg_catalog.pg_class tbl, pg_catalog.pg_class idx, pg_catalog.pg_index i, pg_catalog.pg_attribute att
-		 WHERE tbl.oid = a.attrelid AND idx.oid = i.indexrelid AND att.attrelid = tbl.oid
-								 AND i.indrelid = tbl.oid AND att.attnum = any(i.indkey) AND att.attname = a.attname LIMIT 1) as IsPrimaryKey,
-		CASE a.attidentity
-				WHEN 'd' THEN 'By Default'
-				WHEN 'a' THEN 'Always'
-				ELSE 'No'
-		END as IsIdentity,
-		CASE a.attnotnull
-				WHEN false THEN 'YES'
-				ELSE 'NO'
-		END as IsNullable,
-		(SELECT true
-		 FROM pg_catalog.pg_enum e
-		 WHERE e.enumtypid = a.atttypid FETCH FIRST ROW ONLY) as IsEnum${
-				table
-					? ''
-					: `,
-    ns.nspname AS schema_name,
-    c.relname AS table_name`
-			}
-	FROM pg_catalog.pg_attribute a${
-		table
-			? `
-	WHERE a.attrelid = (SELECT c.oid FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid WHERE relname = '${
-		table.split('.').reverse()[0]
-	}' AND ns.nspname = '${table.split('.').reverse()[1] ?? 'public'}')
-		AND a.attnum > 0 AND NOT a.attisdropped
-		`
-			: `
-	JOIN pg_catalog.pg_class c ON a.attrelid = c.oid
-	JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid
-	WHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped
-		AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'`
-	}
-	ORDER BY ${table ? 'a.attnum' : 'ns.nspname, c.relname, a.attnum'};
-	
-	`
-	} else if (input.resourceType === 'ms_sql_server') {
-		return `
-		SELECT 
-    COLUMN_NAME as field,
-    DATA_TYPE as DataType,
-    COLUMN_DEFAULT as DefaultValue,
-    CASE WHEN COLUMNPROPERTY(OBJECT_ID(TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1 THEN 'By Default' ELSE 'No' END as IsIdentity,
-    CASE WHEN COLUMNPROPERTY(OBJECT_ID(TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1 THEN 1 ELSE 0 END as IsPrimaryKey, -- This line still needs correction for primary key identification
-    CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
-    CASE WHEN DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum${
-			table
-				? ''
-				: `,
-		TABLE_NAME as table_name`
-		}
-FROM    
-    INFORMATION_SCHEMA.COLUMNS${
-			table
-				? `
-WHERE   
-    TABLE_NAME = '${table}'`
-				: ''
-		}
-ORDER BY
-    ORDINAL_POSITION;
-	`
-	} else if (
-		input.resourceType === 'snowflake' ||
-		(input.resourceType as any) === 'snowflake_oauth'
-	) {
-		return `
-		select COLUMN_NAME as field,
-		DATA_TYPE as DataType,
-		COLUMN_DEFAULT as DefaultValue,
-		CASE WHEN COLUMN_DEFAULT like 'AUTOINCREMENT%' THEN 'By Default' ELSE 'No' END as IsIdentity,
-		CASE WHEN COLUMN_DEFAULT like 'AUTOINCREMENT%' THEN 1 ELSE 0 END as IsPrimaryKey,
-		CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
-		CASE WHEN DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum${
-			table
-				? ''
-				: `,
-		table_name as table_name,
-		table_schema as schema_name`
-		}
-	from information_schema.columns${
-		table
-			? `
-	where table_name = '${table.split('.').reverse()[0]}' and table_schema = '${
-		table.split('.').reverse()[1] ?? 'PUBLIC'
-	}'`
-			: "\nwhere table_schema <> 'INFORMATION_SCHEMA'\n"
-	}
-	order by ORDINAL_POSITION;
-	`
-	} else if (input.resourceType === 'bigquery') {
-		// TODO: find a solution for this (query uses hardcoded dataset name)
-		if (!table) throw new Error('Table name is required for BigQuery')
-		return `SELECT 
-    c.COLUMN_NAME as field,
-    DATA_TYPE as DataType,
-    CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue,
-    CASE WHEN constraint_name is not null THEN true ELSE false END as IsPrimaryKey,
-    'No' as IsIdentity,
-    IS_NULLABLE as IsNullable,
-    false as IsEnum
-FROM
-    ${table.split('.')[0]}.INFORMATION_SCHEMA.COLUMNS c
-    LEFT JOIN
-    ${table.split('.')[0]}.INFORMATION_SCHEMA.KEY_COLUMN_USAGE p
-    on c.table_name = p.table_name AND c.column_name = p.COLUMN_NAME
-WHERE   
-    c.TABLE_NAME = '${table.split('.')[1]}'
-order by c.ORDINAL_POSITION;`
-	} else {
-		throw new Error('Unsupported database type:' + input.resourceType)
-	}
-}
-
 export function resourceTypeToLang(rt: string) {
 	if (rt === 'ms_sql_server') {
 		return 'mssql'
@@ -326,14 +60,6 @@ export function resourceTypeToLang(rt: string) {
 	}
 }
 
-function lowercaseKeys(obj: Record<string, any>): any {
-	const newObj = {}
-	Object.keys(obj).forEach((key) => {
-		newObj[key.toLowerCase()] = obj[key]
-	})
-	return newObj
-}
-
 const scripts: Record<
 	string,
 	{
@@ -541,119 +267,6 @@ return schema
 }
 
 export { scripts }
-export async function getDbSchemas(
-	resourceType: string,
-	resourcePath: string,
-	workspace: string | undefined,
-	dbSchemas: DBSchemas,
-	errorCallback: (message: string) => void
-): Promise<void> {
-	if (!scripts[resourceType]) return
-
-	return new Promise(async (resolve, reject) => {
-		if (!resourceType || !resourcePath || !workspace) {
-			resolve()
-			return
-		}
-
-		const job = await JobService.runScriptPreview({
-			workspace: workspace,
-			requestBody: {
-				language: scripts[resourceType].lang as Preview['language'],
-				content: scripts[resourceType].code,
-				args: {
-					[scripts[resourceType].argName]: resourcePath.startsWith('datatable://')
-						? resourcePath
-						: '$res:' + resourcePath
-				}
-			}
-		})
-
-		tryEvery({
-			tryCode: async () => {
-				if (resourcePath) {
-					const testResult = await JobService.getCompletedJob({
-						workspace,
-						id: job
-					})
-					if (!testResult.success) {
-						console.error(testResult.result?.['error']?.['message'])
-					} else {
-						if (testResult.result === 'WINDMILL_TOO_BIG') {
-							console.info('Result is too big, fetching result separately')
-							const data = await JobService.getCompletedJobResult({
-								workspace,
-								id: job
-							})
-							testResult.result = data
-						}
-						if (resourceType !== undefined) {
-							if (resourceType !== 'graphql') {
-								const { processingFn } = scripts[resourceType]
-								let schema: any
-								try {
-									schema =
-										processingFn !== undefined ? processingFn(testResult.result) : testResult.result
-								} catch (e) {
-									console.error(e)
-									errorCallback('Error processing schema')
-									resolve()
-									return
-								}
-								const dbSchema = {
-									lang: resourceTypeToLang(resourceType) as SQLSchema['lang'],
-									schema,
-									publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
-								}
-								dbSchemas[resourcePath] = {
-									...dbSchema,
-									stringified: stringifySchema(dbSchema)
-								}
-							} else {
-								if (
-									typeof testResult.result !== 'object' ||
-									!('__schema' in (testResult?.result ?? {}))
-								) {
-									console.error('Invalid GraphQL schema')
-
-									errorCallback('Invalid GraphQL schema')
-								} else {
-									const dbSchema = {
-										lang: 'graphql' as GraphqlSchema['lang'],
-										schema: testResult.result
-									}
-									dbSchemas[resourcePath] = {
-										...(dbSchema as any),
-										stringified: stringifySchema(dbSchema as any)
-									}
-								}
-							}
-						}
-					}
-					resolve()
-				}
-			},
-			timeoutCode: async () => {
-				console.error('Could not query schema within 5s')
-				errorCallback('Could not query schema within 5s')
-				try {
-					await JobService.cancelQueuedJob({
-						workspace,
-						id: job,
-						requestBody: {
-							reason: 'Could not query schema within 5s'
-						}
-					})
-				} catch (err) {
-					console.error(err)
-				}
-				reject()
-			},
-			interval: 500,
-			timeout: 5000
-		})
-	})
-}
 
 export function formatSchema(dbSchema: {
 	lang: SQLSchema['lang']
@@ -671,17 +284,6 @@ export function formatGraphqlSchema(schema: IntrospectionQuery): string {
 	return printSchema(buildClientSchema(schema))
 }
 
-export type DbType = (typeof dbTypes)[number]
-export const dbTypes = [
-	'mysql',
-	'ms_sql_server',
-	'postgresql',
-	'snowflake',
-	'bigquery',
-	'duckdb'
-] as const
-export const isDbType = (str?: string): str is DbType => !!str && dbTypes.includes(str as DbType)
-
 export function buildVisibleFieldList(columnDefs: ColumnDef[], dbType: DbType) {
 	// Filter out hidden columns to avoid counting the wrong number of rows
 	return columnDefs
@@ -754,92 +356,6 @@ export function getPrimaryKeys(tableMetadata?: TableMetadata): string[] {
 	return r ?? []
 }
 
-export async function getTablesByResource(
-	schema: Partial<Record<string, DBSchema>>,
-	dbType: DbType | undefined,
-	dbPath: string,
-	workspace: string
-): Promise<string[]> {
-	const s = Object.values(schema)?.[0]
-	switch (dbType) {
-		case 'ms_sql_server': {
-			const paths: string[] = []
-			for (const key in s?.schema) {
-				for (const subKey in s.schema[key]) {
-					if (key === 'dbo') {
-						paths.push(`${subKey}`)
-					}
-				}
-			}
-			return paths
-		}
-		case 'mysql': {
-			const resourceObj = (await ResourceService.getResourceValue({
-				workspace,
-				path: dbPath.split('$res:')[1]
-			})) as any
-			const paths: string[] = []
-			for (const key in s?.schema) {
-				for (const subKey in s.schema[key]) {
-					if (key === resourceObj?.database) {
-						paths.push(`${subKey}`)
-					} else {
-						paths.push(`${key}.${subKey}`)
-					}
-				}
-			}
-			return paths
-		}
-		case 'snowflake': {
-			const paths: string[] = []
-			for (const key in s?.schema) {
-				for (const subKey in s.schema[key]) {
-					if (key === 'PUBLIC') {
-						paths.push(`${subKey}`)
-					} else {
-						paths.push(`${key}.${subKey}`)
-					}
-				}
-			}
-			return paths
-		}
-		case 'postgresql': {
-			const paths: string[] = []
-			for (const key in s?.schema) {
-				for (const subKey in s.schema[key]) {
-					if (key === 'public') {
-						paths.push(`${subKey}`)
-					} else {
-						paths.push(`${key}.${subKey}`)
-					}
-				}
-			}
-			return paths
-		}
-		case 'bigquery': {
-			const paths: string[] = []
-			for (const key in s?.schema) {
-				for (const subKey in s.schema[key]) {
-					paths.push(`${key}.${subKey}`)
-				}
-			}
-			return paths
-		}
-		case 'duckdb': {
-			const paths: string[] = []
-			for (const key in s?.schema) {
-				for (const subKey in s.schema[key]) {
-					paths.push(`${subKey}`)
-				}
-			}
-
-			return paths
-		}
-		default:
-			return []
-	}
-}
-
 export function dbSupportsSchemas(dbType: DbType): boolean {
 	return dbType === 'postgresql' || dbType === 'snowflake' || dbType === 'bigquery'
 }

frontend/src/lib/components/apps/components/display/table/SyncColumnDefs.svelte

@@ -1,13 +1,13 @@
 <script lang="ts">
 	import { getContext, tick } from 'svelte'
 	import type { AppViewerContext } from '../../../types'
-	import { findGridItem } from '$lib/components/apps/editor/appUtils'
 	import Button from '$lib/components/common/button/Button.svelte'
 	import { RefreshCw } from 'lucide-svelte'
 	import Badge from '$lib/components/common/badge/Badge.svelte'
 	import Alert from '$lib/components/common/alert/Alert.svelte'
 	import { isObject } from '$lib/utils'
 	import type { WindmillColumnDef } from './utils'
+	import { findGridItem } from '$lib/components/apps/editor/appUtilsCore'
 
 	type ColumnDefsConfiguration =
 		| { type: 'static'; value: WindmillColumnDef[] }

frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte

@@ -19,7 +19,8 @@
 	} from '../../types'
 	import { computeGlobalContext, eval_like } from './eval'
 	import InputValue from './InputValue.svelte'
-	import { collectOneOfFields, selectId } from '../../editor/appUtils'
+	import { selectId } from '../../editor/appUtils'
+	import { collectOneOfFields } from '../../editor/appUtilsCore'
 	import { userStore } from '$lib/stores'
 	import { get } from 'svelte/store'
 	import RefreshButton from '$lib/components/apps/components/helpers/RefreshButton.svelte'
@@ -195,7 +196,8 @@
 				id: string
 				result_stream?: string
 			}) {
-				setResult(nresult_stream, id)
+				setResult(nresult_stream, id, false)
+				dispatch('streamupdate', { id, result_stream: nresult_stream })
 			},
 			cancel({ id }: { id: string }) {
 				onCancel?.()
@@ -441,7 +443,8 @@
 						$appPath,
 						id,
 						await buildRequestBody(dynamicArgsOverride),
-						inlineScriptOverride
+						inlineScriptOverride,
+						extraQueryParams
 					)
 					if (isEditor) {
 						addJob(uuid)
@@ -472,35 +475,36 @@
 	}
 
 	export async function buildRequestBody(dynamicArgsOverride: Record<string, any> | undefined) {
-		const nonStaticRunnableInputs = dynamicArgsOverride ?? {}
-		const staticRunnableInputs = {}
+		const nonStaticRunnableInputs: Record<string, any> = dynamicArgsOverride ?? {}
+		const staticRunnableInputs: Record<string, any> = {}
 		const allowUserResources: string[] = []
+
 		for (const k of Object.keys(fields ?? {})) {
-			let field = fields[k]
-			if (field?.type == 'static' && fields[k]) {
+			const field = fields[k]
+
+			if (
+				isEditor &&
+				['user', 'evalv2', 'connected'].includes(field.type) &&
+				'allowUserResources' in field &&
+				field.allowUserResources
+			) {
+				allowUserResources.push(k)
+			}
+
+			if (field?.type == 'static') {
 				if (isEditor) {
 					staticRunnableInputs[k] = field.value
 				}
 			} else if (field?.type == 'user') {
 				nonStaticRunnableInputs[k] = args?.[k]
-				if (isEditor && field.allowUserResources) {
-					allowUserResources.push(k)
-				}
 			} else if (field?.type == 'eval' || (field?.type == 'evalv2' && inputValues[k])) {
 				const ctxMatch = field?.expr?.match(ctxRegex)
 				if (ctxMatch) {
 					nonStaticRunnableInputs[k] = '$ctx:' + ctxMatch[1]
 				} else {
-					// console.log('k', k)
 					nonStaticRunnableInputs[k] = await inputValues[k]?.computeExpr()
 				}
-				if (isEditor && field?.type == 'evalv2' && field.allowUserResources) {
-					allowUserResources.push(k)
-				}
 			} else {
-				if (isEditor && field?.type == 'connected' && field.allowUserResources) {
-					allowUserResources.push(k)
-				}
 				nonStaticRunnableInputs[k] = runnableInputValues[k]
 			}
 		}
@@ -629,7 +633,7 @@
 		result = res
 	}
 
-	async function setResult(res: any, jobId: string | undefined) {
+	async function setResult(res: any, jobId: string | undefined, dispatchSuccess: boolean = true) {
 		dispatch('resultSet', res)
 		const errors = getResultErrors(res)
 
@@ -668,7 +672,9 @@
 		recordJob(jobId, result, undefined, transformerResult)
 		delete $errorByComponent[id]
 
-		dispatch('success', result)
+		if (dispatchSuccess) {
+			dispatch('success', result)
+		}
 		// callbacks?.done(res)
 	}
 

frontend/src/lib/components/apps/components/helpers/RunnableWrapper.svelte

@@ -360,6 +360,7 @@
 		on:cancel
 		on:recompute
 		on:argsChanged
+		on:streamupdate
 		on:resultSet={(e) => {
 			const res = e.detail
 			if ($initialized?.runnableInitialized?.[fullId] === undefined) {

frontend/src/lib/components/apps/components/helpers/eval.ts

@@ -31,7 +31,7 @@ function create_context_function_template(
 ) {
 	let hasReturnAsLastLine = noReturn || eval_string.split('\n').some((x) => x.startsWith('return '))
 	return `
-return async function (context, state, createProxy, goto, setTab, recompute, globalRecompute, getAgGrid, setValue, setSelectedIndex, openModal, closeModal, open, close, validate, invalidate, validateAll, clearFiles, showToast, waitJob, askNewResource, downloadFile) {
+return async function (context, state, createProxy, goto, setTab, recompute, globalRecompute, getAgGrid, setValue, setSelectedIndex, openModal, closeModal, open, close, validate, invalidate, validateAll, clearFiles, sendMessage, showToast, waitJob, askNewResource, downloadFile) {
 "use strict";
 ${
 	contextKeys && contextKeys.length > 0
@@ -68,6 +68,7 @@ type WmFunctor = (
 	invalidate,
 	validateAll,
 	clearFiles,
+	sendMessage,
 	showToast,
 	waitJob,
 	askNewResource,
@@ -123,6 +124,7 @@ export async function eval_like(
 			waitJob?: (jobId: string) => void
 			askNewResource?: () => void
 			setGroupValue?: (key: string, value: any) => void
+			sendMessage?: (message: string) => void
 		}
 	>,
 	worldStore: World | undefined,
@@ -246,6 +248,9 @@ export async function eval_like(
 		(id) => {
 			controlComponents[id]?.clearFiles?.()
 		},
+		(id, message) => {
+			controlComponents[id]?.sendMessage?.(message)
+		},
 		(message, error) => {
 			sendUserToast(message, error)
 		},

frontend/src/lib/components/apps/components/helpers/executeRunnable.ts

@@ -11,7 +11,8 @@ export async function executeRunnable(
 	path: string,
 	id: string,
 	requestBody: ExecuteComponentData['requestBody'],
-	inlineScriptOverride?: InlineScript
+	inlineScriptOverride?: InlineScript,
+	queryParams?: Record<string, any>
 ) {
 	let appPath = defaultIfEmptyString(path, `u/${username ?? 'unknown'}/newapp`)
 	if (runnable?.type === 'runnableByName') {
@@ -40,6 +41,10 @@ export async function executeRunnable(
 		requestBody['version'] = version
 	}
 
+	if (queryParams && Object.keys(queryParams).length > 0) {
+		requestBody['run_query_params'] = queryParams
+	}
+
 	const uuid = await AppService.executeComponent({
 		workspace,
 		path: appPath,

frontend/src/lib/components/apps/editor/AppEditor.svelte

@@ -43,7 +43,7 @@
 	import { VariableService } from '$lib/gen'
 	import { initHistory } from '$lib/history.svelte'
 	import { Component, Minus, Paintbrush, Plus, Smartphone, Scan, Hand, Grab } from 'lucide-svelte'
-	import { animateTo, findGridItem, findGridItemParentGrid } from './appUtils'
+	import { animateTo, findGridItemParentGrid } from './appUtils'
 	import ComponentNavigation from './component/ComponentNavigation.svelte'
 	import CssSettings from './componentsPanel/CssSettings.svelte'
 	import SettingsPanel from './SettingsPanel.svelte'
@@ -54,13 +54,14 @@
 		secondaryMenuRightStore
 	} from './settingsPanel/secondaryMenu'
 	import Popover from '../../Popover.svelte'
-	import { BG_PREFIX, migrateApp } from '../utils'
+	import { migrateApp } from '../utils'
 	import DarkModeObserver from '$lib/components/DarkModeObserver.svelte'
 	import { getTheme } from './componentsPanel/themeUtils'
 	import StylePanel from './settingsPanel/StylePanel.svelte'
 	import HideButton from './settingsPanel/HideButton.svelte'
 	import AppEditorBottomPanel from './AppEditorBottomPanel.svelte'
 	import panzoom from 'panzoom'
+	import { BG_PREFIX, findGridItem } from './appUtilsCore'
 
 	let {
 		app,

frontend/src/lib/components/apps/editor/AppEditorHeader.svelte

@@ -34,12 +34,10 @@
 		type Value,
 		replaceFalseWithUndefined
 	} from '../../../utils'
-	import type { AppInput, Runnable } from '../inputType'
 	import type { App, AppEditorContext, AppViewerContext } from '../types'
-	import { BG_PREFIX, allItems, toStatic } from '../utils'
+	import { toStatic } from '../utils'
 	import AppExportButton from './AppExportButton.svelte'
 	import AppInputs from './AppInputs.svelte'
-	import type { AppComponent } from './component/components'
 	import PreviewToggle from './PreviewToggle.svelte'
 
 	import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
@@ -52,48 +50,19 @@
 	import Dropdown from '$lib/components/DropdownV2.svelte'
 	import AppEditorTutorial from './AppEditorTutorial.svelte'
 	import AppReportsDrawer from './AppReportsDrawer.svelte'
-	import { type ColumnDef, type DbType, getPrimaryKeys } from '../components/display/dbtable/utils'
 	import DebugPanel from './contextPanel/DebugPanel.svelte'
-	import { getCountInput } from '../components/display/dbtable/queries/count'
-	import { getSelectInput } from '../components/display/dbtable/queries/select'
-	import { getInsertInput } from '../components/display/dbtable/queries/insert'
-	import { getUpdateInput } from '../components/display/dbtable/queries/update'
-	import { getDeleteInput } from '../components/display/dbtable/queries/delete'
-	import { collectOneOfFields } from './appUtils'
+
 	import Summary from '$lib/components/Summary.svelte'
 	import HideButton from './settingsPanel/HideButton.svelte'
 	import DeployOverrideConfirmationModal from '$lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte'
-	import {
-		computeS3FileInputPolicy,
-		computeWorkspaceS3FileInputPolicy,
-		computeS3ImageViewerPolicy
-	} from './appUtilsS3'
+
 	import AppJobsDrawer from './AppJobsDrawer.svelte'
-	import { collectStaticFields, type TriggerableV2 } from './commonAppUtils'
 	import LazyModePanel from './contextPanel/LazyModePanel.svelte'
-	import { Sha256 } from '@aws-crypto/sha256-js'
 	import type { DiffDrawerI } from '$lib/components/diff_drawer'
 	import AppEditorHeaderDeploy from './AppEditorHeaderDeploy.svelte'
 	import AppEditorHeaderDeployInitialDraft from './AppEditorHeaderDeployInitialDraft.svelte'
 	import { computeSecretUrl } from './appDeploy.svelte'
-	import type { DbInput } from '$lib/components/dbOps'
-
-	async function hash(message) {
-		try {
-			const msgUint8 = new TextEncoder().encode(message) // encode as (utf-8) Uint8Array
-			const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8) // hash the message
-			const hashArray = Array.from(new Uint8Array(hashBuffer)) // convert buffer to byte array
-			const hashHex = hashArray.map((b) => b.toString(16).padStart(2, '0')).join('') // convert bytes to hex string
-			return hashHex
-		} catch {
-			//subtle not available, trying pure js
-			const hash = new Sha256()
-			hash.update(message ?? '')
-			const result = Array.from(await hash.digest())
-			const hex = result.map((b) => b.toString(16).padStart(2, '0')).join('') // convert bytes to hex string
-			return hex
-		}
-	}
+	import { updatePolicy } from './appPolicy'
 
 	interface Props {
 		policy: Policy
@@ -198,208 +167,8 @@
 		draftDrawerOpen = false
 	}
 
-	async function computeTriggerables() {
-		const items = allItems($app.grid, $app.subgrids)
-
-		const allTriggers: ([string, TriggerableV2] | undefined)[] = (await Promise.all(
-			items
-				.flatMap((x) => {
-					let c = x.data as AppComponent
-					let r: { input: AppInput | undefined; id: string }[] = [
-						{ input: c.componentInput, id: x.id }
-					]
-					if (c.type === 'tablecomponent') {
-						r.push(...c.actionButtons.map((x) => ({ input: x.componentInput, id: x.id })))
-					}
-					if (
-						(c.type === 'aggridcomponent' ||
-							c.type === 'aggridcomponentee' ||
-							c.type === 'dbexplorercomponent' ||
-							c.type === 'aggridinfinitecomponent' ||
-							c.type === 'aggridinfinitecomponentee') &&
-						Array.isArray(c.actions)
-					) {
-						r.push(...c.actions.map((x) => ({ input: x.componentInput, id: x.id })))
-					}
-					if (c.type === 'menucomponent') {
-						r.push(...c.menuItems.map((x) => ({ input: x.componentInput, id: x.id })))
-					}
-					if (c.type === 'dbexplorercomponent') {
-						let nr: { id: string; input: AppInput }[] = []
-						let config = c.configuration as any
-
-						const dbType = config?.type?.selected as DbType
-						let pg = config?.type?.configuration?.[dbType]
-
-						if (pg && dbType) {
-							const { table, resource, ducklake } = pg
-							const tableValue = table.value
-							const dbPath =
-								resource?.value.split('$res:')[1] ??
-								(ducklake?.value as string | undefined)?.split('ducklake://')[1]
-							const columnDefs = (c.configuration.columnDefs as any).value as ColumnDef[]
-							const whereClause = (c.configuration.whereClause as any).value as unknown as
-								| string
-								| undefined
-							if (tableValue && dbPath && columnDefs) {
-								let dbInput: DbInput = ducklake
-									? { type: 'ducklake', ducklake: dbPath }
-									: { type: 'database', resourcePath: dbPath, resourceType: dbType }
-								r.push({
-									input: getSelectInput(dbInput, tableValue, columnDefs, whereClause),
-									id: x.id
-								})
-
-								r.push({
-									input: getCountInput(dbInput, tableValue, columnDefs, whereClause),
-									id: x.id + '_count'
-								})
-
-								r.push({
-									input: getInsertInput(dbInput, tableValue, columnDefs),
-									id: x.id + '_insert'
-								})
-
-								let primaryColumns = getPrimaryKeys(columnDefs)
-								let columns = columnDefs?.filter((x) => primaryColumns.includes(x.field))
-
-								r.push({
-									input: getDeleteInput(dbInput, tableValue, columns),
-									id: x.id + '_delete'
-								})
-
-								columnDefs
-									.filter((col) => col.editable || config.allEditable.value)
-									.forEach((column) => {
-										r.push({
-											input: getUpdateInput(dbInput, tableValue, column, columns),
-											id: x.id + '_update'
-										})
-									})
-							}
-						}
-						r.push(...nr)
-					}
-
-					const processed = r
-						.filter((x) => x.input)
-						.map(async (o) => {
-							if (o.input?.type == 'runnable') {
-								return await processRunnable(o.id, o.input.runnable, o.input.fields)
-							}
-						})
-
-					return processed as Promise<[string, TriggerableV2] | undefined>[]
-				})
-				.concat(
-					Object.values($app.hiddenInlineScripts ?? {}).map(async (v, i) => {
-						return await processRunnable(BG_PREFIX + i, v, v.fields)
-					}) as Promise<[string, TriggerableV2] | undefined>[]
-				)
-		)) as ([string, TriggerableV2] | undefined)[]
-
-		delete policy.triggerables
-		const ntriggerables: Record<string, TriggerableV2> = Object.fromEntries(
-			allTriggers.filter(Boolean) as [string, TriggerableV2][]
-		)
-		policy.triggerables_v2 = ntriggerables
-
-		const s3_inputs = items
-			.filter((x) => (x.data as AppComponent).type === 's3fileinputcomponent')
-			.map((x) => {
-				const c = x.data as AppComponent
-				const config = c.configuration as any
-				return computeS3FileInputPolicy(config?.type?.configuration?.s3, $app)
-			})
-			.filter(Boolean) as {
-			allowed_resources: string[]
-			allow_user_resources: boolean
-			file_key_regex: string
-		}[]
-
-		if (
-			items.findIndex((x) => {
-				const c = x.data as AppComponent
-				if (
-					c.type === 'schemaformcomponent' ||
-					c.type === 'formbuttoncomponent' ||
-					c.type === 'formcomponent'
-				) {
-					const props =
-						c.type === 'schemaformcomponent'
-							? (c.componentInput as any)?.value?.properties
-							: (c.componentInput as any)?.runnable?.type === 'runnableByName'
-								? (c.componentInput as any)?.runnable?.inlineScript?.schema?.properties
-								: (c.componentInput as any)?.runnable?.schema?.properties
-					return (
-						Object.values(props ?? {}).findIndex(
-							(p: any) =>
-								(p?.type === 'object' && p?.format === 'resource-s3_object') ||
-								(p?.type === 'array' &&
-									(p?.items?.resourceType === 's3object' || p?.items?.resourceType === 's3_object'))
-						) !== -1
-					)
-				} else {
-					return false
-				}
-			}) !== -1
-		) {
-			s3_inputs.push(computeWorkspaceS3FileInputPolicy())
-		}
-
-		policy.s3_inputs = s3_inputs
-
-		const s3FileKeys = items
-			.filter((x) => (x.data as AppComponent).type === 'imagecomponent')
-			.map((x) => {
-				const c = x.data as AppComponent
-				const config = c.configuration
-				return computeS3ImageViewerPolicy(config)
-			})
-			.filter(Boolean) as { s3_path: string; storage?: string | undefined }[]
-
-		policy.allowed_s3_keys = s3FileKeys
-	}
-
-	async function processRunnable(
-		id: string,
-		runnable: Runnable,
-		fields: Record<string, any>
-	): Promise<[string, TriggerableV2] | undefined> {
-		const staticInputs = collectStaticFields(fields)
-		const oneOfInputs = collectOneOfFields(fields, $app)
-		const allowUserResources: string[] = Object.entries(fields)
-			.map(([k, v]) => {
-				return v['allowUserResources'] ? k : undefined
-			})
-			.filter(Boolean) as string[]
-
-		if (runnable?.type == 'runnableByName') {
-			let hex = await hash(runnable.inlineScript?.content)
-			console.log('hex', hex, id)
-			return [
-				`${id}:rawscript/${hex}`,
-				{
-					static_inputs: staticInputs,
-					one_of_inputs: oneOfInputs,
-					allow_user_resources: allowUserResources
-				}
-			]
-		} else if (runnable?.type == 'runnableByPath') {
-			let prefix = runnable.runType !== 'hubscript' ? runnable.runType : 'script'
-			return [
-				`${id}:${prefix}/${runnable.path}`,
-				{
-					static_inputs: staticInputs,
-					one_of_inputs: oneOfInputs,
-					allow_user_resources: allowUserResources
-				}
-			]
-		}
-	}
-
 	async function createApp(path: string) {
-		await computeTriggerables()
+		policy = await updatePolicy($app, policy)
 		try {
 			await AppService.createApp({
 				workspace: $workspaceStore!,
@@ -492,7 +261,7 @@
 	}
 
 	async function updateApp(npath: string) {
-		await computeTriggerables()
+		policy = await updatePolicy($app, policy)
 		await AppService.updateApp({
 			workspace: $workspaceStore!,
 			path: $appPath!,
@@ -534,7 +303,7 @@
 	}
 
 	async function setPublishState() {
-		await computeTriggerables()
+		policy = await updatePolicy($app, policy)
 		await AppService.updateApp({
 			workspace: $workspaceStore!,
 			path: $appPath,
@@ -556,7 +325,7 @@
 	}
 
 	async function saveInitialDraft() {
-		await computeTriggerables()
+		policy = await updatePolicy($app, policy)
 		try {
 			await AppService.createApp({
 				workspace: $workspaceStore!,
@@ -636,7 +405,7 @@
 		}
 		loading.saveDraft = true
 		try {
-			await computeTriggerables()
+			policy = await updatePolicy($app, policy)
 			let path = $appPath
 			if (savedApp.draft_only) {
 				await AppService.deleteApp({
@@ -907,6 +676,7 @@
 			document.documentElement.classList.remove('dark')
 		}
 		$darkMode = newDarkMode ?? globalDarkMode
+		$app.darkMode = newDarkMode
 	}
 
 	let priorDarkMode = document.documentElement.classList.contains('dark')

frontend/src/lib/components/apps/editor/AppInputs.svelte

@@ -3,9 +3,9 @@
 	import Toggle from '$lib/components/Toggle.svelte'
 	import { getContext } from 'svelte'
 	import type { AppViewerContext } from '../types'
-	import { BG_PREFIX, allItems } from '../utils'
 	import AppComponentInput from './AppComponentInput.svelte'
 	import InputsSpecsEditor from './settingsPanel/InputsSpecsEditor.svelte'
+	import { allItems, BG_PREFIX } from './appUtilsCore'
 
 	const { app } = getContext<AppViewerContext>('AppViewerContext')